core/stdarch/crates/core_arch/src/s390x/
vector.rs

1//! s390x vector intrinsics.
2//!
3//! For more info see the [Reference Summary] or the online [IBM docs].
4//!
5//! [Reference Summary]: https://www.ibm.com/support/pages/sites/default/files/2021-05/SA22-7871-10.pdf
6//! [IBM docs]: https://www.ibm.com/docs/en/zos/2.4.0?topic=support-vector-built-in-functions
7
8#![allow(non_camel_case_types)]
9
10use crate::{core_arch::simd::*, intrinsics::simd::*, mem::MaybeUninit, mem::transmute};
11
12#[cfg(test)]
13use stdarch_test::assert_instr;
14
15use super::macros::*;
16
17types! {
18    #![unstable(feature = "stdarch_s390x", issue = "135681")]
19
20    /// s390x-specific 128-bit wide vector of sixteen packed `i8`
21    pub struct vector_signed_char(16 x i8);
22    /// s390x-specific 128-bit wide vector of sixteen packed `u8`
23    pub struct vector_unsigned_char(16 x u8);
24    /// s390x-specific 128-bit wide vector mask of sixteen packed elements
25    pub struct vector_bool_char(16 x i8);
26
27    /// s390x-specific 128-bit wide vector of eight packed `i16`
28    pub struct vector_signed_short(8 x i16);
29    /// s390x-specific 128-bit wide vector of eight packed `u16`
30    pub struct vector_unsigned_short(8 x u16);
31    /// s390x-specific 128-bit wide vector mask of eight packed elements
32    pub struct vector_bool_short(8 x i16);
33
34    /// s390x-specific 128-bit wide vector of four packed `i32`
35    pub struct vector_signed_int(4 x i32);
36    /// s390x-specific 128-bit wide vector of four packed `u32`
37    pub struct vector_unsigned_int(4 x u32);
38    /// s390x-specific 128-bit wide vector mask of four packed elements
39    pub struct vector_bool_int(4 x i32);
40
41    /// s390x-specific 128-bit wide vector of two packed `i64`
42    pub struct vector_signed_long_long(2 x i64);
43    /// s390x-specific 128-bit wide vector of two packed `u64`
44    pub struct vector_unsigned_long_long(2 x u64);
45    /// s390x-specific 128-bit wide vector mask of two packed elements
46    pub struct vector_bool_long_long(2 x i64);
47
48    /// s390x-specific 128-bit wide vector of four packed `f32`
49    pub struct vector_float(4 x f32);
50    /// s390x-specific 128-bit wide vector of two packed `f64`
51    pub struct vector_double(2 x f64);
52}
53
54#[repr(C, packed)]
55struct PackedTuple<T, U> {
56    x: T,
57    y: U,
58}
59
60#[allow(improper_ctypes)]
61#[rustfmt::skip]
62unsafe extern "unadjusted" {
63    #[link_name = "llvm.smax.v16i8"] fn vmxb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
64    #[link_name = "llvm.smax.v8i16"] fn vmxh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
65    #[link_name = "llvm.smax.v4i32"] fn vmxf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
66    #[link_name = "llvm.smax.v2i64"] fn vmxg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
67
68    #[link_name = "llvm.umax.v16i8"] fn vmxlb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
69    #[link_name = "llvm.umax.v8i16"] fn vmxlh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
70    #[link_name = "llvm.umax.v4i32"] fn vmxlf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
71    #[link_name = "llvm.umax.v2i64"] fn vmxlg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
72
73    #[link_name = "llvm.smin.v16i8"] fn vmnb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
74    #[link_name = "llvm.smin.v8i16"] fn vmnh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
75    #[link_name = "llvm.smin.v4i32"] fn vmnf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
76    #[link_name = "llvm.smin.v2i64"] fn vmng(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
77
78    #[link_name = "llvm.umin.v16i8"] fn vmnlb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
79    #[link_name = "llvm.umin.v8i16"] fn vmnlh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
80    #[link_name = "llvm.umin.v4i32"] fn vmnlf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
81    #[link_name = "llvm.umin.v2i64"] fn vmnlg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
82
83    #[link_name = "llvm.nearbyint.v4f32"] fn nearbyint_v4f32(a: vector_float) -> vector_float;
84    #[link_name = "llvm.nearbyint.v2f64"] fn nearbyint_v2f64(a: vector_double) -> vector_double;
85
86    #[link_name = "llvm.roundeven.v4f32"] fn roundeven_v4f32(a: vector_float) -> vector_float;
87    #[link_name = "llvm.roundeven.v2f64"] fn roundeven_v2f64(a: vector_double) -> vector_double;
88
89    #[link_name = "llvm.s390.vsra"] fn vsra(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
90    #[link_name = "llvm.s390.vsrl"] fn vsrl(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
91    #[link_name = "llvm.s390.vsl"] fn vsl(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
92
93    #[link_name = "llvm.s390.vsrab"] fn vsrab(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
94    #[link_name = "llvm.s390.vsrlb"] fn vsrlb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
95    #[link_name = "llvm.s390.vslb"] fn vslb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
96
97    #[link_name = "llvm.s390.vsldb"] fn vsldb(a: i8x16, b: i8x16, c: u32) -> i8x16;
98    #[link_name = "llvm.s390.vsld"] fn vsld(a: i8x16, b: i8x16, c: u32) -> i8x16;
99    #[link_name = "llvm.s390.vsrd"] fn vsrd(a: i8x16, b: i8x16, c: u32) -> i8x16;
100
101    #[link_name = "llvm.s390.verimb"] fn verimb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char, d: i32) -> vector_signed_char;
102    #[link_name = "llvm.s390.verimh"] fn verimh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short, d: i32) -> vector_signed_short;
103    #[link_name = "llvm.s390.verimf"] fn verimf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int, d: i32) -> vector_signed_int;
104    #[link_name = "llvm.s390.verimg"] fn verimg(a: vector_signed_long_long, b: vector_signed_long_long, c: vector_signed_long_long, d: i32) -> vector_signed_long_long;
105
106    #[link_name = "llvm.s390.vperm"] fn vperm(a: vector_signed_char, b: vector_signed_char, c: vector_unsigned_char) -> vector_signed_char;
107
108    #[link_name = "llvm.s390.vsumb"] fn vsumb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_int;
109    #[link_name = "llvm.s390.vsumh"] fn vsumh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
110
111    #[link_name = "llvm.s390.vsumgh"] fn vsumgh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_long_long;
112    #[link_name = "llvm.s390.vsumgf"] fn vsumgf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
113
114    #[link_name = "llvm.s390.vsumqf"] fn vsumqf(a: vector_unsigned_int, b: vector_unsigned_int) -> u128;
115    #[link_name = "llvm.s390.vsumqg"] fn vsumqg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128;
116
117    #[link_name = "llvm.s390.vscbiq"] fn vscbiq(a: u128, b: u128) -> u128;
118    #[link_name = "llvm.s390.vsbiq"] fn vsbiq(a: u128, b: u128, c: u128) -> u128;
119    #[link_name = "llvm.s390.vsbcbiq"] fn vsbcbiq(a: u128, b: u128, c: u128) -> u128;
120
121    #[link_name = "llvm.s390.vacq"] fn vacq(a: u128, b: u128, c: u128) -> u128;
122
123    #[link_name = "llvm.s390.vscbib"] fn vscbib(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
124    #[link_name = "llvm.s390.vscbih"] fn vscbih(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
125    #[link_name = "llvm.s390.vscbif"] fn vscbif(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
126    #[link_name = "llvm.s390.vscbig"] fn vscbig(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
127
128    #[link_name = "llvm.s390.vfaeb"] fn vfaeb(a: vector_signed_char, b: vector_signed_char, c: i32) -> vector_signed_char;
129    #[link_name = "llvm.s390.vfaeh"] fn vfaeh(a: vector_signed_short, b: vector_signed_short, c: i32) -> vector_signed_short;
130    #[link_name = "llvm.s390.vfaef"] fn vfaef(a: vector_signed_int, b: vector_signed_int, c: i32) -> vector_signed_int;
131
132    #[link_name = "llvm.s390.vfaezb"] fn vfaezb(a: vector_signed_char, b: vector_signed_char, c: i32) -> vector_signed_char;
133    #[link_name = "llvm.s390.vfaezh"] fn vfaezh(a: vector_signed_short, b: vector_signed_short, c: i32) -> vector_signed_short;
134    #[link_name = "llvm.s390.vfaezf"] fn vfaezf(a: vector_signed_int, b: vector_signed_int, c: i32) -> vector_signed_int;
135
136    #[link_name = "llvm.s390.vfaebs"] fn vfaebs(a: vector_signed_char, b: vector_signed_char, c: i32) -> PackedTuple<vector_signed_char, i32>;
137    #[link_name = "llvm.s390.vfaehs"] fn vfaehs(a: vector_signed_short, b: vector_signed_short, c: i32) -> PackedTuple<vector_signed_short, i32>;
138    #[link_name = "llvm.s390.vfaefs"] fn vfaefs(a: vector_signed_int, b: vector_signed_int, c: i32) -> PackedTuple<vector_signed_int, i32>;
139
140    #[link_name = "llvm.s390.vfaezbs"] fn vfaezbs(a: vector_signed_char, b: vector_signed_char, c: i32) -> PackedTuple<vector_signed_char, i32>;
141    #[link_name = "llvm.s390.vfaezhs"] fn vfaezhs(a: vector_signed_short, b: vector_signed_short, c: i32) -> PackedTuple<vector_signed_short, i32>;
142    #[link_name = "llvm.s390.vfaezfs"] fn vfaezfs(a: vector_signed_int, b: vector_signed_int, c: i32) -> PackedTuple<vector_signed_int, i32>;
143
144    #[link_name = "llvm.s390.vll"] fn vll(a: u32, b: *const u8) -> vector_signed_char;
145    #[link_name = "llvm.s390.vstl"] fn vstl(a: vector_signed_char, b: u32, c: *mut u8);
146
147    #[link_name = "llvm.s390.vlrl"] fn vlrl(a: u32, b: *const u8) -> vector_unsigned_char;
148    #[link_name = "llvm.s390.vstrl"] fn vstrl(a: vector_unsigned_char, b: u32, c: *mut u8);
149
150    #[link_name = "llvm.s390.lcbb"] fn lcbb(a: *const u8, b: u32) -> u32;
151    #[link_name = "llvm.s390.vlbb"] fn vlbb(a: *const u8, b: u32) -> MaybeUninit<vector_signed_char>;
152
153    #[link_name = "llvm.s390.vpksh"] fn vpksh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_char;
154    #[link_name = "llvm.s390.vpksf"] fn vpksf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_short;
155    #[link_name = "llvm.s390.vpksg"] fn vpksg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_int;
156
157    #[link_name = "llvm.s390.vpklsh"] fn vpklsh(a: vector_signed_short, b: vector_signed_short) -> vector_unsigned_char;
158    #[link_name = "llvm.s390.vpklsf"] fn vpklsf(a: vector_signed_int, b: vector_signed_int) -> vector_unsigned_short;
159    #[link_name = "llvm.s390.vpklsg"] fn vpklsg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_unsigned_int;
160
161    #[link_name = "llvm.s390.vpkshs"] fn vpkshs(a: vector_signed_short, b: vector_signed_short) -> PackedTuple<vector_signed_char, i32>;
162    #[link_name = "llvm.s390.vpksfs"] fn vpksfs(a: vector_signed_int, b: vector_signed_int) -> PackedTuple<vector_signed_short, i32>;
163    #[link_name = "llvm.s390.vpksgs"] fn vpksgs(a: vector_signed_long_long, b: vector_signed_long_long) -> PackedTuple<vector_signed_int, i32>;
164
165    #[link_name = "llvm.s390.vpklshs"] fn vpklshs(a: vector_unsigned_short, b: vector_unsigned_short) -> PackedTuple<vector_unsigned_char, i32>;
166    #[link_name = "llvm.s390.vpklsfs"] fn vpklsfs(a: vector_unsigned_int, b: vector_unsigned_int) -> PackedTuple<vector_unsigned_short, i32>;
167    #[link_name = "llvm.s390.vpklsgs"] fn vpklsgs(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> PackedTuple<vector_unsigned_int, i32>;
168
169    #[link_name = "llvm.s390.vuplb"] fn vuplb (a: vector_signed_char) -> vector_signed_short;
170    #[link_name = "llvm.s390.vuplhw"] fn vuplhw (a: vector_signed_short) -> vector_signed_int;
171    #[link_name = "llvm.s390.vuplf"] fn vuplf (a: vector_signed_int) -> vector_signed_long_long;
172    #[link_name = "llvm.s390.vupllb"] fn vupllb (a: vector_unsigned_char) -> vector_unsigned_short;
173    #[link_name = "llvm.s390.vupllh"] fn vupllh (a: vector_unsigned_short) -> vector_unsigned_int;
174    #[link_name = "llvm.s390.vupllf"] fn vupllf (a: vector_unsigned_int) -> vector_unsigned_long_long;
175
176    #[link_name = "llvm.s390.vavgb"] fn vavgb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
177    #[link_name = "llvm.s390.vavgh"] fn vavgh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
178    #[link_name = "llvm.s390.vavgf"] fn vavgf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
179    #[link_name = "llvm.s390.vavgg"] fn vavgg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
180
181    #[link_name = "llvm.s390.vavglb"] fn vavglb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
182    #[link_name = "llvm.s390.vavglh"] fn vavglh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
183    #[link_name = "llvm.s390.vavglf"] fn vavglf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
184    #[link_name = "llvm.s390.vavglg"] fn vavglg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
185
186    #[link_name = "llvm.s390.vcksm"] fn vcksm(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
187
188    #[link_name = "llvm.s390.vmeb"] fn vmeb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short;
189    #[link_name = "llvm.s390.vmeh"] fn vmeh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int;
190    #[link_name = "llvm.s390.vmef"] fn vmef(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long;
191
192    #[link_name = "llvm.s390.vmleb"] fn vmleb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
193    #[link_name = "llvm.s390.vmleh"] fn vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
194    #[link_name = "llvm.s390.vmlef"] fn vmlef(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
195
196    #[link_name = "llvm.s390.vmob"] fn vmob(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short;
197    #[link_name = "llvm.s390.vmoh"] fn vmoh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int;
198    #[link_name = "llvm.s390.vmof"] fn vmof(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long;
199
200    #[link_name = "llvm.s390.vmlob"] fn vmlob(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
201    #[link_name = "llvm.s390.vmloh"] fn vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
202    #[link_name = "llvm.s390.vmlof"] fn vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
203
204    #[link_name = "llvm.s390.vmhb"] fn vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
205    #[link_name = "llvm.s390.vmhh"] fn vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
206    #[link_name = "llvm.s390.vmhf"] fn vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
207
208    #[link_name = "llvm.s390.vmlhb"] fn vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
209    #[link_name = "llvm.s390.vmlhh"] fn vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
210    #[link_name = "llvm.s390.vmlhf"] fn vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
211
212    #[link_name = "llvm.s390.vmaeb"] fn vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short;
213    #[link_name = "llvm.s390.vmaeh"] fn vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int;
214    #[link_name = "llvm.s390.vmaef"] fn vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long;
215
216    #[link_name = "llvm.s390.vmaleb"] fn vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
217    #[link_name = "llvm.s390.vmaleh"] fn vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
218    #[link_name = "llvm.s390.vmalef"] fn vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
219
220    #[link_name = "llvm.s390.vmaob"] fn vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short;
221    #[link_name = "llvm.s390.vmaoh"] fn vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int;
222    #[link_name = "llvm.s390.vmaof"] fn vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long;
223
224    #[link_name = "llvm.s390.vmalob"] fn vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
225    #[link_name = "llvm.s390.vmaloh"] fn vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
226    #[link_name = "llvm.s390.vmalof"] fn vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
227
228    #[link_name = "llvm.s390.vmahb"] fn vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char;
229    #[link_name = "llvm.s390.vmahh"] fn vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short;
230    #[link_name = "llvm.s390.vmahf"] fn vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int;
231
232    #[link_name = "llvm.s390.vmalhb"] fn vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char;
233    #[link_name = "llvm.s390.vmalhh"] fn vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short;
234    #[link_name = "llvm.s390.vmalhf"] fn vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int;
235
236    #[link_name = "llvm.s390.vmalb"] fn vmalb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char;
237    #[link_name = "llvm.s390.vmalh"] fn vmalh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short;
238    #[link_name = "llvm.s390.vmalf"] fn vmalf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int;
239
240    #[link_name = "llvm.s390.vmallb"] fn vmallb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char;
241    #[link_name = "llvm.s390.vmallh"] fn vmallh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short;
242    #[link_name = "llvm.s390.vmallf"] fn vmallf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int;
243
244    #[link_name = "llvm.s390.vgfmb"] fn vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
245    #[link_name = "llvm.s390.vgfmh"] fn vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
246    #[link_name = "llvm.s390.vgfmf"] fn vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
247    #[link_name = "llvm.s390.vgfmg"] fn vgfmg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128;
248
249    #[link_name = "llvm.s390.vgfmab"] fn vgfmab(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
250    #[link_name = "llvm.s390.vgfmah"] fn vgfmah(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
251    #[link_name = "llvm.s390.vgfmaf"] fn vgfmaf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
252    #[link_name = "llvm.s390.vgfmag"] fn vgfmag(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128) -> u128;
253
254    #[link_name = "llvm.s390.vbperm"] fn vbperm(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_long_long;
255
256    #[link_name = "llvm.s390.vftcisb"] fn vftcisb(a: vector_float, b: u32) -> PackedTuple<vector_bool_int, i32>;
257    #[link_name = "llvm.s390.vftcidb"] fn vftcidb(a: vector_double, b: u32) -> PackedTuple<vector_bool_long_long, i32>;
258
259    #[link_name = "llvm.s390.vtm"] fn vtm(a: i8x16, b: i8x16) -> i32;
260
261    #[link_name = "llvm.s390.vstrsb"] fn vstrsb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
262    #[link_name = "llvm.s390.vstrsh"] fn vstrsh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
263    #[link_name = "llvm.s390.vstrsf"] fn vstrsf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
264
265    #[link_name = "llvm.s390.vstrszb"] fn vstrszb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
266    #[link_name = "llvm.s390.vstrszh"] fn vstrszh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
267    #[link_name = "llvm.s390.vstrszf"] fn vstrszf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
268
269    #[link_name = "llvm.s390.vistrb"] fn vistrb(a: vector_unsigned_char) -> vector_unsigned_char;
270    #[link_name = "llvm.s390.vistrh"] fn vistrh(a: vector_unsigned_short) -> vector_unsigned_short;
271    #[link_name = "llvm.s390.vistrf"] fn vistrf(a: vector_unsigned_int) -> vector_unsigned_int;
272
273    #[link_name = "llvm.s390.vistrbs"] fn vistrbs(a: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
274    #[link_name = "llvm.s390.vistrhs"] fn vistrhs(a: vector_unsigned_short) -> PackedTuple<vector_unsigned_short, i32>;
275    #[link_name = "llvm.s390.vistrfs"] fn vistrfs(a: vector_unsigned_int) -> PackedTuple<vector_unsigned_int, i32>;
276
277    #[link_name = "llvm.s390.vmslg"] fn vmslg(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128, d: u32) -> u128;
278
279    #[link_name = "llvm.s390.vstrcb"] fn vstrcb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char;
280    #[link_name = "llvm.s390.vstrch"] fn vstrch(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short;
281    #[link_name = "llvm.s390.vstrcf"] fn vstrcf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int;
282
283    #[link_name = "llvm.s390.vstrcbs"] fn vstrcbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple<vector_bool_char, i32>;
284    #[link_name = "llvm.s390.vstrchs"] fn vstrchs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple<vector_bool_short, i32>;
285    #[link_name = "llvm.s390.vstrcfs"] fn vstrcfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple<vector_bool_int, i32>;
286
287    #[link_name = "llvm.s390.vstrczb"] fn vstrczb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char;
288    #[link_name = "llvm.s390.vstrczh"] fn vstrczh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short;
289    #[link_name = "llvm.s390.vstrczf"] fn vstrczf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int;
290
291    #[link_name = "llvm.s390.vstrczbs"] fn vstrczbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple<vector_bool_char, i32>;
292    #[link_name = "llvm.s390.vstrczhs"] fn vstrczhs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple<vector_bool_short, i32>;
293    #[link_name = "llvm.s390.vstrczfs"] fn vstrczfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple<vector_bool_int, i32>;
294
295    #[link_name = "llvm.s390.vfeeb"] fn vfeeb(a: i8x16, b: i8x16) -> i8x16;
296    #[link_name = "llvm.s390.vfeeh"] fn vfeeh(a: i16x8, b: i16x8) -> i16x8;
297    #[link_name = "llvm.s390.vfeef"] fn vfeef(a: i32x4, b: i32x4) -> i32x4;
298
299    #[link_name = "llvm.s390.vfeezb"] fn vfeezb(a: i8x16, b: i8x16) -> i8x16;
300    #[link_name = "llvm.s390.vfeezh"] fn vfeezh(a: i16x8, b: i16x8) -> i16x8;
301    #[link_name = "llvm.s390.vfeezf"] fn vfeezf(a: i32x4, b: i32x4) -> i32x4;
302
303    #[link_name = "llvm.s390.vfeebs"] fn vfeebs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
304    #[link_name = "llvm.s390.vfeehs"] fn vfeehs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
305    #[link_name = "llvm.s390.vfeefs"] fn vfeefs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
306
307    #[link_name = "llvm.s390.vfeezbs"] fn vfeezbs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
308    #[link_name = "llvm.s390.vfeezhs"] fn vfeezhs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
309    #[link_name = "llvm.s390.vfeezfs"] fn vfeezfs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
310
311    #[link_name = "llvm.s390.vfeneb"] fn vfeneb(a: i8x16, b: i8x16) -> i8x16;
312    #[link_name = "llvm.s390.vfeneh"] fn vfeneh(a: i16x8, b: i16x8) -> i16x8;
313    #[link_name = "llvm.s390.vfenef"] fn vfenef(a: i32x4, b: i32x4) -> i32x4;
314
315    #[link_name = "llvm.s390.vfenezb"] fn vfenezb(a: i8x16, b: i8x16) -> i8x16;
316    #[link_name = "llvm.s390.vfenezh"] fn vfenezh(a: i16x8, b: i16x8) -> i16x8;
317    #[link_name = "llvm.s390.vfenezf"] fn vfenezf(a: i32x4, b: i32x4) -> i32x4;
318
319    #[link_name = "llvm.s390.vfenebs"] fn vfenebs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
320    #[link_name = "llvm.s390.vfenehs"] fn vfenehs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
321    #[link_name = "llvm.s390.vfenefs"] fn vfenefs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
322
323    #[link_name = "llvm.s390.vfenezbs"] fn vfenezbs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
324    #[link_name = "llvm.s390.vfenezhs"] fn vfenezhs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
325    #[link_name = "llvm.s390.vfenezfs"] fn vfenezfs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
326}
327
328impl_from! { i8x16, u8x16,  i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 }
329
330impl_neg! { i8x16 : 0 }
331impl_neg! { i16x8 : 0 }
332impl_neg! { i32x4 : 0 }
333impl_neg! { i64x2 : 0 }
334impl_neg! { f32x4 : 0f32 }
335impl_neg! { f64x2 : 0f64 }
336
337#[repr(simd)]
338struct ShuffleMask<const N: usize>([u32; N]);
339
340impl<const N: usize> ShuffleMask<N> {
341    const fn reverse() -> Self {
342        let mut index = [0; N];
343        let mut i = 0;
344        while i < N {
345            index[i] = (N - i - 1) as u32;
346            i += 1;
347        }
348        ShuffleMask(index)
349    }
350
351    const fn merge_low() -> Self {
352        let mut mask = [0; N];
353        let mut i = N / 2;
354        let mut index = 0;
355        while index < N {
356            mask[index] = i as u32;
357            mask[index + 1] = (i + N) as u32;
358
359            i += 1;
360            index += 2;
361        }
362        ShuffleMask(mask)
363    }
364
365    const fn merge_high() -> Self {
366        let mut mask = [0; N];
367        let mut i = 0;
368        let mut index = 0;
369        while index < N {
370            mask[index] = i as u32;
371            mask[index + 1] = (i + N) as u32;
372
373            i += 1;
374            index += 2;
375        }
376        ShuffleMask(mask)
377    }
378
379    const fn pack() -> Self {
380        let mut mask = [0; N];
381        let mut i = 1;
382        let mut index = 0;
383        while index < N {
384            mask[index] = i as u32;
385
386            i += 2;
387            index += 1;
388        }
389        ShuffleMask(mask)
390    }
391
392    const fn unpack_low() -> Self {
393        let mut mask = [0; N];
394        let mut i = 0;
395        while i < N {
396            mask[i] = (N + i) as u32;
397            i += 1;
398        }
399        ShuffleMask(mask)
400    }
401
402    const fn unpack_high() -> Self {
403        let mut mask = [0; N];
404        let mut i = 0;
405        while i < N {
406            mask[i] = i as u32;
407            i += 1;
408        }
409        ShuffleMask(mask)
410    }
411}
412
413const fn genmask<const MASK: u16>() -> [u8; 16] {
414    let mut bits = MASK;
415    let mut elements = [0u8; 16];
416
417    let mut i = 0;
418    while i < 16 {
419        elements[i] = match bits & (1u16 << 15) {
420            0 => 0,
421            _ => 0xFF,
422        };
423
424        bits <<= 1;
425        i += 1;
426    }
427
428    elements
429}
430
431const fn genmasks(bit_width: u32, a: u8, b: u8) -> u64 {
432    let bit_width = bit_width as u8;
433    let a = a % bit_width;
434    let mut b = b % bit_width;
435    if a > b {
436        b = bit_width - 1;
437    }
438
439    // of course these indices start from the left
440    let a = (bit_width - 1) - a;
441    let b = (bit_width - 1) - b;
442
443    ((1u64.wrapping_shl(a as u32 + 1)) - 1) & !((1u64.wrapping_shl(b as u32)) - 1)
444}
445
446const fn validate_block_boundary(block_boundary: u16) -> u32 {
447    assert!(
448        block_boundary.is_power_of_two() && block_boundary >= 64 && block_boundary <= 4096,
449        "block boundary must be a constant power of 2 from 64 to 4096",
450    );
451
452    // so that 64 is encoded as 0, 128 as 1, ect.
453    block_boundary as u32 >> 7
454}
455
456enum FindImm {
457    Eq = 4,
458    Ne = 12,
459    EqIdx = 0,
460    NeIdx = 8,
461}
462
463#[macro_use]
464mod sealed {
465    use super::*;
466
467    #[unstable(feature = "stdarch_s390x", issue = "135681")]
468    pub trait VectorAdd<Other> {
469        type Result;
470        unsafe fn vec_add(self, other: Other) -> Self::Result;
471    }
472
473    macro_rules! impl_add {
474        ($name:ident, $a:ty, $instr:ident) => {
475            impl_add!($name, $a, $a, $a, $instr);
476        };
477        ($name:ident, $a:ty, $b:ty, $c:ty, $instr:ident) => {
478            #[inline]
479            #[target_feature(enable = "vector")]
480            #[cfg_attr(test, assert_instr($instr))]
481            pub unsafe fn $name(a: $a, b: $b) -> $c {
482                transmute(simd_add(transmute(a), b))
483            }
484
485            #[unstable(feature = "stdarch_s390x", issue = "135681")]
486            impl VectorAdd<$b> for $a {
487                type Result = $c;
488
489                #[inline]
490                #[target_feature(enable = "vector")]
491                unsafe fn vec_add(self, other: $b) -> Self::Result {
492                    $name(self, other)
493                }
494            }
495        };
496    }
497
498    #[rustfmt::skip]
499    mod impl_add {
500        use super::*;
501
502        impl_add!(va_sc, vector_signed_char, vab);
503        impl_add!(va_uc, vector_unsigned_char, vab);
504        impl_add!(va_sh, vector_signed_short, vah);
505        impl_add!(va_uh, vector_unsigned_short, vah);
506        impl_add!(va_sf, vector_signed_int, vaf);
507        impl_add!(va_uf, vector_unsigned_int, vaf);
508        impl_add!(va_sg, vector_signed_long_long, vag);
509        impl_add!(va_ug, vector_unsigned_long_long, vag);
510
511        impl_add!(va_sc_bc, vector_signed_char, vector_bool_char, vector_signed_char, vab);
512        impl_add!(va_uc_bc, vector_unsigned_char, vector_bool_char, vector_unsigned_char, vab);
513        impl_add!(va_sh_bh, vector_signed_short, vector_bool_short, vector_signed_short, vah);
514        impl_add!(va_uh_bh, vector_unsigned_short, vector_bool_short, vector_unsigned_short, vah);
515        impl_add!(va_sf_bf, vector_signed_int, vector_bool_int, vector_signed_int, vaf);
516        impl_add!(va_uf_bf, vector_unsigned_int, vector_bool_int, vector_unsigned_int, vaf);
517        impl_add!(va_sg_bg, vector_signed_long_long, vector_bool_long_long, vector_signed_long_long, vag);
518        impl_add!(va_ug_bg, vector_unsigned_long_long, vector_bool_long_long, vector_unsigned_long_long, vag);
519
520        impl_add!(va_bc_sc, vector_bool_char, vector_signed_char, vector_signed_char, vab);
521        impl_add!(va_bc_uc, vector_bool_char, vector_unsigned_char, vector_unsigned_char, vab);
522        impl_add!(va_bh_sh, vector_bool_short, vector_signed_short, vector_signed_short, vah);
523        impl_add!(va_bh_uh, vector_bool_short, vector_unsigned_short, vector_unsigned_short, vah);
524        impl_add!(va_bf_sf, vector_bool_int, vector_signed_int, vector_signed_int, vaf);
525        impl_add!(va_bf_uf, vector_bool_int, vector_unsigned_int, vector_unsigned_int, vaf);
526        impl_add!(va_bg_sg, vector_bool_long_long, vector_signed_long_long, vector_signed_long_long, vag);
527        impl_add!(va_bg_ug, vector_bool_long_long, vector_unsigned_long_long, vector_unsigned_long_long, vag);
528
529        impl_add!(va_double, vector_double, vfadb);
530
531        #[inline]
532        #[target_feature(enable = "vector")]
533        #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vfasb))]
534        pub unsafe fn va_float(a: vector_float, b: vector_float) -> vector_float {
535            transmute(simd_add(a, b))
536        }
537
538        #[unstable(feature = "stdarch_s390x", issue = "135681")]
539        impl VectorAdd<Self> for vector_float {
540            type Result = Self;
541
542            #[inline]
543            #[target_feature(enable = "vector")]
544            unsafe fn vec_add(self, other: Self) -> Self::Result {
545                va_float(self, other)
546            }
547        }
548    }
549
550    #[unstable(feature = "stdarch_s390x", issue = "135681")]
551    pub trait VectorSub<Other> {
552        type Result;
553        unsafe fn vec_sub(self, other: Other) -> Self::Result;
554    }
555
556    macro_rules! impl_sub {
557        ($name:ident, $a:ty, $instr:ident) => {
558            impl_sub!($name, $a, $a, $a, $instr);
559        };
560        ($name:ident, $a:ty, $b:ty, $c:ty, $instr:ident) => {
561            #[inline]
562            #[target_feature(enable = "vector")]
563            #[cfg_attr(test, assert_instr($instr))]
564            pub unsafe fn $name(a: $a, b: $b) -> $c {
565                transmute(simd_sub(transmute(a), b))
566            }
567
568            #[unstable(feature = "stdarch_s390x", issue = "135681")]
569            impl VectorSub<$b> for $a {
570                type Result = $c;
571
572                #[inline]
573                #[target_feature(enable = "vector")]
574                unsafe fn vec_sub(self, other: $b) -> Self::Result {
575                    $name(self, other)
576                }
577            }
578        };
579    }
580
581    #[rustfmt::skip]
582    mod impl_sub {
583        use super::*;
584
585        impl_sub!(vs_sc, vector_signed_char, vsb);
586        impl_sub!(vs_uc, vector_unsigned_char, vsb);
587        impl_sub!(vs_sh, vector_signed_short, vsh);
588        impl_sub!(vs_uh, vector_unsigned_short, vsh);
589        impl_sub!(vs_sf, vector_signed_int, vsf);
590        impl_sub!(vs_uf, vector_unsigned_int, vsf);
591        impl_sub!(vs_sg, vector_signed_long_long, vsg);
592        impl_sub!(vs_ug, vector_unsigned_long_long, vsg);
593
594        impl_sub!(vs_sc_bc, vector_signed_char, vector_bool_char, vector_signed_char, vsb);
595        impl_sub!(vs_uc_bc, vector_unsigned_char, vector_bool_char, vector_unsigned_char, vsb);
596        impl_sub!(vs_sh_bh, vector_signed_short, vector_bool_short, vector_signed_short, vsh);
597        impl_sub!(vs_uh_bh, vector_unsigned_short, vector_bool_short, vector_unsigned_short, vsh);
598        impl_sub!(vs_sf_bf, vector_signed_int, vector_bool_int, vector_signed_int, vsf);
599        impl_sub!(vs_uf_bf, vector_unsigned_int, vector_bool_int, vector_unsigned_int, vsf);
600        impl_sub!(vs_sg_bg, vector_signed_long_long, vector_bool_long_long, vector_signed_long_long, vsg);
601        impl_sub!(vs_ug_bg, vector_unsigned_long_long, vector_bool_long_long, vector_unsigned_long_long, vsg);
602
603        impl_sub!(vs_bc_sc, vector_bool_char, vector_signed_char, vector_signed_char, vsb);
604        impl_sub!(vs_bc_uc, vector_bool_char, vector_unsigned_char, vector_unsigned_char, vsb);
605        impl_sub!(vs_bh_sh, vector_bool_short, vector_signed_short, vector_signed_short, vsh);
606        impl_sub!(vs_bh_uh, vector_bool_short, vector_unsigned_short, vector_unsigned_short, vsh);
607        impl_sub!(vs_bf_sf, vector_bool_int, vector_signed_int, vector_signed_int, vsf);
608        impl_sub!(vs_bf_uf, vector_bool_int, vector_unsigned_int, vector_unsigned_int, vsf);
609        impl_sub!(vs_bg_sg, vector_bool_long_long, vector_signed_long_long, vector_signed_long_long, vsg);
610        impl_sub!(vs_bg_ug, vector_bool_long_long, vector_unsigned_long_long, vector_unsigned_long_long, vsg);
611
612        impl_sub!(vs_double, vector_double, vfsdb);
613
614        #[inline]
615        #[target_feature(enable = "vector")]
616        #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vfssb))]
617        pub unsafe fn vs_float(a: vector_float, b: vector_float) -> vector_float {
618            transmute(simd_sub(a, b))
619        }
620
621        #[unstable(feature = "stdarch_s390x", issue = "135681")]
622        impl VectorSub<Self> for vector_float {
623            type Result = Self;
624
625            #[inline]
626            #[target_feature(enable = "vector")]
627            unsafe fn vec_sub(self, other: Self) -> Self::Result {
628                vs_float(self, other)
629            }
630        }
631    }
632
633    #[unstable(feature = "stdarch_s390x", issue = "135681")]
634    pub trait VectorMul {
635        unsafe fn vec_mul(self, b: Self) -> Self;
636    }
637
638    macro_rules! impl_mul {
639        ($name:ident, $a:ty, std_simd) => {
640            #[unstable(feature = "stdarch_s390x", issue = "135681")]
641            impl VectorMul for $a {
642                #[inline]
643                #[target_feature(enable = "vector")]
644                unsafe fn vec_mul(self, other: Self) -> Self {
645                    transmute(simd_mul(transmute(self), other))
646                }
647            }
648        };
649        ($name:ident, $a:ty, $instr:ident) => {
650            #[inline]
651            #[target_feature(enable = "vector")]
652            #[cfg_attr(test, assert_instr($instr))]
653            pub unsafe fn $name(a: $a, b: $a) -> $a {
654                transmute(simd_mul(transmute(a), b))
655            }
656
657            #[unstable(feature = "stdarch_s390x", issue = "135681")]
658            impl VectorMul for $a {
659                #[inline]
660                #[target_feature(enable = "vector")]
661                unsafe fn vec_mul(self, other: Self) -> Self {
662                    $name(self, other)
663                }
664            }
665        };
666    }
667
668    #[rustfmt::skip]
669    mod impl_mul {
670        use super::*;
671
672        impl_mul!(vml_sc, vector_signed_char, vmlb);
673        impl_mul!(vml_uc, vector_unsigned_char, vmlb);
674        impl_mul!(vml_sh, vector_signed_short, vmlhw);
675        impl_mul!(vml_uh, vector_unsigned_short, vmlhw);
676        impl_mul!(vml_sf, vector_signed_int, vmlf);
677        impl_mul!(vml_uf, vector_unsigned_int, vmlf);
678        impl_mul!(vml_sg, vector_signed_long_long, std_simd);
679        impl_mul!(vml_ug, vector_unsigned_long_long, std_simd);
680
681        impl_mul!(vml_float, vector_float, std_simd);
682        impl_mul!(vml_double, vector_double, vfmdb);
683    }
684
685    #[unstable(feature = "stdarch_s390x", issue = "135681")]
686    pub trait VectorMax<Other> {
687        type Result;
688        unsafe fn vec_max(self, b: Other) -> Self::Result;
689    }
690
691    test_impl! { vec_vmxsb (a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [vmxb, vmxb] }
692    test_impl! { vec_vmxsh (a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [vmxh, vmxh] }
693    test_impl! { vec_vmxsf (a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [vmxf, vmxf] }
694    test_impl! { vec_vmxsg (a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [vmxg, vmxg] }
695
696    test_impl! { vec_vmxslb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vmxlb, vmxlb] }
697    test_impl! { vec_vmxslh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vmxlh, vmxlh] }
698    test_impl! { vec_vmxslf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vmxlf, vmxlf] }
699    test_impl! { vec_vmxslg (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vmxlg, vmxlg] }
700
701    impl_vec_trait! { [VectorMax vec_max] ~(vmxlb, vmxb, vmxlh, vmxh, vmxlf, vmxf, vmxlg, vmxg) }
702
703    test_impl! { vec_vfmaxsb (a: vector_float, b: vector_float) -> vector_float [simd_fmax, "vector-enhancements-1" vfmaxsb ] }
704    test_impl! { vec_vfmaxdb (a: vector_double, b: vector_double) -> vector_double [simd_fmax, "vector-enhancements-1" vfmaxdb] }
705
706    impl_vec_trait!([VectorMax vec_max] vec_vfmaxsb (vector_float, vector_float) -> vector_float);
707    impl_vec_trait!([VectorMax vec_max] vec_vfmaxdb (vector_double, vector_double) -> vector_double);
708
709    #[unstable(feature = "stdarch_s390x", issue = "135681")]
710    pub trait VectorMin<Other> {
711        type Result;
712        unsafe fn vec_min(self, b: Other) -> Self::Result;
713    }
714
715    test_impl! { vec_vmnsb (a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [vmnb, vmnb] }
716    test_impl! { vec_vmnsh (a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [vmnh, vmnh] }
717    test_impl! { vec_vmnsf (a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [vmnf, vmnf] }
718    test_impl! { vec_vmnsg (a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [vmng, vmng] }
719
720    test_impl! { vec_vmnslb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vmnlb, vmnlb] }
721    test_impl! { vec_vmnslh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vmnlh, vmnlh] }
722    test_impl! { vec_vmnslf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vmnlf, vmnlf] }
723    test_impl! { vec_vmnslg (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vmnlg, vmnlg] }
724
725    impl_vec_trait! { [VectorMin vec_min] ~(vmxlb, vmxb, vmxlh, vmxh, vmxlf, vmxf, vmxlg, vmxg) }
726
727    test_impl! { vec_vfminsb (a: vector_float, b: vector_float) -> vector_float [simd_fmin, "vector-enhancements-1" vfminsb]  }
728    test_impl! { vec_vfmindb (a: vector_double, b: vector_double) -> vector_double [simd_fmin, "vector-enhancements-1" vfmindb]  }
729
730    impl_vec_trait!([VectorMin vec_min] vec_vfminsb (vector_float, vector_float) -> vector_float);
731    impl_vec_trait!([VectorMin vec_min] vec_vfmindb (vector_double, vector_double) -> vector_double);
732
733    #[unstable(feature = "stdarch_s390x", issue = "135681")]
734    pub trait VectorAbs {
735        unsafe fn vec_abs(self) -> Self;
736    }
737
738    macro_rules! impl_abs {
739        ($name:ident, $ty:ident) => {
740            #[inline]
741            #[target_feature(enable = "vector")]
742            unsafe fn $name(v: s_t_l!($ty)) -> s_t_l!($ty) {
743                v.vec_max(-v)
744            }
745
746            impl_vec_trait! { [VectorAbs vec_abs] $name (s_t_l!($ty)) }
747        };
748    }
749
750    impl_abs! { vec_abs_i8, i8x16 }
751    impl_abs! { vec_abs_i16, i16x8 }
752    impl_abs! { vec_abs_i32, i32x4 }
753    impl_abs! { vec_abs_i64, i64x2 }
754
755    test_impl! { vec_abs_f32 (v: vector_float) -> vector_float [ simd_fabs, "vector-enhancements-1" vflpsb ] }
756    test_impl! { vec_abs_f64 (v: vector_double) -> vector_double [ simd_fabs, vflpdb ] }
757
758    impl_vec_trait! { [VectorAbs vec_abs] vec_abs_f32 (vector_float) }
759    impl_vec_trait! { [VectorAbs vec_abs] vec_abs_f64 (vector_double) }
760
761    #[unstable(feature = "stdarch_s390x", issue = "135681")]
762    pub trait VectorNabs {
763        unsafe fn vec_nabs(self) -> Self;
764    }
765
766    #[inline]
767    #[target_feature(enable = "vector")]
768    #[cfg_attr(
769        all(test, target_feature = "vector-enhancements-1"),
770        assert_instr(vflnsb)
771    )]
772    unsafe fn vec_nabs_f32(a: vector_float) -> vector_float {
773        simd_neg(simd_fabs(a))
774    }
775
776    #[inline]
777    #[target_feature(enable = "vector")]
778    #[cfg_attr(test, assert_instr(vflndb))]
779    unsafe fn vec_nabs_f64(a: vector_double) -> vector_double {
780        simd_neg(simd_fabs(a))
781    }
782
783    impl_vec_trait! { [VectorNabs vec_nabs] vec_nabs_f32 (vector_float) }
784    impl_vec_trait! { [VectorNabs vec_nabs] vec_nabs_f64 (vector_double) }
785
786    #[unstable(feature = "stdarch_s390x", issue = "135681")]
787    pub trait VectorNmsub {
788        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self;
789    }
790
791    #[inline]
792    #[target_feature(enable = "vector")]
793    #[cfg_attr(
794        all(test, target_feature = "vector-enhancements-2"),
795        assert_instr(vfnmssb)
796    )]
797    unsafe fn vec_nmsub_f32(a: vector_float, b: vector_float, c: vector_float) -> vector_float {
798        simd_neg(simd_fma(a, b, simd_neg(c)))
799    }
800
801    #[unstable(feature = "stdarch_s390x", issue = "135681")]
802    impl VectorNmsub for vector_float {
803        #[target_feature(enable = "vector")]
804        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self {
805            vec_nmsub_f32(self, b, c)
806        }
807    }
808
809    #[inline]
810    #[target_feature(enable = "vector")]
811    #[cfg_attr(
812        all(test, target_feature = "vector-enhancements-2"),
813        assert_instr(vfnmsdb)
814    )]
815    unsafe fn vec_nmsub_f64(a: vector_double, b: vector_double, c: vector_double) -> vector_double {
816        simd_neg(simd_fma(a, b, simd_neg(c)))
817    }
818
819    #[unstable(feature = "stdarch_s390x", issue = "135681")]
820    impl VectorNmsub for vector_double {
821        #[target_feature(enable = "vector")]
822        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self {
823            vec_nmsub_f64(self, b, c)
824        }
825    }
826
827    #[unstable(feature = "stdarch_s390x", issue = "135681")]
828    pub trait VectorNmadd {
829        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self;
830    }
831
832    #[inline]
833    #[target_feature(enable = "vector")]
834    #[cfg_attr(
835        all(test, target_feature = "vector-enhancements-2"),
836        assert_instr(vfnmasb)
837    )]
838    unsafe fn vec_nmadd_f32(a: vector_float, b: vector_float, c: vector_float) -> vector_float {
839        simd_neg(simd_fma(a, b, c))
840    }
841
842    #[unstable(feature = "stdarch_s390x", issue = "135681")]
843    impl VectorNmadd for vector_float {
844        #[target_feature(enable = "vector")]
845        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self {
846            vec_nmadd_f32(self, b, c)
847        }
848    }
849
850    #[inline]
851    #[target_feature(enable = "vector")]
852    #[cfg_attr(
853        all(test, target_feature = "vector-enhancements-2"),
854        assert_instr(vfnmadb)
855    )]
856    unsafe fn vec_nmadd_f64(a: vector_double, b: vector_double, c: vector_double) -> vector_double {
857        simd_neg(simd_fma(a, b, c))
858    }
859
860    #[unstable(feature = "stdarch_s390x", issue = "135681")]
861    impl VectorNmadd for vector_double {
862        #[target_feature(enable = "vector")]
863        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self {
864            vec_nmadd_f64(self, b, c)
865        }
866    }
867
868    #[unstable(feature = "stdarch_s390x", issue = "135681")]
869    pub trait VectorSplat {
870        unsafe fn vec_splat<const IMM: u32>(self) -> Self;
871    }
872
873    #[inline]
874    #[target_feature(enable = "vector")]
875    #[cfg_attr(test, assert_instr(vrepb, IMM2 = 1))]
876    unsafe fn vrepb<const IMM2: u32>(a: vector_signed_char) -> vector_signed_char {
877        static_assert_uimm_bits!(IMM2, 4);
878        simd_shuffle(a, a, const { u32x16::from_array([IMM2; 16]) })
879    }
880
881    #[inline]
882    #[target_feature(enable = "vector")]
883    #[cfg_attr(test, assert_instr(vreph, IMM2 = 1))]
884    unsafe fn vreph<const IMM2: u32>(a: vector_signed_short) -> vector_signed_short {
885        static_assert_uimm_bits!(IMM2, 3);
886        simd_shuffle(a, a, const { u32x8::from_array([IMM2; 8]) })
887    }
888
889    #[inline]
890    #[target_feature(enable = "vector")]
891    #[cfg_attr(test, assert_instr(vrepf, IMM2 = 1))]
892    unsafe fn vrepf<const IMM2: u32>(a: vector_signed_int) -> vector_signed_int {
893        static_assert_uimm_bits!(IMM2, 2);
894        simd_shuffle(a, a, const { u32x4::from_array([IMM2; 4]) })
895    }
896
897    #[inline]
898    #[target_feature(enable = "vector")]
899    #[cfg_attr(test, assert_instr(vrepg, IMM2 = 1))]
900    unsafe fn vrepg<const IMM2: u32>(a: vector_signed_long_long) -> vector_signed_long_long {
901        static_assert_uimm_bits!(IMM2, 1);
902        simd_shuffle(a, a, const { u32x2::from_array([IMM2; 2]) })
903    }
904
905    macro_rules! impl_vec_splat {
906        ($ty:ty, $fun:ident) => {
907            #[unstable(feature = "stdarch_s390x", issue = "135681")]
908            impl VectorSplat for $ty {
909                #[inline]
910                #[target_feature(enable = "vector")]
911                unsafe fn vec_splat<const IMM: u32>(self) -> Self {
912                    transmute($fun::<IMM>(transmute(self)))
913                }
914            }
915        };
916    }
917
918    impl_vec_splat! { vector_signed_char, vrepb }
919    impl_vec_splat! { vector_unsigned_char, vrepb }
920    impl_vec_splat! { vector_bool_char, vrepb }
921    impl_vec_splat! { vector_signed_short, vreph }
922    impl_vec_splat! { vector_unsigned_short, vreph }
923    impl_vec_splat! { vector_bool_short, vreph }
924    impl_vec_splat! { vector_signed_int, vrepf }
925    impl_vec_splat! { vector_unsigned_int, vrepf }
926    impl_vec_splat! { vector_bool_int, vrepf }
927    impl_vec_splat! { vector_signed_long_long, vrepg }
928    impl_vec_splat! { vector_unsigned_long_long, vrepg }
929    impl_vec_splat! { vector_bool_long_long, vrepg }
930
931    impl_vec_splat! { vector_float, vrepf }
932    impl_vec_splat! { vector_double, vrepg }
933
934    #[unstable(feature = "stdarch_s390x", issue = "135681")]
935    pub trait VectorSplats<Output> {
936        unsafe fn vec_splats(self) -> Output;
937    }
938
939    macro_rules! impl_vec_splats {
940        ($(($fn:ident ($ty:ty, $shortty:tt) $instr:ident)),*) => {
941            $(
942                #[inline]
943                #[target_feature(enable = "vector")]
944                #[cfg_attr(test, assert_instr($instr))]
945                pub unsafe fn $fn(v: $ty) -> s_t_l!($shortty) {
946                    transmute($shortty::splat(v))
947                }
948
949                #[unstable(feature = "stdarch_s390x", issue = "135681")]
950                impl VectorSplats<s_t_l!($shortty)> for $ty {
951                    #[inline]
952                    #[target_feature(enable = "vector")]
953                    unsafe fn vec_splats(self) -> s_t_l!($shortty) {
954                        $fn (self)
955                    }
956                }
957            )*
958        }
959    }
960
961    impl_vec_splats! {
962        (vec_splats_u8 (u8, u8x16) vrepb),
963        (vec_splats_i8 (i8, i8x16) vrepb),
964        (vec_splats_u16 (u16, u16x8) vreph),
965        (vec_splats_i16 (i16, i16x8) vreph),
966        (vec_splats_u32 (u32, u32x4) vrepf),
967        (vec_splats_i32 (i32, i32x4) vrepf),
968        (vec_splats_u64 (u64, u64x2) vlvgp),
969        (vec_splats_i64 (i64, i64x2) vlvgp),
970        (vec_splats_f32 (f32, f32x4) vrepf),
971        (vec_splats_f64 (f64, f64x2) vrepg)
972    }
973
974    macro_rules! impl_bool_vec_splats {
975        ($(($ty:ty, $shortty:tt, $boolty:ty)),*) => {
976            $(
977                #[unstable(feature = "stdarch_s390x", issue = "135681")]
978                impl VectorSplats<$boolty> for $ty {
979                    #[inline]
980                    #[target_feature(enable = "vector")]
981                    unsafe fn vec_splats(self) -> $boolty {
982                        transmute($shortty::splat(self))
983                    }
984                }
985            )*
986        }
987    }
988
989    impl_bool_vec_splats! {
990        (u8, u8x16, vector_bool_char),
991        (i8, i8x16, vector_bool_char),
992        (u16, u16x8, vector_bool_short),
993        (i16, i16x8, vector_bool_short),
994        (u32, u32x4, vector_bool_int),
995        (i32, i32x4, vector_bool_int),
996        (u64, u64x2, vector_bool_long_long),
997        (i64, i64x2, vector_bool_long_long)
998    }
999
1000    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1001    pub trait CountBits {
1002        type Result;
1003
1004        unsafe fn vec_cntlz(self) -> Self::Result;
1005        unsafe fn vec_cnttz(self) -> Self::Result;
1006        unsafe fn vec_popcnt(self) -> Self::Result;
1007    }
1008
1009    macro_rules! impl_count_bits {
1010        ($ty:tt) => {
1011            #[unstable(feature = "stdarch_s390x", issue = "135681")]
1012            impl CountBits for $ty {
1013                type Result = t_u!($ty);
1014
1015                #[inline]
1016                #[target_feature(enable = "vector")]
1017                unsafe fn vec_cntlz(self) -> Self::Result {
1018                    transmute(simd_ctlz(self))
1019                }
1020
1021                #[inline]
1022                #[target_feature(enable = "vector")]
1023                unsafe fn vec_cnttz(self) -> Self::Result {
1024                    transmute(simd_cttz(self))
1025                }
1026
1027                #[inline]
1028                #[target_feature(enable = "vector")]
1029                unsafe fn vec_popcnt(self) -> Self::Result {
1030                    transmute(simd_ctpop(self))
1031                }
1032            }
1033        };
1034    }
1035
1036    impl_count_bits!(vector_signed_char);
1037    impl_count_bits!(vector_unsigned_char);
1038    impl_count_bits!(vector_signed_short);
1039    impl_count_bits!(vector_unsigned_short);
1040    impl_count_bits!(vector_signed_int);
1041    impl_count_bits!(vector_unsigned_int);
1042    impl_count_bits!(vector_signed_long_long);
1043    impl_count_bits!(vector_unsigned_long_long);
1044
1045    test_impl! { vec_clzb_signed +(a: vector_signed_char) -> vector_unsigned_char [simd_ctlz, vclzb] }
1046    test_impl! { vec_clzh_signed +(a: vector_signed_short) -> vector_unsigned_short [simd_ctlz, vclzh] }
1047    test_impl! { vec_clzf_signed +(a: vector_signed_int) -> vector_unsigned_int [simd_ctlz, vclzf] }
1048    test_impl! { vec_clzg_signed +(a: vector_signed_long_long) -> vector_unsigned_long_long [simd_ctlz, vclzg] }
1049
1050    test_impl! { vec_clzb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_ctlz, vclzb] }
1051    test_impl! { vec_clzh_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_ctlz, vclzh] }
1052    test_impl! { vec_clzf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_ctlz, vclzf] }
1053    test_impl! { vec_clzg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_ctlz, vclzg] }
1054
1055    test_impl! { vec_ctzb_signed +(a: vector_signed_char) -> vector_unsigned_char [simd_cttz, vctzb] }
1056    test_impl! { vec_ctzh_signed +(a: vector_signed_short) -> vector_unsigned_short [simd_cttz, vctzh] }
1057    test_impl! { vec_ctzf_signed +(a: vector_signed_int) -> vector_unsigned_int [simd_cttz, vctzf] }
1058    test_impl! { vec_ctzg_signed +(a: vector_signed_long_long) -> vector_unsigned_long_long [simd_cttz, vctzg] }
1059
1060    test_impl! { vec_ctzb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_cttz, vctzb] }
1061    test_impl! { vec_ctzh_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_cttz, vctzh] }
1062    test_impl! { vec_ctzf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_cttz, vctzf] }
1063    test_impl! { vec_ctzg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_cttz, vctzg] }
1064
1065    test_impl! { vec_vpopctb_signed +(a: vector_signed_char) -> vector_signed_char [simd_ctpop, vpopctb] }
1066    test_impl! { vec_vpopcth_signed +(a: vector_signed_short) -> vector_signed_short [simd_ctpop, "vector-enhancements-1" vpopcth] }
1067    test_impl! { vec_vpopctf_signed +(a: vector_signed_int) -> vector_signed_int [simd_ctpop, "vector-enhancements-1" vpopctf] }
1068    test_impl! { vec_vpopctg_signed +(a: vector_signed_long_long) -> vector_signed_long_long [simd_ctpop, "vector-enhancements-1" vpopctg] }
1069
1070    test_impl! { vec_vpopctb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_ctpop, vpopctb] }
1071    test_impl! { vec_vpopcth_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_ctpop, "vector-enhancements-1" vpopcth] }
1072    test_impl! { vec_vpopctf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_ctpop, "vector-enhancements-1" vpopctf] }
1073    test_impl! { vec_vpopctg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_ctpop, "vector-enhancements-1" vpopctg] }
1074
1075    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1076    pub trait VectorAnd<Other> {
1077        type Result;
1078        unsafe fn vec_and(self, b: Other) -> Self::Result;
1079    }
1080
1081    impl_vec_trait! { [VectorAnd vec_and] ~(simd_and) }
1082
1083    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1084    pub trait VectorOr<Other> {
1085        type Result;
1086        unsafe fn vec_or(self, b: Other) -> Self::Result;
1087    }
1088
1089    impl_vec_trait! { [VectorOr vec_or] ~(simd_or) }
1090
1091    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1092    pub trait VectorXor<Other> {
1093        type Result;
1094        unsafe fn vec_xor(self, b: Other) -> Self::Result;
1095    }
1096
1097    impl_vec_trait! { [VectorXor vec_xor] ~(simd_xor) }
1098
1099    #[inline]
1100    #[target_feature(enable = "vector")]
1101    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vno))]
1102    unsafe fn nor(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1103        let a: u8x16 = transmute(a);
1104        let b: u8x16 = transmute(b);
1105        transmute(simd_xor(simd_or(a, b), u8x16::splat(0xff)))
1106    }
1107
1108    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1109    pub trait VectorNor<Other> {
1110        type Result;
1111        unsafe fn vec_nor(self, b: Other) -> Self::Result;
1112    }
1113
1114    impl_vec_trait! { [VectorNor vec_nor]+ 2c (nor) }
1115
1116    #[inline]
1117    #[target_feature(enable = "vector")]
1118    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnn))]
1119    unsafe fn nand(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1120        let a: u8x16 = transmute(a);
1121        let b: u8x16 = transmute(b);
1122        transmute(simd_xor(simd_and(a, b), u8x16::splat(0xff)))
1123    }
1124
1125    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1126    pub trait VectorNand<Other> {
1127        type Result;
1128        unsafe fn vec_nand(self, b: Other) -> Self::Result;
1129    }
1130
1131    impl_vec_trait! { [VectorNand vec_nand]+ 2c (nand) }
1132
1133    #[inline]
1134    #[target_feature(enable = "vector")]
1135    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnx))]
1136    unsafe fn eqv(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1137        let a: u8x16 = transmute(a);
1138        let b: u8x16 = transmute(b);
1139        transmute(simd_xor(simd_xor(a, b), u8x16::splat(0xff)))
1140    }
1141
1142    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1143    pub trait VectorEqv<Other> {
1144        type Result;
1145        unsafe fn vec_eqv(self, b: Other) -> Self::Result;
1146    }
1147
1148    impl_vec_trait! { [VectorEqv vec_eqv]+ 2c (eqv) }
1149
1150    #[inline]
1151    #[target_feature(enable = "vector")]
1152    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnc))]
1153    unsafe fn andc(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1154        let a = transmute(a);
1155        let b = transmute(b);
1156        transmute(simd_and(simd_xor(u8x16::splat(0xff), b), a))
1157    }
1158
1159    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1160    pub trait VectorAndc<Other> {
1161        type Result;
1162        unsafe fn vec_andc(self, b: Other) -> Self::Result;
1163    }
1164
1165    impl_vec_trait! { [VectorAndc vec_andc]+ 2c (andc) }
1166
1167    #[inline]
1168    #[target_feature(enable = "vector")]
1169    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(voc))]
1170    unsafe fn orc(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1171        let a = transmute(a);
1172        let b = transmute(b);
1173        transmute(simd_or(simd_xor(u8x16::splat(0xff), b), a))
1174    }
1175
1176    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1177    pub trait VectorOrc<Other> {
1178        type Result;
1179        unsafe fn vec_orc(self, b: Other) -> Self::Result;
1180    }
1181
1182    impl_vec_trait! { [VectorOrc vec_orc]+ 2c (orc) }
1183
1184    // Z vector intrinsic      C23 math.h  LLVM IR         ISO/IEC 60559 operation        inexact  vfidb parameters
1185    //
1186    // vec_rint                rint        llvm.rint       roundToIntegralExact           yes      0, 0
1187    // vec_roundc              nearbyint   llvm.nearbyint  n/a                            no       4, 0
1188    // vec_floor / vec_roundm  floor       llvm.floor      roundToIntegralTowardNegative  no       4, 7
1189    // vec_ceil / vec_roundp   ceil        llvm.ceil       roundToIntegralTowardPositive  no       4, 6
1190    // vec_trunc / vec_roundz  trunc       llvm.trunc      roundToIntegralTowardZero      no       4, 5
1191    // vec_round               roundeven   llvm.roundeven  roundToIntegralTiesToEven      no       4, 4
1192    // n/a                     round       llvm.round      roundToIntegralTiesAway        no       4, 1
1193
1194    // `simd_round_ties_even` is implemented as `llvm.rint`.
1195    test_impl! { vec_rint_f32 (a: vector_float) -> vector_float [simd_round_ties_even, "vector-enhancements-1" vfisb] }
1196    test_impl! { vec_rint_f64 (a: vector_double) -> vector_double [simd_round_ties_even, vfidb] }
1197
1198    test_impl! { vec_roundc_f32 (a: vector_float) -> vector_float [nearbyint_v4f32,  "vector-enhancements-1" vfisb] }
1199    test_impl! { vec_roundc_f64 (a: vector_double) -> vector_double [nearbyint_v2f64, vfidb] }
1200
1201    // FIXME(llvm) llvm trunk already lowers roundeven to vfidb, but rust does not use it yet
1202    // use https://godbolt.org/z/cWq95fexe to check, and enable the instruction test when it works
1203    test_impl! { vec_round_f32 (a: vector_float) -> vector_float [roundeven_v4f32, _] }
1204    test_impl! { vec_round_f64 (a: vector_double) -> vector_double [roundeven_v2f64, _] }
1205
1206    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1207    pub trait VectorRoundc {
1208        unsafe fn vec_roundc(self) -> Self;
1209    }
1210
1211    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1212    pub trait VectorRound {
1213        unsafe fn vec_round(self) -> Self;
1214    }
1215
1216    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1217    pub trait VectorRint {
1218        unsafe fn vec_rint(self) -> Self;
1219    }
1220
1221    impl_vec_trait! { [VectorRoundc vec_roundc] vec_roundc_f32 (vector_float) }
1222    impl_vec_trait! { [VectorRoundc vec_roundc] vec_roundc_f64 (vector_double) }
1223
1224    impl_vec_trait! { [VectorRound vec_round] vec_round_f32 (vector_float) }
1225    impl_vec_trait! { [VectorRound vec_round] vec_round_f64 (vector_double) }
1226
1227    impl_vec_trait! { [VectorRint vec_rint] simd_round_ties_even (vector_float) }
1228    impl_vec_trait! { [VectorRint vec_rint] simd_round_ties_even (vector_double) }
1229
1230    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1231    pub trait VectorTrunc {
1232        // same as vec_roundz
1233        unsafe fn vec_trunc(self) -> Self;
1234    }
1235
1236    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1237    pub trait VectorCeil {
1238        // same as vec_roundp
1239        unsafe fn vec_ceil(self) -> Self;
1240    }
1241
1242    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1243    pub trait VectorFloor {
1244        // same as vec_roundm
1245        unsafe fn vec_floor(self) -> Self;
1246    }
1247
1248    impl_vec_trait! { [VectorTrunc vec_trunc] simd_trunc (vector_float) }
1249    impl_vec_trait! { [VectorTrunc vec_trunc] simd_trunc (vector_double) }
1250
1251    impl_vec_trait! { [VectorCeil vec_ceil] simd_ceil (vector_float) }
1252    impl_vec_trait! { [VectorCeil vec_ceil] simd_ceil (vector_double) }
1253
1254    impl_vec_trait! { [VectorFloor vec_floor] simd_floor (vector_float) }
1255    impl_vec_trait! { [VectorFloor vec_floor] simd_floor (vector_double) }
1256
1257    macro_rules! impl_vec_shift {
1258        ([$Trait:ident $m:ident] ($b:ident, $h:ident, $w:ident, $g:ident)) => {
1259            impl_vec_trait!{ [$Trait $m]+ $b (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1260            impl_vec_trait!{ [$Trait $m]+ $b (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1261            impl_vec_trait!{ [$Trait $m]+ $h (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1262            impl_vec_trait!{ [$Trait $m]+ $h (vector_signed_short, vector_unsigned_short) -> vector_signed_short }
1263            impl_vec_trait!{ [$Trait $m]+ $w (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1264            impl_vec_trait!{ [$Trait $m]+ $w (vector_signed_int, vector_unsigned_int) -> vector_signed_int }
1265            impl_vec_trait!{ [$Trait $m]+ $g (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1266            impl_vec_trait!{ [$Trait $m]+ $g (vector_signed_long_long, vector_unsigned_long_long) -> vector_signed_long_long }
1267        };
1268    }
1269
1270    macro_rules! impl_shift {
1271        ($fun:ident $intr:ident $ty:ident) => {
1272            #[inline]
1273            #[target_feature(enable = "vector")]
1274            #[cfg_attr(test, assert_instr($fun))]
1275            unsafe fn $fun(a: t_t_l!($ty), b: t_t_l!($ty)) -> t_t_l!($ty) {
1276                let a = transmute(a);
1277                // use the remainder of b by the width of a's elements to prevent UB
1278                let b = simd_rem(transmute(b), <t_t_s!($ty)>::splat($ty::BITS as $ty));
1279
1280                transmute($intr(a, b))
1281            }
1282        };
1283    }
1284
1285    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1286    pub trait VectorSl<Other> {
1287        type Result;
1288        unsafe fn vec_sl(self, b: Other) -> Self::Result;
1289    }
1290
1291    impl_shift! { veslvb simd_shl u8 }
1292    impl_shift! { veslvh simd_shl u16 }
1293    impl_shift! { veslvf simd_shl u32 }
1294    impl_shift! { veslvg simd_shl u64 }
1295
1296    impl_vec_shift! { [VectorSl vec_sl] (veslvb, veslvh, veslvf, veslvg) }
1297
1298    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1299    pub trait VectorSr<Other> {
1300        type Result;
1301        unsafe fn vec_sr(self, b: Other) -> Self::Result;
1302    }
1303
1304    impl_shift! { vesrlvb simd_shr u8 }
1305    impl_shift! { vesrlvh simd_shr u16 }
1306    impl_shift! { vesrlvf simd_shr u32 }
1307    impl_shift! { vesrlvg simd_shr u64 }
1308
1309    impl_vec_shift! { [VectorSr vec_sr] (vesrlvb, vesrlvh, vesrlvf, vesrlvg) }
1310
1311    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1312    pub trait VectorSra<Other> {
1313        type Result;
1314        unsafe fn vec_sra(self, b: Other) -> Self::Result;
1315    }
1316
1317    impl_shift! { vesravb simd_shr i8 }
1318    impl_shift! { vesravh simd_shr i16 }
1319    impl_shift! { vesravf simd_shr i32 }
1320    impl_shift! { vesravg simd_shr i64 }
1321
1322    impl_vec_shift! { [VectorSra vec_sra] (vesravb, vesravh, vesravf, vesravg) }
1323
1324    macro_rules! impl_vec_shift_byte {
1325        ([$trait:ident $m:ident] ($f:ident)) => {
1326            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_signed_char) -> vector_unsigned_char }
1327            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1328            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_signed_char) -> vector_signed_char }
1329            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1330            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_signed_short) -> vector_unsigned_short }
1331            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1332            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_signed_short) -> vector_signed_short }
1333            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_unsigned_short) -> vector_signed_short }
1334            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_signed_int) -> vector_unsigned_int }
1335            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1336            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_signed_int) -> vector_signed_int }
1337            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_unsigned_int) -> vector_signed_int }
1338            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_signed_long_long) -> vector_unsigned_long_long }
1339            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1340            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_signed_long_long) -> vector_signed_long_long }
1341            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_unsigned_long_long) -> vector_signed_long_long }
1342            impl_vec_trait!{ [$trait $m]+ $f (vector_float, vector_signed_int) -> vector_float }
1343            impl_vec_trait!{ [$trait $m]+ $f (vector_float, vector_unsigned_int) -> vector_float }
1344            impl_vec_trait!{ [$trait $m]+ $f (vector_double, vector_signed_long_long) -> vector_double }
1345            impl_vec_trait!{ [$trait $m]+ $f (vector_double, vector_unsigned_long_long) -> vector_double }
1346        };
1347    }
1348
1349    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1350    pub trait VectorSlb<Other> {
1351        type Result;
1352        unsafe fn vec_slb(self, b: Other) -> Self::Result;
1353    }
1354
1355    impl_vec_shift_byte! { [VectorSlb vec_slb] (vslb) }
1356
1357    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1358    pub trait VectorSrab<Other> {
1359        type Result;
1360        unsafe fn vec_srab(self, b: Other) -> Self::Result;
1361    }
1362
1363    impl_vec_shift_byte! { [VectorSrab vec_srab] (vsrab) }
1364
1365    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1366    pub trait VectorSrb<Other> {
1367        type Result;
1368        unsafe fn vec_srb(self, b: Other) -> Self::Result;
1369    }
1370
1371    impl_vec_shift_byte! { [VectorSrb vec_srb] (vsrlb) }
1372
1373    macro_rules! impl_vec_shift_long {
1374        ([$trait:ident $m:ident] ($f:ident)) => {
1375            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1376            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1377            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_unsigned_char) -> vector_unsigned_short }
1378            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_unsigned_char) -> vector_signed_short }
1379            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_unsigned_char) -> vector_unsigned_int }
1380            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_unsigned_char) -> vector_signed_int }
1381            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_unsigned_char) -> vector_unsigned_long_long }
1382            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_unsigned_char) -> vector_signed_long_long }
1383        };
1384    }
1385
1386    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1387    pub trait VectorSrl<Other> {
1388        type Result;
1389        unsafe fn vec_srl(self, b: Other) -> Self::Result;
1390    }
1391
1392    impl_vec_shift_long! { [VectorSrl vec_srl] (vsrl) }
1393
1394    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1395    pub trait VectorSral<Other> {
1396        type Result;
1397        unsafe fn vec_sral(self, b: Other) -> Self::Result;
1398    }
1399
1400    impl_vec_shift_long! { [VectorSral vec_sral] (vsra) }
1401
1402    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1403    pub trait VectorSll<Other> {
1404        type Result;
1405        unsafe fn vec_sll(self, b: Other) -> Self::Result;
1406    }
1407
1408    impl_vec_shift_long! { [VectorSll vec_sll] (vsl) }
1409
1410    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1411    pub trait VectorRl<Other> {
1412        type Result;
1413        unsafe fn vec_rl(self, b: Other) -> Self::Result;
1414    }
1415
1416    macro_rules! impl_rot {
1417        ($fun:ident $ty:ident) => {
1418            #[inline]
1419            #[target_feature(enable = "vector")]
1420            #[cfg_attr(test, assert_instr($fun))]
1421            unsafe fn $fun(a: t_t_l!($ty), b: t_t_l!($ty)) -> t_t_l!($ty) {
1422                simd_funnel_shl(a, a, b)
1423            }
1424        };
1425    }
1426
1427    impl_rot! { verllvb u8 }
1428    impl_rot! { verllvh u16 }
1429    impl_rot! { verllvf u32 }
1430    impl_rot! { verllvg u64 }
1431
1432    impl_vec_shift! { [VectorRl vec_rl] (verllvb, verllvh, verllvf, verllvg) }
1433
1434    macro_rules! test_rot_imm {
1435        ($fun:ident $instr:ident $ty:ident) => {
1436            #[inline]
1437            #[target_feature(enable = "vector")]
1438            #[cfg_attr(test, assert_instr($instr))]
1439            unsafe fn $fun(a: t_t_l!($ty), bits: core::ffi::c_ulong) -> t_t_l!($ty) {
1440                // mod by the number of bits in a's element type to prevent UB
1441                let bits = (bits % $ty::BITS as core::ffi::c_ulong) as $ty;
1442                let b = <t_t_s!($ty)>::splat(bits);
1443
1444                simd_funnel_shl(a, a, transmute(b))
1445            }
1446        };
1447    }
1448
1449    test_rot_imm! { verllvb_imm verllb u8 }
1450    test_rot_imm! { verllvh_imm verllh u16 }
1451    test_rot_imm! { verllvf_imm verllf u32 }
1452    test_rot_imm! { verllvg_imm verllg u64 }
1453
1454    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1455    pub trait VectorRli {
1456        unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self;
1457    }
1458
1459    macro_rules! impl_rot_imm {
1460        ($($ty:ident, $intr:ident),*) => {
1461            $(
1462                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1463                impl VectorRli for $ty {
1464                    #[inline]
1465                    #[target_feature(enable = "vector")]
1466                    unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self {
1467                        transmute($intr(transmute(self), bits))
1468                    }
1469                }
1470
1471                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1472                impl VectorRli for t_u!($ty) {
1473                    #[inline]
1474                    #[target_feature(enable = "vector")]
1475                    unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self {
1476                        $intr(self, bits)
1477                    }
1478                }
1479            )*
1480        }
1481    }
1482
1483    impl_rot_imm! {
1484        vector_signed_char, verllvb_imm,
1485        vector_signed_short, verllvh_imm,
1486        vector_signed_int, verllvf_imm,
1487        vector_signed_long_long, verllvg_imm
1488    }
1489
1490    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1491    pub trait VectorRlMask<Other> {
1492        unsafe fn vec_rl_mask<const IMM8: u8>(self, other: Other) -> Self;
1493    }
1494
1495    macro_rules! impl_rl_mask {
1496        ($($ty:ident, $intr:ident, $fun:ident),*) => {
1497            $(
1498                #[inline]
1499                #[target_feature(enable = "vector")]
1500                #[cfg_attr(test, assert_instr($intr, IMM8 = 6))]
1501                unsafe fn $fun<const IMM8: u8>(a: $ty, b: t_u!($ty)) -> $ty {
1502                    // mod by the number of bits in a's element type to prevent UB
1503                    $intr(a, a, transmute(b), const { (IMM8 % <l_t_t!($ty)>::BITS as u8) as i32 })
1504                }
1505
1506                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1507                impl VectorRlMask<t_u!($ty)> for $ty {
1508                    #[inline]
1509                    #[target_feature(enable = "vector")]
1510                    unsafe fn vec_rl_mask<const IMM8: u8>(self, other: t_u!($ty)) -> Self {
1511                        $fun::<IMM8>(self, other)
1512                    }
1513                }
1514
1515                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1516                impl VectorRlMask<t_u!($ty)> for t_u!($ty) {
1517                    #[inline]
1518                    #[target_feature(enable = "vector")]
1519                    unsafe fn vec_rl_mask<const IMM8: u8>(self, other: t_u!($ty)) -> Self {
1520                        transmute($fun::<IMM8>(transmute(self), transmute(other)))
1521                    }
1522                }
1523            )*
1524        }
1525    }
1526
1527    impl_rl_mask! {
1528        vector_signed_char, verimb, test_verimb,
1529        vector_signed_short, verimh, test_verimh,
1530        vector_signed_int, verimf, test_verimf,
1531        vector_signed_long_long, verimg, test_verimg
1532    }
1533
1534    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1535    pub trait VectorReve {
1536        unsafe fn vec_reve(self) -> Self;
1537    }
1538
1539    #[repr(simd)]
1540    struct ReverseMask<const N: usize>([u32; N]);
1541
1542    impl<const N: usize> ReverseMask<N> {
1543        const fn new() -> Self {
1544            let mut index = [0; N];
1545            let mut i = 0;
1546            while i < N {
1547                index[i] = (N - i - 1) as u32;
1548                i += 1;
1549            }
1550            ReverseMask(index)
1551        }
1552    }
1553
1554    macro_rules! impl_reve {
1555        ($($ty:ident, $fun:ident, $instr:ident),*) => {
1556            $(
1557                #[inline]
1558                #[target_feature(enable = "vector")]
1559                #[cfg_attr(test, assert_instr($instr))]
1560                unsafe fn $fun(a: $ty) -> $ty {
1561                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1562                    simd_shuffle(a, a, const { ShuffleMask::<N>::reverse() })
1563                }
1564
1565                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1566                impl VectorReve for $ty {
1567                    #[inline]
1568                    #[target_feature(enable = "vector")]
1569                    unsafe fn vec_reve(self) -> Self {
1570                        $fun(self)
1571                    }
1572                }
1573
1574                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1575                impl VectorReve for t_u!($ty) {
1576                    #[inline]
1577                    #[target_feature(enable = "vector")]
1578                    unsafe fn vec_reve(self) -> Self {
1579                        transmute($fun(transmute(self)))
1580                    }
1581                }
1582
1583                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1584                impl VectorReve for t_b!($ty) {
1585                    #[inline]
1586                    #[target_feature(enable = "vector")]
1587                    unsafe fn vec_reve(self) -> Self {
1588                        transmute($fun(transmute(self)))
1589                    }
1590                }
1591            )*
1592        }
1593    }
1594
1595    impl_reve! {
1596        vector_signed_char, reveb, vperm,
1597        vector_signed_short, reveh, vperm,
1598        vector_signed_int, revef, vperm,
1599        vector_signed_long_long, reveg, vpdi
1600    }
1601
1602    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1603    impl VectorReve for vector_float {
1604        #[inline]
1605        #[target_feature(enable = "vector")]
1606        unsafe fn vec_reve(self) -> Self {
1607            transmute(transmute::<_, vector_signed_int>(self).vec_reve())
1608        }
1609    }
1610
1611    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1612    impl VectorReve for vector_double {
1613        #[inline]
1614        #[target_feature(enable = "vector")]
1615        unsafe fn vec_reve(self) -> Self {
1616            transmute(transmute::<_, vector_signed_long_long>(self).vec_reve())
1617        }
1618    }
1619
1620    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1621    pub trait VectorRevb {
1622        unsafe fn vec_revb(self) -> Self;
1623    }
1624
1625    test_impl! { bswapb (a: vector_signed_char) -> vector_signed_char [simd_bswap, _] }
1626    test_impl! { bswaph (a: vector_signed_short) -> vector_signed_short [simd_bswap, vperm] }
1627    test_impl! { bswapf (a: vector_signed_int) -> vector_signed_int [simd_bswap, vperm] }
1628    test_impl! { bswapg (a: vector_signed_long_long) -> vector_signed_long_long [simd_bswap, vperm] }
1629
1630    impl_vec_trait! { [VectorRevb vec_revb]+ bswapb (vector_unsigned_char) }
1631    impl_vec_trait! { [VectorRevb vec_revb]+ bswapb (vector_signed_char) }
1632    impl_vec_trait! { [VectorRevb vec_revb]+ bswaph (vector_unsigned_short) }
1633    impl_vec_trait! { [VectorRevb vec_revb]+ bswaph (vector_signed_short) }
1634    impl_vec_trait! { [VectorRevb vec_revb]+ bswapf (vector_unsigned_int) }
1635    impl_vec_trait! { [VectorRevb vec_revb]+ bswapf (vector_signed_int) }
1636    impl_vec_trait! { [VectorRevb vec_revb]+ bswapg (vector_unsigned_long_long) }
1637    impl_vec_trait! { [VectorRevb vec_revb]+ bswapg (vector_signed_long_long) }
1638
1639    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1640    impl VectorRevb for vector_float {
1641        #[inline]
1642        #[target_feature(enable = "vector")]
1643        unsafe fn vec_revb(self) -> Self {
1644            transmute(transmute::<_, vector_signed_int>(self).vec_revb())
1645        }
1646    }
1647
1648    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1649    impl VectorRevb for vector_double {
1650        #[inline]
1651        #[target_feature(enable = "vector")]
1652        unsafe fn vec_revb(self) -> Self {
1653            transmute(transmute::<_, vector_signed_long_long>(self).vec_revb())
1654        }
1655    }
1656
1657    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1658    pub trait VectorMergel {
1659        unsafe fn vec_mergel(self, other: Self) -> Self;
1660    }
1661
1662    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1663    pub trait VectorMergeh {
1664        unsafe fn vec_mergeh(self, other: Self) -> Self;
1665    }
1666
1667    macro_rules! impl_merge {
1668        ($($ty:ident, $mergel:ident, $mergeh:ident),*) => {
1669            $(
1670                #[inline]
1671                #[target_feature(enable = "vector")]
1672                #[cfg_attr(test, assert_instr($mergel))]
1673                unsafe fn $mergel(a: $ty, b: $ty) -> $ty {
1674                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1675                    simd_shuffle(a, b, const { ShuffleMask::<N>::merge_low() })
1676                }
1677
1678                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1679                impl VectorMergel for $ty {
1680                    #[inline]
1681                    #[target_feature(enable = "vector")]
1682                    unsafe fn vec_mergel(self, other: Self) -> Self {
1683                        $mergel(self, other)
1684                    }
1685                }
1686
1687                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1688                impl VectorMergel for t_u!($ty) {
1689                    #[inline]
1690                    #[target_feature(enable = "vector")]
1691                    unsafe fn vec_mergel(self, other: Self) -> Self {
1692                        transmute($mergel(transmute(self), transmute(other)))
1693                    }
1694                }
1695
1696                #[inline]
1697                #[target_feature(enable = "vector")]
1698                #[cfg_attr(test, assert_instr($mergeh))]
1699                unsafe fn $mergeh(a: $ty, b: $ty) -> $ty {
1700                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1701                    simd_shuffle(a, b, const { ShuffleMask::<N>::merge_high() })
1702                }
1703
1704                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1705                impl VectorMergeh for $ty {
1706                    #[inline]
1707                    #[target_feature(enable = "vector")]
1708                    unsafe fn vec_mergeh(self, other: Self) -> Self {
1709                        $mergeh(self, other)
1710                    }
1711                }
1712
1713                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1714                impl VectorMergeh for t_u!($ty) {
1715                    #[inline]
1716                    #[target_feature(enable = "vector")]
1717                    unsafe fn vec_mergeh(self, other: Self) -> Self {
1718                        transmute($mergeh(transmute(self), transmute(other)))
1719                    }
1720                }
1721            )*
1722        }
1723    }
1724
1725    impl_merge! {
1726        vector_signed_char, vmrlb, vmrhb,
1727        vector_signed_short, vmrlh, vmrhh,
1728        vector_signed_int, vmrlf, vmrhf,
1729        vector_signed_long_long, vmrlg, vmrhg
1730    }
1731
1732    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1733    pub trait VectorPerm {
1734        unsafe fn vec_perm(self, other: Self, c: vector_unsigned_char) -> Self;
1735    }
1736
1737    macro_rules! impl_merge {
1738        ($($ty:ident),*) => {
1739            $(
1740                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1741                impl VectorPerm for $ty {
1742                    #[inline]
1743                    #[target_feature(enable = "vector")]
1744                    unsafe fn vec_perm(self, other: Self, c: vector_unsigned_char) -> Self {
1745                        transmute(vperm(transmute(self), transmute(other), c))
1746                    }
1747                }
1748            )*
1749        }
1750    }
1751
1752    impl_merge! {
1753        vector_signed_char,
1754        vector_signed_short,
1755        vector_signed_int,
1756        vector_signed_long_long,
1757        vector_unsigned_char,
1758        vector_unsigned_short,
1759        vector_unsigned_int,
1760        vector_unsigned_long_long,
1761        vector_bool_char,
1762        vector_bool_short,
1763        vector_bool_int,
1764        vector_bool_long_long,
1765        vector_float,
1766        vector_double
1767    }
1768
1769    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1770    pub trait VectorSumU128 {
1771        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char;
1772    }
1773
1774    #[inline]
1775    #[target_feature(enable = "vector")]
1776    #[cfg_attr(test, assert_instr(vsumqf))]
1777    pub unsafe fn vec_vsumqf(a: vector_unsigned_int, b: vector_unsigned_int) -> u128 {
1778        transmute(vsumqf(a, b))
1779    }
1780
1781    #[inline]
1782    #[target_feature(enable = "vector")]
1783    #[cfg_attr(test, assert_instr(vsumqg))]
1784    pub unsafe fn vec_vsumqg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128 {
1785        transmute(vsumqg(a, b))
1786    }
1787
1788    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1789    impl VectorSumU128 for vector_unsigned_int {
1790        #[inline]
1791        #[target_feature(enable = "vector")]
1792        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char {
1793            transmute(vec_vsumqf(self, other))
1794        }
1795    }
1796
1797    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1798    impl VectorSumU128 for vector_unsigned_long_long {
1799        #[inline]
1800        #[target_feature(enable = "vector")]
1801        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char {
1802            transmute(vec_vsumqg(self, other))
1803        }
1804    }
1805
1806    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1807    pub trait VectorSum2 {
1808        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long;
1809    }
1810
1811    test_impl! { vec_vsumgh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_long_long [vsumgh, vsumgh] }
1812    test_impl! { vec_vsumgf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [vsumgf, vsumgf] }
1813
1814    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1815    impl VectorSum2 for vector_unsigned_short {
1816        #[inline]
1817        #[target_feature(enable = "vector")]
1818        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long {
1819            vec_vsumgh(self, other)
1820        }
1821    }
1822
1823    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1824    impl VectorSum2 for vector_unsigned_int {
1825        #[inline]
1826        #[target_feature(enable = "vector")]
1827        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long {
1828            vec_vsumgf(self, other)
1829        }
1830    }
1831
1832    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1833    pub trait VectorSum4 {
1834        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int;
1835    }
1836
1837    test_impl! { vec_vsumb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_int [vsumb, vsumb] }
1838    test_impl! { vec_vsumh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int [vsumh, vsumh] }
1839
1840    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1841    impl VectorSum4 for vector_unsigned_char {
1842        #[inline]
1843        #[target_feature(enable = "vector")]
1844        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int {
1845            vec_vsumb(self, other)
1846        }
1847    }
1848
1849    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1850    impl VectorSum4 for vector_unsigned_short {
1851        #[inline]
1852        #[target_feature(enable = "vector")]
1853        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int {
1854            vec_vsumh(self, other)
1855        }
1856    }
1857
1858    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1859    pub trait VectorSubc<Other> {
1860        type Result;
1861        unsafe fn vec_subc(self, b: Other) -> Self::Result;
1862    }
1863
1864    test_impl! { vec_vscbib (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vscbib, vscbib] }
1865    test_impl! { vec_vscbih (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vscbih, vscbih] }
1866    test_impl! { vec_vscbif (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vscbif, vscbif] }
1867    test_impl! { vec_vscbig (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vscbig, vscbig] }
1868
1869    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbib (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1870    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbih (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1871    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbif (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1872    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbig (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1873
1874    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1875    pub trait VectorSqrt {
1876        unsafe fn vec_sqrt(self) -> Self;
1877    }
1878
1879    test_impl! { vec_sqrt_f32 (v: vector_float) -> vector_float [ simd_fsqrt, "vector-enhancements-1" vfsqsb ] }
1880    test_impl! { vec_sqrt_f64 (v: vector_double) -> vector_double [ simd_fsqrt, vfsqdb ] }
1881
1882    impl_vec_trait! { [VectorSqrt vec_sqrt] vec_sqrt_f32 (vector_float) }
1883    impl_vec_trait! { [VectorSqrt vec_sqrt] vec_sqrt_f64 (vector_double) }
1884
1885    macro_rules! vfae_wrapper {
1886        ($($name:ident $ty:ident)*) => {
1887            $(
1888                #[inline]
1889                #[target_feature(enable = "vector")]
1890                #[cfg_attr(test, assert_instr($name, IMM = 0))]
1891                unsafe fn $name<const IMM: i32>(
1892                    a: $ty,
1893                    b: $ty,
1894                ) -> $ty {
1895                    super::$name(a, b, IMM)
1896                }
1897            )*
1898        }
1899     }
1900
1901    vfae_wrapper! {
1902       vfaeb vector_signed_char
1903       vfaeh vector_signed_short
1904       vfaef vector_signed_int
1905
1906       vfaezb vector_signed_char
1907       vfaezh vector_signed_short
1908       vfaezf vector_signed_int
1909    }
1910
1911    macro_rules! impl_vfae {
1912        ([idx_cc $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1913            impl_vfae! { [idx_cc $Trait $m] $imm
1914                $b vector_signed_char vector_signed_char
1915                $b vector_unsigned_char vector_unsigned_char
1916                $b vector_bool_char vector_unsigned_char
1917
1918                $h vector_signed_short vector_signed_short
1919                $h vector_unsigned_short vector_unsigned_short
1920                $h vector_bool_short vector_unsigned_short
1921
1922                $f vector_signed_int vector_signed_int
1923                $f vector_unsigned_int vector_unsigned_int
1924                $f vector_bool_int vector_unsigned_int
1925            }
1926        };
1927        ([idx_cc $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident $r:ident)*) => {
1928            $(
1929                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1930                impl $Trait<Self> for $ty {
1931                    type Result = $r;
1932                    #[inline]
1933                    #[target_feature(enable = "vector")]
1934                    unsafe fn $m(self, b: Self) -> (Self::Result, i32) {
1935                        let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b));
1936                        (transmute(x), y)
1937                    }
1938                }
1939            )*
1940        };
1941        ([cc $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1942            impl_vfae! { [cc $Trait $m] $imm
1943                $b vector_signed_char
1944                $b vector_unsigned_char
1945                $b vector_bool_char
1946
1947                $h vector_signed_short
1948                $h vector_unsigned_short
1949                $h vector_bool_short
1950
1951                $f vector_signed_int
1952                $f vector_unsigned_int
1953                $f vector_bool_int
1954            }
1955        };
1956        ([cc $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident)*) => {
1957            $(
1958                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1959                impl $Trait<Self> for $ty {
1960                    type Result = t_b!($ty);
1961                    #[inline]
1962                    #[target_feature(enable = "vector")]
1963                    unsafe fn $m(self, b: Self) -> (Self::Result, i32) {
1964                        let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b));
1965                        (transmute(x), y)
1966                    }
1967                }
1968            )*
1969        };
1970        ([idx $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1971            impl_vfae! { [idx $Trait $m] $imm
1972                $b vector_signed_char vector_signed_char
1973                $b vector_unsigned_char vector_unsigned_char
1974                $b vector_bool_char vector_unsigned_char
1975
1976                $h vector_signed_short vector_signed_short
1977                $h vector_unsigned_short vector_unsigned_short
1978                $h vector_bool_short vector_unsigned_short
1979
1980                $f vector_signed_int vector_signed_int
1981                $f vector_unsigned_int vector_unsigned_int
1982                $f vector_bool_int vector_unsigned_int
1983            }
1984        };
1985        ([idx $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident $r:ident)*) => {
1986            $(
1987                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1988                impl $Trait<Self> for $ty {
1989                    type Result = $r;
1990                    #[inline]
1991                    #[target_feature(enable = "vector")]
1992                    unsafe fn $m(self, b: Self) -> Self::Result {
1993                        transmute($fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)))
1994                    }
1995                }
1996            )*
1997        };
1998        ([$Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1999            impl_vfae! { [$Trait $m] $imm
2000                $b vector_signed_char
2001                $b vector_unsigned_char
2002                $b vector_bool_char
2003
2004                $h vector_signed_short
2005                $h vector_unsigned_short
2006                $h vector_bool_short
2007
2008                $f vector_signed_int
2009                $f vector_unsigned_int
2010                $f vector_bool_int
2011            }
2012        };
2013        ([$Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident)*) => {
2014            $(
2015                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2016                impl $Trait<Self> for $ty {
2017                    type Result = t_b!($ty);
2018                    #[inline]
2019                    #[target_feature(enable = "vector")]
2020                    unsafe fn $m(self, b: Self) -> Self::Result {
2021                        transmute($fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)))
2022                    }
2023                }
2024            )*
2025        };
2026    }
2027
2028    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2029    pub trait VectorFindAnyEq<Other> {
2030        type Result;
2031        unsafe fn vec_find_any_eq(self, other: Other) -> Self::Result;
2032    }
2033
2034    impl_vfae! { [VectorFindAnyEq vec_find_any_eq] Eq vfaeb vfaeh vfaef }
2035
2036    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2037    pub trait VectorFindAnyNe<Other> {
2038        type Result;
2039        unsafe fn vec_find_any_ne(self, other: Other) -> Self::Result;
2040    }
2041
2042    impl_vfae! { [VectorFindAnyNe vec_find_any_ne] Ne vfaeb vfaeh vfaef }
2043
2044    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2045    pub trait VectorFindAnyEqOrZeroIdx<Other> {
2046        type Result;
2047        unsafe fn vec_find_any_eq_or_0_idx(self, other: Other) -> Self::Result;
2048    }
2049
2050    impl_vfae! { [idx VectorFindAnyEqOrZeroIdx vec_find_any_eq_or_0_idx] EqIdx
2051        vfaezb vector_signed_char vector_signed_char
2052        vfaezb vector_unsigned_char vector_unsigned_char
2053        vfaezb vector_bool_char vector_unsigned_char
2054
2055        vfaezh vector_signed_short vector_signed_short
2056        vfaezh vector_unsigned_short vector_unsigned_short
2057        vfaezh vector_bool_short vector_unsigned_short
2058
2059        vfaezf vector_signed_int vector_signed_int
2060        vfaezf vector_unsigned_int vector_unsigned_int
2061        vfaezf vector_bool_int vector_unsigned_int
2062    }
2063
2064    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2065    pub trait VectorFindAnyNeOrZeroIdx<Other> {
2066        type Result;
2067        unsafe fn vec_find_any_ne_or_0_idx(self, other: Other) -> Self::Result;
2068    }
2069
2070    impl_vfae! { [idx VectorFindAnyNeOrZeroIdx vec_find_any_ne_or_0_idx] NeIdx
2071        vfaezb vector_signed_char vector_signed_char
2072        vfaezb vector_unsigned_char vector_unsigned_char
2073        vfaezb vector_bool_char vector_unsigned_char
2074
2075        vfaezh vector_signed_short vector_signed_short
2076        vfaezh vector_unsigned_short vector_unsigned_short
2077        vfaezh vector_bool_short vector_unsigned_short
2078
2079        vfaezf vector_signed_int vector_signed_int
2080        vfaezf vector_unsigned_int vector_unsigned_int
2081        vfaezf vector_bool_int vector_unsigned_int
2082    }
2083
2084    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2085    pub trait VectorFindAnyEqIdx<Other> {
2086        type Result;
2087        unsafe fn vec_find_any_eq_idx(self, other: Other) -> Self::Result;
2088    }
2089
2090    impl_vfae! { [idx VectorFindAnyEqIdx vec_find_any_eq_idx] EqIdx vfaeb vfaeh vfaef }
2091
2092    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2093    pub trait VectorFindAnyNeIdx<Other> {
2094        type Result;
2095        unsafe fn vec_find_any_ne_idx(self, other: Other) -> Self::Result;
2096    }
2097
2098    impl_vfae! { [idx VectorFindAnyNeIdx vec_find_any_ne_idx] NeIdx vfaeb vfaeh vfaef }
2099
2100    macro_rules! vfaes_wrapper {
2101        ($($name:ident $ty:ident)*) => {
2102            $(
2103                #[inline]
2104                #[target_feature(enable = "vector")]
2105                #[cfg_attr(test, assert_instr($name, IMM = 0))]
2106                unsafe fn $name<const IMM: i32>(
2107                    a: $ty,
2108                    b: $ty,
2109                ) -> PackedTuple<$ty, i32> {
2110                    super::$name(a, b, IMM)
2111                }
2112            )*
2113        }
2114     }
2115
2116    vfaes_wrapper! {
2117        vfaebs vector_signed_char
2118        vfaehs vector_signed_short
2119        vfaefs vector_signed_int
2120
2121        vfaezbs vector_signed_char
2122        vfaezhs vector_signed_short
2123        vfaezfs vector_signed_int
2124    }
2125
2126    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2127    pub trait VectorFindAnyEqCC<Other> {
2128        type Result;
2129        unsafe fn vec_find_any_eq_cc(self, other: Other) -> (Self::Result, i32);
2130    }
2131
2132    impl_vfae! { [cc VectorFindAnyEqCC vec_find_any_eq_cc] Eq vfaebs vfaehs vfaefs }
2133
2134    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2135    pub trait VectorFindAnyNeCC<Other> {
2136        type Result;
2137        unsafe fn vec_find_any_ne_cc(self, other: Other) -> (Self::Result, i32);
2138    }
2139
2140    impl_vfae! { [cc VectorFindAnyNeCC vec_find_any_ne_cc] Ne vfaebs vfaehs vfaefs }
2141
2142    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2143    pub trait VectorFindAnyEqIdxCC<Other> {
2144        type Result;
2145        unsafe fn vec_find_any_eq_idx_cc(self, other: Other) -> (Self::Result, i32);
2146    }
2147
2148    impl_vfae! { [idx_cc VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc] EqIdx vfaebs vfaehs vfaefs }
2149
2150    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2151    pub trait VectorFindAnyNeIdxCC<Other> {
2152        type Result;
2153        unsafe fn vec_find_any_ne_idx_cc(self, other: Other) -> (Self::Result, i32);
2154    }
2155
2156    impl_vfae! { [idx_cc VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc] NeIdx vfaebs vfaehs vfaefs }
2157
2158    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2159    pub trait VectorFindAnyEqOrZeroIdxCC<Other> {
2160        type Result;
2161        unsafe fn vec_find_any_eq_or_0_idx_cc(self, other: Other) -> (Self::Result, i32);
2162    }
2163
2164    impl_vfae! { [idx_cc VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc] EqIdx vfaezbs vfaezhs vfaezfs }
2165
2166    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2167    pub trait VectorFindAnyNeOrZeroIdxCC<Other> {
2168        type Result;
2169        unsafe fn vec_find_any_ne_or_0_idx_cc(self, other: Other) -> (Self::Result, i32);
2170    }
2171
2172    impl_vfae! { [idx_cc VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc] NeIdx vfaezbs vfaezhs vfaezfs }
2173
2174    #[inline]
2175    #[target_feature(enable = "vector")]
2176    #[cfg_attr(test, assert_instr(vl))]
2177    unsafe fn test_vector_load(offset: isize, ptr: *const i32) -> vector_signed_int {
2178        ptr.byte_offset(offset)
2179            .cast::<vector_signed_int>()
2180            .read_unaligned()
2181    }
2182
2183    #[inline]
2184    #[target_feature(enable = "vector")]
2185    #[cfg_attr(test, assert_instr(vst))]
2186    unsafe fn test_vector_store(vector: vector_signed_int, offset: isize, ptr: *mut i32) {
2187        ptr.byte_offset(offset)
2188            .cast::<vector_signed_int>()
2189            .write_unaligned(vector)
2190    }
2191
2192    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2193    pub trait VectorLoad: Sized {
2194        type ElementType;
2195
2196        #[inline]
2197        #[target_feature(enable = "vector")]
2198        unsafe fn vec_xl(offset: isize, ptr: *const Self::ElementType) -> Self {
2199            ptr.byte_offset(offset).cast::<Self>().read_unaligned()
2200        }
2201
2202        unsafe fn vec_load_len(ptr: *const Self::ElementType, byte_count: u32) -> Self;
2203
2204        unsafe fn vec_load_bndry<const BLOCK_BOUNDARY: u16>(
2205            ptr: *const Self::ElementType,
2206        ) -> MaybeUninit<Self>;
2207    }
2208
2209    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2210    pub trait VectorStore: Sized {
2211        type ElementType;
2212
2213        #[inline]
2214        #[target_feature(enable = "vector")]
2215        unsafe fn vec_xst(self, offset: isize, ptr: *mut Self::ElementType) {
2216            ptr.byte_offset(offset).cast::<Self>().write_unaligned(self)
2217        }
2218
2219        unsafe fn vec_store_len(self, ptr: *mut Self::ElementType, byte_count: u32);
2220    }
2221
2222    macro_rules! impl_load_store {
2223        ($($ty:ident)*) => {
2224            $(
2225                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2226                impl VectorLoad for t_t_l!($ty) {
2227                    type ElementType = $ty;
2228
2229                    #[inline]
2230                    #[target_feature(enable = "vector")]
2231                    unsafe fn vec_load_len(ptr: *const Self::ElementType, byte_count: u32) -> Self {
2232                        transmute(vll( byte_count, ptr.cast(),))
2233                    }
2234
2235                    #[inline]
2236                    #[target_feature(enable = "vector")]
2237                    unsafe fn vec_load_bndry<const BLOCK_BOUNDARY: u16>(ptr: *const Self::ElementType) -> MaybeUninit<Self> {
2238                        transmute(vlbb(ptr.cast(), const { validate_block_boundary(BLOCK_BOUNDARY) }))
2239                    }
2240
2241                }
2242
2243                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2244                impl VectorStore for t_t_l!($ty) {
2245                    type ElementType = $ty;
2246
2247                    #[inline]
2248                    #[target_feature(enable = "vector")]
2249                    unsafe fn vec_store_len(self, ptr: *mut Self::ElementType, byte_count: u32) {
2250                        vstl(transmute(self), byte_count, ptr.cast())
2251                    }
2252                }
2253            )*
2254        }
2255    }
2256
2257    impl_load_store! { i8 u8 i16 u16 i32 u32 i64 u64 f32 f64 }
2258
2259    #[inline]
2260    #[target_feature(enable = "vector")]
2261    #[cfg_attr(test, assert_instr(vll))]
2262    unsafe fn test_vec_load_len(ptr: *const i32, byte_count: u32) -> vector_signed_int {
2263        vector_signed_int::vec_load_len(ptr, byte_count)
2264    }
2265
2266    #[inline]
2267    #[target_feature(enable = "vector")]
2268    #[cfg_attr(test, assert_instr(vlbb))]
2269    unsafe fn test_vec_load_bndry(ptr: *const i32) -> MaybeUninit<vector_signed_int> {
2270        vector_signed_int::vec_load_bndry::<512>(ptr)
2271    }
2272
2273    #[inline]
2274    #[target_feature(enable = "vector")]
2275    #[cfg_attr(test, assert_instr(vstl))]
2276    unsafe fn test_vec_store_len(vector: vector_signed_int, ptr: *mut i32, byte_count: u32) {
2277        vector.vec_store_len(ptr, byte_count)
2278    }
2279
2280    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2281    pub trait VectorLoadPair: Sized {
2282        type ElementType;
2283
2284        unsafe fn vec_load_pair(a: Self::ElementType, b: Self::ElementType) -> Self;
2285    }
2286
2287    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2288    impl VectorLoadPair for vector_signed_long_long {
2289        type ElementType = i64;
2290
2291        #[inline]
2292        #[target_feature(enable = "vector")]
2293        unsafe fn vec_load_pair(a: i64, b: i64) -> Self {
2294            vector_signed_long_long([a, b])
2295        }
2296    }
2297
2298    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2299    impl VectorLoadPair for vector_unsigned_long_long {
2300        type ElementType = u64;
2301
2302        #[inline]
2303        #[target_feature(enable = "vector")]
2304        unsafe fn vec_load_pair(a: u64, b: u64) -> Self {
2305            vector_unsigned_long_long([a, b])
2306        }
2307    }
2308
2309    #[inline]
2310    #[target_feature(enable = "vector")]
2311    unsafe fn pack<T, const N: usize>(a: T, b: T) -> T {
2312        simd_shuffle(a, b, const { ShuffleMask::<N>::pack() })
2313    }
2314
2315    #[inline]
2316    #[target_feature(enable = "vector")]
2317    #[cfg_attr(test, assert_instr(vpkh))]
2318    unsafe fn vpkh(a: i16x8, b: i16x8) -> i8x16 {
2319        let a: i8x16 = transmute(a);
2320        let b: i8x16 = transmute(b);
2321        simd_shuffle(a, b, const { ShuffleMask::<16>::pack() })
2322    }
2323    #[inline]
2324    #[target_feature(enable = "vector")]
2325    #[cfg_attr(test, assert_instr(vpkf))]
2326    unsafe fn vpkf(a: i32x4, b: i32x4) -> i16x8 {
2327        let a: i16x8 = transmute(a);
2328        let b: i16x8 = transmute(b);
2329        simd_shuffle(a, b, const { ShuffleMask::<8>::pack() })
2330    }
2331    #[inline]
2332    #[target_feature(enable = "vector")]
2333    #[cfg_attr(test, assert_instr(vpkg))]
2334    unsafe fn vpkg(a: i64x2, b: i64x2) -> i32x4 {
2335        let a: i32x4 = transmute(a);
2336        let b: i32x4 = transmute(b);
2337        simd_shuffle(a, b, const { ShuffleMask::<4>::pack() })
2338    }
2339
2340    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2341    pub trait VectorPack<Other> {
2342        type Result;
2343        unsafe fn vec_pack(self, b: Other) -> Self::Result;
2344    }
2345
2346    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_signed_short, vector_signed_short) -> vector_signed_char }
2347    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2348    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_bool_short, vector_bool_short) -> vector_bool_char }
2349    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_signed_int, vector_signed_int) -> vector_signed_short }
2350    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2351    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_bool_int, vector_bool_int) -> vector_bool_short }
2352    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_signed_long_long, vector_signed_long_long) -> vector_signed_int }
2353    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2354    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_bool_long_long, vector_bool_long_long) -> vector_bool_int }
2355
2356    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2357    pub trait VectorPacks<Other> {
2358        type Result;
2359        unsafe fn vec_packs(self, b: Other) -> Self::Result;
2360    }
2361
2362    impl_vec_trait! { [VectorPacks vec_packs] vpksh (vector_signed_short, vector_signed_short) -> vector_signed_char }
2363    impl_vec_trait! { [VectorPacks vec_packs] vpklsh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2364    impl_vec_trait! { [VectorPacks vec_packs] vpksf (vector_signed_int, vector_signed_int) -> vector_signed_short }
2365    impl_vec_trait! { [VectorPacks vec_packs] vpklsf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2366    impl_vec_trait! { [VectorPacks vec_packs] vpksg (vector_signed_long_long, vector_signed_long_long) -> vector_signed_int }
2367    impl_vec_trait! { [VectorPacks vec_packs] vpklsg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2368
2369    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2370    pub trait VectorPacksu<Other> {
2371        type Result;
2372        unsafe fn vec_packsu(self, b: Other) -> Self::Result;
2373    }
2374
2375    unsafe fn simd_smax<T: Copy>(a: T, b: T) -> T {
2376        simd_select::<T, T>(simd_gt::<T, T>(a, b), a, b)
2377    }
2378
2379    #[inline]
2380    #[target_feature(enable = "vector")]
2381    #[cfg_attr(test, assert_instr(vpklsh))]
2382    unsafe fn vpacksuh(a: vector_signed_short, b: vector_signed_short) -> vector_unsigned_char {
2383        vpklsh(
2384            simd_smax(a, vector_signed_short([0; 8])),
2385            simd_smax(b, vector_signed_short([0; 8])),
2386        )
2387    }
2388    #[inline]
2389    #[target_feature(enable = "vector")]
2390    #[cfg_attr(test, assert_instr(vpklsf))]
2391    unsafe fn vpacksuf(a: vector_signed_int, b: vector_signed_int) -> vector_unsigned_short {
2392        vpklsf(
2393            simd_smax(a, vector_signed_int([0; 4])),
2394            simd_smax(b, vector_signed_int([0; 4])),
2395        )
2396    }
2397    #[inline]
2398    #[target_feature(enable = "vector")]
2399    #[cfg_attr(test, assert_instr(vpklsg))]
2400    unsafe fn vpacksug(
2401        a: vector_signed_long_long,
2402        b: vector_signed_long_long,
2403    ) -> vector_unsigned_int {
2404        vpklsg(
2405            simd_smax(a, vector_signed_long_long([0; 2])),
2406            simd_smax(b, vector_signed_long_long([0; 2])),
2407        )
2408    }
2409
2410    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksuh (vector_signed_short, vector_signed_short) -> vector_unsigned_char }
2411    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2412    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksuf (vector_signed_int, vector_signed_int) -> vector_unsigned_short }
2413    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2414    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksug (vector_signed_long_long, vector_signed_long_long) -> vector_unsigned_int }
2415    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2416
2417    macro_rules! impl_vector_packs_cc {
2418        ($($intr:ident $ty:ident $outty:ident)*) => {
2419            $(
2420                #[inline]
2421                #[target_feature(enable = "vector")]
2422                #[cfg_attr(test, assert_instr($intr))]
2423                unsafe fn $intr(
2424                    a: $ty,
2425                    b: $ty,
2426                ) -> ($outty, i32) {
2427                    let PackedTuple { x, y } = super::$intr(a, b);
2428                    (x, y)
2429                }
2430
2431                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2432                impl VectorPacksCC for $ty {
2433                    type Result = $outty;
2434
2435                    #[inline]
2436                    #[target_feature(enable = "vector")]
2437                    unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32) {
2438                        $intr(self, b)
2439                    }
2440                }
2441            )*
2442        }
2443    }
2444
2445    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2446    pub trait VectorPacksCC {
2447        type Result;
2448        unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32);
2449    }
2450
2451    impl_vector_packs_cc! {
2452        vpkshs vector_signed_short vector_signed_char
2453        vpklshs vector_unsigned_short vector_unsigned_char
2454        vpksfs vector_signed_int vector_signed_short
2455        vpklsfs vector_unsigned_int vector_unsigned_short
2456        vpksgs vector_signed_long_long vector_signed_int
2457        vpklsgs vector_unsigned_long_long vector_unsigned_int
2458    }
2459
2460    macro_rules! impl_vector_packsu_cc {
2461        ($($intr:ident $ty:ident $outty:ident)*) => {
2462            $(
2463                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2464                impl VectorPacksuCC for $ty {
2465                    type Result = $outty;
2466
2467                    #[inline]
2468                    #[target_feature(enable = "vector")]
2469                    unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32) {
2470                        $intr(self, b)
2471                    }
2472                }
2473            )*
2474        }
2475    }
2476
2477    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2478    pub trait VectorPacksuCC {
2479        type Result;
2480        unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32);
2481    }
2482
2483    impl_vector_packsu_cc! {
2484        vpklshs vector_unsigned_short vector_unsigned_char
2485        vpklsfs vector_unsigned_int vector_unsigned_short
2486        vpklsgs vector_unsigned_long_long vector_unsigned_int
2487    }
2488
2489    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2490    pub trait VectorMadd {
2491        unsafe fn vec_madd(self, b: Self, c: Self) -> Self;
2492        unsafe fn vec_msub(self, b: Self, c: Self) -> Self;
2493    }
2494
2495    test_impl! { vfmasb (a: vector_float, b: vector_float, c: vector_float) -> vector_float [simd_fma, "vector-enhancements-1" vfmasb] }
2496    test_impl! { vfmadb (a: vector_double, b: vector_double, c: vector_double) -> vector_double [simd_fma, vfmadb] }
2497
2498    #[inline]
2499    unsafe fn simd_fms<T>(a: T, b: T, c: T) -> T {
2500        simd_fma(a, b, simd_neg(c))
2501    }
2502
2503    test_impl! { vfmssb (a: vector_float, b: vector_float, c: vector_float) -> vector_float [simd_fms, "vector-enhancements-1" vfmssb] }
2504    test_impl! { vfmsdb (a: vector_double, b: vector_double, c: vector_double) -> vector_double [simd_fms, vfmsdb] }
2505
2506    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2507    impl VectorMadd for vector_float {
2508        #[inline]
2509        #[target_feature(enable = "vector")]
2510        unsafe fn vec_madd(self, b: Self, c: Self) -> Self {
2511            vfmasb(self, b, c)
2512        }
2513
2514        #[inline]
2515        #[target_feature(enable = "vector")]
2516        unsafe fn vec_msub(self, b: Self, c: Self) -> Self {
2517            vfmssb(self, b, c)
2518        }
2519    }
2520
2521    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2522    impl VectorMadd for vector_double {
2523        #[inline]
2524        #[target_feature(enable = "vector")]
2525        unsafe fn vec_madd(self, b: Self, c: Self) -> Self {
2526            vfmadb(self, b, c)
2527        }
2528
2529        #[inline]
2530        #[target_feature(enable = "vector")]
2531        unsafe fn vec_msub(self, b: Self, c: Self) -> Self {
2532            vfmsdb(self, b, c)
2533        }
2534    }
2535
2536    macro_rules! impl_vec_unpack {
2537        ($mask:ident $instr:ident $src:ident $shuffled:ident $dst:ident $width:literal) => {
2538            #[inline]
2539            #[target_feature(enable = "vector")]
2540            #[cfg_attr(test, assert_instr($instr))]
2541            unsafe fn $instr(a: $src) -> $dst {
2542                simd_as(simd_shuffle::<_, _, $shuffled>(
2543                    a,
2544                    a,
2545                    const { ShuffleMask::<$width>::$mask() },
2546                ))
2547            }
2548        };
2549    }
2550
2551    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2552    pub trait VectorUnpackh {
2553        type Result;
2554        unsafe fn vec_unpackh(self) -> Self::Result;
2555    }
2556
2557    impl_vec_unpack!(unpack_high vuphb vector_signed_char i8x8 vector_signed_short 8);
2558    impl_vec_unpack!(unpack_high vuphh vector_signed_short i16x4 vector_signed_int 4);
2559    impl_vec_unpack!(unpack_high vuphf vector_signed_int i32x2 vector_signed_long_long 2);
2560
2561    impl_vec_unpack!(unpack_high vuplhb vector_unsigned_char u8x8 vector_unsigned_short 8);
2562    impl_vec_unpack!(unpack_high vuplhh vector_unsigned_short u16x4 vector_unsigned_int 4);
2563    impl_vec_unpack!(unpack_high vuplhf vector_unsigned_int u32x2 vector_unsigned_long_long 2);
2564
2565    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphb (vector_signed_char) -> vector_signed_short}
2566    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphh (vector_signed_short) -> vector_signed_int}
2567    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphf (vector_signed_int) -> vector_signed_long_long}
2568
2569    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhb (vector_unsigned_char) -> vector_unsigned_short}
2570    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhh (vector_unsigned_short) -> vector_unsigned_int}
2571    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhf (vector_unsigned_int) -> vector_unsigned_long_long}
2572
2573    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhb (vector_bool_char) -> vector_bool_short}
2574    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhh (vector_bool_short) -> vector_bool_int}
2575    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhf (vector_bool_int) -> vector_bool_long_long}
2576
2577    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2578    pub trait VectorUnpackl {
2579        type Result;
2580        unsafe fn vec_unpackl(self) -> Self::Result;
2581    }
2582
2583    // FIXME(llvm): a shuffle + simd_as does not currently optimize into a single instruction like
2584    // unpachk above. Tracked in https://github.com/llvm/llvm-project/issues/129576.
2585
2586    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplb (vector_signed_char) -> vector_signed_short}
2587    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplhw (vector_signed_short) -> vector_signed_int}
2588    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplf (vector_signed_int) -> vector_signed_long_long}
2589
2590    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllb (vector_unsigned_char) -> vector_unsigned_short}
2591    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllh (vector_unsigned_short) -> vector_unsigned_int}
2592    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllf (vector_unsigned_int) -> vector_unsigned_long_long}
2593
2594    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllb (vector_bool_char) -> vector_bool_short}
2595    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllh (vector_bool_short) -> vector_bool_int}
2596    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllf (vector_bool_int) -> vector_bool_long_long}
2597
2598    test_impl! { vec_vavgb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [ vavgb, vavgb ] }
2599    test_impl! { vec_vavgh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [ vavgh, vavgh ] }
2600    test_impl! { vec_vavgf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [ vavgf, vavgf ] }
2601    test_impl! { vec_vavgg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [ vavgg, vavgg ] }
2602
2603    test_impl! { vec_vavglb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [ vavglb, vavglb ] }
2604    test_impl! { vec_vavglh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [ vavglh, vavglh ] }
2605    test_impl! { vec_vavglf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [ vavglf, vavglf ] }
2606    test_impl! { vec_vavglg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [ vavglg, vavglg ] }
2607
2608    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2609    pub trait VectorAvg<Other> {
2610        type Result;
2611        unsafe fn vec_avg(self, b: Other) -> Self::Result;
2612    }
2613
2614    impl_vec_trait! { [VectorAvg vec_avg] 2 (vec_vavglb, vec_vavgb, vec_vavglh, vec_vavgh, vec_vavglf, vec_vavgf, vec_vavglg, vec_vavgg) }
2615
2616    macro_rules! impl_mul {
2617        ([$Trait:ident $m:ident] $fun:ident ($a:ty, $b:ty) -> $r:ty) => {
2618            #[unstable(feature = "stdarch_s390x", issue = "135681")]
2619            impl $Trait<$r> for $a {
2620                #[inline]
2621                #[target_feature(enable = "vector")]
2622                unsafe fn $m(self, b: $b) -> $r {
2623                    $fun(transmute(self), transmute(b))
2624                }
2625            }
2626        };
2627        ([$Trait:ident $m:ident] $fun:ident ($a:ty, $b:ty, $c:ty) -> $r:ty) => {
2628            #[unstable(feature = "stdarch_s390x", issue = "135681")]
2629            impl $Trait for $a {
2630                type Result = $r;
2631                #[inline]
2632                #[target_feature(enable = "vector")]
2633                unsafe fn $m(self, b: $b, c: $c) -> $r {
2634                    $fun(self, b, c)
2635                }
2636            }
2637        };
2638    }
2639
2640    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2641    pub trait VectorMule<Result> {
2642        unsafe fn vec_mule(self, b: Self) -> Result;
2643    }
2644
2645    // FIXME(llvm) sadly this does not yet work https://github.com/llvm/llvm-project/issues/129705
2646    //    #[target_feature(enable = "vector")]
2647    //    #[cfg_attr(test, assert_instr(vmleh))]
2648    //    unsafe fn vec_vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int {
2649    //        let even_a: vector_unsigned_int = simd_as(simd_shuffle::<_, _, u16x4>(
2650    //            a,
2651    //            a,
2652    //            const { ShuffleMask([0, 2, 4, 6]) },
2653    //        ));
2654    //
2655    //        let even_b: vector_unsigned_int = simd_as(simd_shuffle::<_, _, u16x4>(
2656    //            b,
2657    //            b,
2658    //            const { ShuffleMask([0, 2, 4, 6]) },
2659    //        ));
2660    //
2661    //        simd_mul(even_a, even_b)
2662    //    }
2663
2664    test_impl! { vec_vmeb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short [ vmeb, vmeb ] }
2665    test_impl! { vec_vmeh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int[ vmeh, vmeh ] }
2666    test_impl! { vec_vmef(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long [ vmef, vmef ] }
2667
2668    test_impl! { vec_vmleb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vmleb, vmleb ] }
2669    test_impl! { vec_vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vmleh, vmleh ] }
2670    test_impl! { vec_vmlef(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vmlef, vmlef ] }
2671
2672    impl_mul!([VectorMule vec_mule] vec_vmeb (vector_signed_char, vector_signed_char) -> vector_signed_short );
2673    impl_mul!([VectorMule vec_mule] vec_vmeh (vector_signed_short, vector_signed_short) -> vector_signed_int);
2674    impl_mul!([VectorMule vec_mule] vec_vmef (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
2675
2676    impl_mul!([VectorMule vec_mule] vec_vmleb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2677    impl_mul!([VectorMule vec_mule] vec_vmleh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2678    impl_mul!([VectorMule vec_mule] vec_vmlef (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2679
2680    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2681    pub trait VectorMulo<Result> {
2682        unsafe fn vec_mulo(self, b: Self) -> Result;
2683    }
2684
2685    test_impl! { vec_vmob(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short [ vmob, vmob ] }
2686    test_impl! { vec_vmoh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int[ vmoh, vmoh ] }
2687    test_impl! { vec_vmof(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long [ vmof, vmof ] }
2688
2689    test_impl! { vec_vmlob(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vmlob, vmlob ] }
2690    test_impl! { vec_vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vmloh, vmloh ] }
2691    test_impl! { vec_vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vmlof, vmlof ] }
2692
2693    impl_mul!([VectorMulo vec_mulo] vec_vmob (vector_signed_char, vector_signed_char) -> vector_signed_short );
2694    impl_mul!([VectorMulo vec_mulo] vec_vmoh (vector_signed_short, vector_signed_short) -> vector_signed_int);
2695    impl_mul!([VectorMulo vec_mulo] vec_vmof (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
2696
2697    impl_mul!([VectorMulo vec_mulo] vec_vmlob (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2698    impl_mul!([VectorMulo vec_mulo] vec_vmloh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2699    impl_mul!([VectorMulo vec_mulo] vec_vmlof (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2700
2701    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2702    pub trait VectorMulh<Result> {
2703        unsafe fn vec_mulh(self, b: Self) -> Result;
2704    }
2705
2706    test_impl! { vec_vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [ vmhb, vmhb ] }
2707    test_impl! { vec_vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [ vmhh, vmhh ] }
2708    test_impl! { vec_vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [ vmhf, vmhf ] }
2709
2710    test_impl! { vec_vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [ vmlhb, vmlhb ] }
2711    test_impl! { vec_vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [ vmlhh, vmlhh ] }
2712    test_impl! { vec_vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [ vmlhf, vmlhf ] }
2713
2714    impl_mul!([VectorMulh vec_mulh] vec_vmhb (vector_signed_char, vector_signed_char) -> vector_signed_char);
2715    impl_mul!([VectorMulh vec_mulh] vec_vmhh (vector_signed_short, vector_signed_short) -> vector_signed_short);
2716    impl_mul!([VectorMulh vec_mulh] vec_vmhf (vector_signed_int, vector_signed_int) -> vector_signed_int);
2717
2718    impl_mul!([VectorMulh vec_mulh] vec_vmlhb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char);
2719    impl_mul!([VectorMulh vec_mulh] vec_vmlhh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2720    impl_mul!([VectorMulh vec_mulh] vec_vmlhf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int);
2721
2722    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2723    pub trait VectorMeadd {
2724        type Result;
2725        unsafe fn vec_meadd(self, b: Self, c: Self::Result) -> Self::Result;
2726    }
2727
2728    test_impl! { vec_vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaeb, vmaeb ] }
2729    test_impl! { vec_vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaeh, vmaeh ] }
2730    test_impl! { vec_vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaef, vmaef ] }
2731
2732    test_impl! { vec_vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmaleb, vmaleb ] }
2733    test_impl! { vec_vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaleh, vmaleh ] }
2734    test_impl! { vec_vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalef, vmalef ] }
2735
2736    impl_mul!([VectorMeadd vec_meadd] vec_vmaeb (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short );
2737    impl_mul!([VectorMeadd vec_meadd] vec_vmaeh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int);
2738    impl_mul!([VectorMeadd vec_meadd] vec_vmaef (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long );
2739
2740    impl_mul!([VectorMeadd vec_meadd] vec_vmaleb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short );
2741    impl_mul!([VectorMeadd vec_meadd] vec_vmaleh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int);
2742    impl_mul!([VectorMeadd vec_meadd] vec_vmalef (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long );
2743
2744    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2745    pub trait VectorMoadd {
2746        type Result;
2747        unsafe fn vec_moadd(self, b: Self, c: Self::Result) -> Self::Result;
2748    }
2749
2750    test_impl! { vec_vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaob, vmaob ] }
2751    test_impl! { vec_vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaoh, vmaoh ] }
2752    test_impl! { vec_vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaof, vmaof ] }
2753
2754    test_impl! { vec_vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmalob, vmalob ] }
2755    test_impl! { vec_vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaloh, vmaloh ] }
2756    test_impl! { vec_vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalof, vmalof ] }
2757
2758    impl_mul!([VectorMoadd vec_moadd] vec_vmaob (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short );
2759    impl_mul!([VectorMoadd vec_moadd] vec_vmaoh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int);
2760    impl_mul!([VectorMoadd vec_moadd] vec_vmaof (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long );
2761
2762    impl_mul!([VectorMoadd vec_moadd] vec_vmalob (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short );
2763    impl_mul!([VectorMoadd vec_moadd] vec_vmaloh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int);
2764    impl_mul!([VectorMoadd vec_moadd] vec_vmalof (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long );
2765
2766    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2767    pub trait VectorMhadd {
2768        type Result;
2769        unsafe fn vec_mhadd(self, b: Self, c: Self::Result) -> Self::Result;
2770    }
2771
2772    test_impl! { vec_vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [ vmahb, vmahb ] }
2773    test_impl! { vec_vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[ vmahh, vmahh ] }
2774    test_impl! { vec_vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [ vmahf, vmahf ] }
2775
2776    test_impl! { vec_vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [ vmalhb, vmalhb ] }
2777    test_impl! { vec_vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[ vmalhh, vmalhh ] }
2778    test_impl! { vec_vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [ vmalhf, vmalhf ] }
2779
2780    impl_mul!([VectorMhadd vec_mhadd] vec_vmahb (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char );
2781    impl_mul!([VectorMhadd vec_mhadd] vec_vmahh (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short);
2782    impl_mul!([VectorMhadd vec_mhadd] vec_vmahf (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int );
2783
2784    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char );
2785    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2786    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int );
2787
2788    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2789    pub trait VectorMladd {
2790        type Result;
2791        unsafe fn vec_mladd(self, b: Self, c: Self::Result) -> Self::Result;
2792    }
2793
2794    #[inline]
2795    #[target_feature(enable = "vector")]
2796    unsafe fn simd_mladd<T>(a: T, b: T, c: T) -> T {
2797        simd_add(simd_mul(a, b), c)
2798    }
2799
2800    test_impl! { vec_vmal_ib(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [simd_mladd, vmalb ] }
2801    test_impl! { vec_vmal_ih(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[simd_mladd, vmalhw ] }
2802    test_impl! { vec_vmal_if(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [simd_mladd, vmalf ] }
2803
2804    test_impl! { vec_vmal_ub(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [simd_mladd, vmalb ] }
2805    test_impl! { vec_vmal_uh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[simd_mladd, vmalhw ] }
2806    test_impl! { vec_vmal_uf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [simd_mladd, vmalf ] }
2807
2808    impl_mul!([VectorMladd vec_mladd] vec_vmal_ib (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char );
2809    impl_mul!([VectorMladd vec_mladd] vec_vmal_ih (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short);
2810    impl_mul!([VectorMladd vec_mladd] vec_vmal_if (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int );
2811
2812    impl_mul!([VectorMladd vec_mladd] vec_vmal_ub (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char );
2813    impl_mul!([VectorMladd vec_mladd] vec_vmal_uh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2814    impl_mul!([VectorMladd vec_mladd] vec_vmal_uf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int );
2815
2816    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2817    pub trait VectorGfmsum<Result> {
2818        unsafe fn vec_gfmsum(self, b: Self) -> Result;
2819    }
2820
2821    test_impl! { vec_vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vgfmb, vgfmb ] }
2822    test_impl! { vec_vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vgfmh, vgfmh] }
2823    test_impl! { vec_vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vgfmf, vgfmf ] }
2824
2825    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2826    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2827    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2828
2829    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2830    pub trait VectorGfmsumAccum {
2831        type Result;
2832        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result;
2833    }
2834
2835    test_impl! { vec_vgfmab(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vgfmab, vgfmab ] }
2836    test_impl! { vec_vgfmah(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vgfmah, vgfmah] }
2837    test_impl! { vec_vgfmaf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vgfmaf, vgfmaf ] }
2838
2839    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2840    impl VectorGfmsumAccum for vector_unsigned_char {
2841        type Result = vector_unsigned_short;
2842        #[inline]
2843        #[target_feature(enable = "vector")]
2844        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2845            vec_vgfmab(self, b, c)
2846        }
2847    }
2848    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2849    impl VectorGfmsumAccum for vector_unsigned_short {
2850        type Result = vector_unsigned_int;
2851        #[inline]
2852        #[target_feature(enable = "vector")]
2853        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2854            vec_vgfmah(self, b, c)
2855        }
2856    }
2857    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2858    impl VectorGfmsumAccum for vector_unsigned_int {
2859        type Result = vector_unsigned_long_long;
2860        #[inline]
2861        #[target_feature(enable = "vector")]
2862        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2863            vec_vgfmaf(self, b, c)
2864        }
2865    }
2866
2867    #[inline]
2868    #[target_feature(enable = "vector")]
2869    #[cfg_attr(test, assert_instr(vgef, D = 3))]
2870    unsafe fn vgef<const D: u32>(
2871        a: vector_unsigned_int,
2872        b: vector_unsigned_int,
2873        c: *const u32,
2874    ) -> vector_unsigned_int {
2875        static_assert_uimm_bits!(D, 2);
2876        let offset: u32 = simd_extract(b, D);
2877        let ptr = c.byte_add(offset as usize);
2878        let value = ptr.read();
2879        simd_insert(a, D, value)
2880    }
2881
2882    #[inline]
2883    #[target_feature(enable = "vector")]
2884    #[cfg_attr(test, assert_instr(vgeg, D = 1))]
2885    unsafe fn vgeg<const D: u32>(
2886        a: vector_unsigned_long_long,
2887        b: vector_unsigned_long_long,
2888        c: *const u64,
2889    ) -> vector_unsigned_long_long {
2890        static_assert_uimm_bits!(D, 1);
2891        let offset: u64 = simd_extract(b, D);
2892        let ptr = c.byte_add(offset as usize);
2893        let value = ptr.read();
2894        simd_insert(a, D, value)
2895    }
2896
2897    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2898    pub trait VectorGatherElement {
2899        type Element;
2900        type Offset;
2901        unsafe fn vec_gather_element<const D: u32>(
2902            self,
2903            b: Self::Offset,
2904            c: *const Self::Element,
2905        ) -> Self;
2906    }
2907
2908    macro_rules! impl_vec_gather_element {
2909        ($($instr:ident $ty:ident)*) => {
2910            $(
2911                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2912                impl VectorGatherElement for $ty {
2913                    type Element = l_t_t!($ty);
2914                    type Offset = t_u!($ty);
2915
2916                    #[inline]
2917                    #[target_feature(enable = "vector")]
2918                    unsafe fn vec_gather_element<const D: u32>(self, b: Self::Offset, c: *const Self::Element) -> Self {
2919                        transmute($instr::<D>(transmute(self), b, c.cast()))
2920                    }
2921                }
2922            )*
2923        }
2924    }
2925
2926    impl_vec_gather_element! {
2927        vgef vector_signed_int
2928        vgef vector_bool_int
2929        vgef vector_unsigned_int
2930
2931        vgeg vector_signed_long_long
2932        vgeg vector_bool_long_long
2933        vgeg vector_unsigned_long_long
2934
2935        vgef vector_float
2936        vgeg vector_double
2937    }
2938
2939    #[inline]
2940    #[target_feature(enable = "vector")]
2941    #[cfg_attr(test, assert_instr(vscef, D = 3))]
2942    unsafe fn vscef<const D: u32>(a: vector_unsigned_int, b: vector_unsigned_int, c: *mut u32) {
2943        static_assert_uimm_bits!(D, 2);
2944        let value = simd_extract(a, D);
2945        let offset: u32 = simd_extract(b, D);
2946        let ptr = c.byte_add(offset as usize);
2947        ptr.write(value);
2948    }
2949
2950    #[inline]
2951    #[target_feature(enable = "vector")]
2952    #[cfg_attr(test, assert_instr(vsceg, D = 1))]
2953    unsafe fn vsceg<const D: u32>(
2954        a: vector_unsigned_long_long,
2955        b: vector_unsigned_long_long,
2956        c: *mut u64,
2957    ) {
2958        static_assert_uimm_bits!(D, 1);
2959        let value = simd_extract(a, D);
2960        let offset: u64 = simd_extract(b, D);
2961        let ptr = c.byte_add(offset as usize);
2962        ptr.write(value);
2963    }
2964
2965    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2966    pub trait VectorScatterElement {
2967        type Element;
2968        type Offset;
2969        unsafe fn vec_scatter_element<const D: u32>(self, b: Self::Offset, c: *mut Self::Element);
2970    }
2971
2972    macro_rules! impl_vec_scatter_element {
2973        ($($instr:ident $ty:ident)*) => {
2974            $(
2975                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2976                impl VectorScatterElement for $ty {
2977                    type Element = l_t_t!($ty);
2978                    type Offset = t_u!($ty);
2979
2980                    #[inline]
2981                    #[target_feature(enable = "vector")]
2982                    unsafe fn vec_scatter_element<const D: u32>(self, b: Self::Offset, c: *mut Self::Element) {
2983                        $instr::<D>(transmute(self), b, c.cast())
2984                    }
2985                }
2986            )*
2987        }
2988    }
2989
2990    impl_vec_scatter_element! {
2991        vscef vector_signed_int
2992        vscef vector_bool_int
2993        vscef vector_unsigned_int
2994
2995        vsceg vector_signed_long_long
2996        vsceg vector_bool_long_long
2997        vsceg vector_unsigned_long_long
2998
2999        vscef vector_float
3000        vsceg vector_double
3001    }
3002
3003    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3004    pub trait VectorSel<Mask>: Sized {
3005        unsafe fn vec_sel(self, b: Self, c: Mask) -> Self;
3006    }
3007
3008    macro_rules! impl_vec_sel {
3009        ($($ty:ident)*) => {
3010            $(
3011                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3012                impl VectorSel<t_u!($ty)> for $ty {
3013                    #[inline]
3014                    #[target_feature(enable = "vector")]
3015                    unsafe fn vec_sel(self, b: Self, c: t_u!($ty)) -> Self {
3016                        let b = simd_and(transmute(b), c);
3017                        let a = simd_and(transmute(self), simd_xor(c, transmute(vector_signed_char([!0; 16]))));
3018                        transmute(simd_or(a, b))
3019                    }
3020                }
3021
3022                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3023                impl VectorSel<t_b!($ty)> for $ty {
3024                    #[inline]
3025                    #[target_feature(enable = "vector")]
3026                    unsafe fn vec_sel(self, b: Self, c: t_b!($ty)) -> Self {
3027                        // defer to the implementation with an unsigned mask
3028                        self.vec_sel(b, transmute::<_, t_u!($ty)>(c))
3029                    }
3030                }
3031            )*
3032        }
3033    }
3034
3035    impl_vec_sel! {
3036        vector_signed_char
3037        vector_signed_short
3038        vector_signed_int
3039        vector_signed_long_long
3040
3041        vector_unsigned_char
3042        vector_unsigned_short
3043        vector_unsigned_int
3044        vector_unsigned_long_long
3045
3046        vector_bool_char
3047        vector_bool_short
3048        vector_bool_int
3049        vector_bool_long_long
3050
3051        vector_float
3052        vector_double
3053    }
3054
3055    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3056    pub trait VectorFpTestDataClass {
3057        type Result;
3058        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32);
3059    }
3060
3061    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3062    impl VectorFpTestDataClass for vector_float {
3063        type Result = vector_bool_int;
3064
3065        #[inline]
3066        #[target_feature(enable = "vector")]
3067        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32) {
3068            let PackedTuple { x, y } = vftcisb(self, CLASS);
3069            (x, y)
3070        }
3071    }
3072
3073    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3074    impl VectorFpTestDataClass for vector_double {
3075        type Result = vector_bool_long_long;
3076
3077        #[inline]
3078        #[target_feature(enable = "vector")]
3079        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32) {
3080            let PackedTuple { x, y } = vftcidb(self, CLASS);
3081            (x, y)
3082        }
3083    }
3084
3085    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3086    pub trait VectorCompare {
3087        unsafe fn vec_all_lt(self, other: Self) -> i32;
3088        unsafe fn vec_all_le(self, other: Self) -> i32;
3089        unsafe fn vec_all_gt(self, other: Self) -> i32;
3090        unsafe fn vec_all_ge(self, other: Self) -> i32;
3091    }
3092
3093    // NOTE: this implementation is currently non-optimal, but it does work for floats even with
3094    // only `vector` enabled.
3095    //
3096    // - https://github.com/llvm/llvm-project/issues/129434
3097    // - https://github.com/llvm/llvm-project/issues/130424
3098    macro_rules! impl_vec_compare {
3099        ($($ty:ident)*) => {
3100            $(
3101                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3102                impl VectorCompare for $ty {
3103                    #[inline]
3104                    #[target_feature(enable = "vector")]
3105                    unsafe fn vec_all_lt(self, other: Self) -> i32 {
3106                        simd_reduce_all(simd_lt::<_, t_b!($ty)>(self, other)) as i32
3107                    }
3108                    #[inline]
3109                    #[target_feature(enable = "vector")]
3110                    unsafe fn vec_all_le(self, other: Self) -> i32 {
3111                        simd_reduce_all(simd_le::<_, t_b!($ty)>(self, other)) as i32
3112                    }
3113                    #[inline]
3114                    #[target_feature(enable = "vector")]
3115                    unsafe fn vec_all_gt(self, other: Self) -> i32 {
3116                        simd_reduce_all(simd_gt::<_, t_b!($ty)>(self, other)) as i32
3117                    }
3118                    #[inline]
3119                    #[target_feature(enable = "vector")]
3120                    unsafe fn vec_all_ge(self, other: Self) -> i32 {
3121                        simd_reduce_all(simd_ge::<_, t_b!($ty)>(self, other)) as i32
3122                    }
3123                }
3124            )*
3125        }
3126    }
3127
3128    impl_vec_compare! {
3129        vector_signed_char
3130        vector_unsigned_char
3131
3132        vector_signed_short
3133        vector_unsigned_short
3134
3135        vector_signed_int
3136        vector_unsigned_int
3137        vector_float
3138
3139        vector_signed_long_long
3140        vector_unsigned_long_long
3141        vector_double
3142    }
3143
3144    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3145    pub trait VectorTestMask {
3146        type Mask;
3147        unsafe fn vec_test_mask(self, other: Self::Mask) -> i32;
3148    }
3149
3150    macro_rules! impl_vec_test_mask {
3151        ($($instr:ident $ty:ident)*) => {
3152            $(
3153                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3154                impl VectorTestMask for $ty {
3155                    type Mask = t_u!($ty);
3156
3157                    #[inline]
3158                    #[target_feature(enable = "vector")]
3159                    unsafe fn vec_test_mask(self, other: Self::Mask) -> i32 {
3160                        vtm(transmute(self), transmute(other))
3161                    }
3162                }
3163            )*
3164        }
3165    }
3166
3167    impl_vec_test_mask! {
3168        vector_signed_char
3169        vector_signed_short
3170        vector_signed_int
3171        vector_signed_long_long
3172
3173        vector_unsigned_char
3174        vector_unsigned_short
3175        vector_unsigned_int
3176        vector_unsigned_long_long
3177
3178        vector_float
3179        vector_double
3180    }
3181
3182    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3183    pub trait VectorSearchString {
3184        unsafe fn vec_search_string_cc(
3185            self,
3186            b: Self,
3187            c: vector_unsigned_char,
3188        ) -> (vector_unsigned_char, i32);
3189
3190        unsafe fn vec_search_string_until_zero_cc(
3191            self,
3192            b: Self,
3193            c: vector_unsigned_char,
3194        ) -> (vector_unsigned_char, i32);
3195    }
3196
3197    macro_rules! impl_vec_search_string{
3198        ($($intr_s:ident $intr_sz:ident $ty:ident)*) => {
3199            $(
3200                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3201                impl VectorSearchString for $ty {
3202                    #[inline]
3203                    #[target_feature(enable = "vector-enhancements-2")]
3204                    unsafe fn vec_search_string_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) {
3205                        let PackedTuple { x,y } = $intr_s(transmute(self), transmute(b), c);
3206                        (x, y)
3207                    }
3208
3209                    #[inline]
3210                    #[target_feature(enable = "vector-enhancements-2")]
3211                    unsafe fn vec_search_string_until_zero_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) {
3212                        let PackedTuple { x,y } = $intr_sz(transmute(self), transmute(b), c);
3213                        (x, y)
3214                    }
3215                }
3216
3217            )*
3218        }
3219    }
3220
3221    impl_vec_search_string! {
3222        vstrsb vstrszb vector_signed_char
3223        vstrsb vstrszb vector_bool_char
3224        vstrsb vstrszb vector_unsigned_char
3225
3226        vstrsh vstrszh vector_signed_short
3227        vstrsh vstrszh vector_bool_short
3228        vstrsh vstrszh vector_unsigned_short
3229
3230        vstrsf vstrszf vector_signed_int
3231        vstrsf vstrszf vector_bool_int
3232        vstrsf vstrszf vector_unsigned_int
3233    }
3234
3235    #[inline]
3236    #[target_feature(enable = "vector")]
3237    #[cfg_attr(test, assert_instr(vcdgb))]
3238    pub unsafe fn vcdgb(a: vector_signed_long_long) -> vector_double {
3239        simd_as(a)
3240    }
3241
3242    #[inline]
3243    #[target_feature(enable = "vector")]
3244    #[cfg_attr(test, assert_instr(vcdlgb))]
3245    pub unsafe fn vcdlgb(a: vector_unsigned_long_long) -> vector_double {
3246        simd_as(a)
3247    }
3248
3249    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3250    pub trait VectorDouble {
3251        unsafe fn vec_double(self) -> vector_double;
3252    }
3253
3254    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3255    impl VectorDouble for vector_signed_long_long {
3256        #[inline]
3257        #[target_feature(enable = "vector")]
3258        unsafe fn vec_double(self) -> vector_double {
3259            vcdgb(self)
3260        }
3261    }
3262
3263    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3264    impl VectorDouble for vector_unsigned_long_long {
3265        #[inline]
3266        #[target_feature(enable = "vector")]
3267        unsafe fn vec_double(self) -> vector_double {
3268            vcdlgb(self)
3269        }
3270    }
3271
3272    #[inline]
3273    #[target_feature(enable = "vector")]
3274    #[cfg_attr(
3275        all(test, target_feature = "vector-enhancements-2"),
3276        assert_instr(vcefb)
3277    )]
3278    pub unsafe fn vcefb(a: vector_signed_int) -> vector_float {
3279        simd_as(a)
3280    }
3281
3282    #[inline]
3283    #[target_feature(enable = "vector")]
3284    #[cfg_attr(
3285        all(test, target_feature = "vector-enhancements-2"),
3286        assert_instr(vcelfb)
3287    )]
3288    pub unsafe fn vcelfb(a: vector_unsigned_int) -> vector_float {
3289        simd_as(a)
3290    }
3291
3292    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3293    pub trait VectorFloat {
3294        unsafe fn vec_float(self) -> vector_float;
3295    }
3296
3297    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3298    impl VectorFloat for vector_signed_int {
3299        #[inline]
3300        #[target_feature(enable = "vector")]
3301        unsafe fn vec_float(self) -> vector_float {
3302            vcefb(self)
3303        }
3304    }
3305
3306    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3307    impl VectorFloat for vector_unsigned_int {
3308        #[inline]
3309        #[target_feature(enable = "vector")]
3310        unsafe fn vec_float(self) -> vector_float {
3311            vcelfb(self)
3312        }
3313    }
3314
3315    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3316    pub trait VectorExtendSigned64 {
3317        unsafe fn vec_extend_s64(self) -> vector_signed_long_long;
3318    }
3319
3320    #[inline]
3321    #[target_feature(enable = "vector")]
3322    // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
3323    // #[cfg_attr(test, assert_instr(vsegb))]
3324    pub unsafe fn vsegb(a: vector_signed_char) -> vector_signed_long_long {
3325        simd_as(simd_shuffle::<_, _, i8x2>(
3326            a,
3327            a,
3328            const { u32x2::from_array([7, 15]) },
3329        ))
3330    }
3331
3332    #[inline]
3333    #[target_feature(enable = "vector")]
3334    // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
3335    // #[cfg_attr(test, assert_instr(vsegh))]
3336    pub unsafe fn vsegh(a: vector_signed_short) -> vector_signed_long_long {
3337        simd_as(simd_shuffle::<_, _, i16x2>(
3338            a,
3339            a,
3340            const { u32x2::from_array([3, 7]) },
3341        ))
3342    }
3343
3344    #[inline]
3345    #[target_feature(enable = "vector")]
3346    // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
3347    // #[cfg_attr(test, assert_instr(vsegf))]
3348    pub unsafe fn vsegf(a: vector_signed_int) -> vector_signed_long_long {
3349        simd_as(simd_shuffle::<_, _, i32x2>(
3350            a,
3351            a,
3352            const { u32x2::from_array([1, 3]) },
3353        ))
3354    }
3355
3356    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3357    impl VectorExtendSigned64 for vector_signed_char {
3358        #[inline]
3359        #[target_feature(enable = "vector")]
3360        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3361            vsegb(self)
3362        }
3363    }
3364    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3365    impl VectorExtendSigned64 for vector_signed_short {
3366        #[inline]
3367        #[target_feature(enable = "vector")]
3368        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3369            vsegh(self)
3370        }
3371    }
3372    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3373    impl VectorExtendSigned64 for vector_signed_int {
3374        #[inline]
3375        #[target_feature(enable = "vector")]
3376        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3377            vsegf(self)
3378        }
3379    }
3380
3381    // NOTE: VectorSigned and VectorUnsigned make strong safety assumptions around floats.
3382    // This is what C provides, but even IBM does not clearly document these constraints.
3383    //
3384    // https://doc.rust-lang.org/std/intrinsics/simd/fn.simd_cast.html
3385
3386    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3387    pub trait VectorSigned {
3388        type Result;
3389        unsafe fn vec_signed(self) -> Self::Result;
3390    }
3391
3392    test_impl! { vcgsb (a: vector_float) -> vector_signed_int [simd_cast, "vector-enhancements-2" vcgsb] }
3393    test_impl! { vcgdb (a: vector_double) -> vector_signed_long_long [simd_cast, vcgdb] }
3394
3395    impl_vec_trait! { [VectorSigned vec_signed] vcgsb (vector_float) -> vector_signed_int }
3396    impl_vec_trait! { [VectorSigned vec_signed] vcgdb (vector_double) -> vector_signed_long_long }
3397
3398    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3399    pub trait VectorUnsigned {
3400        type Result;
3401        unsafe fn vec_unsigned(self) -> Self::Result;
3402    }
3403
3404    test_impl! { vclgsb (a: vector_float) -> vector_unsigned_int [simd_cast, "vector-enhancements-2" vclgsb] }
3405    test_impl! { vclgdb (a: vector_double) -> vector_unsigned_long_long [simd_cast, vclgdb] }
3406
3407    impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgsb (vector_float) -> vector_unsigned_int }
3408    impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgdb (vector_double) -> vector_unsigned_long_long }
3409
3410    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3411    pub trait VectorCopyUntilZero {
3412        unsafe fn vec_cp_until_zero(self) -> Self;
3413    }
3414
3415    test_impl! { vec_vistrb (a: vector_unsigned_char) -> vector_unsigned_char [vistrb, vistrb] }
3416    test_impl! { vec_vistrh (a: vector_unsigned_short) -> vector_unsigned_short [vistrh, vistrh] }
3417    test_impl! { vec_vistrf (a: vector_unsigned_int) -> vector_unsigned_int [vistrf, vistrf] }
3418
3419    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_signed_char) }
3420    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_bool_char) }
3421    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_unsigned_char) }
3422
3423    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_signed_short) }
3424    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_bool_short) }
3425    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_unsigned_short) }
3426
3427    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_signed_int) }
3428    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_bool_int) }
3429    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_unsigned_int) }
3430
3431    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3432    pub trait VectorCopyUntilZeroCC: Sized {
3433        unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32);
3434    }
3435
3436    test_impl! { vec_vistrbs (a: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32> [vistrbs, vistrbs] }
3437    test_impl! { vec_vistrhs (a: vector_unsigned_short) -> PackedTuple<vector_unsigned_short, i32> [vistrhs, vistrhs] }
3438    test_impl! { vec_vistrfs (a: vector_unsigned_int) -> PackedTuple<vector_unsigned_int, i32> [vistrfs, vistrfs] }
3439
3440    macro_rules! impl_vec_copy_until_zero_cc {
3441        ($($intr:ident $ty:ident)*) => {
3442            $(
3443                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3444                impl VectorCopyUntilZeroCC for $ty {
3445                    #[inline]
3446                    #[target_feature(enable = "vector")]
3447                    unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32) {
3448                        let PackedTuple { x,y } = $intr(transmute(self));
3449                        (transmute(x), y)
3450                    }
3451                }
3452
3453            )*
3454        }
3455    }
3456
3457    impl_vec_copy_until_zero_cc! {
3458        vec_vistrbs vector_signed_char
3459        vec_vistrbs vector_bool_char
3460        vec_vistrbs vector_unsigned_char
3461
3462        vec_vistrhs vector_signed_short
3463        vec_vistrhs vector_bool_short
3464        vec_vistrhs vector_unsigned_short
3465
3466        vec_vistrfs vector_signed_int
3467        vec_vistrfs vector_bool_int
3468        vec_vistrfs vector_unsigned_int
3469    }
3470
3471    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3472    pub trait VectorSrdb {
3473        unsafe fn vec_srdb<const C: u32>(self, b: Self) -> Self;
3474    }
3475
3476    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3477    pub trait VectorSld {
3478        unsafe fn vec_sld<const C: u32>(self, b: Self) -> Self;
3479
3480        unsafe fn vec_sldw<const C: u32>(self, b: Self) -> Self;
3481
3482        unsafe fn vec_sldb<const C: u32>(self, b: Self) -> Self;
3483    }
3484
3485    // FIXME(llvm) https://github.com/llvm/llvm-project/issues/129955
3486    // ideally we could implement this in terms of llvm.fshl.i128
3487    // #[link_name = "llvm.fshl.i128"] fn fshl_i128(a: u128, b: u128, c: u128) -> u128;
3488    // transmute(fshl_i128(transmute(a), transmute(b), const { C * 8 } ))
3489
3490    macro_rules! impl_vec_sld {
3491        ($($ty:ident)*) => {
3492            $(
3493                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3494                impl VectorSld for $ty {
3495                    #[inline]
3496                    #[target_feature(enable = "vector")]
3497                    unsafe fn vec_sld<const C: u32>(self, b: Self) -> Self {
3498                        static_assert_uimm_bits!(C, 4);
3499                        transmute(vsldb(transmute(self), transmute(b), C))
3500                    }
3501
3502                    #[inline]
3503                    #[target_feature(enable = "vector")]
3504                    unsafe fn vec_sldw<const C: u32>(self, b: Self) -> Self {
3505                        static_assert_uimm_bits!(C, 2);
3506                        transmute(vsldb(transmute(self), transmute(b), const { 4 * C }))
3507                    }
3508
3509                    #[inline]
3510                    #[target_feature(enable = "vector-enhancements-2")]
3511                    unsafe fn vec_sldb<const C: u32>(self, b: Self) -> Self {
3512                        static_assert_uimm_bits!(C, 3);
3513                        transmute(vsld(transmute(self), transmute(b), C))
3514                    }
3515                }
3516
3517                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3518                impl VectorSrdb for $ty {
3519                    #[inline]
3520                    #[target_feature(enable = "vector-enhancements-2")]
3521                    unsafe fn vec_srdb<const C: u32>(self, b: Self) -> Self {
3522                        static_assert_uimm_bits!(C, 3);
3523                        transmute(vsrd(transmute(self), transmute(b), C))
3524                    }
3525                }
3526            )*
3527        }
3528    }
3529
3530    impl_vec_sld! {
3531        vector_signed_char
3532        vector_bool_char
3533        vector_unsigned_char
3534
3535        vector_signed_short
3536        vector_bool_short
3537        vector_unsigned_short
3538
3539        vector_signed_int
3540        vector_bool_int
3541        vector_unsigned_int
3542
3543        vector_signed_long_long
3544        vector_bool_long_long
3545        vector_unsigned_long_long
3546
3547        vector_float
3548        vector_double
3549    }
3550
3551    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3552    pub trait VectorCompareRange: Sized {
3553        type Result;
3554
3555        unsafe fn vstrc<const IMM: u32>(self, b: Self, c: Self) -> Self::Result;
3556        unsafe fn vstrcz<const IMM: u32>(self, b: Self, c: Self) -> Self::Result;
3557        unsafe fn vstrcs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32);
3558        unsafe fn vstrczs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32);
3559    }
3560
3561    const fn validate_compare_range_imm(imm: u32) {
3562        if !matches!(imm, 0 | 4 | 8 | 12) {
3563            panic!("IMM needs to be one of 0, 4, 8, 12");
3564        }
3565    }
3566
3567    macro_rules! impl_compare_range {
3568        ($($ty:ident $vstrc:ident $vstrcs:ident $vstrcz:ident $vstrczs:ident)*) => {
3569            $(
3570                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3571                impl VectorCompareRange for $ty {
3572                    type Result = t_b!($ty);
3573
3574                    #[inline]
3575                    #[target_feature(enable = "vector")]
3576                    unsafe fn vstrc<const IMM: u32>(self, b: Self, c: Self) -> Self::Result {
3577                        const { validate_compare_range_imm };
3578                        $vstrc(self, b, c, IMM)
3579                    }
3580
3581                    #[inline]
3582                    #[target_feature(enable = "vector")]
3583                    unsafe fn vstrcz<const IMM: u32>(self, b: Self, c: Self) -> Self::Result {
3584                        const { validate_compare_range_imm };
3585                        $vstrcz(self, b, c, IMM)
3586                    }
3587
3588                    #[inline]
3589                    #[target_feature(enable = "vector")]
3590                    unsafe fn vstrcs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32) {
3591                        const { validate_compare_range_imm };
3592                        let PackedTuple { x, y } = $vstrcs(self, b, c, IMM);
3593                        (x,y)
3594                    }
3595
3596                    #[inline]
3597                    #[target_feature(enable = "vector")]
3598                    unsafe fn vstrczs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32) {
3599                        const { validate_compare_range_imm };
3600                        let PackedTuple { x, y } = $vstrczs(self, b, c, IMM);
3601                        (x,y)
3602                    }
3603                }
3604            )*
3605        }
3606    }
3607
3608    impl_compare_range! {
3609        vector_unsigned_char    vstrcb vstrcbs vstrczb vstrczbs
3610        vector_unsigned_short   vstrch vstrchs vstrczh vstrczhs
3611        vector_unsigned_int     vstrcf vstrcfs vstrczf vstrczfs
3612    }
3613
3614    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3615    pub trait VectorComparePredicate: Sized {
3616        type Result;
3617
3618        #[inline]
3619        #[target_feature(enable = "vector")]
3620        unsafe fn vec_cmpgt(self, other: Self) -> Self::Result {
3621            simd_gt(self, other)
3622        }
3623
3624        #[inline]
3625        #[target_feature(enable = "vector")]
3626        unsafe fn vec_cmpge(self, other: Self) -> Self::Result {
3627            simd_ge(self, other)
3628        }
3629
3630        #[inline]
3631        #[target_feature(enable = "vector")]
3632        unsafe fn vec_cmplt(self, other: Self) -> Self::Result {
3633            simd_lt(self, other)
3634        }
3635
3636        #[inline]
3637        #[target_feature(enable = "vector")]
3638        unsafe fn vec_cmple(self, other: Self) -> Self::Result {
3639            simd_le(self, other)
3640        }
3641    }
3642
3643    macro_rules! impl_compare_predicate {
3644        ($($ty:ident)*) => {
3645            $(
3646                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3647                impl VectorComparePredicate for $ty {
3648                    type Result = t_b!($ty);
3649                }
3650            )*
3651        }
3652    }
3653
3654    impl_compare_predicate! {
3655        vector_signed_char
3656        vector_unsigned_char
3657
3658        vector_signed_short
3659        vector_unsigned_short
3660
3661        vector_signed_int
3662        vector_unsigned_int
3663        vector_float
3664
3665        vector_signed_long_long
3666        vector_unsigned_long_long
3667        vector_double
3668    }
3669
3670    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3671    pub trait VectorEquality: Sized {
3672        type Result;
3673
3674        #[inline]
3675        #[target_feature(enable = "vector")]
3676        unsafe fn vec_cmpeq(self, other: Self) -> Self::Result {
3677            simd_eq(self, other)
3678        }
3679
3680        #[inline]
3681        #[target_feature(enable = "vector")]
3682        unsafe fn vec_cmpne(self, other: Self) -> Self::Result {
3683            simd_ne(self, other)
3684        }
3685    }
3686
3687    macro_rules! impl_compare_equality {
3688        ($($ty:ident)*) => {
3689            $(
3690                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3691                impl VectorEquality for $ty {
3692                    type Result = t_b!($ty);
3693                }
3694            )*
3695        }
3696    }
3697
3698    impl_compare_equality! {
3699        vector_bool_char
3700        vector_signed_char
3701        vector_unsigned_char
3702
3703        vector_bool_short
3704        vector_signed_short
3705        vector_unsigned_short
3706
3707        vector_bool_int
3708        vector_signed_int
3709        vector_unsigned_int
3710        vector_float
3711
3712        vector_bool_long_long
3713        vector_signed_long_long
3714        vector_unsigned_long_long
3715        vector_double
3716    }
3717
3718    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3719    pub trait VectorEqualityIdx: Sized {
3720        type Result;
3721
3722        unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result;
3723        unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result;
3724
3725        unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32);
3726        unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32);
3727
3728        unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result;
3729        unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result;
3730
3731        unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32);
3732        unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32);
3733    }
3734
3735    macro_rules! impl_compare_equality_idx {
3736        ($($ty:ident $ret:ident
3737                $cmpeq:ident $cmpne:ident
3738                $cmpeq_or_0:ident $cmpne_or_0:ident
3739                $cmpeq_cc:ident $cmpne_cc:ident
3740                $cmpeq_or_0_cc:ident $cmpne_or_0_cc:ident
3741        )*) => {
3742            $(
3743                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3744                impl VectorEqualityIdx for $ty {
3745                    type Result = $ret;
3746
3747                    #[inline]
3748                    #[target_feature(enable = "vector")]
3749                    unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result {
3750                        transmute($cmpeq(transmute(self), transmute(other)))
3751                    }
3752
3753                    #[inline]
3754                    #[target_feature(enable = "vector")]
3755                    unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result {
3756                        transmute($cmpne(transmute(self), transmute(other)))
3757                    }
3758
3759                    #[inline]
3760                    #[target_feature(enable = "vector")]
3761                    unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result {
3762                        transmute($cmpeq_or_0(transmute(self), transmute(other)))
3763                    }
3764
3765                    #[inline]
3766                    #[target_feature(enable = "vector")]
3767                    unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result {
3768                        transmute($cmpne_or_0(transmute(self), transmute(other)))
3769                    }
3770
3771                    #[inline]
3772                    #[target_feature(enable = "vector")]
3773                    unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32) {
3774                        let PackedTuple { x, y } = $cmpeq_cc(transmute(self), transmute(other));
3775                        (transmute(x), y)
3776                    }
3777
3778                    #[inline]
3779                    #[target_feature(enable = "vector")]
3780                    unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32) {
3781                        let PackedTuple { x, y } = $cmpne_cc(transmute(self), transmute(other));
3782                        (transmute(x),y)
3783                    }
3784
3785                    #[inline]
3786                    #[target_feature(enable = "vector")]
3787                    unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) {
3788                        let PackedTuple { x, y } = $cmpeq_or_0_cc(transmute(self), transmute(other));
3789                        (transmute(x), y)
3790                    }
3791
3792                    #[inline]
3793                    #[target_feature(enable = "vector")]
3794                    unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) {
3795                        let PackedTuple { x, y } = $cmpne_or_0_cc(transmute(self), transmute(other));
3796                        (transmute(x),y)
3797                    }
3798                }
3799            )*
3800        }
3801    }
3802
3803    impl_compare_equality_idx! {
3804        vector_signed_char vector_signed_char               vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3805        vector_bool_char vector_unsigned_char               vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3806        vector_unsigned_char vector_unsigned_char           vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3807        vector_signed_short vector_signed_short             vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3808        vector_bool_short  vector_unsigned_short            vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3809        vector_unsigned_short vector_unsigned_short         vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3810        vector_signed_int vector_signed_int                 vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3811        vector_bool_int  vector_unsigned_int                vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3812        vector_unsigned_int vector_unsigned_int             vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3813    }
3814
3815    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3816    pub trait VectorExtract {
3817        type ElementType;
3818
3819        unsafe fn vec_extract(a: Self, b: i32) -> Self::ElementType;
3820    }
3821
3822    #[inline]
3823    #[target_feature(enable = "vector")]
3824    #[cfg_attr(test, assert_instr(vlgvb))]
3825    unsafe fn vlgvb(a: vector_unsigned_char, b: i32) -> u8 {
3826        simd_extract_dyn(a, b as u32 % 16)
3827    }
3828
3829    #[inline]
3830    #[target_feature(enable = "vector")]
3831    #[cfg_attr(test, assert_instr(vlgvh))]
3832    unsafe fn vlgvh(a: vector_unsigned_short, b: i32) -> u16 {
3833        simd_extract_dyn(a, b as u32 % 8)
3834    }
3835
3836    #[inline]
3837    #[target_feature(enable = "vector")]
3838    #[cfg_attr(test, assert_instr(vlgvf))]
3839    unsafe fn vlgvf(a: vector_unsigned_int, b: i32) -> u32 {
3840        simd_extract_dyn(a, b as u32 % 4)
3841    }
3842
3843    #[inline]
3844    #[target_feature(enable = "vector")]
3845    #[cfg_attr(test, assert_instr(vlgvg))]
3846    unsafe fn vlgvg(a: vector_unsigned_long_long, b: i32) -> u64 {
3847        simd_extract_dyn(a, b as u32 % 2)
3848    }
3849
3850    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3851    pub trait VectorInsert {
3852        type ElementType;
3853
3854        unsafe fn vec_insert(a: Self::ElementType, b: Self, c: i32) -> Self;
3855    }
3856
3857    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3858    pub trait VectorPromote: Sized {
3859        type ElementType;
3860
3861        unsafe fn vec_promote(a: Self::ElementType, b: i32) -> MaybeUninit<Self>;
3862    }
3863
3864    #[inline]
3865    #[target_feature(enable = "vector")]
3866    #[cfg_attr(test, assert_instr(vlvgb))]
3867    unsafe fn vlvgb(a: u8, b: vector_unsigned_char, c: i32) -> vector_unsigned_char {
3868        simd_insert_dyn(b, c as u32 % 16, a)
3869    }
3870
3871    #[inline]
3872    #[target_feature(enable = "vector")]
3873    #[cfg_attr(test, assert_instr(vlvgh))]
3874    unsafe fn vlvgh(a: u16, b: vector_unsigned_short, c: i32) -> vector_unsigned_short {
3875        simd_insert_dyn(b, c as u32 % 8, a)
3876    }
3877
3878    #[inline]
3879    #[target_feature(enable = "vector")]
3880    #[cfg_attr(test, assert_instr(vlvgf))]
3881    unsafe fn vlvgf(a: u32, b: vector_unsigned_int, c: i32) -> vector_unsigned_int {
3882        simd_insert_dyn(b, c as u32 % 4, a)
3883    }
3884
3885    #[inline]
3886    #[target_feature(enable = "vector")]
3887    #[cfg_attr(test, assert_instr(vlvgg))]
3888    unsafe fn vlvgg(a: u64, b: vector_unsigned_long_long, c: i32) -> vector_unsigned_long_long {
3889        simd_insert_dyn(b, c as u32 % 2, a)
3890    }
3891
3892    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3893    pub trait VectorInsertAndZero {
3894        type ElementType;
3895
3896        unsafe fn vec_insert_and_zero(a: *const Self::ElementType) -> Self;
3897    }
3898
3899    #[inline]
3900    #[target_feature(enable = "vector")]
3901    #[cfg_attr(test, assert_instr(vllezb))]
3902    unsafe fn vllezb(x: *const u8) -> vector_unsigned_char {
3903        vector_unsigned_char([0, 0, 0, 0, 0, 0, 0, *x, 0, 0, 0, 0, 0, 0, 0, 0])
3904    }
3905
3906    #[inline]
3907    #[target_feature(enable = "vector")]
3908    #[cfg_attr(test, assert_instr(vllezh))]
3909    unsafe fn vllezh(x: *const u16) -> vector_unsigned_short {
3910        vector_unsigned_short([0, 0, 0, *x, 0, 0, 0, 0])
3911    }
3912
3913    #[inline]
3914    #[target_feature(enable = "vector")]
3915    #[cfg_attr(test, assert_instr(vllezf))]
3916    unsafe fn vllezf(x: *const u32) -> vector_unsigned_int {
3917        vector_unsigned_int([0, *x, 0, 0])
3918    }
3919
3920    #[inline]
3921    #[target_feature(enable = "vector")]
3922    #[cfg_attr(test, assert_instr(vllezg))]
3923    unsafe fn vllezg(x: *const u64) -> vector_unsigned_long_long {
3924        vector_unsigned_long_long([*x, 0])
3925    }
3926
3927    macro_rules! impl_extract_insert {
3928        ($($ty:ident $extract_intr:ident $insert_intr:ident $insert_and_zero_intr:ident)*) => {
3929            $(
3930                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3931                impl VectorExtract for $ty {
3932                    type ElementType = l_t_t!($ty);
3933
3934                    #[inline]
3935                    #[target_feature(enable = "vector")]
3936                    unsafe fn vec_extract(a: Self, b: i32) -> Self::ElementType {
3937                        transmute($extract_intr(transmute(a), b))
3938                    }
3939                }
3940
3941                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3942                impl VectorInsert for $ty {
3943                    type ElementType = l_t_t!($ty);
3944
3945                    #[inline]
3946                    #[target_feature(enable = "vector")]
3947                    unsafe fn vec_insert(a: Self::ElementType, b: Self, c: i32) -> Self {
3948                        transmute($insert_intr(transmute(a), transmute(b), c))
3949                    }
3950                }
3951
3952                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3953                impl VectorInsertAndZero for $ty {
3954                    type ElementType = l_t_t!($ty);
3955
3956                    #[inline]
3957                    #[target_feature(enable = "vector")]
3958                    unsafe fn vec_insert_and_zero(a: *const Self::ElementType) -> Self {
3959                        transmute($insert_and_zero_intr(a.cast()))
3960                    }
3961                }
3962
3963                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3964                impl VectorPromote for $ty {
3965                    type ElementType = l_t_t!($ty);
3966
3967                    #[inline]
3968                    #[target_feature(enable = "vector")]
3969                    unsafe fn vec_promote(a: Self::ElementType, c: i32) -> MaybeUninit<Self> {
3970                        // Rust does not currently support `MaybeUninit` element types to simd
3971                        // vectors. In C/LLVM that is allowed (using poison values). So rust will
3972                        // use an extra instruction to zero the memory.
3973                        let b = MaybeUninit::<$ty>::zeroed();
3974                        MaybeUninit::new(transmute($insert_intr(transmute(a), transmute(b), c)))
3975                    }
3976                }
3977            )*
3978        }
3979
3980    }
3981
3982    impl_extract_insert! {
3983        vector_signed_char          vlgvb vlvgb vllezb
3984        vector_unsigned_char        vlgvb vlvgb vllezb
3985        vector_signed_short         vlgvh vlvgh vllezh
3986        vector_unsigned_short       vlgvh vlvgh vllezh
3987        vector_signed_int           vlgvf vlvgf vllezf
3988        vector_unsigned_int         vlgvf vlvgf vllezf
3989        vector_signed_long_long     vlgvg vlvgg vllezg
3990        vector_unsigned_long_long   vlgvg vlvgg vllezg
3991        vector_float                vlgvf vlvgf vllezf
3992        vector_double               vlgvg vlvgg vllezg
3993    }
3994}
3995
3996/// Load Count to Block Boundary
3997#[inline]
3998#[target_feature(enable = "vector")]
3999#[unstable(feature = "stdarch_s390x", issue = "135681")]
4000#[cfg_attr(test, assert_instr(lcbb, BLOCK_BOUNDARY = 512))]
4001unsafe fn __lcbb<const BLOCK_BOUNDARY: u16>(ptr: *const u8) -> u32 {
4002    lcbb(ptr, const { validate_block_boundary(BLOCK_BOUNDARY) })
4003}
4004
4005/// Vector Add
4006#[inline]
4007#[target_feature(enable = "vector")]
4008#[unstable(feature = "stdarch_s390x", issue = "135681")]
4009pub unsafe fn vec_add<T: sealed::VectorAdd<U>, U>(a: T, b: U) -> T::Result {
4010    a.vec_add(b)
4011}
4012
4013/// Vector Subtract
4014#[inline]
4015#[target_feature(enable = "vector")]
4016#[unstable(feature = "stdarch_s390x", issue = "135681")]
4017pub unsafe fn vec_sub<T: sealed::VectorSub<U>, U>(a: T, b: U) -> T::Result {
4018    a.vec_sub(b)
4019}
4020
4021/// Vector Multiply
4022///
4023/// ## Purpose
4024/// Compute the products of corresponding elements of two vectors.
4025///
4026/// ## Result value
4027/// Each element of r receives the product of the corresponding elements of a and b.
4028#[inline]
4029#[target_feature(enable = "vector")]
4030#[unstable(feature = "stdarch_s390x", issue = "135681")]
4031pub unsafe fn vec_mul<T: sealed::VectorMul>(a: T, b: T) -> T {
4032    a.vec_mul(b)
4033}
4034
4035/// Vector Count Leading Zeros
4036#[inline]
4037#[target_feature(enable = "vector")]
4038#[unstable(feature = "stdarch_s390x", issue = "135681")]
4039pub unsafe fn vec_cntlz<T: sealed::CountBits>(a: T) -> T::Result {
4040    a.vec_cntlz()
4041}
4042
4043/// Vector Count Trailing Zeros
4044#[inline]
4045#[target_feature(enable = "vector")]
4046#[unstable(feature = "stdarch_s390x", issue = "135681")]
4047pub unsafe fn vec_cnttz<T: sealed::CountBits>(a: T) -> T::Result {
4048    a.vec_cnttz()
4049}
4050
4051/// Vector Population Count
4052///
4053/// Computes the population count (number of set bits) in each element of the input.
4054#[inline]
4055#[target_feature(enable = "vector")]
4056#[unstable(feature = "stdarch_s390x", issue = "135681")]
4057pub unsafe fn vec_popcnt<T: sealed::CountBits>(a: T) -> T::Result {
4058    a.vec_popcnt()
4059}
4060
4061/// Vector Maximum
4062#[inline]
4063#[target_feature(enable = "vector")]
4064#[unstable(feature = "stdarch_s390x", issue = "135681")]
4065pub unsafe fn vec_max<T: sealed::VectorMax<U>, U>(a: T, b: U) -> T::Result {
4066    a.vec_max(b)
4067}
4068
4069/// Vector  Minimum
4070#[inline]
4071#[target_feature(enable = "vector")]
4072#[unstable(feature = "stdarch_s390x", issue = "135681")]
4073pub unsafe fn vec_min<T: sealed::VectorMin<U>, U>(a: T, b: U) -> T::Result {
4074    a.vec_min(b)
4075}
4076
4077/// Vector Absolute
4078#[inline]
4079#[target_feature(enable = "vector")]
4080#[unstable(feature = "stdarch_s390x", issue = "135681")]
4081pub unsafe fn vec_abs<T: sealed::VectorAbs>(a: T) -> T {
4082    a.vec_abs()
4083}
4084
4085/// Vector Negative Absolute
4086#[inline]
4087#[target_feature(enable = "vector")]
4088#[unstable(feature = "stdarch_s390x", issue = "135681")]
4089pub unsafe fn vec_nabs<T: sealed::VectorNabs>(a: T) -> T {
4090    a.vec_nabs()
4091}
4092
4093/// Vector Negative Multiply Add
4094#[inline]
4095#[target_feature(enable = "vector")]
4096#[unstable(feature = "stdarch_s390x", issue = "135681")]
4097pub unsafe fn vec_nmadd<T: sealed::VectorNmadd>(a: T, b: T, c: T) -> T {
4098    a.vec_nmadd(b, c)
4099}
4100
4101/// Vector Negative Multiply Subtract
4102#[inline]
4103#[target_feature(enable = "vector")]
4104#[unstable(feature = "stdarch_s390x", issue = "135681")]
4105pub unsafe fn vec_nmsub<T: sealed::VectorNmsub>(a: T, b: T, c: T) -> T {
4106    a.vec_nmsub(b, c)
4107}
4108
4109/// Vector Square Root
4110#[inline]
4111#[target_feature(enable = "vector")]
4112#[unstable(feature = "stdarch_s390x", issue = "135681")]
4113pub unsafe fn vec_sqrt<T: sealed::VectorSqrt>(a: T) -> T {
4114    a.vec_sqrt()
4115}
4116
4117/// Vector Splat
4118#[inline]
4119#[target_feature(enable = "vector")]
4120#[unstable(feature = "stdarch_s390x", issue = "135681")]
4121pub unsafe fn vec_splat<T: sealed::VectorSplat, const IMM: u32>(a: T) -> T {
4122    a.vec_splat::<IMM>()
4123}
4124
4125/// Vector Splats
4126#[inline]
4127#[target_feature(enable = "vector")]
4128#[unstable(feature = "stdarch_s390x", issue = "135681")]
4129pub unsafe fn vec_splats<T: sealed::VectorSplats<U>, U>(a: T) -> U {
4130    a.vec_splats()
4131}
4132
4133/// Vector AND
4134#[inline]
4135#[target_feature(enable = "vector")]
4136#[unstable(feature = "stdarch_s390x", issue = "135681")]
4137pub unsafe fn vec_and<T: sealed::VectorAnd<U>, U>(a: T, b: U) -> T::Result {
4138    a.vec_and(b)
4139}
4140
4141/// Vector OR
4142#[inline]
4143#[target_feature(enable = "vector")]
4144#[unstable(feature = "stdarch_s390x", issue = "135681")]
4145pub unsafe fn vec_or<T: sealed::VectorOr<U>, U>(a: T, b: U) -> T::Result {
4146    a.vec_or(b)
4147}
4148
4149/// Vector XOR
4150#[inline]
4151#[target_feature(enable = "vector")]
4152#[unstable(feature = "stdarch_s390x", issue = "135681")]
4153pub unsafe fn vec_xor<T: sealed::VectorXor<U>, U>(a: T, b: U) -> T::Result {
4154    a.vec_xor(b)
4155}
4156
4157/// Vector NOR
4158#[inline]
4159#[target_feature(enable = "vector")]
4160#[unstable(feature = "stdarch_s390x", issue = "135681")]
4161pub unsafe fn vec_nor<T: sealed::VectorNor<U>, U>(a: T, b: U) -> T::Result {
4162    a.vec_nor(b)
4163}
4164
4165/// Vector NAND
4166#[inline]
4167#[target_feature(enable = "vector")]
4168#[unstable(feature = "stdarch_s390x", issue = "135681")]
4169pub unsafe fn vec_nand<T: sealed::VectorNand<U>, U>(a: T, b: U) -> T::Result {
4170    a.vec_nand(b)
4171}
4172
4173/// Vector XNOR
4174#[inline]
4175#[target_feature(enable = "vector")]
4176#[unstable(feature = "stdarch_s390x", issue = "135681")]
4177pub unsafe fn vec_eqv<T: sealed::VectorEqv<U>, U>(a: T, b: U) -> T::Result {
4178    a.vec_eqv(b)
4179}
4180
4181/// Vector ANDC
4182#[inline]
4183#[target_feature(enable = "vector")]
4184#[unstable(feature = "stdarch_s390x", issue = "135681")]
4185pub unsafe fn vec_andc<T: sealed::VectorAndc<U>, U>(a: T, b: U) -> T::Result {
4186    a.vec_andc(b)
4187}
4188
4189/// Vector OR with Complement
4190///
4191/// ## Purpose
4192/// Performs a bitwise OR of the first vector with the bitwise-complemented second vector.
4193///
4194/// ## Result value
4195/// r is the bitwise OR of a and the bitwise complement of b.
4196#[inline]
4197#[target_feature(enable = "vector")]
4198#[unstable(feature = "stdarch_s390x", issue = "135681")]
4199pub unsafe fn vec_orc<T: sealed::VectorOrc<U>, U>(a: T, b: U) -> T::Result {
4200    a.vec_orc(b)
4201}
4202
4203/// Vector Floor
4204#[inline]
4205#[target_feature(enable = "vector")]
4206#[unstable(feature = "stdarch_s390x", issue = "135681")]
4207pub unsafe fn vec_floor<T: sealed::VectorFloor>(a: T) -> T {
4208    a.vec_floor()
4209}
4210
4211/// Vector Ceil
4212#[inline]
4213#[target_feature(enable = "vector")]
4214#[unstable(feature = "stdarch_s390x", issue = "135681")]
4215pub unsafe fn vec_ceil<T: sealed::VectorCeil>(a: T) -> T {
4216    a.vec_ceil()
4217}
4218
4219/// Vector Truncate
4220///
4221/// Returns a vector containing the truncated values of the corresponding elements of the given vector.
4222/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value.
4223#[inline]
4224#[target_feature(enable = "vector")]
4225#[unstable(feature = "stdarch_s390x", issue = "135681")]
4226pub unsafe fn vec_trunc<T: sealed::VectorTrunc>(a: T) -> T {
4227    a.vec_trunc()
4228}
4229
4230/// Vector Round
4231///
4232/// Returns a vector containing the rounded values to the nearest representable floating-point integer,
4233/// using IEEE round-to-nearest rounding, of the corresponding elements of the given vector
4234#[inline]
4235#[target_feature(enable = "vector")]
4236#[unstable(feature = "stdarch_s390x", issue = "135681")]
4237pub unsafe fn vec_round<T: sealed::VectorRound>(a: T) -> T {
4238    a.vec_round()
4239}
4240
4241/// Vector Round to Current
4242///
4243/// Returns a vector by using the current rounding mode to round every
4244/// floating-point element in the given vector to integer.
4245#[inline]
4246#[target_feature(enable = "vector")]
4247#[unstable(feature = "stdarch_s390x", issue = "135681")]
4248pub unsafe fn vec_roundc<T: sealed::VectorRoundc>(a: T) -> T {
4249    a.vec_roundc()
4250}
4251
4252/// Vector Round toward Negative Infinity
4253///
4254/// Returns a vector containing the largest representable floating-point integral values less
4255/// than or equal to the values of the corresponding elements of the given vector.
4256#[inline]
4257#[target_feature(enable = "vector")]
4258#[unstable(feature = "stdarch_s390x", issue = "135681")]
4259pub unsafe fn vec_roundm<T: sealed::VectorFloor>(a: T) -> T {
4260    // the IBM docs note
4261    //
4262    // > vec_roundm provides the same functionality as vec_floor, except that vec_roundz would not trigger the IEEE-inexact exception.
4263    //
4264    // but in practice `vec_floor` also does not trigger that exception, so both are equivalent
4265    a.vec_floor()
4266}
4267
4268/// Vector Round toward Positive Infinity
4269///
4270/// Returns a vector containing the smallest representable floating-point integral values greater
4271/// than or equal to the values of the corresponding elements of the given vector.
4272#[inline]
4273#[target_feature(enable = "vector")]
4274#[unstable(feature = "stdarch_s390x", issue = "135681")]
4275pub unsafe fn vec_roundp<T: sealed::VectorCeil>(a: T) -> T {
4276    // the IBM docs note
4277    //
4278    // > vec_roundp provides the same functionality as vec_ceil, except that vec_roundz would not trigger the IEEE-inexact exception.
4279    //
4280    // but in practice `vec_ceil` also does not trigger that exception, so both are equivalent
4281    a.vec_ceil()
4282}
4283
4284/// Vector Round toward Zero
4285///
4286/// Returns a vector containing the truncated values of the corresponding elements of the given vector.
4287/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value.
4288#[inline]
4289#[target_feature(enable = "vector")]
4290#[unstable(feature = "stdarch_s390x", issue = "135681")]
4291pub unsafe fn vec_roundz<T: sealed::VectorTrunc>(a: T) -> T {
4292    // the IBM docs note
4293    //
4294    // > vec_roundz provides the same functionality as vec_trunc, except that vec_roundz would not trigger the IEEE-inexact exception.
4295    //
4296    // but in practice `vec_trunc` also does not trigger that exception, so both are equivalent
4297    a.vec_trunc()
4298}
4299
4300/// Vector Round to Integer
4301///
4302/// Returns a vector by using the current rounding mode to round every floating-point element in the given vector to integer.
4303#[inline]
4304#[target_feature(enable = "vector")]
4305#[unstable(feature = "stdarch_s390x", issue = "135681")]
4306pub unsafe fn vec_rint<T: sealed::VectorRint>(a: T) -> T {
4307    a.vec_rint()
4308}
4309
4310/// Vector Average
4311#[inline]
4312#[target_feature(enable = "vector")]
4313#[unstable(feature = "stdarch_s390x", issue = "135681")]
4314pub unsafe fn vec_avg<T: sealed::VectorAvg<U>, U>(a: T, b: U) -> T::Result {
4315    a.vec_avg(b)
4316}
4317
4318/// Vector Shift Left
4319#[inline]
4320#[target_feature(enable = "vector")]
4321#[unstable(feature = "stdarch_s390x", issue = "135681")]
4322pub unsafe fn vec_sl<T: sealed::VectorSl<U>, U>(a: T, b: U) -> T::Result {
4323    a.vec_sl(b)
4324}
4325
4326/// Vector Shift Right
4327#[inline]
4328#[target_feature(enable = "vector")]
4329#[unstable(feature = "stdarch_s390x", issue = "135681")]
4330pub unsafe fn vec_sr<T: sealed::VectorSr<U>, U>(a: T, b: U) -> T::Result {
4331    a.vec_sr(b)
4332}
4333
4334/// Vector Shift Right Algebraic
4335#[inline]
4336#[target_feature(enable = "vector")]
4337#[unstable(feature = "stdarch_s390x", issue = "135681")]
4338pub unsafe fn vec_sra<T: sealed::VectorSra<U>, U>(a: T, b: U) -> T::Result {
4339    a.vec_sra(b)
4340}
4341
4342/// Vector Shift Left by Byte
4343#[inline]
4344#[target_feature(enable = "vector")]
4345#[unstable(feature = "stdarch_s390x", issue = "135681")]
4346pub unsafe fn vec_slb<T: sealed::VectorSlb<U>, U>(a: T, b: U) -> T::Result {
4347    a.vec_slb(b)
4348}
4349
4350/// Vector Shift Right by Byte
4351#[inline]
4352#[target_feature(enable = "vector")]
4353#[unstable(feature = "stdarch_s390x", issue = "135681")]
4354pub unsafe fn vec_srb<T: sealed::VectorSrb<U>, U>(a: T, b: U) -> T::Result {
4355    a.vec_srb(b)
4356}
4357
4358/// Vector Shift Right Algebraic by Byte
4359#[inline]
4360#[target_feature(enable = "vector")]
4361#[unstable(feature = "stdarch_s390x", issue = "135681")]
4362pub unsafe fn vec_srab<T: sealed::VectorSrab<U>, U>(a: T, b: U) -> T::Result {
4363    a.vec_srab(b)
4364}
4365
4366/// Vector Element Rotate Left
4367#[inline]
4368#[target_feature(enable = "vector")]
4369#[unstable(feature = "stdarch_s390x", issue = "135681")]
4370pub unsafe fn vec_rl<T: sealed::VectorRl<U>, U>(a: T, b: U) -> T::Result {
4371    a.vec_rl(b)
4372}
4373
4374/// Vector Shift Left
4375///
4376/// Performs a left shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4377/// element of a left by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros.
4378#[inline]
4379#[target_feature(enable = "vector")]
4380#[unstable(feature = "stdarch_s390x", issue = "135681")]
4381pub unsafe fn vec_sll<T>(a: T, b: vector_unsigned_char) -> T
4382where
4383    T: sealed::VectorSll<vector_unsigned_char, Result = T>,
4384{
4385    a.vec_sll(b)
4386}
4387
4388/// Vector Shift Right
4389///
4390/// Performs a right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4391/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros.
4392#[inline]
4393#[target_feature(enable = "vector")]
4394#[unstable(feature = "stdarch_s390x", issue = "135681")]
4395pub unsafe fn vec_srl<T>(a: T, b: vector_unsigned_char) -> T
4396where
4397    T: sealed::VectorSrl<vector_unsigned_char, Result = T>,
4398{
4399    a.vec_srl(b)
4400}
4401
4402/// Vector Shift Right Arithmetic
4403///
4404/// Performs an algebraic right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4405/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by copies of
4406/// the most significant bit of the element of a.
4407#[inline]
4408#[target_feature(enable = "vector")]
4409#[unstable(feature = "stdarch_s390x", issue = "135681")]
4410pub unsafe fn vec_sral<T>(a: T, b: vector_unsigned_char) -> T
4411where
4412    T: sealed::VectorSral<vector_unsigned_char, Result = T>,
4413{
4414    a.vec_sral(b)
4415}
4416
4417/// Vector Element Rotate Left Immediate
4418///
4419/// Rotates each element of a vector left by a given number of bits. Each element of the result is obtained by rotating the corresponding element
4420/// of a left by the number of bits specified by b, modulo the number of bits in the element.
4421#[inline]
4422#[target_feature(enable = "vector")]
4423#[unstable(feature = "stdarch_s390x", issue = "135681")]
4424pub unsafe fn vec_rli<T: sealed::VectorRli>(a: T, bits: core::ffi::c_ulong) -> T {
4425    a.vec_rli(bits)
4426}
4427
4428/// Vector Reverse Elements
4429///
4430/// Returns a vector with the elements of the input vector in reversed order.
4431#[inline]
4432#[target_feature(enable = "vector")]
4433#[unstable(feature = "stdarch_s390x", issue = "135681")]
4434pub unsafe fn vec_reve<T: sealed::VectorReve>(a: T) -> T {
4435    a.vec_reve()
4436}
4437
4438/// Vector Byte Reverse
4439///
4440/// Returns a vector where each vector element contains the corresponding byte-reversed vector element of the input vector.
4441#[inline]
4442#[target_feature(enable = "vector")]
4443#[unstable(feature = "stdarch_s390x", issue = "135681")]
4444pub unsafe fn vec_revb<T: sealed::VectorRevb>(a: T) -> T {
4445    a.vec_revb()
4446}
4447
4448/// Vector Merge High
4449///
4450/// Merges the most significant ("high") halves of two vectors.
4451#[inline]
4452#[target_feature(enable = "vector")]
4453#[unstable(feature = "stdarch_s390x", issue = "135681")]
4454pub unsafe fn vec_mergeh<T: sealed::VectorMergeh>(a: T, b: T) -> T {
4455    a.vec_mergeh(b)
4456}
4457
4458/// Vector Merge Low
4459///
4460/// Merges the least significant ("low") halves of two vectors.
4461#[inline]
4462#[target_feature(enable = "vector")]
4463#[unstable(feature = "stdarch_s390x", issue = "135681")]
4464pub unsafe fn vec_mergel<T: sealed::VectorMergel>(a: T, b: T) -> T {
4465    a.vec_mergel(b)
4466}
4467
4468/// Vector Pack
4469#[inline]
4470#[target_feature(enable = "vector")]
4471#[unstable(feature = "stdarch_s390x", issue = "135681")]
4472pub unsafe fn vec_pack<T: sealed::VectorPack<U>, U>(a: T, b: U) -> T::Result {
4473    a.vec_pack(b)
4474}
4475
4476/// Vector Pack Saturated
4477#[inline]
4478#[target_feature(enable = "vector")]
4479#[unstable(feature = "stdarch_s390x", issue = "135681")]
4480pub unsafe fn vec_packs<T: sealed::VectorPacks<U>, U>(a: T, b: U) -> T::Result {
4481    a.vec_packs(b)
4482}
4483
4484/// Vector Pack Saturated Condition Code
4485#[inline]
4486#[target_feature(enable = "vector")]
4487#[unstable(feature = "stdarch_s390x", issue = "135681")]
4488pub unsafe fn vec_packs_cc<T: sealed::VectorPacksCC>(a: T, b: T) -> (T::Result, i32) {
4489    a.vec_packs_cc(b)
4490}
4491
4492/// Vector Pack Saturated Unsigned
4493#[inline]
4494#[target_feature(enable = "vector")]
4495#[unstable(feature = "stdarch_s390x", issue = "135681")]
4496pub unsafe fn vec_packsu<T: sealed::VectorPacksu<U>, U>(a: T, b: U) -> T::Result {
4497    a.vec_packsu(b)
4498}
4499
4500/// Vector Pack Saturated Unsigned Condition Code
4501#[inline]
4502#[target_feature(enable = "vector")]
4503#[unstable(feature = "stdarch_s390x", issue = "135681")]
4504pub unsafe fn vec_packsu_cc<T: sealed::VectorPacksuCC>(a: T, b: T) -> (T::Result, i32) {
4505    a.vec_packsu_cc(b)
4506}
4507
4508/// Vector Unpack High
4509#[inline]
4510#[target_feature(enable = "vector")]
4511#[unstable(feature = "stdarch_s390x", issue = "135681")]
4512pub unsafe fn vec_unpackh<T: sealed::VectorUnpackh>(a: T) -> <T as sealed::VectorUnpackh>::Result {
4513    a.vec_unpackh()
4514}
4515
4516/// Vector Unpack Low
4517#[inline]
4518#[target_feature(enable = "vector")]
4519#[unstable(feature = "stdarch_s390x", issue = "135681")]
4520pub unsafe fn vec_unpackl<T: sealed::VectorUnpackl>(a: T) -> <T as sealed::VectorUnpackl>::Result {
4521    a.vec_unpackl()
4522}
4523
4524/// Vector Generate Byte Mask
4525///
4526/// Generates byte masks for elements in the return vector. For each bit in a, if the bit is one, all bit positions
4527/// in the corresponding byte element of d are set to ones. Otherwise, if the bit is zero, the corresponding byte element is set to zero.
4528#[inline]
4529#[target_feature(enable = "vector")]
4530#[unstable(feature = "stdarch_s390x", issue = "135681")]
4531#[cfg_attr(test, assert_instr(vgbm, MASK = 0x00FF))]
4532pub unsafe fn vec_genmask<const MASK: u16>() -> vector_unsigned_char {
4533    vector_unsigned_char(const { genmask::<MASK>() })
4534}
4535
4536/// Vector Generate Mask (Byte)
4537#[inline]
4538#[target_feature(enable = "vector")]
4539#[unstable(feature = "stdarch_s390x", issue = "135681")]
4540#[cfg_attr(test, assert_instr(vrepib, L = 3, H = 5))]
4541pub unsafe fn vec_genmasks_8<const L: u8, const H: u8>() -> vector_unsigned_char {
4542    vector_unsigned_char(const { [genmasks(u8::BITS, L, H) as u8; 16] })
4543}
4544
4545/// Vector Generate Mask (Halfword)
4546#[inline]
4547#[target_feature(enable = "vector")]
4548#[unstable(feature = "stdarch_s390x", issue = "135681")]
4549#[cfg_attr(test, assert_instr(vrepih, L = 3, H = 5))]
4550pub unsafe fn vec_genmasks_16<const L: u8, const H: u8>() -> vector_unsigned_short {
4551    vector_unsigned_short(const { [genmasks(u16::BITS, L, H) as u16; 8] })
4552}
4553
4554/// Vector Generate Mask (Word)
4555#[inline]
4556#[target_feature(enable = "vector")]
4557#[unstable(feature = "stdarch_s390x", issue = "135681")]
4558#[cfg_attr(test, assert_instr(vgmf, L = 3, H = 5))]
4559pub unsafe fn vec_genmasks_32<const L: u8, const H: u8>() -> vector_unsigned_int {
4560    vector_unsigned_int(const { [genmasks(u32::BITS, L, H) as u32; 4] })
4561}
4562
4563/// Vector Generate Mask (Doubleword)
4564#[inline]
4565#[target_feature(enable = "vector")]
4566#[unstable(feature = "stdarch_s390x", issue = "135681")]
4567#[cfg_attr(test, assert_instr(vgmg, L = 3, H = 5))]
4568pub unsafe fn vec_genmasks_64<const L: u8, const H: u8>() -> vector_unsigned_long_long {
4569    vector_unsigned_long_long(const { [genmasks(u64::BITS, L, H); 2] })
4570}
4571
4572/// Vector Permute
4573///
4574/// Returns a vector that contains some elements of two vectors, in the order specified by a third vector.
4575/// Each byte of the result is selected by using the least significant 5 bits of the corresponding byte of c as an index into the concatenated bytes of a and b.
4576/// Note: The vector generate mask built-in function [`vec_genmask`] could help generate the mask c.
4577#[inline]
4578#[target_feature(enable = "vector")]
4579#[unstable(feature = "stdarch_s390x", issue = "135681")]
4580pub unsafe fn vec_perm<T: sealed::VectorPerm>(a: T, b: T, c: vector_unsigned_char) -> T {
4581    a.vec_perm(b, c)
4582}
4583
4584/// Vector Sum Across Quadword
4585///
4586/// Returns a vector containing the results of performing a sum across all the elements in each of the quadword of vector a,
4587/// and the rightmost word or doubleword element of the b. The result is an unsigned 128-bit integer.
4588#[inline]
4589#[target_feature(enable = "vector")]
4590#[unstable(feature = "stdarch_s390x", issue = "135681")]
4591pub unsafe fn vec_sum_u128<T: sealed::VectorSumU128>(a: T, b: T) -> vector_unsigned_char {
4592    a.vec_sum_u128(b)
4593}
4594
4595/// Vector Sum Across Doubleword
4596///
4597/// Returns a vector containing the results of performing a sum across all the elements in each of the doubleword of vector a,
4598/// and the rightmost sub-element of the corresponding doubleword of b.
4599#[inline]
4600#[target_feature(enable = "vector")]
4601#[unstable(feature = "stdarch_s390x", issue = "135681")]
4602pub unsafe fn vec_sum2<T: sealed::VectorSum2>(a: T, b: T) -> vector_unsigned_long_long {
4603    a.vec_sum2(b)
4604}
4605
4606/// Vector Sum Across Word
4607///
4608/// Returns a vector containing the results of performing a sum across all the elements in each of the word of vector a,
4609/// and the rightmost sub-element of the corresponding word of b.
4610#[inline]
4611#[target_feature(enable = "vector")]
4612#[unstable(feature = "stdarch_s390x", issue = "135681")]
4613pub unsafe fn vec_sum4<T: sealed::VectorSum4>(a: T, b: T) -> vector_unsigned_int {
4614    a.vec_sum4(b)
4615}
4616
4617/// Vector Addition unsigned 128-bits
4618///
4619/// Adds unsigned quadword values.
4620///
4621/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a + b.
4622#[inline]
4623#[target_feature(enable = "vector")]
4624#[unstable(feature = "stdarch_s390x", issue = "135681")]
4625#[cfg_attr(test, assert_instr(vaq))]
4626pub unsafe fn vec_add_u128(
4627    a: vector_unsigned_char,
4628    b: vector_unsigned_char,
4629) -> vector_unsigned_char {
4630    let a: u128 = transmute(a);
4631    let b: u128 = transmute(b);
4632    transmute(a.wrapping_add(b))
4633}
4634
4635/// Vector Subtract unsigned 128-bits
4636///
4637/// Subtracts unsigned quadword values.
4638///
4639/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a - b.
4640#[inline]
4641#[target_feature(enable = "vector")]
4642#[unstable(feature = "stdarch_s390x", issue = "135681")]
4643#[cfg_attr(test, assert_instr(vsq))]
4644pub unsafe fn vec_sub_u128(
4645    a: vector_unsigned_char,
4646    b: vector_unsigned_char,
4647) -> vector_unsigned_char {
4648    let a: u128 = transmute(a);
4649    let b: u128 = transmute(b);
4650
4651    transmute(a.wrapping_sub(b))
4652}
4653
4654/// Vector Subtract Carryout
4655///
4656/// Returns a vector containing the borrow produced by subtracting each of corresponding elements of b from a.
4657///
4658/// On each resulting element, the value is 0 if a borrow occurred, or 1 if no borrow occurred.
4659#[inline]
4660#[target_feature(enable = "vector")]
4661#[unstable(feature = "stdarch_s390x", issue = "135681")]
4662pub unsafe fn vec_subc<T: sealed::VectorSubc<U>, U>(a: T, b: U) -> T::Result {
4663    a.vec_subc(b)
4664}
4665
4666/// Vector Subtract Carryout unsigned 128-bits
4667///
4668/// Gets the carry bit of the 128-bit subtraction of two quadword values.
4669/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the borrow produced by subtracting b from a, as unsigned 128-bits integers.
4670/// If no borrow occurred, the bit 127 of d is 1; otherwise it is set to 0. All other bits of d are 0.
4671#[inline]
4672#[target_feature(enable = "vector")]
4673#[unstable(feature = "stdarch_s390x", issue = "135681")]
4674#[cfg_attr(test, assert_instr(vscbiq))]
4675pub unsafe fn vec_subc_u128(
4676    a: vector_unsigned_char,
4677    b: vector_unsigned_char,
4678) -> vector_unsigned_char {
4679    // FIXME(llvm) sadly this does not work https://github.com/llvm/llvm-project/issues/129608
4680    // let a: u128 = transmute(a);
4681    // let b: u128 = transmute(b);
4682    // transmute(!a.overflowing_sub(b).1 as u128)
4683    transmute(vscbiq(transmute(a), transmute(b)))
4684}
4685
4686/// Vector Add Compute Carryout unsigned 128-bits
4687#[inline]
4688#[target_feature(enable = "vector")]
4689#[unstable(feature = "stdarch_s390x", issue = "135681")]
4690#[cfg_attr(test, assert_instr(vaccq))]
4691pub unsafe fn vec_addc_u128(
4692    a: vector_unsigned_char,
4693    b: vector_unsigned_char,
4694) -> vector_unsigned_char {
4695    let a: u128 = transmute(a);
4696    let b: u128 = transmute(b);
4697    transmute(a.overflowing_add(b).1 as u128)
4698}
4699
4700/// Vector Add With Carry unsigned 128-bits
4701#[inline]
4702#[target_feature(enable = "vector")]
4703#[unstable(feature = "stdarch_s390x", issue = "135681")]
4704#[cfg_attr(test, assert_instr(vacq))]
4705pub unsafe fn vec_adde_u128(
4706    a: vector_unsigned_char,
4707    b: vector_unsigned_char,
4708    c: vector_unsigned_char,
4709) -> vector_unsigned_char {
4710    let a: u128 = transmute(a);
4711    let b: u128 = transmute(b);
4712    let c: u128 = transmute(c);
4713    // FIXME(llvm) sadly this does not work
4714    //     let (d, _carry) = a.carrying_add(b, c & 1 != 0);
4715    //     transmute(d)
4716    transmute(vacq(a, b, c))
4717}
4718
4719/// Vector Add With Carry Compute Carry unsigned 128-bits
4720#[inline]
4721#[target_feature(enable = "vector")]
4722#[unstable(feature = "stdarch_s390x", issue = "135681")]
4723#[cfg_attr(test, assert_instr(vacccq))]
4724pub unsafe fn vec_addec_u128(
4725    a: vector_unsigned_char,
4726    b: vector_unsigned_char,
4727    c: vector_unsigned_char,
4728) -> vector_unsigned_char {
4729    let a: u128 = transmute(a);
4730    let b: u128 = transmute(b);
4731    let c: u128 = transmute(c);
4732    let (_d, carry) = a.carrying_add(b, c & 1 != 0);
4733    transmute(carry as u128)
4734}
4735
4736/// Vector Subtract with Carryout
4737///
4738/// Subtracts unsigned quadword values with carry bit from a previous operation.
4739///
4740/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the result of subtracting of b from a,
4741/// and the carryout bit from a previous operation.
4742///
4743/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored.
4744#[inline]
4745#[target_feature(enable = "vector")]
4746#[unstable(feature = "stdarch_s390x", issue = "135681")]
4747#[cfg_attr(test, assert_instr(vsbiq))]
4748pub unsafe fn vec_sube_u128(
4749    a: vector_unsigned_char,
4750    b: vector_unsigned_char,
4751    c: vector_unsigned_char,
4752) -> vector_unsigned_char {
4753    transmute(vsbiq(transmute(a), transmute(b), transmute(c)))
4754}
4755
4756/// Vector Subtract with Carryout, Carryout
4757///
4758/// Gets the carry bit of the 128-bit subtraction of two quadword values with carry bit from the previous operation.
4759///
4760/// It returns a vector containing the carryout produced from the result of subtracting of b from a,
4761/// and the carryout bit from a previous operation. If no borrow occurred, the 127-bit of d is 1, otherwise 0.
4762/// All other bits of d are 0.
4763///
4764/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored.
4765#[inline]
4766#[target_feature(enable = "vector")]
4767#[unstable(feature = "stdarch_s390x", issue = "135681")]
4768#[cfg_attr(test, assert_instr(vsbcbiq))]
4769pub unsafe fn vec_subec_u128(
4770    a: vector_unsigned_char,
4771    b: vector_unsigned_char,
4772    c: vector_unsigned_char,
4773) -> vector_unsigned_char {
4774    transmute(vsbcbiq(transmute(a), transmute(b), transmute(c)))
4775}
4776
4777/// Vector Splat Signed Byte
4778#[inline]
4779#[target_feature(enable = "vector")]
4780#[unstable(feature = "stdarch_s390x", issue = "135681")]
4781#[cfg_attr(test, assert_instr(vrepib, IMM = 42))]
4782pub unsafe fn vec_splat_s8<const IMM: i8>() -> vector_signed_char {
4783    vector_signed_char([IMM; 16])
4784}
4785
4786/// Vector Splat Signed Halfword
4787#[inline]
4788#[target_feature(enable = "vector")]
4789#[unstable(feature = "stdarch_s390x", issue = "135681")]
4790#[cfg_attr(test, assert_instr(vrepih, IMM = 42))]
4791pub unsafe fn vec_splat_s16<const IMM: i16>() -> vector_signed_short {
4792    vector_signed_short([IMM; 8])
4793}
4794
4795/// Vector Splat Signed Word
4796#[inline]
4797#[target_feature(enable = "vector")]
4798#[unstable(feature = "stdarch_s390x", issue = "135681")]
4799#[cfg_attr(test, assert_instr(vrepif, IMM = 42))]
4800pub unsafe fn vec_splat_s32<const IMM: i16>() -> vector_signed_int {
4801    vector_signed_int([IMM as i32; 4])
4802}
4803
4804/// Vector Splat Signed Doubleword
4805#[inline]
4806#[target_feature(enable = "vector")]
4807#[unstable(feature = "stdarch_s390x", issue = "135681")]
4808#[cfg_attr(test, assert_instr(vrepig, IMM = 42))]
4809pub unsafe fn vec_splat_s64<const IMM: i16>() -> vector_signed_long_long {
4810    vector_signed_long_long([IMM as i64; 2])
4811}
4812
4813/// Vector Splat Unsigned Byte
4814#[inline]
4815#[target_feature(enable = "vector")]
4816#[unstable(feature = "stdarch_s390x", issue = "135681")]
4817#[cfg_attr(test, assert_instr(vrepib, IMM = 42))]
4818pub unsafe fn vec_splat_u8<const IMM: u8>() -> vector_unsigned_char {
4819    vector_unsigned_char([IMM; 16])
4820}
4821
4822/// Vector Splat Unsigned Halfword
4823#[inline]
4824#[target_feature(enable = "vector")]
4825#[unstable(feature = "stdarch_s390x", issue = "135681")]
4826#[cfg_attr(test, assert_instr(vrepih, IMM = 42))]
4827pub unsafe fn vec_splat_u16<const IMM: i16>() -> vector_unsigned_short {
4828    vector_unsigned_short([IMM as u16; 8])
4829}
4830
4831/// Vector Splat Unsigned Word
4832#[inline]
4833#[target_feature(enable = "vector")]
4834#[unstable(feature = "stdarch_s390x", issue = "135681")]
4835#[cfg_attr(test, assert_instr(vrepif, IMM = 42))]
4836pub unsafe fn vec_splat_u32<const IMM: i16>() -> vector_unsigned_int {
4837    vector_unsigned_int([IMM as u32; 4])
4838}
4839
4840/// Vector Splat Unsigned Doubleword
4841#[inline]
4842#[target_feature(enable = "vector")]
4843#[unstable(feature = "stdarch_s390x", issue = "135681")]
4844#[cfg_attr(test, assert_instr(vrepig, IMM = 42))]
4845pub unsafe fn vec_splat_u64<const IMM: i16>() -> vector_unsigned_long_long {
4846    vector_unsigned_long_long([IMM as u64; 2])
4847}
4848
4849macro_rules! vec_find_any {
4850    ($($Trait:ident $fun:ident $doc:literal)*) => {
4851        $(
4852            #[inline]
4853            #[target_feature(enable = "vector")]
4854            #[unstable(feature = "stdarch_s390x", issue = "135681")]
4855            #[doc = $doc]
4856            pub unsafe fn $fun<T: sealed::$Trait<U>, U>(a: T, b: U) -> T::Result {
4857                a.$fun(b)
4858            }
4859        )*
4860    }
4861}
4862
4863vec_find_any! {
4864    VectorFindAnyEq vec_find_any_eq "Vector Find Any Element Equal with Condition Code"
4865    VectorFindAnyNe vec_find_any_ne "Vector Find Any Element Not Equal with Condition Code"
4866    VectorFindAnyEqIdx vec_find_any_eq_idx "Vector Find Any Element Equal Index with Condition Code"
4867    VectorFindAnyNeIdx vec_find_any_ne_idx "Vector Find Any Element Not Equal Index with Condition Code"
4868    VectorFindAnyEqOrZeroIdx vec_find_any_eq_or_0_idx "Vector Find Any Element Equal or Zero Index with Condition Code"
4869    VectorFindAnyNeOrZeroIdx vec_find_any_ne_or_0_idx "Vector Find Any Element Not Equal or Zero Index with Condition Code"
4870}
4871
4872macro_rules! vec_find_any_cc {
4873    ($($Trait:ident $fun:ident $doc:literal)*) => {
4874        $(
4875            #[inline]
4876            #[target_feature(enable = "vector")]
4877            #[unstable(feature = "stdarch_s390x", issue = "135681")]
4878            #[doc = $doc]
4879            pub unsafe fn $fun<T: sealed::$Trait<U>, U>(a: T, b: U) -> (T::Result, i32) {
4880                a.$fun(b)
4881            }
4882        )*
4883    }
4884}
4885
4886vec_find_any_cc! {
4887    VectorFindAnyEqCC vec_find_any_eq_cc "Vector Find Any Element Equal with Condition Code"
4888    VectorFindAnyNeCC vec_find_any_ne_cc "Vector Find Any Element Not Equal with Condition Code"
4889    VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc "Vector Find Any Element Equal Index with Condition Code"
4890    VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc "Vector Find Any Element Not Equal Index with Condition Code"
4891    VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc "Vector Find Any Element Equal or Zero Index with Condition Code"
4892    VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc "Vector Find Any Element Not Equal or Zero Index with Condition Code"
4893}
4894
4895/// Vector Load
4896#[inline]
4897#[target_feature(enable = "vector")]
4898#[unstable(feature = "stdarch_s390x", issue = "135681")]
4899pub unsafe fn vec_xl<T: sealed::VectorLoad>(offset: isize, ptr: *const T::ElementType) -> T {
4900    T::vec_xl(offset, ptr)
4901}
4902
4903/// Vector Load Pair
4904#[inline]
4905#[target_feature(enable = "vector")]
4906#[unstable(feature = "stdarch_s390x", issue = "135681")]
4907pub unsafe fn vec_load_pair<T: sealed::VectorLoadPair>(a: T::ElementType, b: T::ElementType) -> T {
4908    T::vec_load_pair(a, b)
4909}
4910
4911/// Vector Load to Block Boundary
4912#[inline]
4913#[target_feature(enable = "vector")]
4914#[unstable(feature = "stdarch_s390x", issue = "135681")]
4915pub unsafe fn vec_load_bndry<T: sealed::VectorLoad, const BLOCK_BOUNDARY: u16>(
4916    ptr: *const T::ElementType,
4917) -> MaybeUninit<T> {
4918    T::vec_load_bndry::<BLOCK_BOUNDARY>(ptr)
4919}
4920
4921/// Vector Store
4922#[inline]
4923#[target_feature(enable = "vector")]
4924#[unstable(feature = "stdarch_s390x", issue = "135681")]
4925pub unsafe fn vec_xst<T: sealed::VectorStore>(vector: T, offset: isize, ptr: *mut T::ElementType) {
4926    vector.vec_xst(offset, ptr)
4927}
4928
4929/// Vector Load with Length
4930#[inline]
4931#[target_feature(enable = "vector")]
4932#[unstable(feature = "stdarch_s390x", issue = "135681")]
4933pub unsafe fn vec_load_len<T: sealed::VectorLoad>(
4934    ptr: *const T::ElementType,
4935    byte_count: u32,
4936) -> T {
4937    T::vec_load_len(ptr, byte_count)
4938}
4939
4940/// Vector Store with Length
4941#[inline]
4942#[target_feature(enable = "vector")]
4943#[unstable(feature = "stdarch_s390x", issue = "135681")]
4944pub unsafe fn vec_store_len<T: sealed::VectorStore>(
4945    vector: T,
4946    ptr: *mut T::ElementType,
4947    byte_count: u32,
4948) {
4949    vector.vec_store_len(ptr, byte_count)
4950}
4951
4952/// Vector Load Rightmost with Length
4953#[inline]
4954#[target_feature(enable = "vector-packed-decimal")]
4955#[unstable(feature = "stdarch_s390x", issue = "135681")]
4956#[cfg_attr(test, assert_instr(vlrlr))]
4957pub unsafe fn vec_load_len_r(ptr: *const u8, byte_count: u32) -> vector_unsigned_char {
4958    vlrl(byte_count, ptr)
4959}
4960
4961/// Vector Store Rightmost with Length
4962#[inline]
4963#[target_feature(enable = "vector-packed-decimal")]
4964#[unstable(feature = "stdarch_s390x", issue = "135681")]
4965#[cfg_attr(test, assert_instr(vstrlr))]
4966pub unsafe fn vec_store_len_r(vector: vector_unsigned_char, ptr: *mut u8, byte_count: u32) {
4967    vstrl(vector, byte_count, ptr)
4968}
4969
4970/// Vector Multiply Add
4971#[inline]
4972#[target_feature(enable = "vector-packed-decimal")]
4973#[unstable(feature = "stdarch_s390x", issue = "135681")]
4974pub unsafe fn vec_madd<T: sealed::VectorMadd>(a: T, b: T, c: T) -> T {
4975    a.vec_madd(b, c)
4976}
4977
4978/// Vector Multiply Add
4979#[inline]
4980#[target_feature(enable = "vector-packed-decimal")]
4981#[unstable(feature = "stdarch_s390x", issue = "135681")]
4982pub unsafe fn vec_msub<T: sealed::VectorMadd>(a: T, b: T, c: T) -> T {
4983    a.vec_msub(b, c)
4984}
4985
4986/// Vector Multiply and Add Even
4987#[inline]
4988#[target_feature(enable = "vector-packed-decimal")]
4989#[unstable(feature = "stdarch_s390x", issue = "135681")]
4990pub unsafe fn vec_meadd<T: sealed::VectorMeadd>(a: T, b: T, c: T::Result) -> T::Result {
4991    a.vec_meadd(b, c)
4992}
4993
4994/// Vector Multiply and Add Odd
4995#[inline]
4996#[target_feature(enable = "vector-packed-decimal")]
4997#[unstable(feature = "stdarch_s390x", issue = "135681")]
4998pub unsafe fn vec_moadd<T: sealed::VectorMoadd>(a: T, b: T, c: T::Result) -> T::Result {
4999    a.vec_moadd(b, c)
5000}
5001
5002/// Vector Multiply and Add High
5003#[inline]
5004#[target_feature(enable = "vector-packed-decimal")]
5005#[unstable(feature = "stdarch_s390x", issue = "135681")]
5006pub unsafe fn vec_mhadd<T: sealed::VectorMhadd>(a: T, b: T, c: T::Result) -> T::Result {
5007    a.vec_mhadd(b, c)
5008}
5009
5010/// Vector Multiply and Add Low
5011#[inline]
5012#[target_feature(enable = "vector-packed-decimal")]
5013#[unstable(feature = "stdarch_s390x", issue = "135681")]
5014pub unsafe fn vec_mladd<T: sealed::VectorMladd>(a: T, b: T, c: T::Result) -> T::Result {
5015    a.vec_mladd(b, c)
5016}
5017
5018/// Vector Checksum
5019#[inline]
5020#[target_feature(enable = "vector")]
5021#[unstable(feature = "stdarch_s390x", issue = "135681")]
5022#[cfg_attr(test, assert_instr(vcksm))]
5023pub unsafe fn vec_checksum(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int {
5024    vcksm(a, b)
5025}
5026
5027/// Vector Multiply Even
5028#[inline]
5029#[target_feature(enable = "vector")]
5030#[unstable(feature = "stdarch_s390x", issue = "135681")]
5031pub unsafe fn vec_mule<T: sealed::VectorMule<U>, U>(a: T, b: T) -> U {
5032    a.vec_mule(b)
5033}
5034
5035/// Vector Multiply Odd
5036#[inline]
5037#[target_feature(enable = "vector")]
5038#[unstable(feature = "stdarch_s390x", issue = "135681")]
5039pub unsafe fn vec_mulo<T: sealed::VectorMulo<U>, U>(a: T, b: T) -> U {
5040    a.vec_mulo(b)
5041}
5042
5043/// Vector Multiply High
5044#[inline]
5045#[target_feature(enable = "vector")]
5046#[unstable(feature = "stdarch_s390x", issue = "135681")]
5047pub unsafe fn vec_mulh<T: sealed::VectorMulh<U>, U>(a: T, b: T) -> U {
5048    a.vec_mulh(b)
5049}
5050
5051/// Vector Galois Field Multiply Sum
5052#[inline]
5053#[target_feature(enable = "vector")]
5054#[unstable(feature = "stdarch_s390x", issue = "135681")]
5055pub unsafe fn vec_gfmsum<T: sealed::VectorGfmsum<U>, U>(a: T, b: T) -> U {
5056    a.vec_gfmsum(b)
5057}
5058
5059/// Vector Galois Field Multiply Sum
5060#[inline]
5061#[target_feature(enable = "vector")]
5062#[unstable(feature = "stdarch_s390x", issue = "135681")]
5063pub unsafe fn vec_gfmsum_accum<T: sealed::VectorGfmsumAccum>(
5064    a: T,
5065    b: T,
5066    c: T::Result,
5067) -> T::Result {
5068    a.vec_gfmsum_accum(b, c)
5069}
5070
5071/// Vector Galois Field Multiply Sum 128-bits
5072#[inline]
5073#[target_feature(enable = "vector")]
5074#[unstable(feature = "stdarch_s390x", issue = "135681")]
5075#[cfg_attr(test, assert_instr(vgfmg))]
5076pub unsafe fn vec_gfmsum_128(
5077    a: vector_unsigned_long_long,
5078    b: vector_unsigned_long_long,
5079) -> vector_unsigned_char {
5080    transmute(vgfmg(a, b))
5081}
5082
5083/// Vector Galois Field Multiply Sum and Accumulate 128-bits
5084#[inline]
5085#[target_feature(enable = "vector")]
5086#[unstable(feature = "stdarch_s390x", issue = "135681")]
5087#[cfg_attr(test, assert_instr(vgfmag))]
5088pub unsafe fn vec_gfmsum_accum_128(
5089    a: vector_unsigned_long_long,
5090    b: vector_unsigned_long_long,
5091    c: vector_unsigned_char,
5092) -> vector_unsigned_char {
5093    transmute(vgfmag(a, b, transmute(c)))
5094}
5095
5096/// Vector Bit Permute
5097#[inline]
5098#[target_feature(enable = "vector-enhancements-1")]
5099#[unstable(feature = "stdarch_s390x", issue = "135681")]
5100#[cfg_attr(test, assert_instr(vbperm))]
5101pub unsafe fn vec_bperm_u128(
5102    a: vector_unsigned_char,
5103    b: vector_unsigned_char,
5104) -> vector_unsigned_long_long {
5105    vbperm(a, b)
5106}
5107
5108/// Vector Gather Element
5109#[inline]
5110#[target_feature(enable = "vector")]
5111#[unstable(feature = "stdarch_s390x", issue = "135681")]
5112pub unsafe fn vec_gather_element<T: sealed::VectorGatherElement, const D: u32>(
5113    a: T,
5114    b: T::Offset,
5115    c: *const T::Element,
5116) -> T {
5117    a.vec_gather_element::<D>(b, c)
5118}
5119
5120/// Vector Select
5121#[inline]
5122#[target_feature(enable = "vector")]
5123#[unstable(feature = "stdarch_s390x", issue = "135681")]
5124pub unsafe fn vec_sel<T: sealed::VectorSel<U>, U>(a: T, b: T, c: U) -> T {
5125    a.vec_sel(b, c)
5126}
5127
5128#[unstable(feature = "stdarch_s390x", issue = "135681")]
5129pub const __VEC_CLASS_FP_ZERO_P: u32 = 1 << 11;
5130#[unstable(feature = "stdarch_s390x", issue = "135681")]
5131pub const __VEC_CLASS_FP_ZERO_N: u32 = 1 << 10;
5132#[unstable(feature = "stdarch_s390x", issue = "135681")]
5133pub const __VEC_CLASS_FP_ZERO: u32 = __VEC_CLASS_FP_ZERO_P | __VEC_CLASS_FP_ZERO_N;
5134#[unstable(feature = "stdarch_s390x", issue = "135681")]
5135pub const __VEC_CLASS_FP_NORMAL_P: u32 = 1 << 9;
5136#[unstable(feature = "stdarch_s390x", issue = "135681")]
5137pub const __VEC_CLASS_FP_NORMAL_N: u32 = 1 << 8;
5138#[unstable(feature = "stdarch_s390x", issue = "135681")]
5139pub const __VEC_CLASS_FP_NORMAL: u32 = __VEC_CLASS_FP_NORMAL_P | __VEC_CLASS_FP_NORMAL_N;
5140#[unstable(feature = "stdarch_s390x", issue = "135681")]
5141pub const __VEC_CLASS_FP_SUBNORMAL_P: u32 = 1 << 7;
5142#[unstable(feature = "stdarch_s390x", issue = "135681")]
5143pub const __VEC_CLASS_FP_SUBNORMAL_N: u32 = 1 << 6;
5144#[unstable(feature = "stdarch_s390x", issue = "135681")]
5145pub const __VEC_CLASS_FP_SUBNORMAL: u32 = __VEC_CLASS_FP_SUBNORMAL_P | __VEC_CLASS_FP_SUBNORMAL_N;
5146#[unstable(feature = "stdarch_s390x", issue = "135681")]
5147pub const __VEC_CLASS_FP_INFINITY_P: u32 = 1 << 5;
5148#[unstable(feature = "stdarch_s390x", issue = "135681")]
5149pub const __VEC_CLASS_FP_INFINITY_N: u32 = 1 << 4;
5150#[unstable(feature = "stdarch_s390x", issue = "135681")]
5151pub const __VEC_CLASS_FP_INFINITY: u32 = __VEC_CLASS_FP_INFINITY_P | __VEC_CLASS_FP_INFINITY_N;
5152#[unstable(feature = "stdarch_s390x", issue = "135681")]
5153pub const __VEC_CLASS_FP_QNAN_P: u32 = 1 << 3;
5154#[unstable(feature = "stdarch_s390x", issue = "135681")]
5155pub const __VEC_CLASS_FP_QNAN_N: u32 = 1 << 2;
5156#[unstable(feature = "stdarch_s390x", issue = "135681")]
5157pub const __VEC_CLASS_FP_QNAN: u32 = __VEC_CLASS_FP_QNAN_P | __VEC_CLASS_FP_QNAN_N;
5158#[unstable(feature = "stdarch_s390x", issue = "135681")]
5159pub const __VEC_CLASS_FP_SNAN_P: u32 = 1 << 1;
5160#[unstable(feature = "stdarch_s390x", issue = "135681")]
5161pub const __VEC_CLASS_FP_SNAN_N: u32 = 1 << 0;
5162#[unstable(feature = "stdarch_s390x", issue = "135681")]
5163pub const __VEC_CLASS_FP_SNAN: u32 = __VEC_CLASS_FP_SNAN_P | __VEC_CLASS_FP_SNAN_N;
5164#[unstable(feature = "stdarch_s390x", issue = "135681")]
5165pub const __VEC_CLASS_FP_NAN: u32 = __VEC_CLASS_FP_QNAN | __VEC_CLASS_FP_SNAN;
5166#[unstable(feature = "stdarch_s390x", issue = "135681")]
5167pub const __VEC_CLASS_FP_NOT_NORMAL: u32 =
5168    __VEC_CLASS_FP_NAN | __VEC_CLASS_FP_SUBNORMAL | __VEC_CLASS_FP_ZERO | __VEC_CLASS_FP_INFINITY;
5169
5170/// Vector Floating-Point Test Data Class
5171///
5172/// You can use the `__VEC_CLASS_FP_*` constants as the argument for this operand
5173#[inline]
5174#[target_feature(enable = "vector")]
5175#[unstable(feature = "stdarch_s390x", issue = "135681")]
5176pub unsafe fn vec_fp_test_data_class<T: sealed::VectorFpTestDataClass, const CLASS: u32>(
5177    a: T,
5178    c: *mut i32,
5179) -> T::Result {
5180    let (x, y) = a.vec_fp_test_data_class::<CLASS>();
5181    c.write(y);
5182    x
5183}
5184
5185/// All Elements Not a Number
5186#[inline]
5187#[target_feature(enable = "vector")]
5188#[unstable(feature = "stdarch_s390x", issue = "135681")]
5189pub unsafe fn vec_all_nan<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5190    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 0)
5191}
5192
5193/// All Elements Numeric
5194#[inline]
5195#[target_feature(enable = "vector")]
5196#[unstable(feature = "stdarch_s390x", issue = "135681")]
5197pub unsafe fn vec_all_numeric<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5198    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 3)
5199}
5200
5201/// Any Elements Not a Number
5202#[inline]
5203#[target_feature(enable = "vector")]
5204#[unstable(feature = "stdarch_s390x", issue = "135681")]
5205pub unsafe fn vec_any_nan<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5206    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 3)
5207}
5208
5209/// Any Elements Numeric
5210#[inline]
5211#[target_feature(enable = "vector")]
5212#[unstable(feature = "stdarch_s390x", issue = "135681")]
5213pub unsafe fn vec_any_numeric<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5214    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 0)
5215}
5216
5217/// Vector Test under Mask
5218#[inline]
5219#[target_feature(enable = "vector")]
5220#[unstable(feature = "stdarch_s390x", issue = "135681")]
5221pub unsafe fn vec_test_mask<T: sealed::VectorTestMask>(a: T, b: T::Mask) -> i32 {
5222    // I can't find much information about this, but this might just be a check for whether the
5223    // bitwise and of a and b is non-zero?
5224    a.vec_test_mask(b)
5225}
5226
5227/// Vector Search String
5228#[inline]
5229#[target_feature(enable = "vector")]
5230#[unstable(feature = "stdarch_s390x", issue = "135681")]
5231pub unsafe fn vec_search_string_cc<T: sealed::VectorSearchString>(
5232    a: T,
5233    b: T,
5234    c: vector_unsigned_char,
5235) -> (vector_unsigned_char, i32) {
5236    a.vec_search_string_cc(b, c)
5237}
5238
5239/// Vector Search String Until Zero
5240#[inline]
5241#[target_feature(enable = "vector")]
5242#[unstable(feature = "stdarch_s390x", issue = "135681")]
5243pub unsafe fn vec_search_string_until_zero_cc<T: sealed::VectorSearchString>(
5244    a: T,
5245    b: T,
5246    c: vector_unsigned_char,
5247) -> (vector_unsigned_char, i32) {
5248    a.vec_search_string_until_zero_cc(b, c)
5249}
5250
5251/// Vector Convert from float (even elements) to double
5252#[inline]
5253#[target_feature(enable = "vector-enhancements-1")]
5254#[unstable(feature = "stdarch_s390x", issue = "135681")]
5255// FIXME: this emits `vflls` where `vldeb` is expected
5256// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vldeb))]
5257pub unsafe fn vec_doublee(a: vector_float) -> vector_double {
5258    let even = simd_shuffle::<_, _, f32x2>(a, a, const { u32x2::from_array([0, 2]) });
5259    simd_as(even)
5260}
5261
5262/// Vector Convert from double to float (even elements)
5263#[inline]
5264#[target_feature(enable = "vector-enhancements-1")]
5265#[unstable(feature = "stdarch_s390x", issue = "135681")]
5266// FIXME: the C version uses a shuffle mask with poison; we can't do that
5267// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vledb))]
5268pub unsafe fn vec_floate(a: vector_double) -> vector_float {
5269    let truncated: f32x2 = simd_as(a);
5270    simd_shuffle(
5271        truncated,
5272        truncated,
5273        const { u32x4::from_array([0, 0, 1, 1]) },
5274    )
5275}
5276
5277/// Vector Convert from int to float
5278#[inline]
5279#[target_feature(enable = "vector")]
5280#[unstable(feature = "stdarch_s390x", issue = "135681")]
5281pub unsafe fn vec_float(a: impl sealed::VectorFloat) -> vector_float {
5282    a.vec_float()
5283}
5284
5285/// Vector Convert from long long to double
5286#[inline]
5287#[target_feature(enable = "vector")]
5288#[unstable(feature = "stdarch_s390x", issue = "135681")]
5289pub unsafe fn vec_double(a: impl sealed::VectorDouble) -> vector_double {
5290    a.vec_double()
5291}
5292
5293/// Vector Sign Extend to Doubleword
5294#[inline]
5295#[target_feature(enable = "vector")]
5296#[unstable(feature = "stdarch_s390x", issue = "135681")]
5297pub unsafe fn vec_extend_s64(a: impl sealed::VectorExtendSigned64) -> vector_signed_long_long {
5298    a.vec_extend_s64()
5299}
5300
5301/// Vector Convert floating point to signed
5302#[inline]
5303#[target_feature(enable = "vector")]
5304#[unstable(feature = "stdarch_s390x", issue = "135681")]
5305pub unsafe fn vec_signed<T: sealed::VectorSigned>(a: T) -> T::Result {
5306    a.vec_signed()
5307}
5308
5309/// Vector Convert floating point to unsigned
5310#[inline]
5311#[target_feature(enable = "vector")]
5312#[unstable(feature = "stdarch_s390x", issue = "135681")]
5313pub unsafe fn vec_unsigned<T: sealed::VectorUnsigned>(a: T) -> T::Result {
5314    a.vec_unsigned()
5315}
5316
5317/// Vector Copy Until Zero
5318#[inline]
5319#[target_feature(enable = "vector")]
5320#[unstable(feature = "stdarch_s390x", issue = "135681")]
5321pub unsafe fn vec_cp_until_zero<T: sealed::VectorCopyUntilZero>(a: T) -> T {
5322    a.vec_cp_until_zero()
5323}
5324
5325/// Vector Copy Until Zero
5326#[inline]
5327#[target_feature(enable = "vector")]
5328#[unstable(feature = "stdarch_s390x", issue = "135681")]
5329pub unsafe fn vec_cp_until_zero_cc<T: sealed::VectorCopyUntilZeroCC>(a: T) -> (T, i32) {
5330    a.vec_cp_until_zero_cc()
5331}
5332
5333/// Vector Multiply Sum Logical
5334#[inline]
5335#[target_feature(enable = "vector-enhancements-1")]
5336#[unstable(feature = "stdarch_s390x", issue = "135681")]
5337#[cfg_attr(
5338    all(test, target_feature = "vector-enhancements-1"),
5339    assert_instr(vmslg, D = 4)
5340)]
5341pub unsafe fn vec_msum_u128<const D: u32>(
5342    a: vector_unsigned_long_long,
5343    b: vector_unsigned_long_long,
5344    c: vector_unsigned_char,
5345) -> vector_unsigned_char {
5346    const {
5347        if !matches!(D, 0 | 4 | 8 | 12) {
5348            panic!("D needs to be one of 0, 4, 8, 12");
5349        }
5350    };
5351    transmute(vmslg(a, b, transmute(c), D))
5352}
5353
5354/// Vector Shift Left Double by Byte
5355#[inline]
5356#[target_feature(enable = "vector")]
5357#[unstable(feature = "stdarch_s390x", issue = "135681")]
5358pub unsafe fn vec_sld<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5359    static_assert_uimm_bits!(C, 4);
5360    a.vec_sld::<C>(b)
5361}
5362
5363/// Vector Shift Left Double by Word
5364#[inline]
5365#[target_feature(enable = "vector")]
5366#[unstable(feature = "stdarch_s390x", issue = "135681")]
5367pub unsafe fn vec_sldw<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5368    static_assert_uimm_bits!(C, 2);
5369    a.vec_sldw::<C>(b)
5370}
5371
5372/// Vector Shift Left Double by Bit
5373#[inline]
5374#[target_feature(enable = "vector-enhancements-2")]
5375#[unstable(feature = "stdarch_s390x", issue = "135681")]
5376pub unsafe fn vec_sldb<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5377    static_assert_uimm_bits!(C, 3);
5378    a.vec_sldb::<C>(b)
5379}
5380
5381/// Vector Shift Right Double by Bit
5382#[inline]
5383#[target_feature(enable = "vector-enhancements-2")]
5384#[unstable(feature = "stdarch_s390x", issue = "135681")]
5385pub unsafe fn vec_srdb<T: sealed::VectorSrdb, const C: u32>(a: T, b: T) -> T {
5386    static_assert_uimm_bits!(C, 3);
5387    a.vec_srdb::<C>(b)
5388}
5389
5390/// Vector Compare Ranges
5391#[inline]
5392#[target_feature(enable = "vector")]
5393#[unstable(feature = "stdarch_s390x", issue = "135681")]
5394pub unsafe fn vec_cmprg<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5395    a.vstrc::<{ FindImm::Eq as u32 }>(b, c)
5396}
5397
5398/// Vector Compare Not in Ranges
5399#[inline]
5400#[target_feature(enable = "vector")]
5401#[unstable(feature = "stdarch_s390x", issue = "135681")]
5402pub unsafe fn vec_cmpnrg<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5403    a.vstrc::<{ FindImm::Ne as u32 }>(b, c)
5404}
5405
5406/// Vector Compare Ranges Index
5407#[inline]
5408#[target_feature(enable = "vector")]
5409#[unstable(feature = "stdarch_s390x", issue = "135681")]
5410pub unsafe fn vec_cmprg_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5411    a.vstrc::<{ FindImm::EqIdx as u32 }>(b, c)
5412}
5413
5414/// Vector Compare Not in Ranges Index
5415#[inline]
5416#[target_feature(enable = "vector")]
5417#[unstable(feature = "stdarch_s390x", issue = "135681")]
5418pub unsafe fn vec_cmpnrg_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5419    a.vstrc::<{ FindImm::NeIdx as u32 }>(b, c)
5420}
5421
5422/// Vector Compare Ranges with Condition Code
5423#[inline]
5424#[target_feature(enable = "vector")]
5425#[unstable(feature = "stdarch_s390x", issue = "135681")]
5426pub unsafe fn vec_cmprg_cc<T: sealed::VectorCompareRange>(
5427    a: T,
5428    b: T,
5429    c: T,
5430    d: *mut i32,
5431) -> T::Result {
5432    let (x, y) = a.vstrcs::<{ FindImm::Eq as u32 }>(b, c);
5433    d.write(y);
5434    x
5435}
5436
5437/// Vector Compare Not in Ranges with Condition Code
5438#[inline]
5439#[target_feature(enable = "vector")]
5440#[unstable(feature = "stdarch_s390x", issue = "135681")]
5441pub unsafe fn vec_cmpnrg_cc<T: sealed::VectorCompareRange>(
5442    a: T,
5443    b: T,
5444    c: T,
5445    d: *mut i32,
5446) -> T::Result {
5447    let (x, y) = a.vstrcs::<{ FindImm::Ne as u32 }>(b, c);
5448    d.write(y);
5449    x
5450}
5451
5452/// Vector Compare Ranges Index with Condition Code
5453#[inline]
5454#[target_feature(enable = "vector")]
5455#[unstable(feature = "stdarch_s390x", issue = "135681")]
5456pub unsafe fn vec_cmprg_idx_cc<T: sealed::VectorCompareRange>(
5457    a: T,
5458    b: T,
5459    c: T,
5460    d: *mut i32,
5461) -> T::Result {
5462    let (x, y) = a.vstrcs::<{ FindImm::EqIdx as u32 }>(b, c);
5463    d.write(y);
5464    x
5465}
5466
5467/// Vector Compare Not in Ranges Index with Condition Code
5468#[inline]
5469#[target_feature(enable = "vector")]
5470#[unstable(feature = "stdarch_s390x", issue = "135681")]
5471pub unsafe fn vec_cmpnrg_idx_cc<T: sealed::VectorCompareRange>(
5472    a: T,
5473    b: T,
5474    c: T,
5475    d: *mut i32,
5476) -> T::Result {
5477    let (x, y) = a.vstrcs::<{ FindImm::NeIdx as u32 }>(b, c);
5478    d.write(y);
5479    x
5480}
5481
5482/// Vector Compare Ranges or Zero Index
5483#[inline]
5484#[target_feature(enable = "vector")]
5485#[unstable(feature = "stdarch_s390x", issue = "135681")]
5486pub unsafe fn vec_cmprg_or_0_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5487    a.vstrcz::<{ FindImm::EqIdx as u32 }>(b, c)
5488}
5489
5490/// Vector Compare Not in Ranges or Zero Index
5491#[inline]
5492#[target_feature(enable = "vector")]
5493#[unstable(feature = "stdarch_s390x", issue = "135681")]
5494pub unsafe fn vec_cmpnrg_or_0_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5495    a.vstrcz::<{ FindImm::NeIdx as u32 }>(b, c)
5496}
5497
5498/// Vector Compare Ranges or Zero Index with Condition Code
5499#[inline]
5500#[target_feature(enable = "vector")]
5501#[unstable(feature = "stdarch_s390x", issue = "135681")]
5502pub unsafe fn vec_cmprg_or_0_idx_cc<T: sealed::VectorCompareRange>(
5503    a: T,
5504    b: T,
5505    c: T,
5506    d: *mut i32,
5507) -> T::Result {
5508    let (x, y) = a.vstrczs::<{ FindImm::EqIdx as u32 }>(b, c);
5509    d.write(y);
5510    x
5511}
5512
5513/// Vector Compare Not in Ranges or Zero Index with Condition Code
5514#[inline]
5515#[target_feature(enable = "vector")]
5516#[unstable(feature = "stdarch_s390x", issue = "135681")]
5517pub unsafe fn vec_cmpnrg_or_0_idx_cc<T: sealed::VectorCompareRange>(
5518    a: T,
5519    b: T,
5520    c: T,
5521    d: *mut i32,
5522) -> T::Result {
5523    let (x, y) = a.vstrczs::<{ FindImm::NeIdx as u32 }>(b, c);
5524    d.write(y);
5525    x
5526}
5527
5528/// Vector Compare Equal
5529#[inline]
5530#[target_feature(enable = "vector")]
5531#[unstable(feature = "stdarch_s390x", issue = "135681")]
5532pub unsafe fn vec_cmpeq<T: sealed::VectorEquality>(a: T, b: T) -> T::Result {
5533    a.vec_cmpeq(b)
5534}
5535
5536/// Vector Compare Not Equal
5537#[inline]
5538#[target_feature(enable = "vector")]
5539#[unstable(feature = "stdarch_s390x", issue = "135681")]
5540pub unsafe fn vec_cmpne<T: sealed::VectorEquality>(a: T, b: T) -> T::Result {
5541    a.vec_cmpne(b)
5542}
5543
5544/// Vector Compare Greater Than
5545#[inline]
5546#[target_feature(enable = "vector")]
5547#[unstable(feature = "stdarch_s390x", issue = "135681")]
5548pub unsafe fn vec_cmpgt<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5549    a.vec_cmpgt(b)
5550}
5551
5552/// Vector Compare Greater Than or Equal
5553#[inline]
5554#[target_feature(enable = "vector")]
5555#[unstable(feature = "stdarch_s390x", issue = "135681")]
5556pub unsafe fn vec_cmpge<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5557    a.vec_cmpge(b)
5558}
5559
5560/// Vector Compare Less
5561#[inline]
5562#[target_feature(enable = "vector")]
5563#[unstable(feature = "stdarch_s390x", issue = "135681")]
5564pub unsafe fn vec_cmplt<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5565    a.vec_cmplt(b)
5566}
5567
5568/// Vector Compare Less Than or Equal
5569#[inline]
5570#[target_feature(enable = "vector")]
5571#[unstable(feature = "stdarch_s390x", issue = "135681")]
5572pub unsafe fn vec_cmple<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5573    a.vec_cmple(b)
5574}
5575
5576/// Vector Compare Equal Index
5577#[inline]
5578#[target_feature(enable = "vector")]
5579#[unstable(feature = "stdarch_s390x", issue = "135681")]
5580pub unsafe fn vec_cmpeq_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5581    a.vec_cmpeq_idx(b)
5582}
5583/// Vector Compare Not Equal Index
5584#[inline]
5585#[target_feature(enable = "vector")]
5586#[unstable(feature = "stdarch_s390x", issue = "135681")]
5587pub unsafe fn vec_cmpne_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5588    a.vec_cmpne_idx(b)
5589}
5590/// Vector Compare Equal Index with Condition Code
5591#[inline]
5592#[target_feature(enable = "vector")]
5593#[unstable(feature = "stdarch_s390x", issue = "135681")]
5594pub unsafe fn vec_cmpeq_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5595    a.vec_cmpeq_idx_cc(b)
5596}
5597/// Vector Compare Not Equal Index with Condition Code
5598#[inline]
5599#[target_feature(enable = "vector")]
5600#[unstable(feature = "stdarch_s390x", issue = "135681")]
5601pub unsafe fn vec_cmpne_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5602    a.vec_cmpne_idx_cc(b)
5603}
5604/// Vector Compare Equal or Zero Index
5605#[inline]
5606#[target_feature(enable = "vector")]
5607#[unstable(feature = "stdarch_s390x", issue = "135681")]
5608pub unsafe fn vec_cmpeq_or_0_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5609    a.vec_cmpeq_or_0_idx(b)
5610}
5611/// Vector Compare Not Equal or Zero Index
5612#[inline]
5613#[target_feature(enable = "vector")]
5614#[unstable(feature = "stdarch_s390x", issue = "135681")]
5615pub unsafe fn vec_cmpne_or_0_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5616    a.vec_cmpne_or_0_idx(b)
5617}
5618/// Vector Compare Equal or Zero Index with Condition Code
5619#[inline]
5620#[target_feature(enable = "vector")]
5621#[unstable(feature = "stdarch_s390x", issue = "135681")]
5622pub unsafe fn vec_cmpeq_or_0_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5623    a.vec_cmpeq_or_0_idx_cc(b)
5624}
5625/// Vector Compare Not Equal or Zero Index with Condition Code
5626#[inline]
5627#[target_feature(enable = "vector")]
5628#[unstable(feature = "stdarch_s390x", issue = "135681")]
5629pub unsafe fn vec_cmpne_or_0_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5630    a.vec_cmpne_or_0_idx_cc(b)
5631}
5632
5633/// All Elements Equal
5634#[inline]
5635#[target_feature(enable = "vector")]
5636#[unstable(feature = "stdarch_s390x", issue = "135681")]
5637pub unsafe fn vec_all_eq<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5638    simd_reduce_all(vec_cmpeq(a, b)) as i32 as i32
5639}
5640
5641/// All Elements Not Equal
5642#[inline]
5643#[target_feature(enable = "vector")]
5644#[unstable(feature = "stdarch_s390x", issue = "135681")]
5645pub unsafe fn vec_all_ne<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5646    simd_reduce_all(vec_cmpne(a, b)) as i32
5647}
5648
5649/// Any Element Equal
5650#[inline]
5651#[target_feature(enable = "vector")]
5652#[unstable(feature = "stdarch_s390x", issue = "135681")]
5653pub unsafe fn vec_any_eq<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5654    simd_reduce_any(vec_cmpeq(a, b)) as i32
5655}
5656
5657/// Any Element Not Equal
5658#[inline]
5659#[target_feature(enable = "vector")]
5660#[unstable(feature = "stdarch_s390x", issue = "135681")]
5661pub unsafe fn vec_any_ne<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5662    simd_reduce_any(vec_cmpne(a, b)) as i32
5663}
5664
5665/// All Elements Less Than
5666#[inline]
5667#[target_feature(enable = "vector")]
5668#[unstable(feature = "stdarch_s390x", issue = "135681")]
5669pub unsafe fn vec_all_lt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5670    a.vec_all_lt(b)
5671}
5672
5673/// All Elements Less Than or Equal
5674#[inline]
5675#[target_feature(enable = "vector")]
5676#[unstable(feature = "stdarch_s390x", issue = "135681")]
5677pub unsafe fn vec_all_le<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5678    a.vec_all_le(b)
5679}
5680
5681/// All Elements Greater Than
5682#[inline]
5683#[target_feature(enable = "vector")]
5684#[unstable(feature = "stdarch_s390x", issue = "135681")]
5685pub unsafe fn vec_all_gt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5686    a.vec_all_gt(b)
5687}
5688
5689/// All Elements Greater Than or Equal
5690#[inline]
5691#[target_feature(enable = "vector")]
5692#[unstable(feature = "stdarch_s390x", issue = "135681")]
5693pub unsafe fn vec_all_ge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5694    a.vec_all_ge(b)
5695}
5696
5697/// All Elements Not Less Than
5698#[inline]
5699#[target_feature(enable = "vector")]
5700#[unstable(feature = "stdarch_s390x", issue = "135681")]
5701pub unsafe fn vec_all_nlt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5702    vec_all_ge(a, b)
5703}
5704
5705/// All Elements Not Less Than or Equal
5706#[inline]
5707#[target_feature(enable = "vector")]
5708#[unstable(feature = "stdarch_s390x", issue = "135681")]
5709pub unsafe fn vec_all_nle<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5710    vec_all_gt(a, b)
5711}
5712
5713/// All Elements Not Greater Than
5714#[inline]
5715#[target_feature(enable = "vector")]
5716#[unstable(feature = "stdarch_s390x", issue = "135681")]
5717pub unsafe fn vec_all_ngt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5718    vec_all_le(a, b)
5719}
5720
5721/// All Elements Not Greater Than or Equal
5722#[inline]
5723#[target_feature(enable = "vector")]
5724#[unstable(feature = "stdarch_s390x", issue = "135681")]
5725pub unsafe fn vec_all_nge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5726    vec_all_lt(a, b)
5727}
5728
5729/// Any Elements Less Than
5730#[inline]
5731#[target_feature(enable = "vector")]
5732#[unstable(feature = "stdarch_s390x", issue = "135681")]
5733pub unsafe fn vec_any_lt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5734    !vec_all_ge(a, b)
5735}
5736
5737/// Any Elements Less Than or Equal
5738#[inline]
5739#[target_feature(enable = "vector")]
5740#[unstable(feature = "stdarch_s390x", issue = "135681")]
5741pub unsafe fn vec_any_le<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5742    !vec_all_gt(a, b)
5743}
5744
5745/// Any Elements Greater Than
5746#[inline]
5747#[target_feature(enable = "vector")]
5748#[unstable(feature = "stdarch_s390x", issue = "135681")]
5749pub unsafe fn vec_any_gt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5750    !vec_all_le(a, b)
5751}
5752
5753/// Any Elements Greater Than or Equal
5754#[inline]
5755#[target_feature(enable = "vector")]
5756#[unstable(feature = "stdarch_s390x", issue = "135681")]
5757pub unsafe fn vec_any_ge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5758    !vec_all_lt(a, b)
5759}
5760
5761/// Any Elements Not Less Than
5762#[inline]
5763#[target_feature(enable = "vector")]
5764#[unstable(feature = "stdarch_s390x", issue = "135681")]
5765pub unsafe fn vec_any_nlt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5766    vec_any_ge(a, b)
5767}
5768
5769/// Any Elements Not Less Than or Equal
5770#[inline]
5771#[target_feature(enable = "vector")]
5772#[unstable(feature = "stdarch_s390x", issue = "135681")]
5773pub unsafe fn vec_any_nle<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5774    vec_any_gt(a, b)
5775}
5776
5777/// Any Elements Not Greater Than
5778#[inline]
5779#[target_feature(enable = "vector")]
5780#[unstable(feature = "stdarch_s390x", issue = "135681")]
5781pub unsafe fn vec_any_ngt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5782    vec_any_le(a, b)
5783}
5784
5785/// Any Elements Not Greater Than or Equal
5786#[inline]
5787#[target_feature(enable = "vector")]
5788#[unstable(feature = "stdarch_s390x", issue = "135681")]
5789pub unsafe fn vec_any_nge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5790    vec_any_lt(a, b)
5791}
5792
5793/// Vector Extract
5794#[inline]
5795#[target_feature(enable = "vector")]
5796#[unstable(feature = "stdarch_s390x", issue = "135681")]
5797pub unsafe fn vec_extract<T: sealed::VectorExtract>(a: T, b: i32) -> T::ElementType {
5798    T::vec_extract(a, b)
5799}
5800
5801/// Vector Insert
5802#[inline]
5803#[target_feature(enable = "vector")]
5804#[unstable(feature = "stdarch_s390x", issue = "135681")]
5805pub unsafe fn vec_insert<T: sealed::VectorInsert>(a: T::ElementType, b: T, c: i32) -> T {
5806    T::vec_insert(a, b, c)
5807}
5808
5809/// Vector Insert and Zero
5810#[inline]
5811#[target_feature(enable = "vector")]
5812#[unstable(feature = "stdarch_s390x", issue = "135681")]
5813pub unsafe fn vec_insert_and_zero<T: sealed::VectorInsertAndZero>(a: *const T::ElementType) -> T {
5814    T::vec_insert_and_zero(a)
5815}
5816
5817/// Vector Promote
5818#[inline]
5819#[target_feature(enable = "vector")]
5820#[unstable(feature = "stdarch_s390x", issue = "135681")]
5821pub unsafe fn vec_promote<T: sealed::VectorPromote>(a: T::ElementType, b: i32) -> MaybeUninit<T> {
5822    T::vec_promote(a, b)
5823}
5824
5825#[cfg(test)]
5826mod tests {
5827    use super::*;
5828
5829    use std::mem::transmute;
5830
5831    use crate::core_arch::simd::*;
5832    use stdarch_test::simd_test;
5833
5834    #[test]
5835    fn reverse_mask() {
5836        assert_eq!(ShuffleMask::<4>::reverse().0, [3, 2, 1, 0]);
5837    }
5838
5839    #[test]
5840    fn mergel_mask() {
5841        assert_eq!(ShuffleMask::<4>::merge_low().0, [2, 6, 3, 7]);
5842    }
5843
5844    #[test]
5845    fn mergeh_mask() {
5846        assert_eq!(ShuffleMask::<4>::merge_high().0, [0, 4, 1, 5]);
5847    }
5848
5849    #[test]
5850    fn pack_mask() {
5851        assert_eq!(ShuffleMask::<4>::pack().0, [1, 3, 5, 7]);
5852    }
5853
5854    #[test]
5855    fn test_vec_mask() {
5856        assert_eq!(
5857            genmask::<0x00FF>(),
5858            [
5859                0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
5860            ]
5861        );
5862    }
5863
5864    #[test]
5865    fn test_genmasks() {
5866        assert_eq!(genmasks(u8::BITS, 3, 5), 28);
5867        assert_eq!(genmasks(u8::BITS, 3, 7), 31);
5868
5869        // If a or b is greater than 8, the operation is performed as if the value gets modulo by 8.
5870        assert_eq!(genmasks(u8::BITS, 3 + 8, 7 + 8), 31);
5871        // If a is greater than b, the operation is perform as if b equals 7.
5872        assert_eq!(genmasks(u8::BITS, 5, 4), genmasks(u8::BITS, 5, 7));
5873
5874        assert_eq!(
5875            genmasks(u16::BITS, 4, 12) as u16,
5876            u16::from_be_bytes([15, -8i8 as u8])
5877        );
5878        assert_eq!(
5879            genmasks(u32::BITS, 4, 29) as u32,
5880            u32::from_be_bytes([15, 0xFF, 0xFF, -4i8 as u8])
5881        );
5882    }
5883
5884    macro_rules! test_vec_1 {
5885        { $name: ident, $fn:ident, f32x4, [$($a:expr),+], ~[$($d:expr),+] } => {
5886            #[simd_test(enable = "vector")]
5887            unsafe fn $name() {
5888                let a: vector_float = transmute(f32x4::new($($a),+));
5889
5890                let d: vector_float = transmute(f32x4::new($($d),+));
5891                let r = transmute(vec_cmple(vec_abs(vec_sub($fn(a), d)), vec_splats(f32::EPSILON)));
5892                let e = m32x4::new(true, true, true, true);
5893                assert_eq!(e, r);
5894            }
5895        };
5896        { $name: ident, $fn:ident, $ty: ident, [$($a:expr),+], [$($d:expr),+] } => {
5897            test_vec_1! { $name, $fn, $ty -> $ty, [$($a),+], [$($d),+] }
5898        };
5899        { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($d:expr),+] } => {
5900            #[simd_test(enable = "vector")]
5901            unsafe fn $name() {
5902                let a: s_t_l!($ty) = transmute($ty::new($($a),+));
5903
5904                let d = $ty_out::new($($d),+);
5905                let r : $ty_out = transmute($fn(a));
5906                assert_eq!(d, r);
5907            }
5908        }
5909    }
5910
5911    macro_rules! test_vec_2 {
5912        { $name: ident, $fn:ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5913            test_vec_2! { $name, $fn, $ty -> $ty, [$($a),+], [$($b),+], [$($d),+] }
5914        };
5915        { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5916            test_vec_2! { $name, $fn, $ty, $ty -> $ty, [$($a),+], [$($b),+], [$($d),+] }
5917         };
5918        { $name: ident, $fn:ident, $ty1: ident, $ty2: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5919            #[simd_test(enable = "vector")]
5920            unsafe fn $name() {
5921                let a: s_t_l!($ty1) = transmute($ty1::new($($a),+));
5922                let b: s_t_l!($ty2) = transmute($ty2::new($($b),+));
5923
5924                let d = $ty_out::new($($d),+);
5925                let r : $ty_out = transmute($fn(a, b));
5926                assert_eq!(d, r);
5927            }
5928         };
5929         { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], $d:expr } => {
5930            #[simd_test(enable = "vector")]
5931            unsafe fn $name() {
5932                let a: s_t_l!($ty) = transmute($ty::new($($a),+));
5933                let b: s_t_l!($ty) = transmute($ty::new($($b),+));
5934
5935                let r : $ty_out = transmute($fn(a, b));
5936                assert_eq!($d, r);
5937            }
5938         }
5939   }
5940
5941    #[simd_test(enable = "vector")]
5942    unsafe fn vec_add_i32x4_i32x4() {
5943        let x = i32x4::new(1, 2, 3, 4);
5944        let y = i32x4::new(4, 3, 2, 1);
5945        let x: vector_signed_int = transmute(x);
5946        let y: vector_signed_int = transmute(y);
5947        let z = vec_add(x, y);
5948        assert_eq!(i32x4::splat(5), transmute(z));
5949    }
5950
5951    macro_rules! test_vec_sub {
5952        { $name: ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5953            test_vec_2! {$name, vec_sub, $ty, [$($a),+], [$($b),+], [$($d),+] }
5954        }
5955    }
5956
5957    test_vec_sub! { test_vec_sub_f32x4, f32x4,
5958    [-1.0, 0.0, 1.0, 2.0],
5959    [2.0, 1.0, -1.0, -2.0],
5960    [-3.0, -1.0, 2.0, 4.0] }
5961
5962    test_vec_sub! { test_vec_sub_f64x2, f64x2,
5963    [-1.0, 0.0],
5964    [2.0, 1.0],
5965    [-3.0, -1.0] }
5966
5967    test_vec_sub! { test_vec_sub_i64x2, i64x2,
5968    [-1, 0],
5969    [2, 1],
5970    [-3, -1] }
5971
5972    test_vec_sub! { test_vec_sub_u64x2, u64x2,
5973    [0, 1],
5974    [1, 0],
5975    [u64::MAX, 1] }
5976
5977    test_vec_sub! { test_vec_sub_i32x4, i32x4,
5978    [-1, 0, 1, 2],
5979    [2, 1, -1, -2],
5980    [-3, -1, 2, 4] }
5981
5982    test_vec_sub! { test_vec_sub_u32x4, u32x4,
5983    [0, 0, 1, 2],
5984    [2, 1, 0, 0],
5985    [4294967294, 4294967295, 1, 2] }
5986
5987    test_vec_sub! { test_vec_sub_i16x8, i16x8,
5988    [-1, 0, 1, 2, -1, 0, 1, 2],
5989    [2, 1, -1, -2, 2, 1, -1, -2],
5990    [-3, -1, 2, 4, -3, -1, 2, 4] }
5991
5992    test_vec_sub! { test_vec_sub_u16x8, u16x8,
5993    [0, 0, 1, 2, 0, 0, 1, 2],
5994    [2, 1, 0, 0, 2, 1, 0, 0],
5995    [65534, 65535, 1, 2, 65534, 65535, 1, 2] }
5996
5997    test_vec_sub! { test_vec_sub_i8x16, i8x16,
5998    [-1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2],
5999    [2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2],
6000    [-3, -1, 2, 4, -3, -1, 2, 4, -3, -1, 2, 4, -3, -1, 2, 4] }
6001
6002    test_vec_sub! { test_vec_sub_u8x16, u8x16,
6003    [0, 0, 1, 2, 0, 0, 1, 2, 0, 0, 1, 2, 0, 0, 1, 2],
6004    [2, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 0],
6005    [254, 255, 1, 2, 254, 255, 1, 2, 254, 255, 1, 2, 254, 255, 1, 2] }
6006
6007    macro_rules! test_vec_mul {
6008        { $name: ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
6009            test_vec_2! {$name, vec_mul, $ty, [$($a),+], [$($b),+], [$($d),+] }
6010        }
6011    }
6012
6013    test_vec_mul! { test_vec_mul_f32x4, f32x4,
6014    [-1.0, 0.0, 1.0, 2.0],
6015    [2.0, 1.0, -1.0, -2.0],
6016    [-2.0, 0.0, -1.0, -4.0] }
6017
6018    test_vec_mul! { test_vec_mul_f64x2, f64x2,
6019    [-1.0, 0.0],
6020    [2.0, 1.0],
6021    [-2.0, 0.0] }
6022
6023    test_vec_mul! { test_vec_mul_i64x2, i64x2,
6024    [i64::MAX, -4],
6025    [2, 3],
6026    [i64::MAX.wrapping_mul(2), -12] }
6027
6028    test_vec_mul! { test_vec_mul_u64x2, u64x2,
6029    [u64::MAX, 4],
6030    [2, 3],
6031    [u64::MAX.wrapping_mul(2), 12] }
6032
6033    test_vec_mul! { test_vec_mul_i32x4, i32x4,
6034    [-1, 0, 1, 2],
6035    [2, 1, -1, -2],
6036    [-2, 0, -1, -4] }
6037
6038    test_vec_mul! { test_vec_mul_u32x4, u32x4,
6039    [0, u32::MAX - 1, 1, 2],
6040    [5, 6, 7, 8],
6041    [0, 4294967284, 7, 16] }
6042
6043    test_vec_mul! { test_vec_mul_i16x8, i16x8,
6044    [-1, 0, 1, 2, -1, 0, 1, 2],
6045    [2, 1, -1, -2, 2, 1, -1, -2],
6046    [-2, 0, -1, -4, -2, 0, -1, -4] }
6047
6048    test_vec_mul! { test_vec_mul_u16x8, u16x8,
6049    [0, u16::MAX - 1, 1, 2, 3, 4, 5, 6],
6050    [5, 6, 7, 8, 9, 8, 7, 6],
6051    [0, 65524, 7, 16, 27, 32, 35, 36] }
6052
6053    test_vec_mul! { test_vec_mul_i8x16, i8x16,
6054    [-1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2],
6055    [2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2],
6056    [-2, 0, -1, -4, -2, 0, -1, -4, -2, 0, -1, -4, -2, 0, -1, -4] }
6057
6058    test_vec_mul! { test_vec_mul_u8x16, u8x16,
6059    [0, u8::MAX - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4],
6060    [5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 0, u8::MAX, 1, 2, 3, 4],
6061    [0, 244, 7, 16, 27, 32, 35, 36, 35, 32, 0, 248, 7, 12, 15, 16] }
6062
6063    macro_rules! test_vec_abs {
6064        { $name: ident, $ty: ident, $a: expr, $d: expr } => {
6065            #[simd_test(enable = "vector")]
6066            unsafe fn $name() {
6067                let a: s_t_l!($ty) = vec_splats($a);
6068                let a: s_t_l!($ty) = vec_abs(a);
6069                let d = $ty::splat($d);
6070                assert_eq!(d, transmute(a));
6071            }
6072        }
6073    }
6074
6075    test_vec_abs! { test_vec_abs_i8, i8x16, -42i8, 42i8 }
6076    test_vec_abs! { test_vec_abs_i16, i16x8, -42i16, 42i16 }
6077    test_vec_abs! { test_vec_abs_i32, i32x4, -42i32, 42i32 }
6078    test_vec_abs! { test_vec_abs_i64, i64x2, -42i64, 42i64 }
6079    test_vec_abs! { test_vec_abs_f32, f32x4, -42f32, 42f32 }
6080    test_vec_abs! { test_vec_abs_f64, f64x2, -42f64, 42f64 }
6081
6082    test_vec_1! { test_vec_nabs, vec_nabs, f32x4,
6083    [core::f32::consts::PI, 1.0, 0.0, -1.0],
6084    [-core::f32::consts::PI, -1.0, 0.0, -1.0] }
6085
6086    test_vec_2! { test_vec_andc, vec_andc, i32x4,
6087    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6088    [0b00110011, 0b11110011, 0b00001100, 0b10000000],
6089    [0b11001100, 0b00001100, 0b11000000, 0b01001100] }
6090
6091    test_vec_2! { test_vec_and, vec_and, i32x4,
6092    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6093    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6094    [0b00000000, 0b11000000, 0b00001100, 0b00000000] }
6095
6096    test_vec_2! { test_vec_nand, vec_nand, i32x4,
6097    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6098    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6099    [!0b00000000, !0b11000000, !0b00001100, !0b00000000] }
6100
6101    test_vec_2! { test_vec_orc, vec_orc, u32x4,
6102    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6103    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6104    [0b11001100 | !0b00110011, 0b11001100 | !0b11110011, 0b11001100 | !0b00001100, 0b11001100 | !0b00000000] }
6105
6106    test_vec_2! { test_vec_or, vec_or, i32x4,
6107    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6108    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6109    [0b11111111, 0b11111111, 0b11001100, 0b11001100] }
6110
6111    test_vec_2! { test_vec_nor, vec_nor, i32x4,
6112    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6113    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6114    [!0b11111111, !0b11111111, !0b11001100, !0b11001100] }
6115
6116    test_vec_2! { test_vec_xor, vec_xor, i32x4,
6117    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6118    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6119    [0b11111111, 0b00111111, 0b11000000, 0b11001100] }
6120
6121    test_vec_2! { test_vec_eqv, vec_eqv, i32x4,
6122    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6123    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6124    [!0b11111111, !0b00111111, !0b11000000, !0b11001100] }
6125
6126    test_vec_1! { test_vec_floor_f32, vec_floor, f32x4,
6127        [1.1, 1.9, -0.5, -0.9],
6128        [1.0, 1.0, -1.0, -1.0]
6129    }
6130
6131    test_vec_1! { test_vec_floor_f64_1, vec_floor, f64x2,
6132        [1.1, 1.9],
6133        [1.0, 1.0]
6134    }
6135    test_vec_1! { test_vec_floor_f64_2, vec_floor, f64x2,
6136        [-0.5, -0.9],
6137        [-1.0, -1.0]
6138    }
6139
6140    test_vec_1! { test_vec_ceil_f32, vec_ceil, f32x4,
6141        [0.1, 0.5, 0.6, 0.9],
6142        [1.0, 1.0, 1.0, 1.0]
6143    }
6144    test_vec_1! { test_vec_ceil_f64_1, vec_ceil, f64x2,
6145        [0.1, 0.5],
6146        [1.0, 1.0]
6147    }
6148    test_vec_1! { test_vec_ceil_f64_2, vec_ceil, f64x2,
6149        [0.6, 0.9],
6150        [1.0, 1.0]
6151    }
6152
6153    test_vec_1! { test_vec_round_f32, vec_round, f32x4,
6154        [0.1, 0.5, 0.6, 0.9],
6155        [0.0, 0.0, 1.0, 1.0]
6156    }
6157
6158    test_vec_1! { test_vec_round_f32_even_odd, vec_round, f32x4,
6159        [0.5, 1.5, 2.5, 3.5],
6160        [0.0, 2.0, 2.0, 4.0]
6161    }
6162
6163    test_vec_1! { test_vec_round_f64_1, vec_round, f64x2,
6164        [0.1, 0.5],
6165        [0.0, 0.0]
6166    }
6167    test_vec_1! { test_vec_round_f64_2, vec_round, f64x2,
6168        [0.6, 0.9],
6169        [1.0, 1.0]
6170    }
6171
6172    test_vec_1! { test_vec_roundc_f32, vec_roundc, f32x4,
6173        [0.1, 0.5, 0.6, 0.9],
6174        [0.0, 0.0, 1.0, 1.0]
6175    }
6176
6177    test_vec_1! { test_vec_roundc_f32_even_odd, vec_roundc, f32x4,
6178        [0.5, 1.5, 2.5, 3.5],
6179        [0.0, 2.0, 2.0, 4.0]
6180    }
6181
6182    test_vec_1! { test_vec_roundc_f64_1, vec_roundc, f64x2,
6183        [0.1, 0.5],
6184        [0.0, 0.0]
6185    }
6186    test_vec_1! { test_vec_roundc_f64_2, vec_roundc, f64x2,
6187        [0.6, 0.9],
6188        [1.0, 1.0]
6189    }
6190
6191    test_vec_1! { test_vec_rint_f32, vec_rint, f32x4,
6192        [0.1, 0.5, 0.6, 0.9],
6193        [0.0, 0.0, 1.0, 1.0]
6194    }
6195
6196    test_vec_1! { test_vec_rint_f32_even_odd, vec_rint, f32x4,
6197        [0.5, 1.5, 2.5, 3.5],
6198        [0.0, 2.0, 2.0, 4.0]
6199    }
6200
6201    test_vec_1! { test_vec_rint_f64_1, vec_rint, f64x2,
6202        [0.1, 0.5],
6203        [0.0, 0.0]
6204    }
6205    test_vec_1! { test_vec_rint_f64_2, vec_rint, f64x2,
6206        [0.6, 0.9],
6207        [1.0, 1.0]
6208    }
6209
6210    test_vec_2! { test_vec_sll, vec_sll, i32x4, u8x16 -> i32x4,
6211    [1, 1, 1, 1],
6212    [0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 8],
6213    [1 << 2, 1 << 3, 1 << 4, 1] }
6214
6215    test_vec_2! { test_vec_srl, vec_srl, i32x4, u8x16 -> i32x4,
6216    [0b1000, 0b1000, 0b1000, 0b1000],
6217    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6218    [4, 2, 1, 8] }
6219
6220    test_vec_2! { test_vec_sral_pos, vec_sral, u32x4, u8x16 -> i32x4,
6221    [0b1000, 0b1000, 0b1000, 0b1000],
6222    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6223    [4, 2, 1, 8] }
6224
6225    test_vec_2! { test_vec_sral_neg, vec_sral, i32x4, u8x16 -> i32x4,
6226    [-8, -8, -8, -8],
6227    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6228    [-4, -2, -1, -8] }
6229
6230    test_vec_1! { test_vec_reve_f32, vec_reve, f32x4,
6231        [0.1, 0.5, 0.6, 0.9],
6232        [0.9, 0.6, 0.5, 0.1]
6233    }
6234
6235    test_vec_1! { test_vec_revb_u32, vec_revb, u32x4,
6236        [0xAABBCCDD, 0xEEFF0011, 0x22334455, 0x66778899],
6237        [0xDDCCBBAA, 0x1100FFEE, 0x55443322, 0x99887766]
6238    }
6239
6240    test_vec_2! { test_vec_mergeh_u32, vec_mergeh, u32x4,
6241        [0xAAAAAAAA, 0xBBBBBBBB, 0xCCCCCCCC, 0xDDDDDDDD],
6242        [0x00000000, 0x11111111, 0x22222222, 0x33333333],
6243        [0xAAAAAAAA, 0x00000000, 0xBBBBBBBB, 0x11111111]
6244    }
6245
6246    test_vec_2! { test_vec_mergel_u32, vec_mergel, u32x4,
6247        [0xAAAAAAAA, 0xBBBBBBBB, 0xCCCCCCCC, 0xDDDDDDDD],
6248        [0x00000000, 0x11111111, 0x22222222, 0x33333333],
6249        [0xCCCCCCCC, 0x22222222, 0xDDDDDDDD, 0x33333333]
6250    }
6251
6252    macro_rules! test_vec_perm {
6253        {$name:ident,
6254         $shorttype:ident, $longtype:ident,
6255         [$($a:expr),+], [$($b:expr),+], [$($c:expr),+], [$($d:expr),+]} => {
6256            #[simd_test(enable = "vector")]
6257            unsafe fn $name() {
6258                let a: $longtype = transmute($shorttype::new($($a),+));
6259                let b: $longtype = transmute($shorttype::new($($b),+));
6260                let c: vector_unsigned_char = transmute(u8x16::new($($c),+));
6261                let d = $shorttype::new($($d),+);
6262
6263                let r: $shorttype = transmute(vec_perm(a, b, c));
6264                assert_eq!(d, r);
6265            }
6266        }
6267    }
6268
6269    test_vec_perm! {test_vec_perm_u8x16,
6270    u8x16, vector_unsigned_char,
6271    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
6272    [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115],
6273    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6274     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6275    [0, 1, 100, 101, 2, 3, 102, 103, 4, 5, 104, 105, 6, 7, 106, 107]}
6276    test_vec_perm! {test_vec_perm_i8x16,
6277    i8x16, vector_signed_char,
6278    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
6279    [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115],
6280    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6281     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6282    [0, 1, 100, 101, 2, 3, 102, 103, 4, 5, 104, 105, 6, 7, 106, 107]}
6283
6284    test_vec_perm! {test_vec_perm_m8x16,
6285    m8x16, vector_bool_char,
6286    [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false],
6287    [true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true],
6288    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6289     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6290    [false, false, true, true, false, false, true, true, false, false, true, true, false, false, true, true]}
6291    test_vec_perm! {test_vec_perm_u16x8,
6292    u16x8, vector_unsigned_short,
6293    [0, 1, 2, 3, 4, 5, 6, 7],
6294    [10, 11, 12, 13, 14, 15, 16, 17],
6295    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6296     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6297    [0, 10, 1, 11, 2, 12, 3, 13]}
6298    test_vec_perm! {test_vec_perm_i16x8,
6299    i16x8, vector_signed_short,
6300    [0, 1, 2, 3, 4, 5, 6, 7],
6301    [10, 11, 12, 13, 14, 15, 16, 17],
6302    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6303     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6304    [0, 10, 1, 11, 2, 12, 3, 13]}
6305    test_vec_perm! {test_vec_perm_m16x8,
6306    m16x8, vector_bool_short,
6307    [false, false, false, false, false, false, false, false],
6308    [true, true, true, true, true, true, true, true],
6309    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6310     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6311    [false, true, false, true, false, true, false, true]}
6312
6313    test_vec_perm! {test_vec_perm_u32x4,
6314    u32x4, vector_unsigned_int,
6315    [0, 1, 2, 3],
6316    [10, 11, 12, 13],
6317    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6318     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6319    [0, 10, 1, 11]}
6320    test_vec_perm! {test_vec_perm_i32x4,
6321    i32x4, vector_signed_int,
6322    [0, 1, 2, 3],
6323    [10, 11, 12, 13],
6324    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6325     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6326    [0, 10, 1, 11]}
6327    test_vec_perm! {test_vec_perm_m32x4,
6328    m32x4, vector_bool_int,
6329    [false, false, false, false],
6330    [true, true, true, true],
6331    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6332     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6333    [false, true, false, true]}
6334    test_vec_perm! {test_vec_perm_f32x4,
6335    f32x4, vector_float,
6336    [0.0, 1.0, 2.0, 3.0],
6337    [1.0, 1.1, 1.2, 1.3],
6338    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6339     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6340    [0.0, 1.0, 1.0, 1.1]}
6341
6342    test_vec_1! { test_vec_sqrt, vec_sqrt, f32x4,
6343    [core::f32::consts::PI, 1.0, 25.0, 2.0],
6344    [core::f32::consts::PI.sqrt(), 1.0, 5.0, core::f32::consts::SQRT_2] }
6345
6346    test_vec_2! { test_vec_find_any_eq, vec_find_any_eq, i32x4, i32x4 -> u32x4,
6347        [1, -2, 3, -4],
6348        [-5, 3, -7, 8],
6349        [0, 0, 0xFFFFFFFF, 0]
6350    }
6351
6352    test_vec_2! { test_vec_find_any_ne, vec_find_any_ne, i32x4, i32x4 -> u32x4,
6353        [1, -2, 3, -4],
6354        [-5, 3, -7, 8],
6355        [0xFFFFFFFF, 0xFFFFFFFF, 0, 0xFFFFFFFF]
6356    }
6357
6358    test_vec_2! { test_vec_find_any_eq_idx_1, vec_find_any_eq_idx, i32x4, i32x4 -> u32x4,
6359        [1, 2, 3, 4],
6360        [5, 3, 7, 8],
6361        [0, 8, 0, 0]
6362    }
6363    test_vec_2! { test_vec_find_any_eq_idx_2, vec_find_any_eq_idx, i32x4, i32x4 -> u32x4,
6364        [1, 2, 3, 4],
6365        [5, 6, 7, 8],
6366        [0, 16, 0, 0]
6367    }
6368
6369    test_vec_2! { test_vec_find_any_ne_idx_1, vec_find_any_ne_idx, i32x4, i32x4 -> u32x4,
6370        [1, 2, 3, 4],
6371        [1, 5, 3, 4],
6372        [0, 4, 0, 0]
6373    }
6374    test_vec_2! { test_vec_find_any_ne_idx_2, vec_find_any_ne_idx, i32x4, i32x4 -> u32x4,
6375        [1, 2, 3, 4],
6376        [1, 2, 3, 4],
6377        [0, 16, 0, 0]
6378    }
6379
6380    test_vec_2! { test_vec_find_any_eq_or_0_idx_1, vec_find_any_eq_or_0_idx, i32x4, i32x4 -> u32x4,
6381        [1, 2, 0, 4],
6382        [5, 6, 7, 8],
6383        [0, 8, 0, 0]
6384    }
6385    test_vec_2! { test_vec_find_any_ne_or_0_idx_1, vec_find_any_ne_or_0_idx, i32x4, i32x4 -> u32x4,
6386        [1, 2, 0, 4],
6387        [1, 2, 3, 4],
6388        [0, 8, 0, 0]
6389    }
6390
6391    #[simd_test(enable = "vector")]
6392    fn test_vec_find_any_eq_cc() {
6393        let a = vector_unsigned_int([1, 2, 3, 4]);
6394        let b = vector_unsigned_int([5, 3, 7, 8]);
6395
6396        let (d, c) = unsafe { vec_find_any_eq_cc(a, b) };
6397        assert_eq!(c, 1);
6398        assert_eq!(d.as_array(), &[0, 0, -1, 0]);
6399
6400        let a = vector_unsigned_int([1, 2, 3, 4]);
6401        let b = vector_unsigned_int([5, 6, 7, 8]);
6402        let (d, c) = unsafe { vec_find_any_eq_cc(a, b) };
6403        assert_eq!(c, 3);
6404        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6405    }
6406
6407    #[simd_test(enable = "vector")]
6408    fn test_vec_find_any_ne_cc() {
6409        let a = vector_unsigned_int([1, 2, 3, 4]);
6410        let b = vector_unsigned_int([5, 3, 7, 8]);
6411
6412        let (d, c) = unsafe { vec_find_any_ne_cc(a, b) };
6413        assert_eq!(c, 1);
6414        assert_eq!(d.as_array(), &[-1, -1, 0, -1]);
6415
6416        let a = vector_unsigned_int([1, 2, 3, 4]);
6417        let b = vector_unsigned_int([1, 2, 3, 4]);
6418        let (d, c) = unsafe { vec_find_any_ne_cc(a, b) };
6419        assert_eq!(c, 3);
6420        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6421    }
6422
6423    #[simd_test(enable = "vector")]
6424    fn test_vec_find_any_eq_idx_cc() {
6425        let a = vector_unsigned_int([1, 2, 3, 4]);
6426        let b = vector_unsigned_int([5, 3, 7, 8]);
6427
6428        let (d, c) = unsafe { vec_find_any_eq_idx_cc(a, b) };
6429        assert_eq!(c, 1);
6430        assert_eq!(d.as_array(), &[0, 8, 0, 0]);
6431
6432        let a = vector_unsigned_int([1, 2, 3, 4]);
6433        let b = vector_unsigned_int([5, 6, 7, 8]);
6434        let (d, c) = unsafe { vec_find_any_eq_idx_cc(a, b) };
6435        assert_eq!(c, 3);
6436        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6437    }
6438
6439    #[simd_test(enable = "vector")]
6440    fn test_vec_find_any_ne_idx_cc() {
6441        let a = vector_unsigned_int([5, 2, 3, 4]);
6442        let b = vector_unsigned_int([5, 3, 7, 8]);
6443
6444        let (d, c) = unsafe { vec_find_any_ne_idx_cc(a, b) };
6445        assert_eq!(c, 1);
6446        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6447
6448        let a = vector_unsigned_int([1, 2, 3, 4]);
6449        let b = vector_unsigned_int([1, 2, 3, 4]);
6450        let (d, c) = unsafe { vec_find_any_ne_idx_cc(a, b) };
6451        assert_eq!(c, 3);
6452        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6453    }
6454
6455    #[simd_test(enable = "vector")]
6456    fn test_vec_find_any_eq_or_0_idx_cc() {
6457        // if no element of a matches any element of b with an equal value, and there is at least one element from a with a value of 0
6458        let a = vector_unsigned_int([0, 1, 2, 3]);
6459        let b = vector_unsigned_int([4, 5, 6, 7]);
6460        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6461        assert_eq!(c, 0);
6462        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6463
6464        // if at least one element of a matches any element of b with an equal value, and no elements of a with a value of 0
6465        let a = vector_unsigned_int([1, 2, 3, 4]);
6466        let b = vector_unsigned_int([5, 2, 3, 4]);
6467        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6468        assert_eq!(c, 1);
6469        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6470
6471        // if at least one element of a matches any element of b with an equal value, and there is at least one element from a has a value of 0
6472        let a = vector_unsigned_int([1, 2, 3, 0]);
6473        let b = vector_unsigned_int([1, 2, 3, 4]);
6474        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6475        assert_eq!(c, 2);
6476        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6477
6478        // if no element of a matches any element of b with an equal value, and there is no element from a with a value of 0.
6479        let a = vector_unsigned_int([1, 2, 3, 4]);
6480        let b = vector_unsigned_int([5, 6, 7, 8]);
6481        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6482        assert_eq!(c, 3);
6483        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6484    }
6485
6486    #[simd_test(enable = "vector")]
6487    fn test_vec_find_any_ne_or_0_idx_cc() {
6488        // if no element of a matches any element of b with a not equal value, and there is at least one element from a with a value of 0.
6489        let a = vector_unsigned_int([0, 1, 2, 3]);
6490        let b = vector_unsigned_int([4, 1, 2, 3]);
6491        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6492        assert_eq!(c, 0);
6493        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6494
6495        // if at least one element of a matches any element of b with a not equal value, and no elements of a with a value of 0.
6496        let a = vector_unsigned_int([4, 2, 3, 4]);
6497        let b = vector_unsigned_int([4, 5, 6, 7]);
6498        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6499        assert_eq!(c, 1);
6500        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6501
6502        // if at least one element of a matches any element of b with a not equal value, and there is at least one element from a has a value of 0.
6503        let a = vector_unsigned_int([1, 0, 1, 1]);
6504        let b = vector_unsigned_int([4, 5, 6, 7]);
6505        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6506        assert_eq!(c, 2);
6507        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6508
6509        // if no element of a matches any element of b with a not equal value, and there is no element from a with a value of 0.
6510        let a = vector_unsigned_int([4, 4, 4, 4]);
6511        let b = vector_unsigned_int([4, 5, 6, 7]);
6512        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6513        assert_eq!(c, 3);
6514        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6515    }
6516
6517    #[simd_test(enable = "vector")]
6518    fn test_vector_load() {
6519        let expected = [0xAAAA_AAAA, 0xBBBB_BBBB, 0xCCCC_CCCC, 0xDDDD_DDDD];
6520
6521        let source: [u32; 8] = [
6522            0xAAAA_AAAA,
6523            0xBBBB_BBBB,
6524            0xCCCC_CCCC,
6525            0xDDDD_DDDD,
6526            0,
6527            0,
6528            0,
6529            0,
6530        ];
6531        assert_eq!(
6532            unsafe { vec_xl::<vector_unsigned_int>(0, source.as_ptr()) }.as_array(),
6533            &expected
6534        );
6535
6536        // offset is in bytes
6537        let source: [u32; 8] = [
6538            0x0000_AAAA,
6539            0xAAAA_BBBB,
6540            0xBBBB_CCCC,
6541            0xCCCC_DDDD,
6542            0xDDDD_0000,
6543            0,
6544            0,
6545            0,
6546        ];
6547        assert_eq!(
6548            unsafe { vec_xl::<vector_unsigned_int>(2, source.as_ptr()) }.as_array(),
6549            &expected
6550        );
6551    }
6552
6553    #[simd_test(enable = "vector")]
6554    fn test_vector_store() {
6555        let vec = vector_unsigned_int([0xAAAA_AAAA, 0xBBBB_BBBB, 0xCCCC_CCCC, 0xDDDD_DDDD]);
6556
6557        let mut dest = [0u32; 8];
6558        unsafe { vec_xst(vec, 0, dest.as_mut_ptr()) };
6559        assert_eq!(
6560            dest,
6561            [
6562                0xAAAA_AAAA,
6563                0xBBBB_BBBB,
6564                0xCCCC_CCCC,
6565                0xDDDD_DDDD,
6566                0,
6567                0,
6568                0,
6569                0
6570            ]
6571        );
6572
6573        // offset is in bytes
6574        let mut dest = [0u32; 8];
6575        unsafe { vec_xst(vec, 2, dest.as_mut_ptr()) };
6576        assert_eq!(
6577            dest,
6578            [
6579                0x0000_AAAA,
6580                0xAAAA_BBBB,
6581                0xBBBB_CCCC,
6582                0xCCCC_DDDD,
6583                0xDDDD_0000,
6584                0,
6585                0,
6586                0,
6587            ]
6588        );
6589    }
6590
6591    #[simd_test(enable = "vector")]
6592    fn test_vector_lcbb() {
6593        #[repr(align(64))]
6594        struct Align64<T>(T);
6595
6596        static ARRAY: Align64<[u8; 128]> = Align64([0; 128]);
6597
6598        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[64..].as_ptr()) }, 16);
6599        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[63..].as_ptr()) }, 1);
6600        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[56..].as_ptr()) }, 8);
6601        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[48..].as_ptr()) }, 16);
6602    }
6603
6604    test_vec_2! { test_vec_pack, vec_pack, i16x8, i16x8 -> i8x16,
6605        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6606        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6607        [0, 1, -1, 42, -1, 0, 48, -48, -1, 0, 57, -57, 0, 1, -1, 42]
6608    }
6609
6610    test_vec_2! { test_vec_packs, vec_packs, i16x8, i16x8 -> i8x16,
6611        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6612        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6613        [0, 1, -1, 42, 127, -128, 127, -128, 127, -128, 127, -128, 0, 1, -1, 42]
6614    }
6615
6616    test_vec_2! { test_vec_packsu_signed, vec_packsu, i16x8, i16x8 -> u8x16,
6617        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6618        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6619        [0, 1, 0, 42, 255, 0, 255, 0, 255, 0, 255, 0, 0, 1, 0, 42]
6620    }
6621
6622    test_vec_2! { test_vec_packsu_unsigned, vec_packsu, u16x8, u16x8 -> u8x16,
6623        [65535, 32768, 1234, 5678, 16, 8, 4, 2],
6624        [30000, 25000, 20000, 15000, 31, 63, 127, 255],
6625        [255, 255, 255, 255, 16, 8, 4, 2, 255, 255, 255, 255, 31, 63, 127, 255]
6626    }
6627
6628    test_vec_2! { test_vec_rl, vec_rl, u32x4,
6629        [0x12345678, 0x9ABCDEF0, 0x0F0F0F0F, 0x12345678],
6630        [4, 8, 12, 68],
6631        [0x23456781, 0xBCDEF09A, 0xF0F0F0F0, 0x23456781]
6632    }
6633
6634    test_vec_1! { test_vec_unpackh_i, vec_unpackh, i16x8 -> i32x4,
6635        [0x1234, -2, 0x0F0F, -32768, 0, 0, 0, 0],
6636        [0x1234, -2, 0x0F0F, -32768]
6637    }
6638
6639    test_vec_1! { test_vec_unpackh_u, vec_unpackh, u16x8 -> u32x4,
6640        [0x1234, 0xFFFF, 0x0F0F, 0x8000, 0, 0, 0, 0],
6641        [0x1234, 0xFFFF, 0x0F0F, 0x8000]
6642    }
6643
6644    test_vec_1! { test_vec_unpackl_i, vec_unpackl, i16x8 -> i32x4,
6645        [0, 0, 0, 0, 0x1234, -2, 0x0F0F, -32768],
6646        [0x1234, -2, 0x0F0F, -32768]
6647    }
6648
6649    test_vec_1! { test_vec_unpackl_u, vec_unpackl, u16x8 -> u32x4,
6650        [0, 0, 0, 0, 0x1234, 0xFFFF, 0x0F0F, 0x8000],
6651        [0x1234, 0xFFFF, 0x0F0F, 0x8000]
6652    }
6653
6654    test_vec_2! { test_vec_avg, vec_avg, u32x4,
6655        [2, 1, u32::MAX, 0],
6656        [4, 2, 2, 0],
6657        [3, (1u32 + 2).div_ceil(2), (u32::MAX as u64 + 2u64).div_ceil(2) as u32, 0]
6658    }
6659
6660    test_vec_2! { test_vec_checksum, vec_checksum, u32x4,
6661        [1, 2, 3, u32::MAX],
6662        [5, 6, 7, 8],
6663        [0, 12, 0, 0]
6664    }
6665
6666    test_vec_2! { test_vec_add_u128, vec_add_u128, u8x16,
6667        [0x01, 0x05, 0x0F, 0x1A, 0x2F, 0x3F, 0x50, 0x65,
6668                              0x7A, 0x8F, 0x9A, 0xAD, 0xB0, 0xC3, 0xD5, 0xE8],
6669        [0xF0, 0xEF, 0xC3, 0xB1, 0x92, 0x71, 0x5A, 0x43,
6670                              0x3B, 0x29, 0x13, 0x04, 0xD7, 0xA1, 0x8C, 0x76],
6671        [0xF1, 0xF4, 0xD2, 0xCB, 0xC1, 0xB0, 0xAA, 0xA8, 0xB5, 0xB8, 0xAD, 0xB2, 0x88, 0x65, 0x62, 0x5E]
6672    }
6673
6674    #[simd_test(enable = "vector")]
6675    fn test_vec_addc_u128() {
6676        unsafe {
6677            let a = u128::MAX;
6678            let b = 1u128;
6679
6680            let d: u128 = transmute(vec_addc_u128(transmute(a), transmute(b)));
6681            assert!(a.checked_add(b).is_none());
6682            assert_eq!(d, 1);
6683
6684            let a = 1u128;
6685            let b = 1u128;
6686
6687            let d: u128 = transmute(vec_addc_u128(transmute(a), transmute(b)));
6688            assert!(a.checked_add(b).is_some());
6689            assert_eq!(d, 0);
6690        }
6691    }
6692
6693    #[simd_test(enable = "vector")]
6694    fn test_vec_subc_u128() {
6695        unsafe {
6696            let a = 0u128;
6697            let b = 1u128;
6698
6699            let d: u128 = transmute(vec_subc_u128(transmute(a), transmute(b)));
6700            assert!(a.checked_sub(b).is_none());
6701            assert_eq!(d, 0);
6702
6703            let a = 1u128;
6704            let b = 1u128;
6705
6706            let d: u128 = transmute(vec_subc_u128(transmute(a), transmute(b)));
6707            assert!(a.checked_sub(b).is_some());
6708            assert_eq!(d, 1);
6709        }
6710    }
6711
6712    test_vec_2! { test_vec_mule_u, vec_mule, u16x8, u16x8 -> u32x4,
6713        [0xFFFF, 0, 2, 0, 2, 0, 1, 0],
6714        [0xFFFF, 0, 4, 0, 0xFFFF, 0, 2, 0],
6715        [0xFFFE_0001, 8, 0x0001_FFFE, 2]
6716    }
6717
6718    test_vec_2! { test_vec_mule_i, vec_mule, i16x8, i16x8 -> i32x4,
6719        [i16::MIN, 0, -2, 0, 2, 0, 1, 0],
6720        [i16::MIN, 0, 4, 0, i16::MAX, 0, 2, 0],
6721        [0x4000_0000, -8, 0xFFFE, 2]
6722    }
6723
6724    test_vec_2! { test_vec_mulo_u, vec_mulo, u16x8, u16x8 -> u32x4,
6725        [0, 0xFFFF, 0, 2, 0, 2, 0, 1],
6726        [0, 0xFFFF, 0, 4, 0, 0xFFFF, 0, 2],
6727        [0xFFFE_0001, 8, 0x0001_FFFE, 2]
6728    }
6729
6730    test_vec_2! { test_vec_mulo_i, vec_mulo, i16x8, i16x8 -> i32x4,
6731        [0, i16::MIN, 0, -2, 0, 2, 0, 1],
6732        [0, i16::MIN, 0, 4, 0, i16::MAX, 0, 2],
6733        [0x4000_0000, -8, 0xFFFE, 2]
6734    }
6735
6736    test_vec_2! { test_vec_mulh_u, vec_mulh, u32x4, u32x4 -> u32x4,
6737        [u32::MAX, 2, 2, 1],
6738        [u32::MAX, 4, u32::MAX, 2],
6739        [u32::MAX - 1, 0, 1, 0]
6740    }
6741
6742    test_vec_2! { test_vec_mulh_i, vec_mulh, i32x4, i32x4 -> i32x4,
6743        [i32::MIN, -2, 2, 1],
6744        [i32::MIN, 4, i32::MAX, 2],
6745        [0x4000_0000, -1, 0, 0]
6746    }
6747
6748    test_vec_2! { test_vec_gfmsum_1, vec_gfmsum, u16x8, u16x8 -> u32x4,
6749        [0x1234, 0x5678, 0x9ABC, 0xDEF0, 0x1357, 0x2468, 0xACE0, 0xBDF0],
6750        [0xFFFF, 0x0001, 0x8000, 0x7FFF, 0xAAAA, 0x5555, 0x1234, 0x5678],
6751        [0xE13A794, 0x68764A50, 0x94AA3E, 0x2C93F300]
6752    }
6753
6754    test_vec_2! { test_vec_gfmsum_2, vec_gfmsum, u16x8, u16x8 -> u32x4,
6755        [0x0000, 0xFFFF, 0xAAAA, 0x5555, 0x1234, 0x5678, 0x9ABC, 0xDEF0],
6756        [0xFFFF, 0x0000, 0x5555, 0xAAAA, 0x0001, 0x8000, 0x7FFF, 0x1357],
6757        [0, 0, 0x2B3C1234, 0x3781D244]
6758    }
6759
6760    #[simd_test(enable = "vector")]
6761    fn test_vec_gfmsum_128() {
6762        let a = vector_unsigned_long_long([1, 2]);
6763        let b = vector_unsigned_long_long([3, 4]);
6764
6765        let d: u128 = unsafe { transmute(vec_gfmsum_128(a, b)) };
6766        assert_eq!(d, 11);
6767
6768        let a = vector_unsigned_long_long([0x0101010101010101, 0x0202020202020202]);
6769        let b = vector_unsigned_long_long([0x0404040404040404, 0x0505050505050505]);
6770
6771        let d: u128 = unsafe { transmute(vec_gfmsum_128(a, b)) };
6772        assert_eq!(d, 0xE000E000E000E000E000E000E000E);
6773    }
6774
6775    #[simd_test(enable = "vector-enhancements-1")]
6776    fn test_vec_bperm_u128() {
6777        let a = vector_unsigned_char([65, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]);
6778        let b = vector_unsigned_char([
6779            0, 0, 0, 0, 1, 1, 1, 1, 128, 128, 128, 128, 255, 255, 255, 255,
6780        ]);
6781        let d = unsafe { vec_bperm_u128(a, b) };
6782        assert_eq!(d.as_array(), &[0xF00, 0]);
6783    }
6784
6785    #[simd_test(enable = "vector")]
6786    fn test_vec_sel() {
6787        let a = vector_signed_int([1, 2, 3, 4]);
6788        let b = vector_signed_int([5, 6, 7, 8]);
6789
6790        let e = vector_unsigned_int([9, 10, 11, 12]);
6791        let f = vector_unsigned_int([9, 9, 11, 11]);
6792
6793        let c: vector_bool_int = unsafe { simd_eq(e, f) };
6794        assert_eq!(c.as_array(), &[!0, 0, !0, 0]);
6795        let d: vector_signed_int = unsafe { vec_sel(a, b, c) };
6796        assert_eq!(d.as_array(), &[5, 2, 7, 4]);
6797    }
6798
6799    #[simd_test(enable = "vector")]
6800    fn test_vec_gather_element() {
6801        let a1: [u32; 10] = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19];
6802        let a2: [u32; 10] = [20, 21, 22, 23, 24, 25, 26, 27, 28, 29];
6803
6804        let v1 = vector_unsigned_int([1, 2, 3, 4]);
6805        let v2 = vector_unsigned_int([1, 2, 3, 4]);
6806
6807        let sizeof_int = core::mem::size_of::<u32>() as u32;
6808        let v3 = vector_unsigned_int([
6809            5 * sizeof_int,
6810            8 * sizeof_int,
6811            9 * sizeof_int,
6812            6 * sizeof_int,
6813        ]);
6814
6815        unsafe {
6816            let d1 = vec_gather_element::<_, 0>(v1, v3, a1.as_ptr());
6817            assert_eq!(d1.as_array(), &[15, 2, 3, 4]);
6818            let d2 = vec_gather_element::<_, 0>(v2, v3, a2.as_ptr());
6819            assert_eq!(d2.as_array(), &[25, 2, 3, 4]);
6820        }
6821    }
6822
6823    #[simd_test(enable = "vector")]
6824    fn test_vec_fp_test_data_class() {
6825        let mut cc = 42;
6826
6827        let v1 = vector_double([0.0, f64::NAN]);
6828        let v2 = vector_double([f64::INFINITY, 1.0]);
6829        let v3 = vector_double([1.0, 2.0]);
6830
6831        unsafe {
6832            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_ZERO>(v1, &mut cc);
6833            assert_eq!(cc, 1);
6834            assert_eq!(d.as_array(), &[!0, 0]);
6835
6836            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NAN>(v1, &mut cc);
6837            assert_eq!(cc, 1);
6838            assert_eq!(d.as_array(), &[0, !0]);
6839
6840            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY>(v2, &mut cc);
6841            assert_eq!(cc, 1);
6842            assert_eq!(d.as_array(), &[!0, 0]);
6843
6844            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY_N>(v2, &mut cc);
6845            assert_eq!(cc, 3);
6846            assert_eq!(d.as_array(), &[0, 0]);
6847
6848            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v2, &mut cc);
6849            assert_eq!(cc, 1);
6850            assert_eq!(d.as_array(), &[0, !0]);
6851
6852            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v3, &mut cc);
6853            assert_eq!(cc, 0);
6854            assert_eq!(d.as_array(), &[!0, !0]);
6855        }
6856    }
6857
6858    #[simd_test(enable = "vector")]
6859    fn test_vec_fp_any_all_nan_numeric() {
6860        unsafe {
6861            assert_eq!(
6862                vec_all_nan(vector_double([f64::NAN, f64::NAN])),
6863                i32::from(true)
6864            );
6865            assert_eq!(
6866                vec_all_nan(vector_double([f64::NAN, 1.0])),
6867                i32::from(false)
6868            );
6869            assert_eq!(vec_all_nan(vector_double([0.0, 1.0])), i32::from(false));
6870
6871            assert_eq!(
6872                vec_any_nan(vector_double([f64::NAN, f64::NAN])),
6873                i32::from(true)
6874            );
6875            assert_eq!(vec_any_nan(vector_double([f64::NAN, 1.0])), i32::from(true));
6876            assert_eq!(vec_any_nan(vector_double([0.0, 1.0])), i32::from(false));
6877
6878            assert_eq!(
6879                vec_all_numeric(vector_double([f64::NAN, f64::NAN])),
6880                i32::from(false)
6881            );
6882            assert_eq!(
6883                vec_all_numeric(vector_double([f64::NAN, 1.0])),
6884                i32::from(false)
6885            );
6886            assert_eq!(vec_all_numeric(vector_double([0.0, 1.0])), i32::from(true));
6887
6888            assert_eq!(
6889                vec_any_numeric(vector_double([f64::NAN, f64::NAN])),
6890                i32::from(false)
6891            );
6892            assert_eq!(
6893                vec_any_numeric(vector_double([f64::NAN, 1.0])),
6894                i32::from(true)
6895            );
6896            assert_eq!(vec_any_numeric(vector_double([0.0, 1.0])), i32::from(true));
6897
6898            // "numeric" means "not NaN". infinities are numeric
6899            assert_eq!(
6900                vec_all_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])),
6901                i32::from(true)
6902            );
6903            assert_eq!(
6904                vec_any_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])),
6905                i32::from(true)
6906            );
6907        }
6908    }
6909
6910    #[simd_test(enable = "vector")]
6911    fn test_vec_test_mask() {
6912        unsafe {
6913            let v = vector_unsigned_long_long([0xFF00FF00FF00FF00; 2]);
6914            let m = vector_unsigned_long_long([0x0000FF000000FF00; 2]);
6915            assert_eq!(vec_test_mask(v, m), 3);
6916
6917            let v = vector_unsigned_long_long([u64::MAX; 2]);
6918            let m = vector_unsigned_long_long([0; 2]);
6919            assert_eq!(vec_test_mask(v, m), 0);
6920
6921            let v = vector_unsigned_long_long([0; 2]);
6922            let m = vector_unsigned_long_long([u64::MAX; 2]);
6923            assert_eq!(vec_test_mask(v, m), 0);
6924
6925            let v = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]);
6926            let m = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]);
6927            assert_eq!(vec_test_mask(v, m), 3);
6928        }
6929    }
6930
6931    #[simd_test(enable = "vector-enhancements-2")]
6932    fn test_vec_search_string_cc() {
6933        unsafe {
6934            let b = vector_unsigned_char(*b"ABCD------------");
6935            let c = vector_unsigned_char([4; 16]);
6936
6937            let haystack = vector_unsigned_char(*b"__ABCD__________");
6938            let (result, d) = vec_search_string_cc(haystack, b, c);
6939            assert_eq!(result.as_array()[7], 2);
6940            assert_eq!(d, 2);
6941
6942            let haystack = vector_unsigned_char(*b"___ABCD_________");
6943            let (result, d) = vec_search_string_cc(haystack, b, c);
6944            assert_eq!(result.as_array()[7], 3);
6945            assert_eq!(d, 2);
6946
6947            let haystack = vector_unsigned_char(*b"________________");
6948            let (result, d) = vec_search_string_cc(haystack, b, c);
6949            assert_eq!(result.as_array()[7], 16);
6950            assert_eq!(d, 0);
6951
6952            let haystack = vector_unsigned_char(*b"______\0_________");
6953            let (result, d) = vec_search_string_cc(haystack, b, c);
6954            assert_eq!(result.as_array()[7], 16);
6955            assert_eq!(d, 0);
6956
6957            let haystack = vector_unsigned_char(*b"______\0__ABCD___");
6958            let (result, d) = vec_search_string_cc(haystack, b, c);
6959            assert_eq!(result.as_array()[7], 9);
6960            assert_eq!(d, 2);
6961        }
6962    }
6963
6964    #[simd_test(enable = "vector-enhancements-2")]
6965    fn test_vec_search_string_until_zero_cc() {
6966        unsafe {
6967            let b = vector_unsigned_char(*b"ABCD\0\0\0\0\0\0\0\0\0\0\0\0");
6968            let c = vector_unsigned_char([16; 16]);
6969
6970            let haystack = vector_unsigned_char(*b"__ABCD__________");
6971            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6972            assert_eq!(result.as_array()[7], 2);
6973            assert_eq!(d, 2);
6974
6975            let haystack = vector_unsigned_char(*b"___ABCD_________");
6976            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6977            assert_eq!(result.as_array()[7], 3);
6978            assert_eq!(d, 2);
6979
6980            let haystack = vector_unsigned_char(*b"________________");
6981            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6982            assert_eq!(result.as_array()[7], 16);
6983            assert_eq!(d, 0);
6984
6985            let haystack = vector_unsigned_char(*b"______\0_________");
6986            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6987            assert_eq!(result.as_array()[7], 16);
6988            assert_eq!(d, 1);
6989
6990            let haystack = vector_unsigned_char(*b"______\0__ABCD___");
6991            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6992            assert_eq!(result.as_array()[7], 16);
6993            assert_eq!(d, 1);
6994        }
6995    }
6996
6997    #[simd_test(enable = "vector")]
6998    fn test_vec_doublee() {
6999        unsafe {
7000            let v = vector_float([1.0, 2.0, 3.0, 4.0]);
7001            assert_eq!(vec_doublee(v).as_array(), &[1.0, 3.0]);
7002
7003            let v = vector_float([f32::NAN, 2.0, f32::INFINITY, 4.0]);
7004            let d = vec_doublee(v);
7005            assert!(d.as_array()[0].is_nan());
7006            assert_eq!(d.as_array()[1], f64::INFINITY);
7007        }
7008    }
7009
7010    #[simd_test(enable = "vector")]
7011    fn test_vec_floate() {
7012        // NOTE: indices 1 and 3 can have an arbitrary value. With the C version
7013        // these are poison values, our version initializes the memory but its
7014        // value still should not be relied upon by application code.
7015        unsafe {
7016            let v = vector_double([1.0, 2.0]);
7017            let d = vec_floate(v);
7018            assert_eq!(d.as_array()[0], 1.0);
7019            assert_eq!(d.as_array()[2], 2.0);
7020
7021            let v = vector_double([f64::NAN, f64::INFINITY]);
7022            let d = vec_floate(v);
7023            assert!(d.as_array()[0].is_nan());
7024            assert_eq!(d.as_array()[2], f32::INFINITY);
7025
7026            let v = vector_double([f64::MIN, f64::MAX]);
7027            let d = vec_floate(v);
7028            assert_eq!(d.as_array()[0], f64::MIN as f32);
7029            assert_eq!(d.as_array()[2], f64::MAX as f32);
7030        }
7031    }
7032
7033    #[simd_test(enable = "vector")]
7034    fn test_vec_extend_s64() {
7035        unsafe {
7036            let v = vector_signed_char([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
7037            assert_eq!(vec_extend_s64(v).as_array(), &[7, 15]);
7038
7039            let v = vector_signed_short([0, 1, 2, 3, 4, 5, 6, 7]);
7040            assert_eq!(vec_extend_s64(v).as_array(), &[3, 7]);
7041
7042            let v = vector_signed_int([0, 1, 2, 3]);
7043            assert_eq!(vec_extend_s64(v).as_array(), &[1, 3]);
7044        }
7045    }
7046
7047    #[simd_test(enable = "vector")]
7048    fn test_vec_signed() {
7049        unsafe {
7050            let v = vector_float([1.0, 2.5, -2.5, -0.0]);
7051            assert_eq!(vec_signed(v).as_array(), &[1, 2, -2, 0]);
7052
7053            let v = vector_double([2.5, -2.5]);
7054            assert_eq!(vec_signed(v).as_array(), &[2, -2]);
7055        }
7056    }
7057
7058    #[simd_test(enable = "vector")]
7059    fn test_vec_unsigned() {
7060        // NOTE: converting a negative floating point value is UB!
7061        unsafe {
7062            let v = vector_float([1.0, 2.5, 3.5, 0.0]);
7063            assert_eq!(vec_unsigned(v).as_array(), &[1, 2, 3, 0]);
7064
7065            let v = vector_double([2.5, 3.5]);
7066            assert_eq!(vec_unsigned(v).as_array(), &[2, 3]);
7067        }
7068    }
7069
7070    #[simd_test(enable = "vector")]
7071    fn test_vec_cp_until_zero() {
7072        unsafe {
7073            let v = vector_signed_int([1, 2, 3, 4]);
7074            let d = vec_cp_until_zero(v);
7075            assert_eq!(d.as_array(), &[1, 2, 3, 4]);
7076
7077            let v = vector_signed_int([1, 2, 0, 4]);
7078            let d = vec_cp_until_zero(v);
7079            assert_eq!(d.as_array(), &[1, 2, 0, 0]);
7080        }
7081    }
7082
7083    #[simd_test(enable = "vector")]
7084    fn test_vec_cp_until_zero_cc() {
7085        unsafe {
7086            let v = vector_signed_int([1, 2, 3, 4]);
7087            let (d, cc) = vec_cp_until_zero_cc(v);
7088            assert_eq!(d.as_array(), &[1, 2, 3, 4]);
7089            assert_eq!(cc, 3);
7090
7091            let v = vector_signed_int([1, 2, 0, 4]);
7092            let (d, cc) = vec_cp_until_zero_cc(v);
7093            assert_eq!(d.as_array(), &[1, 2, 0, 0]);
7094            assert_eq!(cc, 0);
7095        }
7096    }
7097
7098    #[simd_test(enable = "vector-enhancements-1")]
7099    fn test_vec_msum_u128() {
7100        let a = vector_unsigned_long_long([1, 2]);
7101        let b = vector_unsigned_long_long([3, 4]);
7102
7103        unsafe {
7104            let c: vector_unsigned_char = transmute(100u128);
7105
7106            let d: u128 = transmute(vec_msum_u128::<0>(a, b, c));
7107            assert_eq!(d, (1 * 3) + (2 * 4) + 100);
7108
7109            let d: u128 = transmute(vec_msum_u128::<4>(a, b, c));
7110            assert_eq!(d, (1 * 3) + (2 * 4) * 2 + 100);
7111
7112            let d: u128 = transmute(vec_msum_u128::<8>(a, b, c));
7113            assert_eq!(d, (1 * 3) * 2 + (2 * 4) + 100);
7114
7115            let d: u128 = transmute(vec_msum_u128::<12>(a, b, c));
7116            assert_eq!(d, (1 * 3) * 2 + (2 * 4) * 2 + 100);
7117        }
7118    }
7119
7120    #[simd_test(enable = "vector")]
7121    fn test_vec_sld() {
7122        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7123        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7124
7125        unsafe {
7126            let d = vec_sld::<_, 4>(a, b);
7127            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]);
7128        }
7129    }
7130
7131    #[simd_test(enable = "vector")]
7132    fn test_vec_sldw() {
7133        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7134        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7135
7136        unsafe {
7137            let d = vec_sldw::<_, 1>(a, b);
7138            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]);
7139        }
7140    }
7141
7142    #[simd_test(enable = "vector-enhancements-2")]
7143    fn test_vec_sldb() {
7144        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7145        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7146
7147        unsafe {
7148            let d = vec_sldb::<_, 4>(a, b);
7149            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAB]);
7150        }
7151    }
7152
7153    #[simd_test(enable = "vector-enhancements-2")]
7154    fn test_vec_srdb() {
7155        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7156        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7157
7158        unsafe {
7159            let d = vec_srdb::<_, 4>(a, b);
7160            assert_eq!(d.as_array(), &[0xABBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7161        }
7162    }
7163
7164    const GT: u32 = 0x20000000;
7165    const LT: u32 = 0x40000000;
7166    const EQ: u32 = 0x80000000;
7167
7168    #[simd_test(enable = "vector")]
7169    fn test_vec_cmprg() {
7170        let a = vector_unsigned_int([11, 22, 33, 44]);
7171        let b = vector_unsigned_int([10, 20, 30, 40]);
7172
7173        let c = vector_unsigned_int([GT, LT, GT, LT]);
7174        let d = unsafe { vec_cmprg(a, b, c) };
7175        assert_eq!(d.as_array(), &[!0, 0, !0, 0]);
7176
7177        let c = vector_unsigned_int([GT, LT, 0, 0]);
7178        let d = unsafe { vec_cmprg(a, b, c) };
7179        assert_eq!(d.as_array(), &[!0, 0, 0, 0]);
7180
7181        let a = vector_unsigned_int([11, 22, 33, 30]);
7182        let b = vector_unsigned_int([10, 20, 30, 30]);
7183
7184        let c = vector_unsigned_int([GT, LT, EQ, EQ]);
7185        let d = unsafe { vec_cmprg(a, b, c) };
7186        assert_eq!(d.as_array(), &[!0, 0, 0, !0]);
7187    }
7188
7189    #[simd_test(enable = "vector")]
7190    fn test_vec_cmpnrg() {
7191        let a = vector_unsigned_int([11, 22, 33, 44]);
7192        let b = vector_unsigned_int([10, 20, 30, 40]);
7193
7194        let c = vector_unsigned_int([GT, LT, GT, LT]);
7195        let d = unsafe { vec_cmpnrg(a, b, c) };
7196        assert_eq!(d.as_array(), &[0, !0, 0, !0]);
7197
7198        let c = vector_unsigned_int([GT, LT, 0, 0]);
7199        let d = unsafe { vec_cmpnrg(a, b, c) };
7200        assert_eq!(d.as_array(), &[0, !0, !0, !0]);
7201
7202        let a = vector_unsigned_int([11, 22, 33, 30]);
7203        let b = vector_unsigned_int([10, 20, 30, 30]);
7204
7205        let c = vector_unsigned_int([GT, LT, EQ, EQ]);
7206        let d = unsafe { vec_cmpnrg(a, b, c) };
7207        assert_eq!(d.as_array(), &[0, !0, !0, 0]);
7208    }
7209
7210    #[simd_test(enable = "vector")]
7211    fn test_vec_cmprg_idx() {
7212        let a = vector_unsigned_int([1, 11, 22, 33]);
7213        let b = vector_unsigned_int([10, 20, 30, 40]);
7214
7215        let c = vector_unsigned_int([GT, LT, GT, LT]);
7216        let d = unsafe { vec_cmprg_idx(a, b, c) };
7217        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
7218    }
7219
7220    #[simd_test(enable = "vector")]
7221    fn test_vec_cmpnrg_idx() {
7222        let a = vector_unsigned_int([1, 11, 22, 33]);
7223        let b = vector_unsigned_int([10, 20, 30, 40]);
7224
7225        let c = vector_unsigned_int([GT, LT, GT, LT]);
7226        let d = unsafe { vec_cmpnrg_idx(a, b, c) };
7227        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
7228    }
7229
7230    #[simd_test(enable = "vector")]
7231    fn test_vec_cmprg_or_0_idx() {
7232        let a = vector_unsigned_int([1, 0, 22, 33]);
7233        let b = vector_unsigned_int([10, 20, 30, 40]);
7234
7235        let c = vector_unsigned_int([GT, LT, GT, LT]);
7236        let d = unsafe { vec_cmprg_or_0_idx(a, b, c) };
7237        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
7238    }
7239
7240    #[simd_test(enable = "vector")]
7241    fn test_vec_cmpnrg_or_0_idx() {
7242        let a = vector_unsigned_int([11, 33, 0, 22]);
7243        let b = vector_unsigned_int([10, 20, 30, 40]);
7244
7245        let c = vector_unsigned_int([GT, LT, GT, LT]);
7246        let d = unsafe { vec_cmpnrg_or_0_idx(a, b, c) };
7247        assert_eq!(d.as_array(), &[0, 8, 0, 0]);
7248    }
7249
7250    test_vec_2! { test_vec_cmpgt, vec_cmpgt, f32x4, f32x4 -> i32x4,
7251        [1.0, f32::NAN, f32::NAN, 3.14],
7252        [2.0, f32::NAN, 5.0, 2.0],
7253        [0, 0, 0, !0]
7254    }
7255
7256    test_vec_2! { test_vec_cmpge, vec_cmpge, f32x4, f32x4 -> i32x4,
7257        [1.0, f32::NAN, f32::NAN, 3.14],
7258        [1.0, f32::NAN, 5.0, 2.0],
7259        [!0, 0, 0, !0]
7260    }
7261
7262    test_vec_2! { test_vec_cmplt, vec_cmplt, f32x4, f32x4 -> i32x4,
7263        [1.0, f32::NAN, f32::NAN, 2.0],
7264        [2.0, f32::NAN, 5.0, 2.0],
7265        [!0, 0, 0, 0]
7266    }
7267
7268    test_vec_2! { test_vec_cmple, vec_cmple, f32x4, f32x4 -> i32x4,
7269        [1.0, f32::NAN, f32::NAN, 2.0],
7270        [1.0, f32::NAN, 5.0, 3.14],
7271        [!0, 0, 0, !0]
7272    }
7273
7274    test_vec_2! { test_vec_cmpeq, vec_cmpeq, f32x4, f32x4 -> i32x4,
7275        [1.0, f32::NAN, f32::NAN, 2.0],
7276        [1.0, f32::NAN, 5.0, 3.14],
7277        [!0, 0, 0, 0]
7278    }
7279
7280    test_vec_2! { test_vec_cmpne, vec_cmpne, f32x4, f32x4 -> i32x4,
7281        [1.0, f32::NAN, f32::NAN, 2.0],
7282        [1.0, f32::NAN, 5.0, 3.14],
7283        [0, !0, !0, !0]
7284    }
7285
7286    #[simd_test(enable = "vector")]
7287    fn test_vec_meadd() {
7288        let a = vector_unsigned_short([1, 0, 2, 0, 3, 0, 4, 0]);
7289        let b = vector_unsigned_short([5, 0, 6, 0, 7, 0, 8, 0]);
7290        let c = vector_unsigned_int([2, 2, 2, 2]);
7291
7292        let d = unsafe { vec_meadd(a, b, c) };
7293        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7294
7295        let a = vector_signed_short([1, 0, 2, 0, 3, 0, 4, 0]);
7296        let b = vector_signed_short([5, 0, 6, 0, 7, 0, 8, 0]);
7297        let c = vector_signed_int([2, -2, 2, -2]);
7298
7299        let d = unsafe { vec_meadd(a, b, c) };
7300        assert_eq!(d.as_array(), &[7, 10, 23, 30]);
7301    }
7302
7303    #[simd_test(enable = "vector")]
7304    fn test_vec_moadd() {
7305        let a = vector_unsigned_short([0, 1, 0, 2, 0, 3, 0, 4]);
7306        let b = vector_unsigned_short([0, 5, 0, 6, 0, 7, 0, 8]);
7307        let c = vector_unsigned_int([2, 2, 2, 2]);
7308
7309        let d = unsafe { vec_moadd(a, b, c) };
7310        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7311
7312        let a = vector_signed_short([0, 1, 0, 2, 0, 3, 0, 4]);
7313        let b = vector_signed_short([0, 5, 0, 6, 0, 7, 0, 8]);
7314        let c = vector_signed_int([2, -2, 2, -2]);
7315
7316        let d = unsafe { vec_moadd(a, b, c) };
7317        assert_eq!(d.as_array(), &[7, 10, 23, 30]);
7318    }
7319
7320    #[simd_test(enable = "vector")]
7321    fn test_vec_mhadd() {
7322        let a = vector_unsigned_int([1, 2, 3, 4]);
7323        let b = vector_unsigned_int([5, 6, 7, 8]);
7324        let c = vector_unsigned_int([u32::MAX; 4]);
7325
7326        let d = unsafe { vec_mhadd(a, b, c) };
7327        assert_eq!(d.as_array(), &[1, 1, 1, 1]);
7328
7329        let a = vector_signed_int([-1, -2, -3, -4]);
7330        let b = vector_signed_int([5, 6, 7, 8]);
7331        let c = vector_signed_int([i32::MIN; 4]);
7332
7333        let d = unsafe { vec_mhadd(a, b, c) };
7334        assert_eq!(d.as_array(), &[-1, -1, -1, -1]);
7335    }
7336
7337    #[simd_test(enable = "vector")]
7338    fn test_vec_mladd() {
7339        let a = vector_unsigned_int([1, 2, 3, 4]);
7340        let b = vector_unsigned_int([5, 6, 7, 8]);
7341        let c = vector_unsigned_int([2, 2, 2, 2]);
7342
7343        let d = unsafe { vec_mladd(a, b, c) };
7344        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7345
7346        let a = vector_signed_int([-1, -2, -3, -4]);
7347        let b = vector_signed_int([5, 6, 7, 8]);
7348        let c = vector_signed_int([2, 2, 2, 2]);
7349
7350        let d = unsafe { vec_mladd(a, b, c) };
7351        assert_eq!(d.as_array(), &[-3, -10, -19, -30]);
7352    }
7353
7354    #[simd_test(enable = "vector")]
7355    fn test_vec_extract() {
7356        let v = vector_unsigned_int([1, 2, 3, 4]);
7357
7358        assert_eq!(unsafe { vec_extract(v, 1) }, 2);
7359        assert_eq!(unsafe { vec_extract(v, 4 + 2) }, 3);
7360    }
7361
7362    #[simd_test(enable = "vector")]
7363    fn test_vec_insert() {
7364        let mut v = vector_unsigned_int([1, 2, 3, 4]);
7365
7366        v = unsafe { vec_insert(42, v, 1) };
7367        assert_eq!(v.as_array(), &[1, 42, 3, 4]);
7368
7369        v = unsafe { vec_insert(64, v, 6) };
7370        assert_eq!(v.as_array(), &[1, 42, 64, 4]);
7371    }
7372
7373    #[simd_test(enable = "vector")]
7374    fn test_vec_promote() {
7375        let v: vector_unsigned_int = unsafe { vec_promote(42, 1).assume_init() };
7376        assert_eq!(v.as_array(), &[0, 42, 0, 0]);
7377    }
7378
7379    #[simd_test(enable = "vector")]
7380    fn test_vec_insert_and_zero() {
7381        let v = unsafe { vec_insert_and_zero::<vector_unsigned_int>(&42u32) };
7382        assert_eq!(v.as_array(), vector_unsigned_int([0, 42, 0, 0]).as_array());
7383    }
7384}