Line data Source code
1 : /* Medium-level subroutines: convert bit-field store and extract
2 : and shifts, multiplies and divides to rtl instructions.
3 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* Work around tree-optimization/91825. */
22 : #pragma GCC diagnostic warning "-Wmaybe-uninitialized"
23 :
24 : #include "config.h"
25 : #include "system.h"
26 : #include "coretypes.h"
27 : #include "backend.h"
28 : #include "target.h"
29 : #include "rtl.h"
30 : #include "tree.h"
31 : #include "predict.h"
32 : #include "memmodel.h"
33 : #include "tm_p.h"
34 : #include "optabs.h"
35 : #include "expmed.h"
36 : #include "regs.h"
37 : #include "emit-rtl.h"
38 : #include "diagnostic-core.h"
39 : #include "fold-const.h"
40 : #include "stor-layout.h"
41 : #include "dojump.h"
42 : #include "explow.h"
43 : #include "expr.h"
44 : #include "langhooks.h"
45 : #include "tree-vector-builder.h"
46 : #include "recog.h"
47 :
48 : struct target_expmed default_target_expmed;
49 : #if SWITCHABLE_TARGET
50 : struct target_expmed *this_target_expmed = &default_target_expmed;
51 : #endif
52 :
53 : static bool store_integral_bit_field (rtx, opt_scalar_int_mode,
54 : unsigned HOST_WIDE_INT,
55 : unsigned HOST_WIDE_INT,
56 : poly_uint64, poly_uint64,
57 : machine_mode, rtx, bool, bool);
58 : /* The constant divisor of the last division expanded, so that a modulo by
59 : the same constant can reuse the quotient. Reset for each function by
60 : prepare_function_start: carrying it across functions would make a
61 : function's code depend on what was expanded before it in the translation
62 : unit. */
63 : HOST_WIDE_INT last_div_const = 0;
64 :
65 : static void store_fixed_bit_field (rtx, opt_scalar_int_mode,
66 : unsigned HOST_WIDE_INT,
67 : unsigned HOST_WIDE_INT,
68 : poly_uint64, poly_uint64,
69 : rtx, scalar_int_mode, bool);
70 : static void store_fixed_bit_field_1 (rtx, scalar_int_mode,
71 : unsigned HOST_WIDE_INT,
72 : unsigned HOST_WIDE_INT,
73 : rtx, scalar_int_mode, bool);
74 : static void store_split_bit_field (rtx, opt_scalar_int_mode,
75 : unsigned HOST_WIDE_INT,
76 : unsigned HOST_WIDE_INT,
77 : poly_uint64, poly_uint64,
78 : rtx, scalar_int_mode, bool);
79 : static rtx extract_integral_bit_field (rtx, opt_scalar_int_mode,
80 : unsigned HOST_WIDE_INT,
81 : unsigned HOST_WIDE_INT, int, rtx,
82 : machine_mode, machine_mode, bool, bool);
83 : static rtx extract_fixed_bit_field (machine_mode, rtx, opt_scalar_int_mode,
84 : unsigned HOST_WIDE_INT,
85 : unsigned HOST_WIDE_INT, rtx, int, bool);
86 : static rtx extract_fixed_bit_field_1 (machine_mode, rtx, scalar_int_mode,
87 : unsigned HOST_WIDE_INT,
88 : unsigned HOST_WIDE_INT, rtx, int, bool);
89 : static rtx lshift_value (machine_mode, unsigned HOST_WIDE_INT, int);
90 : static rtx extract_split_bit_field (rtx, opt_scalar_int_mode,
91 : unsigned HOST_WIDE_INT,
92 : unsigned HOST_WIDE_INT, int, bool);
93 : static void do_cmp_and_jump (rtx, rtx, enum rtx_code, machine_mode, rtx_code_label *);
94 : static rtx expand_smod_pow2 (scalar_int_mode, rtx, HOST_WIDE_INT);
95 : static rtx expand_sdiv_pow2 (scalar_int_mode, rtx, HOST_WIDE_INT);
96 :
97 : /* Return a constant integer mask value of mode MODE with BITSIZE ones
98 : followed by BITPOS zeros, or the complement of that if COMPLEMENT.
99 : The mask is truncated if necessary to the width of mode MODE. The
100 : mask is zero-extended if BITSIZE+BITPOS is too small for MODE. */
101 :
102 : static inline rtx
103 216627 : mask_rtx (scalar_int_mode mode, int bitpos, int bitsize, bool complement)
104 : {
105 216627 : return immed_wide_int_const
106 216627 : (wi::shifted_mask (bitpos, bitsize, complement,
107 216627 : GET_MODE_PRECISION (mode)), mode);
108 : }
109 :
110 : /* Test whether a value is zero of a power of two. */
111 : #define EXACT_POWER_OF_2_OR_ZERO_P(x) \
112 : (((x) & ((x) - HOST_WIDE_INT_1U)) == 0)
113 :
114 : struct init_expmed_rtl
115 : {
116 : rtx reg;
117 : rtx plus;
118 : rtx neg;
119 : rtx mult;
120 : rtx sdiv;
121 : rtx udiv;
122 : rtx sdiv_32;
123 : rtx smod_32;
124 : rtx wide_mult;
125 : rtx wide_lshr;
126 : rtx wide_trunc;
127 : rtx shift;
128 : rtx shift_mult;
129 : rtx shift_add;
130 : rtx shift_sub0;
131 : rtx shift_sub1;
132 : rtx zext;
133 : rtx trunc;
134 :
135 : rtx pow2[MAX_BITS_PER_WORD];
136 : rtx cint[MAX_BITS_PER_WORD];
137 : };
138 :
139 : static void
140 31100440 : init_expmed_one_conv (struct init_expmed_rtl *all, scalar_int_mode to_mode,
141 : scalar_int_mode from_mode, bool speed)
142 : {
143 31100440 : int to_size, from_size;
144 31100440 : rtx which;
145 :
146 31100440 : to_size = GET_MODE_PRECISION (to_mode);
147 31100440 : from_size = GET_MODE_PRECISION (from_mode);
148 :
149 : /* Most partial integers have a precision less than the "full"
150 : integer it requires for storage. In case one doesn't, for
151 : comparison purposes here, reduce the bit size by one in that
152 : case. */
153 31100440 : if (GET_MODE_CLASS (to_mode) == MODE_PARTIAL_INT
154 31100440 : && pow2p_hwi (to_size))
155 6220088 : to_size --;
156 31100440 : if (GET_MODE_CLASS (from_mode) == MODE_PARTIAL_INT
157 31100440 : && pow2p_hwi (from_size))
158 0 : from_size --;
159 :
160 : /* Assume cost of zero-extend and sign-extend is the same. */
161 31100440 : which = (to_size < from_size ? all->trunc : all->zext);
162 :
163 31100440 : PUT_MODE (all->reg, from_mode);
164 31100440 : set_convert_cost (to_mode, from_mode, speed,
165 : set_src_cost (which, to_mode, speed));
166 : /* Restore all->reg's mode. */
167 31100440 : PUT_MODE (all->reg, to_mode);
168 31100440 : }
169 :
170 : static void
171 18215972 : init_expmed_one_mode (struct init_expmed_rtl *all,
172 : machine_mode mode, int speed)
173 : {
174 18215972 : int m, n, mode_bitsize;
175 18215972 : machine_mode mode_from;
176 :
177 18215972 : mode_bitsize = GET_MODE_UNIT_BITSIZE (mode);
178 :
179 18215972 : PUT_MODE (all->reg, mode);
180 18215972 : PUT_MODE (all->plus, mode);
181 18215972 : PUT_MODE (all->neg, mode);
182 18215972 : PUT_MODE (all->mult, mode);
183 18215972 : PUT_MODE (all->sdiv, mode);
184 18215972 : PUT_MODE (all->udiv, mode);
185 18215972 : PUT_MODE (all->sdiv_32, mode);
186 18215972 : PUT_MODE (all->smod_32, mode);
187 18215972 : PUT_MODE (all->wide_trunc, mode);
188 18215972 : PUT_MODE (all->shift, mode);
189 18215972 : PUT_MODE (all->shift_mult, mode);
190 18215972 : PUT_MODE (all->shift_add, mode);
191 18215972 : PUT_MODE (all->shift_sub0, mode);
192 18215972 : PUT_MODE (all->shift_sub1, mode);
193 18215972 : PUT_MODE (all->zext, mode);
194 18215972 : PUT_MODE (all->trunc, mode);
195 :
196 18215972 : set_add_cost (speed, mode, set_src_cost (all->plus, mode, speed));
197 18215972 : set_neg_cost (speed, mode, set_src_cost (all->neg, mode, speed));
198 18215972 : set_mul_cost (speed, mode, set_src_cost (all->mult, mode, speed));
199 18215972 : set_sdiv_cost (speed, mode, set_src_cost (all->sdiv, mode, speed));
200 18215972 : set_udiv_cost (speed, mode, set_src_cost (all->udiv, mode, speed));
201 :
202 18215972 : set_sdiv_pow2_cheap (speed, mode, (set_src_cost (all->sdiv_32, mode, speed)
203 18215972 : <= 2 * add_cost (speed, mode)));
204 18215972 : set_smod_pow2_cheap (speed, mode, (set_src_cost (all->smod_32, mode, speed)
205 18215972 : <= 4 * add_cost (speed, mode)));
206 :
207 18215972 : set_shift_cost (speed, mode, 0, 0);
208 18215972 : {
209 18215972 : int cost = add_cost (speed, mode);
210 18215972 : set_shiftadd_cost (speed, mode, 0, cost);
211 18215972 : set_shiftsub0_cost (speed, mode, 0, cost);
212 18215972 : set_shiftsub1_cost (speed, mode, 0, cost);
213 : }
214 :
215 18215972 : n = MIN (MAX_BITS_PER_WORD, mode_bitsize);
216 611345792 : for (m = 1; m < n; m++)
217 : {
218 593129820 : XEXP (all->shift, 1) = all->cint[m];
219 593129820 : XEXP (all->shift_mult, 1) = all->pow2[m];
220 :
221 593129820 : set_shift_cost (speed, mode, m, set_src_cost (all->shift, mode, speed));
222 593129820 : set_shiftadd_cost (speed, mode, m, set_src_cost (all->shift_add, mode,
223 : speed));
224 593129820 : set_shiftsub0_cost (speed, mode, m, set_src_cost (all->shift_sub0, mode,
225 : speed));
226 593129820 : set_shiftsub1_cost (speed, mode, m, set_src_cost (all->shift_sub1, mode,
227 : speed));
228 : }
229 :
230 18215972 : scalar_int_mode int_mode_to;
231 18215972 : if (is_a <scalar_int_mode> (mode, &int_mode_to))
232 : {
233 35543360 : for (mode_from = MIN_MODE_INT; mode_from <= MAX_MODE_INT;
234 31100440 : mode_from = (machine_mode)(mode_from + 1))
235 31100440 : init_expmed_one_conv (all, int_mode_to,
236 : as_a <scalar_int_mode> (mode_from), speed);
237 :
238 4442920 : scalar_int_mode wider_mode;
239 4442920 : if (GET_MODE_CLASS (int_mode_to) == MODE_INT
240 4442920 : && GET_MODE_WIDER_MODE (int_mode_to).exists (&wider_mode))
241 : {
242 2665752 : PUT_MODE (all->reg, mode);
243 2665752 : PUT_MODE (all->zext, wider_mode);
244 2665752 : PUT_MODE (all->wide_mult, wider_mode);
245 2665752 : PUT_MODE (all->wide_lshr, wider_mode);
246 2665752 : XEXP (all->wide_lshr, 1)
247 2665752 : = gen_int_shift_amount (wider_mode, mode_bitsize);
248 :
249 2665752 : set_mul_widen_cost (speed, wider_mode,
250 : set_src_cost (all->wide_mult, wider_mode, speed));
251 2665752 : set_mul_highpart_cost (speed, int_mode_to,
252 : set_src_cost (all->wide_trunc,
253 : int_mode_to, speed));
254 : }
255 : }
256 18215972 : }
257 :
258 : void
259 222146 : init_expmed (void)
260 : {
261 222146 : struct init_expmed_rtl all;
262 222146 : machine_mode mode = QImode;
263 222146 : int m, speed;
264 :
265 222146 : memset (&all, 0, sizeof all);
266 14217344 : for (m = 1; m < MAX_BITS_PER_WORD; m++)
267 : {
268 13995198 : all.pow2[m] = GEN_INT (HOST_WIDE_INT_1 << m);
269 13995198 : all.cint[m] = GEN_INT (m);
270 : }
271 :
272 : /* Avoid using hard regs in ways which may be unsupported. */
273 222146 : all.reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
274 222146 : all.plus = gen_rtx_PLUS (mode, all.reg, all.reg);
275 222146 : all.neg = gen_rtx_NEG (mode, all.reg);
276 222146 : all.mult = gen_rtx_MULT (mode, all.reg, all.reg);
277 222146 : all.sdiv = gen_rtx_DIV (mode, all.reg, all.reg);
278 222146 : all.udiv = gen_rtx_UDIV (mode, all.reg, all.reg);
279 222146 : all.sdiv_32 = gen_rtx_DIV (mode, all.reg, all.pow2[5]);
280 222146 : all.smod_32 = gen_rtx_MOD (mode, all.reg, all.pow2[5]);
281 222146 : all.zext = gen_rtx_ZERO_EXTEND (mode, all.reg);
282 222146 : all.wide_mult = gen_rtx_MULT (mode, all.zext, all.zext);
283 222146 : all.wide_lshr = gen_rtx_LSHIFTRT (mode, all.wide_mult, all.reg);
284 222146 : all.wide_trunc = gen_rtx_TRUNCATE (mode, all.wide_lshr);
285 222146 : all.shift = gen_rtx_ASHIFT (mode, all.reg, all.reg);
286 222146 : all.shift_mult = gen_rtx_MULT (mode, all.reg, all.reg);
287 222146 : all.shift_add = gen_rtx_PLUS (mode, all.shift_mult, all.reg);
288 222146 : all.shift_sub0 = gen_rtx_MINUS (mode, all.shift_mult, all.reg);
289 222146 : all.shift_sub1 = gen_rtx_MINUS (mode, all.reg, all.shift_mult);
290 222146 : all.trunc = gen_rtx_TRUNCATE (mode, all.reg);
291 :
292 666438 : for (speed = 0; speed < 2; speed++)
293 : {
294 444292 : crtl->maybe_hot_insn_p = speed;
295 444292 : set_zero_cost (speed, set_src_cost (const0_rtx, QImode, speed));
296 :
297 3554336 : for (mode = MIN_MODE_INT; mode <= MAX_MODE_INT;
298 3110044 : mode = (machine_mode)(mode + 1))
299 3110044 : init_expmed_one_mode (&all, mode, speed);
300 :
301 : if (MIN_MODE_PARTIAL_INT != VOIDmode)
302 1777168 : for (mode = MIN_MODE_PARTIAL_INT; mode <= MAX_MODE_PARTIAL_INT;
303 1332876 : mode = (machine_mode)(mode + 1))
304 1332876 : init_expmed_one_mode (&all, mode, speed);
305 :
306 : if (MIN_MODE_VECTOR_INT != VOIDmode)
307 14217344 : for (mode = MIN_MODE_VECTOR_INT; mode <= MAX_MODE_VECTOR_INT;
308 13773052 : mode = (machine_mode)(mode + 1))
309 13773052 : init_expmed_one_mode (&all, mode, speed);
310 : }
311 :
312 222146 : if (alg_hash_used_p ())
313 : {
314 1057 : struct alg_hash_entry *p = alg_hash_entry_ptr (0);
315 1057 : memset (p, 0, sizeof (*p) * NUM_ALG_HASH_ENTRIES);
316 : }
317 : else
318 221089 : set_alg_hash_used_p (true);
319 222146 : default_rtl_profile ();
320 :
321 222146 : ggc_free (all.trunc);
322 222146 : ggc_free (all.shift_sub1);
323 222146 : ggc_free (all.shift_sub0);
324 222146 : ggc_free (all.shift_add);
325 222146 : ggc_free (all.shift_mult);
326 222146 : ggc_free (all.shift);
327 222146 : ggc_free (all.wide_trunc);
328 222146 : ggc_free (all.wide_lshr);
329 222146 : ggc_free (all.wide_mult);
330 222146 : ggc_free (all.zext);
331 222146 : ggc_free (all.smod_32);
332 222146 : ggc_free (all.sdiv_32);
333 222146 : ggc_free (all.udiv);
334 222146 : ggc_free (all.sdiv);
335 222146 : ggc_free (all.mult);
336 222146 : ggc_free (all.neg);
337 222146 : ggc_free (all.plus);
338 222146 : ggc_free (all.reg);
339 222146 : }
340 :
341 : /* Return an rtx representing minus the value of X.
342 : MODE is the intended mode of the result,
343 : useful if X is a CONST_INT. */
344 :
345 : rtx
346 1039149 : negate_rtx (machine_mode mode, rtx x)
347 : {
348 1039149 : rtx result = simplify_unary_operation (NEG, mode, x, mode);
349 :
350 1039149 : if (result == 0)
351 1392 : result = expand_unop (mode, neg_optab, x, NULL_RTX, 0);
352 :
353 1039149 : return result;
354 : }
355 :
356 : /* Whether reverse storage order is supported on the target. */
357 : static int reverse_storage_order_supported = -1;
358 :
359 : /* Check whether reverse storage order is supported on the target. */
360 :
361 : static void
362 311 : check_reverse_storage_order_support (void)
363 : {
364 311 : if (BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
365 : {
366 : reverse_storage_order_supported = 0;
367 : sorry ("reverse scalar storage order");
368 : }
369 : else
370 311 : reverse_storage_order_supported = 1;
371 0 : }
372 :
373 : /* Whether reverse FP storage order is supported on the target. */
374 : static int reverse_float_storage_order_supported = -1;
375 :
376 : /* Check whether reverse FP storage order is supported on the target. */
377 :
378 : static void
379 55 : check_reverse_float_storage_order_support (void)
380 : {
381 55 : if (FLOAT_WORDS_BIG_ENDIAN != WORDS_BIG_ENDIAN)
382 : {
383 : reverse_float_storage_order_supported = 0;
384 : sorry ("reverse floating-point scalar storage order");
385 : }
386 : else
387 55 : reverse_float_storage_order_supported = 1;
388 0 : }
389 :
390 : /* Return an rtx representing value of X with reverse storage order.
391 : MODE is the intended mode of the result,
392 : useful if X is a CONST_INT. */
393 :
394 : rtx
395 3382 : flip_storage_order (machine_mode mode, rtx x)
396 : {
397 3382 : scalar_int_mode int_mode;
398 3382 : rtx result;
399 :
400 3382 : if (mode == QImode)
401 : return x;
402 :
403 2495 : if (COMPLEX_MODE_P (mode))
404 : {
405 52 : rtx real = read_complex_part (x, false);
406 52 : rtx imag = read_complex_part (x, true);
407 :
408 104 : real = flip_storage_order (GET_MODE_INNER (mode), real);
409 104 : imag = flip_storage_order (GET_MODE_INNER (mode), imag);
410 :
411 52 : return gen_rtx_CONCAT (mode, real, imag);
412 : }
413 :
414 2443 : if (UNLIKELY (reverse_storage_order_supported < 0))
415 311 : check_reverse_storage_order_support ();
416 :
417 2443 : if (!is_a <scalar_int_mode> (mode, &int_mode))
418 : {
419 243 : if (FLOAT_MODE_P (mode)
420 243 : && UNLIKELY (reverse_float_storage_order_supported < 0))
421 55 : check_reverse_float_storage_order_support ();
422 :
423 243 : if (!int_mode_for_size (GET_MODE_PRECISION (mode), 0).exists (&int_mode)
424 243 : || !targetm.scalar_mode_supported_p (int_mode))
425 : {
426 0 : sorry ("reverse storage order for %smode", GET_MODE_NAME (mode));
427 0 : return x;
428 : }
429 243 : x = gen_lowpart (int_mode, x);
430 : }
431 :
432 2443 : result = simplify_unary_operation (BSWAP, int_mode, x, int_mode);
433 2443 : if (result == 0)
434 1099 : result = expand_unop (int_mode, bswap_optab, x, NULL_RTX, 1);
435 :
436 2443 : if (int_mode != mode)
437 243 : result = gen_lowpart (mode, result);
438 :
439 : return result;
440 : }
441 :
442 : /* If MODE is set, adjust bitfield memory MEM so that it points to the
443 : first unit of mode MODE that contains a bitfield of size BITSIZE at
444 : bit position BITNUM. If MODE is not set, return a BLKmode reference
445 : to every byte in the bitfield. Set *NEW_BITNUM to the bit position
446 : of the field within the new memory. */
447 :
448 : static rtx
449 425241 : narrow_bit_field_mem (rtx mem, opt_scalar_int_mode mode,
450 : unsigned HOST_WIDE_INT bitsize,
451 : unsigned HOST_WIDE_INT bitnum,
452 : unsigned HOST_WIDE_INT *new_bitnum)
453 : {
454 425241 : scalar_int_mode imode;
455 425241 : if (mode.exists (&imode))
456 : {
457 425241 : unsigned int unit = GET_MODE_BITSIZE (imode);
458 425241 : *new_bitnum = bitnum % unit;
459 425241 : HOST_WIDE_INT offset = (bitnum - *new_bitnum) / BITS_PER_UNIT;
460 425241 : return adjust_bitfield_address (mem, imode, offset);
461 : }
462 : else
463 : {
464 0 : *new_bitnum = bitnum % BITS_PER_UNIT;
465 0 : HOST_WIDE_INT offset = bitnum / BITS_PER_UNIT;
466 0 : HOST_WIDE_INT size = ((*new_bitnum + bitsize + BITS_PER_UNIT - 1)
467 0 : / BITS_PER_UNIT);
468 0 : return adjust_bitfield_address_size (mem, BLKmode, offset, size);
469 : }
470 : }
471 :
472 : /* The caller wants to perform insertion or extraction PATTERN on a
473 : bitfield of size BITSIZE at BITNUM bits into memory operand OP0.
474 : BITREGION_START and BITREGION_END are as for store_bit_field
475 : and FIELDMODE is the natural mode of the field.
476 :
477 : Search for a mode that is compatible with the memory access
478 : restrictions and (where applicable) with a register insertion or
479 : extraction. Return the new memory on success, storing the adjusted
480 : bit position in *NEW_BITNUM. Return null otherwise. */
481 :
482 : static rtx
483 205463 : adjust_bit_field_mem_for_reg (enum extraction_pattern pattern,
484 : rtx op0, HOST_WIDE_INT bitsize,
485 : HOST_WIDE_INT bitnum,
486 : poly_uint64 bitregion_start,
487 : poly_uint64 bitregion_end,
488 : machine_mode fieldmode,
489 : unsigned HOST_WIDE_INT *new_bitnum)
490 : {
491 410926 : bit_field_mode_iterator iter (bitsize, bitnum, bitregion_start,
492 410926 : bitregion_end, MEM_ALIGN (op0),
493 410926 : MEM_VOLATILE_P (op0));
494 205463 : scalar_int_mode best_mode;
495 205463 : if (iter.next_mode (&best_mode))
496 : {
497 : /* We can use a memory in BEST_MODE. See whether this is true for
498 : any wider modes. All other things being equal, we prefer to
499 : use the widest mode possible because it tends to expose more
500 : CSE opportunities. */
501 198979 : if (!iter.prefer_smaller_modes ())
502 : {
503 : /* Limit the search to the mode required by the corresponding
504 : register insertion or extraction instruction, if any. */
505 317 : scalar_int_mode limit_mode = word_mode;
506 317 : extraction_insn insn;
507 634 : if (get_best_reg_extraction_insn (&insn, pattern,
508 317 : GET_MODE_BITSIZE (best_mode),
509 : fieldmode))
510 317 : limit_mode = insn.field_mode;
511 :
512 317 : scalar_int_mode wider_mode;
513 317 : while (iter.next_mode (&wider_mode)
514 1052 : && GET_MODE_SIZE (wider_mode) <= GET_MODE_SIZE (limit_mode))
515 111 : best_mode = wider_mode;
516 : }
517 198979 : return narrow_bit_field_mem (op0, best_mode, bitsize, bitnum,
518 : new_bitnum);
519 : }
520 : return NULL_RTX;
521 : }
522 :
523 : /* Return true if a bitfield of size BITSIZE at bit number BITNUM within
524 : a structure of mode STRUCT_MODE represents a lowpart subreg. The subreg
525 : offset is then BITNUM / BITS_PER_UNIT. */
526 :
527 : static bool
528 836074 : lowpart_bit_field_p (poly_uint64 bitnum, poly_uint64 bitsize,
529 : machine_mode struct_mode)
530 : {
531 836074 : poly_uint64 regsize = REGMODE_NATURAL_SIZE (struct_mode);
532 836074 : if (BYTES_BIG_ENDIAN)
533 : return (multiple_p (bitnum, BITS_PER_UNIT)
534 : && (known_eq (bitnum + bitsize, GET_MODE_BITSIZE (struct_mode))
535 : || multiple_p (bitnum + bitsize,
536 : regsize * BITS_PER_UNIT)));
537 : else
538 836074 : return multiple_p (bitnum, regsize * BITS_PER_UNIT);
539 : }
540 :
541 : /* Return true if -fstrict-volatile-bitfields applies to an access of OP0
542 : containing BITSIZE bits starting at BITNUM, with field mode FIELDMODE.
543 : Return false if the access would touch memory outside the range
544 : BITREGION_START to BITREGION_END for conformance to the C++ memory
545 : model. */
546 :
547 : static bool
548 1612937 : strict_volatile_bitfield_p (rtx op0, unsigned HOST_WIDE_INT bitsize,
549 : unsigned HOST_WIDE_INT bitnum,
550 : scalar_int_mode fieldmode,
551 : poly_uint64 bitregion_start,
552 : poly_uint64 bitregion_end)
553 : {
554 1612937 : unsigned HOST_WIDE_INT modesize = GET_MODE_BITSIZE (fieldmode);
555 :
556 : /* -fstrict-volatile-bitfields must be enabled and we must have a
557 : volatile MEM. */
558 1612937 : if (!MEM_P (op0)
559 166375 : || !MEM_VOLATILE_P (op0)
560 1613149 : || flag_strict_volatile_bitfields <= 0)
561 : return false;
562 :
563 : /* The bit size must not be larger than the field mode, and
564 : the field mode must not be larger than a word. */
565 14 : if (bitsize > modesize || modesize > BITS_PER_WORD)
566 : return false;
567 :
568 : /* Check for cases of unaligned fields that must be split. */
569 14 : if (bitnum % modesize + bitsize > modesize)
570 : return false;
571 :
572 : /* The memory must be sufficiently aligned for a MODESIZE access.
573 : This condition guarantees, that the memory access will not
574 : touch anything after the end of the structure. */
575 11 : if (MEM_ALIGN (op0) < modesize)
576 : return false;
577 :
578 : /* Check for cases where the C++ memory model applies. */
579 11 : if (maybe_ne (bitregion_end, 0U)
580 11 : && (maybe_lt (bitnum - bitnum % modesize, bitregion_start)
581 4 : || maybe_gt (bitnum - bitnum % modesize + modesize - 1,
582 : bitregion_end)))
583 0 : return false;
584 :
585 : return true;
586 : }
587 :
588 : /* Return true if OP is a memory and if a bitfield of size BITSIZE at
589 : bit number BITNUM can be treated as a simple value of mode MODE.
590 : Store the byte offset in *BYTENUM if so. */
591 :
592 : static bool
593 557396 : simple_mem_bitfield_p (rtx op0, poly_uint64 bitsize, poly_uint64 bitnum,
594 : machine_mode mode, poly_uint64 *bytenum)
595 : {
596 557396 : return (MEM_P (op0)
597 260771 : && multiple_p (bitnum, BITS_PER_UNIT, bytenum)
598 208774 : && known_eq (bitsize, GET_MODE_BITSIZE (mode))
599 609808 : && (!targetm.slow_unaligned_access (mode, MEM_ALIGN (op0))
600 0 : || (multiple_p (bitnum, GET_MODE_ALIGNMENT (mode))
601 0 : && MEM_ALIGN (op0) >= GET_MODE_ALIGNMENT (mode))));
602 : }
603 :
604 : /* Try to use instruction INSV to store VALUE into a field of OP0.
605 : If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is a
606 : BLKmode MEM. VALUE_MODE is the mode of VALUE. BITSIZE and BITNUM
607 : are as for store_bit_field. */
608 :
609 : static bool
610 110172 : store_bit_field_using_insv (const extraction_insn *insv, rtx op0,
611 : opt_scalar_int_mode op0_mode,
612 : unsigned HOST_WIDE_INT bitsize,
613 : unsigned HOST_WIDE_INT bitnum,
614 : rtx value, scalar_int_mode value_mode)
615 : {
616 110172 : class expand_operand ops[4];
617 110172 : rtx value1;
618 110172 : rtx xop0 = op0;
619 110172 : rtx_insn *last = get_last_insn ();
620 110172 : bool copy_back = false;
621 :
622 110172 : scalar_int_mode op_mode = insv->field_mode;
623 110172 : unsigned int unit = GET_MODE_BITSIZE (op_mode);
624 110172 : if (bitsize == 0 || bitsize > unit)
625 : return false;
626 :
627 110159 : if (MEM_P (xop0))
628 : /* Get a reference to the first byte of the field. */
629 0 : xop0 = narrow_bit_field_mem (xop0, insv->struct_mode, bitsize, bitnum,
630 : &bitnum);
631 : else
632 : {
633 : /* Convert from counting within OP0 to counting in OP_MODE. */
634 110159 : if (BYTES_BIG_ENDIAN)
635 : bitnum += unit - GET_MODE_BITSIZE (op0_mode.require ());
636 :
637 : /* If xop0 is a register, we need it in OP_MODE
638 : to make it acceptable to the format of insv. */
639 110159 : if (GET_CODE (xop0) == SUBREG)
640 : {
641 : /* If such a SUBREG can't be created, give up. */
642 36060 : if (!validate_subreg (op_mode, GET_MODE (SUBREG_REG (xop0)),
643 36060 : SUBREG_REG (xop0), SUBREG_BYTE (xop0)))
644 : return false;
645 : /* We can't just change the mode, because this might clobber op0,
646 : and we will need the original value of op0 if insv fails. */
647 36060 : xop0 = gen_rtx_SUBREG (op_mode, SUBREG_REG (xop0),
648 36060 : SUBREG_BYTE (xop0));
649 : }
650 110159 : if (REG_P (xop0) && GET_MODE (xop0) != op_mode)
651 27074 : xop0 = gen_lowpart_SUBREG (op_mode, xop0);
652 : }
653 :
654 : /* If the destination is a paradoxical subreg such that we need a
655 : truncate to the inner mode, perform the insertion on a temporary and
656 : truncate the result to the original destination. Note that we can't
657 : just truncate the paradoxical subreg as (truncate:N (subreg:W (reg:N
658 : X) 0)) is (reg:N X). */
659 110159 : if (GET_CODE (xop0) == SUBREG
660 63134 : && REG_P (SUBREG_REG (xop0))
661 173293 : && !TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (SUBREG_REG (xop0)),
662 : op_mode))
663 : {
664 0 : rtx tem = gen_reg_rtx (op_mode);
665 0 : emit_move_insn (tem, xop0);
666 0 : xop0 = tem;
667 0 : copy_back = true;
668 : }
669 :
670 : /* There are similar overflow check at the start of store_bit_field_1,
671 : but that only check the situation where the field lies completely
672 : outside the register, while there do have situation where the field
673 : lies partially in the register, we need to adjust bitsize for this
674 : partial overflow situation. Without this fix, pr48335-2.c on big-endian
675 : will broken on those arch support bit insert instruction, like arm, aarch64
676 : etc. */
677 110159 : if (bitsize + bitnum > unit && bitnum < unit)
678 : {
679 2 : warning (OPT_Wextra, "write of %wu-bit data outside the bound of "
680 : "destination object, data truncated into %wu-bit",
681 : bitsize, unit - bitnum);
682 2 : bitsize = unit - bitnum;
683 : }
684 :
685 : /* If BITS_BIG_ENDIAN is zero on a BYTES_BIG_ENDIAN machine, we count
686 : "backwards" from the size of the unit we are inserting into.
687 : Otherwise, we count bits from the most significant on a
688 : BYTES/BITS_BIG_ENDIAN machine. */
689 :
690 110159 : if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
691 : bitnum = unit - bitsize - bitnum;
692 :
693 : /* Convert VALUE to op_mode (which insv insn wants) in VALUE1. */
694 110159 : value1 = value;
695 110159 : if (value_mode != op_mode)
696 : {
697 142210 : if (GET_MODE_BITSIZE (value_mode) >= bitsize)
698 : {
699 71105 : rtx tmp;
700 : /* Optimization: Don't bother really extending VALUE
701 : if it has all the bits we will actually use. However,
702 : if we must narrow it, be sure we do it correctly. */
703 :
704 213315 : if (GET_MODE_SIZE (value_mode) < GET_MODE_SIZE (op_mode))
705 : {
706 33680 : tmp = simplify_subreg (op_mode, value1, value_mode, 0);
707 33680 : if (! tmp)
708 33093 : tmp = simplify_gen_subreg (op_mode,
709 : force_reg (value_mode, value1),
710 : value_mode, 0);
711 : }
712 : else
713 : {
714 37425 : if (targetm.mode_rep_extended (op_mode, value_mode) != UNKNOWN)
715 0 : tmp = simplify_gen_unary (TRUNCATE, op_mode,
716 : value1, value_mode);
717 : else
718 : {
719 37425 : tmp = gen_lowpart_if_possible (op_mode, value1);
720 37425 : if (! tmp)
721 0 : tmp = gen_lowpart (op_mode, force_reg (value_mode, value1));
722 : }
723 : }
724 : value1 = tmp;
725 : }
726 0 : else if (CONST_INT_P (value))
727 0 : value1 = gen_int_mode (INTVAL (value), op_mode);
728 : else
729 : /* Parse phase is supposed to make VALUE's data type
730 : match that of the component reference, which is a type
731 : at least as wide as the field; so VALUE should have
732 : a mode that corresponds to that type. */
733 0 : gcc_assert (CONSTANT_P (value));
734 : }
735 :
736 110159 : create_fixed_operand (&ops[0], xop0);
737 110159 : create_integer_operand (&ops[1], bitsize);
738 110159 : create_integer_operand (&ops[2], bitnum);
739 110159 : create_input_operand (&ops[3], value1, op_mode);
740 110159 : if (maybe_expand_insn (insv->icode, 4, ops))
741 : {
742 2120 : if (copy_back)
743 0 : convert_move (op0, xop0, true);
744 : return true;
745 : }
746 108039 : delete_insns_since (last);
747 108039 : return false;
748 : }
749 :
750 : /* A subroutine of store_bit_field, with the same arguments. Return true
751 : if the operation could be implemented.
752 :
753 : If FALLBACK_P is true, fall back to store_fixed_bit_field if we have
754 : no other way of implementing the operation. If FALLBACK_P is false,
755 : return false instead.
756 :
757 : if UNDEFINED_P is true then STR_RTX is undefined and may be set using
758 : a subreg instead. */
759 :
760 : static bool
761 906910 : store_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
762 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
763 : machine_mode fieldmode,
764 : rtx value, bool reverse, bool fallback_p, bool undefined_p)
765 : {
766 906910 : rtx op0 = str_rtx;
767 :
768 906916 : while (GET_CODE (op0) == SUBREG)
769 : {
770 6 : bitnum += subreg_memory_offset (op0) * BITS_PER_UNIT;
771 6 : op0 = SUBREG_REG (op0);
772 : }
773 :
774 : /* No action is needed if the target is a register and if the field
775 : lies completely outside that register. This can occur if the source
776 : code contains an out-of-bounds access to a small array. */
777 1738505 : if (REG_P (op0) && known_ge (bitnum, GET_MODE_BITSIZE (GET_MODE (op0))))
778 : return true;
779 :
780 : /* Use vec_set patterns for inserting parts of vectors whenever
781 : available. */
782 906907 : machine_mode outermode = GET_MODE (op0);
783 906907 : scalar_mode innermode = GET_MODE_INNER (outermode);
784 906907 : poly_uint64 pos;
785 904862 : if (VECTOR_MODE_P (outermode)
786 2460 : && !MEM_P (op0)
787 2448 : && optab_handler (vec_set_optab, outermode) != CODE_FOR_nothing
788 1264 : && fieldmode == innermode
789 959 : && known_eq (bitsize, GET_MODE_PRECISION (innermode))
790 907866 : && multiple_p (bitnum, GET_MODE_PRECISION (innermode), &pos))
791 : {
792 959 : class expand_operand ops[3];
793 959 : enum insn_code icode = optab_handler (vec_set_optab, outermode);
794 :
795 959 : create_fixed_operand (&ops[0], op0);
796 959 : create_input_operand (&ops[1], value, innermode);
797 959 : create_integer_operand (&ops[2], pos);
798 959 : if (maybe_expand_insn (icode, 3, ops))
799 959 : return true;
800 : }
801 :
802 : /* If the target is a register, overwriting the entire object, or storing
803 : a full-word or multi-word field can be done with just a SUBREG. */
804 905948 : if (!MEM_P (op0)
805 1736581 : && known_eq (bitsize, GET_MODE_BITSIZE (fieldmode)))
806 : {
807 : /* Use the subreg machinery either to narrow OP0 to the required
808 : words or to cope with mode punning between equal-sized modes.
809 : In the latter case, use subreg on the rhs side, not lhs. */
810 763530 : rtx sub;
811 763530 : poly_uint64 bytenum;
812 763530 : poly_uint64 regsize = REGMODE_NATURAL_SIZE (GET_MODE (op0));
813 763530 : if (known_eq (bitnum, 0U)
814 1149127 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0))))
815 : {
816 61483 : sub = force_subreg (GET_MODE (op0), value, fieldmode, 0);
817 61483 : if (sub)
818 : {
819 61483 : if (reverse)
820 1 : sub = flip_storage_order (GET_MODE (op0), sub);
821 61483 : emit_move_insn (op0, sub);
822 61483 : return true;
823 : }
824 : }
825 892794 : else if (multiple_p (bitnum, BITS_PER_UNIT, &bytenum)
826 702040 : && (undefined_p
827 698386 : || (multiple_p (bitnum, regsize * BITS_PER_UNIT)
828 686843 : && multiple_p (bitsize, regsize * BITS_PER_UNIT)))
829 1307458 : && known_ge (GET_MODE_BITSIZE (GET_MODE (op0)), bitsize))
830 : {
831 653713 : sub = simplify_gen_subreg (fieldmode, op0, GET_MODE (op0), bytenum);
832 653713 : if (sub)
833 : {
834 653711 : if (reverse)
835 0 : value = flip_storage_order (fieldmode, value);
836 653711 : emit_move_insn (sub, value);
837 653711 : return true;
838 : }
839 : }
840 : }
841 :
842 : /* If the target is memory, storing any naturally aligned field can be
843 : done with a simple store. For targets that support fast unaligned
844 : memory, any naturally sized, unit aligned field can be done directly. */
845 190754 : poly_uint64 bytenum;
846 190754 : if (simple_mem_bitfield_p (op0, bitsize, bitnum, fieldmode, &bytenum))
847 : {
848 7037 : op0 = adjust_bitfield_address (op0, fieldmode, bytenum);
849 7037 : if (reverse)
850 0 : value = flip_storage_order (fieldmode, value);
851 7037 : emit_move_insn (op0, value);
852 7037 : return true;
853 : }
854 :
855 : /* It's possible we'll need to handle other cases here for
856 : polynomial bitnum and bitsize. */
857 :
858 : /* From here on we need to be looking at a fixed-size insertion. */
859 183717 : unsigned HOST_WIDE_INT ibitsize = bitsize.to_constant ();
860 183717 : unsigned HOST_WIDE_INT ibitnum = bitnum.to_constant ();
861 :
862 : /* Make sure we are playing with integral modes. Pun with subregs
863 : if we aren't. This must come after the entire register case above,
864 : since that case is valid for any mode. The following cases are only
865 : valid for integral modes. */
866 183717 : opt_scalar_int_mode op0_mode = int_mode_for_mode (GET_MODE (op0));
867 183717 : scalar_int_mode imode;
868 183717 : bool need_stack_p = false;
869 183717 : if (!op0_mode.exists (&imode) || imode != GET_MODE (op0))
870 : {
871 19916 : if (MEM_P (op0))
872 15257 : op0 = adjust_bitfield_address_size (op0, op0_mode.else_blk (),
873 : 0, MEM_SIZE (op0));
874 4659 : else if (!op0_mode.exists ()
875 4659 : || maybe_lt
876 4799 : ((unsigned) UNITS_PER_WORD,
877 4659 : (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
878 : {
879 0 : if (ibitnum == 0
880 0 : && known_eq (ibitsize, GET_MODE_BITSIZE (GET_MODE (op0)))
881 0 : && MEM_P (value)
882 0 : && !reverse)
883 : {
884 0 : value = adjust_address (value, GET_MODE (op0), 0);
885 0 : emit_move_insn (op0, value);
886 0 : return true;
887 : }
888 0 : if (!fallback_p)
889 : return false;
890 : need_stack_p = true;
891 : }
892 : else
893 4659 : op0 = gen_lowpart (op0_mode.require (), op0);
894 : }
895 163801 : else if (!MEM_P (op0)
896 275924 : && maybe_lt ((unsigned) UNITS_PER_WORD,
897 110780 : (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
898 : need_stack_p = true;
899 :
900 : /* With or without punning we might be faced with a mode that we cannot
901 : split into words. If so, spill OP0 to the stack and recurse.
902 : This happens at most once. */
903 19916 : if (need_stack_p)
904 : {
905 0 : rtx temp = assign_stack_temp (GET_MODE (op0),
906 0 : GET_MODE_SIZE (GET_MODE (op0)));
907 0 : emit_move_insn (temp, op0);
908 0 : store_bit_field_1 (temp, bitsize, bitnum, 0, 0, fieldmode, value,
909 : reverse, fallback_p, undefined_p);
910 0 : emit_move_insn (op0, temp);
911 0 : return true;
912 : }
913 :
914 183717 : return store_integral_bit_field (op0, op0_mode, ibitsize, ibitnum,
915 : bitregion_start, bitregion_end,
916 183717 : fieldmode, value, reverse, fallback_p);
917 : }
918 :
919 : /* Subroutine of store_bit_field_1, with the same arguments, except
920 : that BITSIZE and BITNUM are constant. Handle cases specific to
921 : integral modes. If OP0_MODE is defined, it is the mode of OP0,
922 : otherwise OP0 is a BLKmode MEM. */
923 :
924 : static bool
925 183717 : store_integral_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
926 : unsigned HOST_WIDE_INT bitsize,
927 : unsigned HOST_WIDE_INT bitnum,
928 : poly_uint64 bitregion_start,
929 : poly_uint64 bitregion_end,
930 : machine_mode fieldmode,
931 : rtx value, bool reverse, bool fallback_p)
932 : {
933 : /* Storing an lsb-aligned field in a register
934 : can be done with a movstrict instruction. */
935 :
936 183717 : if (!MEM_P (op0)
937 115439 : && !reverse
938 378088 : && lowpart_bit_field_p (bitnum, bitsize, op0_mode.require ())
939 83404 : && known_eq (bitsize, GET_MODE_BITSIZE (fieldmode))
940 220503 : && optab_handler (movstrict_optab, fieldmode) != CODE_FOR_nothing)
941 : {
942 4445 : class expand_operand ops[2];
943 4445 : enum insn_code icode = optab_handler (movstrict_optab, fieldmode);
944 4445 : rtx arg0 = op0;
945 4445 : unsigned HOST_WIDE_INT subreg_off;
946 :
947 4445 : if (GET_CODE (arg0) == SUBREG)
948 : {
949 : /* Else we've got some float mode source being extracted into
950 : a different float mode destination -- this combination of
951 : subregs results in Severe Tire Damage. */
952 459 : gcc_assert (GET_MODE (SUBREG_REG (arg0)) == fieldmode
953 : || GET_MODE_CLASS (fieldmode) == MODE_INT
954 : || GET_MODE_CLASS (fieldmode) == MODE_PARTIAL_INT);
955 : arg0 = SUBREG_REG (arg0);
956 : }
957 :
958 4445 : subreg_off = bitnum / BITS_PER_UNIT;
959 4479 : if (validate_subreg (fieldmode, GET_MODE (arg0), arg0, subreg_off)
960 : /* STRICT_LOW_PART must have a non-paradoxical subreg as
961 : operand. */
962 4445 : && !paradoxical_subreg_p (fieldmode, GET_MODE (arg0)))
963 : {
964 4411 : arg0 = gen_rtx_SUBREG (fieldmode, arg0, subreg_off);
965 :
966 4411 : create_fixed_operand (&ops[0], arg0);
967 : /* Shrink the source operand to FIELDMODE. */
968 4411 : create_convert_operand_to (&ops[1], value, fieldmode, false);
969 4411 : if (maybe_expand_insn (icode, 2, ops))
970 4410 : return true;
971 : }
972 : }
973 :
974 : /* Handle fields bigger than a word. */
975 :
976 180896 : if (bitsize > BITS_PER_WORD)
977 : {
978 : /* Here we transfer the words of the field
979 : in the order least significant first.
980 : This is because the most significant word is the one which may
981 : be less than full.
982 : However, only do that if the value is not BLKmode. */
983 :
984 918 : const bool backwards = WORDS_BIG_ENDIAN && fieldmode != BLKmode;
985 918 : const int nwords = (bitsize + (BITS_PER_WORD - 1)) / BITS_PER_WORD;
986 918 : rtx_insn *last;
987 :
988 : /* This is the mode we must force value to, so that there will be enough
989 : subwords to extract. Note that fieldmode will often (always?) be
990 : VOIDmode, because that is what store_field uses to indicate that this
991 : is a bit field, but passing VOIDmode to operand_subword_force
992 : is not allowed.
993 :
994 : The mode must be fixed-size, since insertions into variable-sized
995 : objects are meant to be handled before calling this function. */
996 918 : fixed_size_mode value_mode = as_a <fixed_size_mode> (GET_MODE (value));
997 918 : if (value_mode == VOIDmode)
998 24 : value_mode
999 24 : = smallest_int_mode_for_size (nwords * BITS_PER_WORD).require ();
1000 :
1001 918 : last = get_last_insn ();
1002 2749 : for (int i = 0; i < nwords; i++)
1003 : {
1004 : /* Number of bits to be stored in this iteration, i.e. BITS_PER_WORD
1005 : except maybe for the last iteration. */
1006 3672 : const unsigned HOST_WIDE_INT new_bitsize
1007 1872 : = MIN (BITS_PER_WORD, bitsize - i * BITS_PER_WORD);
1008 : /* Bit offset from the starting bit number in the target. */
1009 3684 : const unsigned int bit_offset
1010 : = backwards ^ reverse
1011 1836 : ? MAX ((int) bitsize - (i + 1) * BITS_PER_WORD, 0)
1012 : : i * BITS_PER_WORD;
1013 :
1014 : /* No further action is needed if the target is a register and if
1015 : this field lies completely outside that register. */
1016 2110 : if (REG_P (op0) && known_ge (bitnum + bit_offset,
1017 : GET_MODE_BITSIZE (GET_MODE (op0))))
1018 : {
1019 5 : if (backwards ^ reverse)
1020 0 : continue;
1021 : /* For forward operation we are finished. */
1022 183717 : return true;
1023 : }
1024 :
1025 : /* Starting word number in the value. */
1026 1831 : const unsigned int wordnum
1027 : = backwards
1028 1831 : ? GET_MODE_SIZE (value_mode) / UNITS_PER_WORD - (i + 1)
1029 : : i;
1030 : /* The chunk of the value in word_mode. We use bit-field extraction
1031 : in BLKmode to handle unaligned memory references and to shift the
1032 : last chunk right on big-endian machines if need be. */
1033 1831 : rtx value_word
1034 : = (fieldmode == BLKmode
1035 1580 : || maybe_lt ((unsigned) UNITS_PER_WORD,
1036 1580 : (poly_uint64) REGMODE_NATURAL_SIZE (value_mode)))
1037 1867 : ? extract_bit_field (value, new_bitsize, wordnum * BITS_PER_WORD,
1038 : 1, NULL_RTX, word_mode, word_mode, false,
1039 : NULL)
1040 1831 : : operand_subword_force (value, wordnum, value_mode);
1041 :
1042 1831 : if (!store_bit_field_1 (op0, new_bitsize,
1043 1831 : bitnum + bit_offset,
1044 : bitregion_start, bitregion_end,
1045 : word_mode,
1046 : value_word, reverse, fallback_p, false))
1047 : {
1048 0 : delete_insns_since (last);
1049 0 : return false;
1050 : }
1051 : }
1052 : return true;
1053 : }
1054 :
1055 : /* If VALUE has a floating-point or complex mode, access it as an
1056 : integer of the corresponding size. This can occur on a machine
1057 : with 64 bit registers that uses SFmode for float. It can also
1058 : occur for unaligned float or complex fields. */
1059 178389 : rtx orig_value = value;
1060 178389 : scalar_int_mode value_mode;
1061 178389 : if (GET_MODE (value) == VOIDmode)
1062 : /* By this point we've dealt with values that are bigger than a word,
1063 : so word_mode is a conservatively correct choice. */
1064 105884 : value_mode = word_mode;
1065 72505 : else if (!is_a <scalar_int_mode> (GET_MODE (value), &value_mode))
1066 : {
1067 1141 : value_mode = int_mode_for_mode (GET_MODE (value)).require ();
1068 1141 : value = gen_reg_rtx (value_mode);
1069 1141 : emit_move_insn (gen_lowpart (GET_MODE (orig_value), value), orig_value);
1070 : }
1071 :
1072 : /* If OP0 is a multi-word register, narrow it to the affected word.
1073 : If the region spans two words, defer to store_split_bit_field.
1074 : Don't do this if op0 is a single hard register wider than word
1075 : such as a float or vector register. */
1076 178389 : if (!MEM_P (op0)
1077 222685 : && GET_MODE_SIZE (op0_mode.require ()) > UNITS_PER_WORD
1078 214510 : && (!REG_P (op0)
1079 36100 : || !HARD_REGISTER_P (op0)
1080 178389 : || hard_regno_nregs (REGNO (op0), op0_mode.require ()) != 1))
1081 : {
1082 36194 : if (bitnum % BITS_PER_WORD + bitsize > BITS_PER_WORD)
1083 : {
1084 698 : if (!fallback_p)
1085 : return false;
1086 :
1087 71 : store_split_bit_field (op0, op0_mode, bitsize, bitnum,
1088 : bitregion_start, bitregion_end,
1089 : value, value_mode, reverse);
1090 71 : return true;
1091 : }
1092 35421 : rtx new_op0
1093 35421 : = simplify_gen_subreg (word_mode, op0, op0_mode.require (),
1094 35496 : bitnum / BITS_PER_WORD * UNITS_PER_WORD);
1095 35421 : if (!new_op0)
1096 : {
1097 : /* No valid word-mode SUBREG of op0 at this offset. Defer to
1098 : store_split_bit_field, which addresses op0 a word at a time. */
1099 0 : if (!fallback_p)
1100 : return false;
1101 0 : store_split_bit_field (op0, op0_mode, bitsize, bitnum,
1102 : bitregion_start, bitregion_end,
1103 : value, value_mode, reverse);
1104 0 : return true;
1105 : }
1106 35421 : op0 = new_op0;
1107 35421 : op0_mode = word_mode;
1108 35496 : bitnum %= BITS_PER_WORD;
1109 : }
1110 :
1111 : /* From here on we can assume that the field to be stored in fits
1112 : within a word. If the destination is a register, it too fits
1113 : in a word. */
1114 :
1115 177691 : extraction_insn insv;
1116 177691 : if (!MEM_P (op0)
1117 110191 : && !reverse
1118 110174 : && get_best_reg_extraction_insn (&insv, EP_insv,
1119 220348 : GET_MODE_BITSIZE (op0_mode.require ()),
1120 : fieldmode)
1121 287863 : && store_bit_field_using_insv (&insv, op0, op0_mode,
1122 : bitsize, bitnum, value, value_mode))
1123 2120 : return true;
1124 :
1125 : /* If OP0 is a memory, try copying it to a register and seeing if a
1126 : cheap register alternative is available. */
1127 175571 : if (MEM_P (op0) && !reverse)
1128 : {
1129 67098 : if (get_best_mem_extraction_insn (&insv, EP_insv, bitsize, bitnum,
1130 : fieldmode)
1131 67098 : && store_bit_field_using_insv (&insv, op0, op0_mode,
1132 : bitsize, bitnum, value, value_mode))
1133 0 : return true;
1134 :
1135 67098 : rtx_insn *last = get_last_insn ();
1136 :
1137 : /* Try loading part of OP0 into a register, inserting the bitfield
1138 : into that, and then copying the result back to OP0. */
1139 67098 : unsigned HOST_WIDE_INT bitpos;
1140 67098 : rtx xop0 = adjust_bit_field_mem_for_reg (EP_insv, op0, bitsize, bitnum,
1141 : bitregion_start, bitregion_end,
1142 : fieldmode, &bitpos);
1143 67098 : if (xop0)
1144 : {
1145 63261 : rtx tempreg = copy_to_reg (xop0);
1146 63261 : if (store_bit_field_1 (tempreg, bitsize, bitpos,
1147 : bitregion_start, bitregion_end,
1148 : fieldmode, orig_value, reverse, false, false))
1149 : {
1150 0 : emit_move_insn (xop0, tempreg);
1151 0 : return true;
1152 : }
1153 63261 : delete_insns_since (last);
1154 : }
1155 : }
1156 :
1157 175571 : if (!fallback_p)
1158 : return false;
1159 :
1160 112937 : store_fixed_bit_field (op0, op0_mode, bitsize, bitnum, bitregion_start,
1161 : bitregion_end, value, value_mode, reverse);
1162 112937 : return true;
1163 : }
1164 :
1165 : /* Generate code to store value from rtx VALUE
1166 : into a bit-field within structure STR_RTX
1167 : containing BITSIZE bits starting at bit BITNUM.
1168 :
1169 : BITREGION_START is bitpos of the first bitfield in this region.
1170 : BITREGION_END is the bitpos of the ending bitfield in this region.
1171 : These two fields are 0, if the C++ memory model does not apply,
1172 : or we are not interested in keeping track of bitfield regions.
1173 :
1174 : FIELDMODE is the machine-mode of the FIELD_DECL node for this field.
1175 :
1176 : If REVERSE is true, the store is to be done in reverse order.
1177 :
1178 : If UNDEFINED_P is true then STR_RTX is currently undefined. */
1179 :
1180 : void
1181 841818 : store_bit_field (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
1182 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
1183 : machine_mode fieldmode,
1184 : rtx value, bool reverse, bool undefined_p)
1185 : {
1186 : /* Handle -fstrict-volatile-bitfields in the cases where it applies. */
1187 841818 : unsigned HOST_WIDE_INT ibitsize = 0, ibitnum = 0;
1188 841818 : scalar_int_mode int_mode;
1189 841818 : if (bitsize.is_constant (&ibitsize)
1190 841818 : && bitnum.is_constant (&ibitnum)
1191 1589162 : && is_a <scalar_int_mode> (fieldmode, &int_mode)
1192 747348 : && strict_volatile_bitfield_p (str_rtx, ibitsize, ibitnum, int_mode,
1193 : bitregion_start, bitregion_end))
1194 : {
1195 : /* Storing of a full word can be done with a simple store.
1196 : We know here that the field can be accessed with one single
1197 : instruction. For targets that support unaligned memory,
1198 : an unaligned access may be necessary. */
1199 8 : if (ibitsize == GET_MODE_BITSIZE (int_mode))
1200 : {
1201 0 : str_rtx = adjust_bitfield_address (str_rtx, int_mode,
1202 : ibitnum / BITS_PER_UNIT);
1203 0 : if (reverse)
1204 0 : value = flip_storage_order (int_mode, value);
1205 0 : gcc_assert (ibitnum % BITS_PER_UNIT == 0);
1206 0 : emit_move_insn (str_rtx, value);
1207 : }
1208 : else
1209 : {
1210 4 : rtx temp;
1211 :
1212 4 : str_rtx = narrow_bit_field_mem (str_rtx, int_mode, ibitsize,
1213 : ibitnum, &ibitnum);
1214 8 : gcc_assert (ibitnum + ibitsize <= GET_MODE_BITSIZE (int_mode));
1215 4 : temp = copy_to_reg (str_rtx);
1216 4 : if (!store_bit_field_1 (temp, ibitsize, ibitnum, 0, 0,
1217 : int_mode, value, reverse, true, undefined_p))
1218 0 : gcc_unreachable ();
1219 :
1220 4 : emit_move_insn (str_rtx, temp);
1221 : }
1222 :
1223 4 : return;
1224 : }
1225 :
1226 : /* Under the C++0x memory model, we must not touch bits outside the
1227 : bit region. Adjust the address to start at the beginning of the
1228 : bit region. */
1229 841814 : if (MEM_P (str_rtx) && maybe_ne (bitregion_start, 0U))
1230 : {
1231 51185 : scalar_int_mode best_mode;
1232 51185 : machine_mode addr_mode = VOIDmode;
1233 :
1234 51185 : poly_uint64 offset = exact_div (bitregion_start, BITS_PER_UNIT);
1235 51185 : bitnum -= bitregion_start;
1236 51185 : poly_int64 size = bits_to_bytes_round_up (bitnum + bitsize);
1237 51185 : bitregion_end -= bitregion_start;
1238 51185 : bitregion_start = 0;
1239 51185 : if (bitsize.is_constant (&ibitsize)
1240 51185 : && bitnum.is_constant (&ibitnum)
1241 51185 : && get_best_mode (ibitsize, ibitnum,
1242 : bitregion_start, bitregion_end,
1243 51185 : MEM_ALIGN (str_rtx), INT_MAX,
1244 51185 : MEM_VOLATILE_P (str_rtx), &best_mode))
1245 47771 : addr_mode = best_mode;
1246 51185 : str_rtx = adjust_bitfield_address_size (str_rtx, addr_mode,
1247 : offset, size);
1248 : }
1249 :
1250 841814 : if (!store_bit_field_1 (str_rtx, bitsize, bitnum,
1251 : bitregion_start, bitregion_end,
1252 : fieldmode, value, reverse, true, undefined_p))
1253 0 : gcc_unreachable ();
1254 : }
1255 :
1256 : /* Use shifts and boolean operations to store VALUE into a bit field of
1257 : width BITSIZE in OP0, starting at bit BITNUM. If OP0_MODE is defined,
1258 : it is the mode of OP0, otherwise OP0 is a BLKmode MEM. VALUE_MODE is
1259 : the mode of VALUE.
1260 :
1261 : If REVERSE is true, the store is to be done in reverse order. */
1262 :
1263 : static void
1264 130436 : store_fixed_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
1265 : unsigned HOST_WIDE_INT bitsize,
1266 : unsigned HOST_WIDE_INT bitnum,
1267 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
1268 : rtx value, scalar_int_mode value_mode, bool reverse)
1269 : {
1270 : /* There is a case not handled here:
1271 : a structure with a known alignment of just a halfword
1272 : and a field split across two aligned halfwords within the structure.
1273 : Or likewise a structure with a known alignment of just a byte
1274 : and a field split across two bytes.
1275 : Such cases are not supposed to be able to occur. */
1276 :
1277 130436 : scalar_int_mode best_mode;
1278 130436 : if (MEM_P (op0))
1279 : {
1280 84857 : unsigned int max_bitsize = BITS_PER_WORD;
1281 84857 : scalar_int_mode imode;
1282 144884 : if (op0_mode.exists (&imode) && GET_MODE_BITSIZE (imode) < max_bitsize)
1283 91150 : max_bitsize = GET_MODE_BITSIZE (imode);
1284 :
1285 84857 : if (!get_best_mode (bitsize, bitnum, bitregion_start, bitregion_end,
1286 84857 : MEM_ALIGN (op0), max_bitsize, MEM_VOLATILE_P (op0),
1287 : &best_mode))
1288 : {
1289 : /* The only way this should occur is if the field spans word
1290 : boundaries. */
1291 6569 : store_split_bit_field (op0, op0_mode, bitsize, bitnum,
1292 : bitregion_start, bitregion_end,
1293 : value, value_mode, reverse);
1294 6569 : return;
1295 : }
1296 :
1297 78288 : op0 = narrow_bit_field_mem (op0, best_mode, bitsize, bitnum, &bitnum);
1298 : }
1299 : else
1300 45579 : best_mode = op0_mode.require ();
1301 :
1302 123867 : store_fixed_bit_field_1 (op0, best_mode, bitsize, bitnum,
1303 : value, value_mode, reverse);
1304 : }
1305 :
1306 : /* Helper function for store_fixed_bit_field, stores
1307 : the bit field always using MODE, which is the mode of OP0. The other
1308 : arguments are as for store_fixed_bit_field. */
1309 :
1310 : static void
1311 123867 : store_fixed_bit_field_1 (rtx op0, scalar_int_mode mode,
1312 : unsigned HOST_WIDE_INT bitsize,
1313 : unsigned HOST_WIDE_INT bitnum,
1314 : rtx value, scalar_int_mode value_mode, bool reverse)
1315 : {
1316 123867 : rtx temp;
1317 123867 : int all_zero = 0;
1318 123867 : int all_one = 0;
1319 :
1320 : /* Note that bitsize + bitnum can be greater than GET_MODE_BITSIZE (mode)
1321 : for invalid input, such as f5 from gcc.dg/pr48335-2.c. */
1322 :
1323 123867 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
1324 : /* BITNUM is the distance between our msb
1325 : and that of the containing datum.
1326 : Convert it to the distance from the lsb. */
1327 1062 : bitnum = GET_MODE_BITSIZE (mode) - bitsize - bitnum;
1328 :
1329 : /* Now BITNUM is always the distance between our lsb
1330 : and that of OP0. */
1331 :
1332 : /* Shift VALUE left by BITNUM bits. If VALUE is not constant,
1333 : we must first convert its mode to MODE. */
1334 :
1335 123867 : if (CONST_INT_P (value))
1336 : {
1337 73369 : unsigned HOST_WIDE_INT v = UINTVAL (value);
1338 :
1339 73369 : if (bitsize < HOST_BITS_PER_WIDE_INT)
1340 73352 : v &= (HOST_WIDE_INT_1U << bitsize) - 1;
1341 :
1342 73369 : if (v == 0)
1343 : all_zero = 1;
1344 59700 : else if ((bitsize < HOST_BITS_PER_WIDE_INT
1345 59692 : && v == (HOST_WIDE_INT_1U << bitsize) - 1)
1346 51026 : || (bitsize == HOST_BITS_PER_WIDE_INT
1347 51026 : && v == HOST_WIDE_INT_M1U))
1348 8674 : all_one = 1;
1349 :
1350 73369 : value = lshift_value (mode, v, bitnum);
1351 : }
1352 : else
1353 : {
1354 50498 : int must_and = (GET_MODE_BITSIZE (value_mode) != bitsize
1355 78000 : && bitnum + bitsize != GET_MODE_BITSIZE (mode));
1356 :
1357 50498 : if (value_mode != mode)
1358 27913 : value = convert_to_mode (mode, value, 1);
1359 :
1360 50498 : if (must_and)
1361 20698 : value = expand_binop (mode, and_optab, value,
1362 : mask_rtx (mode, 0, bitsize, 0),
1363 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
1364 50498 : if (bitnum > 0)
1365 14320 : value = expand_shift (LSHIFT_EXPR, mode, value,
1366 14320 : bitnum, NULL_RTX, 1);
1367 : }
1368 :
1369 123867 : if (reverse)
1370 531 : value = flip_storage_order (mode, value);
1371 :
1372 : /* A field filling the whole of a MEM has no surrounding bits to preserve,
1373 : so store it directly: the read of a volatile OP0 cannot be removed later
1374 : and would be a spurious access with side effects (PR71048). */
1375 183489 : if (MEM_P (op0) && bitnum == 0 && bitsize == GET_MODE_BITSIZE (mode))
1376 : {
1377 13641 : op0 = copy_rtx (op0);
1378 13641 : emit_move_insn (op0, value);
1379 13641 : return;
1380 : }
1381 :
1382 : /* Now clear the chosen bits in OP0,
1383 : except that if VALUE is -1 we need not bother. */
1384 : /* We keep the intermediates in registers to allow CSE to combine
1385 : consecutive bitfield assignments. */
1386 :
1387 110226 : temp = force_reg (mode, op0);
1388 :
1389 110226 : if (! all_one)
1390 : {
1391 102243 : rtx mask = mask_rtx (mode, bitnum, bitsize, 1);
1392 102243 : if (reverse)
1393 519 : mask = flip_storage_order (mode, mask);
1394 102243 : temp = expand_binop (mode, and_optab, temp, mask,
1395 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
1396 102243 : temp = force_reg (mode, temp);
1397 : }
1398 :
1399 : /* Now logical-or VALUE into OP0, unless it is zero. */
1400 :
1401 110226 : if (! all_zero)
1402 : {
1403 : /* Use aop_optab if the field has been cleared. */
1404 188699 : temp = expand_binop (mode, all_one ? ior_optab : aop_optab,
1405 : temp, value, NULL_RTX, 1, OPTAB_LIB_WIDEN);
1406 98341 : temp = force_reg (mode, temp);
1407 : }
1408 :
1409 110226 : if (op0 != temp)
1410 : {
1411 110226 : op0 = copy_rtx (op0);
1412 110226 : emit_move_insn (op0, temp);
1413 : }
1414 : }
1415 :
1416 : /* Store a bit field that is split across multiple accessible memory objects.
1417 :
1418 : OP0 is the REG, SUBREG or MEM rtx for the first of the objects.
1419 : BITSIZE is the field width; BITPOS the position of its first bit
1420 : (within the word).
1421 : VALUE is the value to store, which has mode VALUE_MODE.
1422 : If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is
1423 : a BLKmode MEM.
1424 :
1425 : If REVERSE is true, the store is to be done in reverse order.
1426 :
1427 : This does not yet handle fields wider than BITS_PER_WORD. */
1428 :
1429 : static void
1430 6640 : store_split_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
1431 : unsigned HOST_WIDE_INT bitsize,
1432 : unsigned HOST_WIDE_INT bitpos,
1433 : poly_uint64 bitregion_start, poly_uint64 bitregion_end,
1434 : rtx value, scalar_int_mode value_mode, bool reverse)
1435 : {
1436 6640 : unsigned int unit, total_bits, bitsdone = 0;
1437 :
1438 : /* Make sure UNIT isn't larger than BITS_PER_WORD, we can only handle that
1439 : much at a time. */
1440 6640 : if (REG_P (op0) || GET_CODE (op0) == SUBREG)
1441 71 : unit = BITS_PER_WORD;
1442 : else
1443 6569 : unit = MIN (MEM_ALIGN (op0), BITS_PER_WORD);
1444 :
1445 : /* If OP0 is a memory with a mode, then UNIT must not be larger than
1446 : OP0's mode as well. Otherwise, store_fixed_bit_field will call us
1447 : again, and we will mutually recurse forever. */
1448 6640 : if (MEM_P (op0) && op0_mode.exists ())
1449 5111 : unit = MIN (unit, GET_MODE_BITSIZE (op0_mode.require ()));
1450 :
1451 : /* If VALUE is a constant other than a CONST_INT, get it into a register in
1452 : WORD_MODE. If we can do this using gen_lowpart_common, do so. Note
1453 : that VALUE might be a floating-point constant. */
1454 6640 : if (CONSTANT_P (value) && !CONST_INT_P (value))
1455 : {
1456 0 : rtx word = gen_lowpart_common (word_mode, value);
1457 :
1458 0 : if (word && (value != word))
1459 : value = word;
1460 : else
1461 0 : value = gen_lowpart_common (word_mode, force_reg (value_mode, value));
1462 0 : value_mode = word_mode;
1463 : }
1464 :
1465 6640 : total_bits = GET_MODE_BITSIZE (value_mode);
1466 :
1467 30943 : while (bitsdone < bitsize)
1468 : {
1469 24303 : unsigned HOST_WIDE_INT thissize;
1470 24303 : unsigned HOST_WIDE_INT thispos;
1471 24303 : unsigned HOST_WIDE_INT offset;
1472 24303 : rtx part;
1473 :
1474 24303 : offset = (bitpos + bitsdone) / unit;
1475 24303 : thispos = (bitpos + bitsdone) % unit;
1476 :
1477 : /* When region of bytes we can touch is restricted, decrease
1478 : UNIT close to the end of the region as needed. If op0 is a REG
1479 : or SUBREG of REG, don't do this, as there can't be data races
1480 : on a register and we can expand shorter code in some cases. */
1481 31107 : if (maybe_ne (bitregion_end, 0U)
1482 24303 : && unit > BITS_PER_UNIT
1483 14219 : && maybe_gt (bitpos + bitsdone - thispos + unit, bitregion_end + 1)
1484 6868 : && !REG_P (op0)
1485 31107 : && (GET_CODE (op0) != SUBREG || !REG_P (SUBREG_REG (op0))))
1486 : {
1487 6804 : unit = unit / 2;
1488 6804 : continue;
1489 : }
1490 :
1491 : /* THISSIZE must not overrun a word boundary. Otherwise,
1492 : store_fixed_bit_field will call us again, and we will mutually
1493 : recurse forever. */
1494 17499 : thissize = MIN (bitsize - bitsdone, BITS_PER_WORD);
1495 17499 : thissize = MIN (thissize, unit - thispos);
1496 :
1497 17499 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
1498 : {
1499 : /* Fetch successively less significant portions. */
1500 214 : if (CONST_INT_P (value))
1501 108 : part = GEN_INT (((unsigned HOST_WIDE_INT) (INTVAL (value))
1502 : >> (bitsize - bitsdone - thissize))
1503 : & ((HOST_WIDE_INT_1 << thissize) - 1));
1504 : /* Likewise, but the source is little-endian. */
1505 106 : else if (reverse)
1506 106 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1507 : thissize,
1508 : bitsize - bitsdone - thissize,
1509 : NULL_RTX, 1, false);
1510 : else
1511 : /* The args are chosen so that the last part includes the
1512 : lsb. Give extract_bit_field the value it needs (with
1513 : endianness compensation) to fetch the piece we want. */
1514 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1515 : thissize,
1516 : total_bits - bitsize + bitsdone,
1517 : NULL_RTX, 1, false);
1518 : }
1519 : else
1520 : {
1521 : /* Fetch successively more significant portions. */
1522 17285 : if (CONST_INT_P (value))
1523 12867 : part = GEN_INT (((unsigned HOST_WIDE_INT) (INTVAL (value))
1524 : >> bitsdone)
1525 : & ((HOST_WIDE_INT_1 << thissize) - 1));
1526 : /* Likewise, but the source is big-endian. */
1527 4418 : else if (reverse)
1528 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1529 : thissize,
1530 : total_bits - bitsdone - thissize,
1531 : NULL_RTX, 1, false);
1532 : else
1533 4418 : part = extract_fixed_bit_field (word_mode, value, value_mode,
1534 : thissize, bitsdone, NULL_RTX,
1535 : 1, false);
1536 : }
1537 :
1538 : /* If OP0 is a register, then handle OFFSET here. */
1539 17499 : rtx op0_piece = op0;
1540 17499 : opt_scalar_int_mode op0_piece_mode = op0_mode;
1541 17499 : if (SUBREG_P (op0) || REG_P (op0))
1542 : {
1543 142 : scalar_int_mode imode;
1544 142 : if (op0_mode.exists (&imode)
1545 142 : && GET_MODE_SIZE (imode) < UNITS_PER_WORD)
1546 : {
1547 0 : if (offset)
1548 0 : op0_piece = const0_rtx;
1549 : }
1550 : else
1551 : {
1552 142 : op0_piece = operand_subword_force (op0,
1553 142 : offset * unit / BITS_PER_WORD,
1554 142 : GET_MODE (op0));
1555 142 : op0_piece_mode = word_mode;
1556 : }
1557 142 : offset &= BITS_PER_WORD / unit - 1;
1558 : }
1559 :
1560 : /* OFFSET is in UNITs, and UNIT is in bits. If WORD is const0_rtx,
1561 : it is just an out-of-bounds access. Ignore it. */
1562 17499 : if (op0_piece != const0_rtx)
1563 17499 : store_fixed_bit_field (op0_piece, op0_piece_mode, thissize,
1564 17499 : offset * unit + thispos, bitregion_start,
1565 : bitregion_end, part, word_mode, reverse);
1566 17499 : bitsdone += thissize;
1567 : }
1568 6640 : }
1569 :
1570 : /* A subroutine of extract_bit_field_1 that converts return value X
1571 : to either MODE or TMODE. MODE, TMODE and UNSIGNEDP are arguments
1572 : to extract_bit_field. */
1573 :
1574 : static rtx
1575 911621 : convert_extracted_bit_field (rtx x, machine_mode mode,
1576 : machine_mode tmode, bool unsignedp)
1577 : {
1578 911621 : if (GET_MODE (x) == tmode || GET_MODE (x) == mode)
1579 : return x;
1580 :
1581 : /* If the x mode is not a scalar integral, first convert to the
1582 : integer mode of that size and then access it as a floating-point
1583 : value via a SUBREG. */
1584 21541 : if (!SCALAR_INT_MODE_P (tmode))
1585 : {
1586 11903 : scalar_int_mode int_mode = int_mode_for_mode (tmode).require ();
1587 11903 : x = convert_to_mode (int_mode, x, unsignedp);
1588 11903 : x = force_reg (int_mode, x);
1589 11903 : return gen_lowpart (tmode, x);
1590 : }
1591 :
1592 9638 : return convert_to_mode (tmode, x, unsignedp);
1593 : }
1594 :
1595 : /* Try to use an ext(z)v pattern to extract a field from OP0.
1596 : Return the extracted value on success, otherwise return null.
1597 : EXTV describes the extraction instruction to use. If OP0_MODE
1598 : is defined, it is the mode of OP0, otherwise OP0 is a BLKmode MEM.
1599 : The other arguments are as for extract_bit_field. */
1600 :
1601 : static rtx
1602 166640 : extract_bit_field_using_extv (const extraction_insn *extv, rtx op0,
1603 : opt_scalar_int_mode op0_mode,
1604 : unsigned HOST_WIDE_INT bitsize,
1605 : unsigned HOST_WIDE_INT bitnum,
1606 : int unsignedp, rtx target,
1607 : machine_mode mode, machine_mode tmode)
1608 : {
1609 166640 : class expand_operand ops[4];
1610 166640 : rtx spec_target = target;
1611 166640 : rtx spec_target_subreg = 0;
1612 166640 : scalar_int_mode ext_mode = extv->field_mode;
1613 166640 : unsigned unit = GET_MODE_BITSIZE (ext_mode);
1614 :
1615 166640 : if (bitsize == 0 || unit < bitsize)
1616 : return NULL_RTX;
1617 :
1618 166640 : if (MEM_P (op0))
1619 : /* Get a reference to the first byte of the field. */
1620 0 : op0 = narrow_bit_field_mem (op0, extv->struct_mode, bitsize, bitnum,
1621 : &bitnum);
1622 : else
1623 : {
1624 : /* Convert from counting within OP0 to counting in EXT_MODE. */
1625 166640 : if (BYTES_BIG_ENDIAN)
1626 : bitnum += unit - GET_MODE_BITSIZE (op0_mode.require ());
1627 :
1628 : /* If op0 is a register, we need it in EXT_MODE to make it
1629 : acceptable to the format of ext(z)v. */
1630 166640 : if (GET_CODE (op0) == SUBREG && op0_mode.require () != ext_mode)
1631 0 : return NULL_RTX;
1632 166640 : if (REG_P (op0) && op0_mode.require () != ext_mode)
1633 50563 : op0 = gen_lowpart_SUBREG (ext_mode, op0);
1634 : }
1635 :
1636 : /* If BITS_BIG_ENDIAN is zero on a BYTES_BIG_ENDIAN machine, we count
1637 : "backwards" from the size of the unit we are extracting from.
1638 : Otherwise, we count bits from the most significant on a
1639 : BYTES/BITS_BIG_ENDIAN machine. */
1640 :
1641 166640 : if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
1642 : bitnum = unit - bitsize - bitnum;
1643 :
1644 166640 : if (target == 0)
1645 46678 : target = spec_target = gen_reg_rtx (tmode);
1646 :
1647 166640 : if (GET_MODE (target) != ext_mode)
1648 : {
1649 99363 : rtx temp;
1650 : /* Don't use LHS paradoxical subreg if explicit truncation is needed
1651 : between the mode of the extraction (word_mode) and the target
1652 : mode. Instead, create a temporary and use convert_move to set
1653 : the target. */
1654 99363 : if (REG_P (target)
1655 98100 : && TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (target), ext_mode)
1656 197463 : && (temp = gen_lowpart_if_possible (ext_mode, target)))
1657 : {
1658 97588 : target = temp;
1659 97588 : if (partial_subreg_p (GET_MODE (spec_target), ext_mode))
1660 96727 : spec_target_subreg = target;
1661 : }
1662 : else
1663 1775 : target = gen_reg_rtx (ext_mode);
1664 : }
1665 :
1666 166640 : create_output_operand (&ops[0], target, ext_mode);
1667 166640 : create_fixed_operand (&ops[1], op0);
1668 166640 : create_integer_operand (&ops[2], bitsize);
1669 166640 : create_integer_operand (&ops[3], bitnum);
1670 166640 : if (maybe_expand_insn (extv->icode, 4, ops))
1671 : {
1672 1675 : target = ops[0].value;
1673 1675 : if (target == spec_target)
1674 : return target;
1675 1675 : if (target == spec_target_subreg)
1676 : return spec_target;
1677 66 : return convert_extracted_bit_field (target, mode, tmode, unsignedp);
1678 : }
1679 : return NULL_RTX;
1680 : }
1681 :
1682 : /* See whether it would be valid to extract the part of OP0 with
1683 : mode OP0_MODE described by BITNUM and BITSIZE into a value of
1684 : mode MODE using a subreg operation.
1685 : Return the subreg if so, otherwise return null. */
1686 :
1687 : static rtx
1688 865042 : extract_bit_field_as_subreg (machine_mode mode, rtx op0,
1689 : machine_mode op0_mode,
1690 : poly_uint64 bitsize, poly_uint64 bitnum)
1691 : {
1692 865042 : poly_uint64 bytenum;
1693 865042 : if (multiple_p (bitnum, BITS_PER_UNIT, &bytenum)
1694 826271 : && known_eq (bitsize, GET_MODE_BITSIZE (mode))
1695 865042 : && lowpart_bit_field_p (bitnum, bitsize, op0_mode)
1696 1691313 : && TRULY_NOOP_TRUNCATION_MODES_P (mode, op0_mode))
1697 690192 : return force_subreg (mode, op0, op0_mode, bytenum);
1698 : return NULL_RTX;
1699 : }
1700 :
1701 : /* Same as extract_vector_from_vector but try with a permutation. */
1702 :
1703 : static rtx
1704 3844 : extract_vector_from_vector_perm (rtx vec, rtx target, machine_mode mode,
1705 : machine_mode subvec_mode, poly_uint64 bitsize,
1706 : poly_uint64 bitnum, rtx *alt_rtl)
1707 : {
1708 3844 : machine_mode vec_mode = GET_MODE (vec);
1709 3844 : poly_uint64 extract_idx;
1710 1349 : if (!VECTOR_MODE_P (vec_mode)
1711 3844 : || MEM_P (vec)
1712 3844 : || !VECTOR_MODE_P (subvec_mode)
1713 3844 : || maybe_ne (bitsize, GET_MODE_PRECISION (subvec_mode))
1714 11529 : || !multiple_p (bitnum, GET_MODE_UNIT_BITSIZE (subvec_mode),
1715 : &extract_idx))
1716 : return NULL_RTX;
1717 :
1718 : /* We don't handle single-element vector modes here, they should
1719 : go through vector-element vec_extract. */
1720 3841 : if (known_le (GET_MODE_NUNITS (subvec_mode), 1)
1721 7669 : || known_le (GET_MODE_NUNITS (vec_mode), 1))
1722 : return NULL_RTX;
1723 :
1724 3812 : machine_mode punned_vec_mode;
1725 3812 : if (!vec_extract_perm_p (vec_mode, subvec_mode, extract_idx).exists
1726 3812 : (&punned_vec_mode))
1727 3812 : return NULL_RTX;
1728 :
1729 0 : if (punned_vec_mode != vec_mode)
1730 0 : vec = gen_lowpart (punned_vec_mode, vec);
1731 :
1732 0 : poly_uint64 out_nunits = GET_MODE_NUNITS (subvec_mode);
1733 :
1734 0 : vec_perm_builder sel (out_nunits, 1, 3);
1735 0 : for (unsigned int i = 0; i < 3; ++i)
1736 0 : sel.quick_push (extract_idx + i);
1737 :
1738 0 : rtx tmp = subvec_mode == mode ? target : NULL_RTX;
1739 0 : rtx res = expand_vec_perm_const (subvec_mode, vec, vec, sel, BLKmode, tmp);
1740 0 : if (!res)
1741 : return NULL_RTX;
1742 :
1743 0 : if (res == target && alt_rtl)
1744 0 : *alt_rtl = target;
1745 :
1746 0 : if (subvec_mode != mode)
1747 0 : res = gen_lowpart (mode, res);
1748 :
1749 : return res;
1750 0 : }
1751 :
1752 : /* Try to extract a subvector of mode SUBVEC_MODE (which has BITSIZE bits)
1753 : from vector VEC, starting at bit BITNUM, then extract the MODE lowpart of
1754 : the subvector. If the vec_extract-based attempt failed, build a vector
1755 : permute to extract the part and hand it to the target.
1756 :
1757 : Return the result on success or NULL_RTX on failure. */
1758 :
1759 : static rtx
1760 1120127 : extract_vector_from_vector (rtx vec, rtx target, machine_mode mode,
1761 : machine_mode subvec_mode, poly_uint64 bitsize,
1762 : poly_uint64 bitnum, rtx *alt_rtl)
1763 : {
1764 1120127 : machine_mode vec_mode = GET_MODE (vec);
1765 1069784 : if (!VECTOR_MODE_P (vec_mode)
1766 72045 : || MEM_P (vec)
1767 70603 : || !VECTOR_MODE_P (subvec_mode)
1768 12875 : || maybe_ne (bitsize, GET_MODE_PRECISION (subvec_mode))
1769 1158752 : || known_le (GET_MODE_SIZE (GET_MODE (vec)),
1770 : GET_MODE_SIZE (subvec_mode)))
1771 : return NULL_RTX;
1772 :
1773 : /* Try to find a fitting mode for extracting. */
1774 38625 : if (GET_MODE_INNER (vec_mode) != GET_MODE_INNER (subvec_mode))
1775 : {
1776 214 : scalar_mode inner_mode = GET_MODE_INNER (subvec_mode);
1777 214 : poly_uint64 nunits;
1778 428 : if (!multiple_p (GET_MODE_BITSIZE (vec_mode),
1779 214 : GET_MODE_UNIT_BITSIZE (subvec_mode), &nunits)
1780 412 : || !related_vector_mode (subvec_mode, inner_mode,
1781 16 : nunits).exists (&vec_mode)
1782 198 : || maybe_ne (GET_MODE_SIZE (vec_mode),
1783 396 : GET_MODE_SIZE (GET_MODE (vec))))
1784 16 : return NULL_RTX;
1785 : }
1786 :
1787 12859 : poly_uint64 pos;
1788 12859 : enum insn_code icode = convert_optab_handler (vec_extract_optab, vec_mode,
1789 : subvec_mode);
1790 12859 : if (icode != CODE_FOR_nothing
1791 21883 : && multiple_p (bitnum, GET_MODE_BITSIZE (subvec_mode), &pos))
1792 : {
1793 9015 : if (vec_mode != GET_MODE (vec))
1794 33 : vec = gen_lowpart (vec_mode, vec);
1795 :
1796 9015 : class expand_operand ops[3];
1797 9015 : create_output_operand (&ops[0], target, subvec_mode);
1798 9015 : ops[0].target = 1;
1799 9015 : create_input_operand (&ops[1], vec, GET_MODE (vec));
1800 9015 : create_integer_operand (&ops[2], pos);
1801 9015 : if (maybe_expand_insn (icode, 3, ops))
1802 : {
1803 9015 : if (alt_rtl && ops[0].target)
1804 207 : *alt_rtl = target;
1805 9015 : target = ops[0].value;
1806 9015 : if (GET_MODE (target) != mode)
1807 9015 : return gen_lowpart (subvec_mode, target);
1808 : return target;
1809 : }
1810 : }
1811 :
1812 3844 : rtx res = extract_vector_from_vector_perm (vec, target, mode, subvec_mode,
1813 : bitsize, bitnum, alt_rtl);
1814 3844 : if (res)
1815 : return res;
1816 :
1817 : return NULL_RTX;
1818 : }
1819 :
1820 : /* Extract an element of OP0 of size BITSIZE, starting at bit position BITNUM.
1821 : Pun the result with mode TMODE if necessary. Return the extracted element
1822 : if successful or NULL_RTX otherwise. */
1823 :
1824 : static rtx
1825 1111112 : extract_element_from_vector (rtx op0, rtx target, machine_mode mode,
1826 : machine_mode tmode, poly_uint64 bitsize,
1827 : poly_uint64 bitnum, rtx *alt_rtl)
1828 : {
1829 : /* See if we can get a better vector mode before extracting. */
1830 1111112 : machine_mode outermode = GET_MODE (op0);
1831 1062369 : if (VECTOR_MODE_P (outermode)
1832 63030 : && !MEM_P (op0)
1833 1172700 : && GET_MODE_INNER (outermode) != tmode)
1834 : {
1835 9609 : machine_mode new_mode;
1836 :
1837 9609 : if (GET_MODE_CLASS (tmode) == MODE_FLOAT)
1838 : new_mode = MIN_MODE_VECTOR_FLOAT;
1839 9052 : else if (GET_MODE_CLASS (tmode) == MODE_FRACT)
1840 : new_mode = MIN_MODE_VECTOR_FRACT;
1841 9052 : else if (GET_MODE_CLASS (tmode) == MODE_UFRACT)
1842 : new_mode = MIN_MODE_VECTOR_UFRACT;
1843 9052 : else if (GET_MODE_CLASS (tmode) == MODE_ACCUM)
1844 : new_mode = MIN_MODE_VECTOR_ACCUM;
1845 9052 : else if (GET_MODE_CLASS (tmode) == MODE_UACCUM)
1846 : new_mode = MIN_MODE_VECTOR_UACCUM;
1847 : else
1848 9609 : new_mode = MIN_MODE_VECTOR_INT;
1849 :
1850 152327 : FOR_EACH_MODE_FROM (new_mode, new_mode)
1851 303948 : if (known_eq (GET_MODE_SIZE (new_mode), GET_MODE_SIZE (outermode))
1852 63324 : && known_eq (GET_MODE_UNIT_SIZE (new_mode), GET_MODE_SIZE (tmode))
1853 19218 : && known_eq (bitsize, GET_MODE_UNIT_PRECISION (new_mode))
1854 28795 : && multiple_p (bitnum, GET_MODE_UNIT_PRECISION (new_mode))
1855 9577 : && targetm.vector_mode_supported_p (new_mode)
1856 161533 : && targetm.modes_tieable_p (outermode, new_mode))
1857 : break;
1858 9609 : if (new_mode != VOIDmode)
1859 9256 : op0 = gen_lowpart (new_mode, op0);
1860 : }
1861 :
1862 1111112 : outermode = GET_MODE (op0);
1863 1111112 : if (VECTOR_MODE_P (outermode) && !MEM_P (op0))
1864 : {
1865 61588 : scalar_mode innermode = GET_MODE_INNER (outermode);
1866 :
1867 61588 : enum insn_code icode
1868 61588 : = convert_optab_handler (vec_extract_optab, outermode, innermode);
1869 :
1870 61588 : poly_uint64 pos;
1871 61588 : if (icode != CODE_FOR_nothing
1872 59924 : && known_eq (bitsize, GET_MODE_PRECISION (innermode))
1873 121230 : && multiple_p (bitnum, GET_MODE_PRECISION (innermode), &pos))
1874 : {
1875 59639 : class expand_operand ops[3];
1876 :
1877 59639 : create_output_operand (&ops[0], target,
1878 59639 : insn_data[icode].operand[0].mode);
1879 59639 : ops[0].target = 1;
1880 59639 : create_input_operand (&ops[1], op0, outermode);
1881 59639 : create_integer_operand (&ops[2], pos);
1882 59639 : if (maybe_expand_insn (icode, 3, ops))
1883 : {
1884 59639 : if (alt_rtl && ops[0].target)
1885 14767 : *alt_rtl = target;
1886 59639 : target = ops[0].value;
1887 59639 : if (GET_MODE (target) != mode)
1888 59639 : return gen_lowpart (tmode, target);
1889 : return target;
1890 : }
1891 : }
1892 : /* Using subregs is useful if we're extracting one register vector
1893 : from a multi-register vector. extract_bit_field_as_subreg checks
1894 : for valid bitsize and bitnum, so we don't need to do that here. */
1895 1949 : if (VECTOR_MODE_P (mode))
1896 : {
1897 45 : rtx sub = extract_bit_field_as_subreg (mode, op0, outermode,
1898 : bitsize, bitnum);
1899 45 : if (sub)
1900 : return sub;
1901 : }
1902 : }
1903 :
1904 : return NULL_RTX;
1905 : }
1906 :
1907 : /* Check whether:
1908 :
1909 : (a) OP0 is a vector that spans multiple registers (which might
1910 : mean that it is a tuple mode) and
1911 :
1912 : (b) a sequence of BITSIZE bits, starting at bit *BITNUM, fits within
1913 : a single vector register.
1914 :
1915 : Return the vector register if so and update *BITNUM so that it
1916 : is relative to the returned register. Return NULL_RTX otherwise. */
1917 :
1918 : static rtx
1919 1051452 : extract_vector_from_multireg (rtx op0, machine_mode tmode,
1920 : poly_uint64 bitsize, poly_uint64 *bitnum)
1921 : {
1922 1051452 : machine_mode outermode = GET_MODE (op0);
1923 1051452 : poly_uint64 outer_natural_sz = REGMODE_NATURAL_SIZE (outermode);
1924 1051452 : scalar_mode elmode = GET_MODE_INNER (outermode);
1925 1051452 : HOST_WIDE_INT vec_idx;
1926 1051452 : poly_uint64 off;
1927 1051452 : poly_uint64 nelts;
1928 1048211 : if (VECTOR_MODE_P (outermode)
1929 3370 : && !MEM_P (op0)
1930 1928 : && known_gt (GET_MODE_SIZE (outermode), outer_natural_sz)
1931 1023 : && known_eq (bitsize, GET_MODE_PRECISION (tmode))
1932 1023 : && can_div_trunc_p (*bitnum, outer_natural_sz * BITS_PER_UNIT,
1933 : &vec_idx, &off)
1934 1023 : && known_le (off + bitsize, outer_natural_sz * BITS_PER_UNIT)
1935 1052453 : && multiple_p (outer_natural_sz, GET_MODE_SIZE (elmode), &nelts))
1936 : {
1937 : /* Get a single-vector mode from the outer multi-vector mode.
1938 : If it exists and we can pun the outer mode with a subreg,
1939 : do so and return it. */
1940 1978 : multiple_p (outer_natural_sz, GET_MODE_SIZE (elmode), &nelts);
1941 989 : opt_machine_mode vmode
1942 989 : = related_vector_mode (outermode, GET_MODE_INNER (outermode), nelts);
1943 :
1944 989 : machine_mode vmode_onevec;
1945 989 : rtx subreg;
1946 989 : if (vmode.exists (&vmode_onevec)
1947 989 : && (subreg = simplify_gen_subreg
1948 989 : (vmode_onevec, op0, outermode,
1949 1978 : vec_idx * GET_MODE_SIZE (vmode_onevec))))
1950 : {
1951 989 : *bitnum = off;
1952 989 : return subreg;
1953 : }
1954 : }
1955 :
1956 : return NULL_RTX;
1957 : }
1958 :
1959 : /* A subroutine of extract_bit_field, with the same arguments.
1960 : If UNSIGNEDP is -1, the result need not be sign or zero extended.
1961 : If FALLBACK_P is true, fall back to extract_fixed_bit_field
1962 : if we can find no other means of implementing the operation.
1963 : if FALLBACK_P is false, return NULL instead. */
1964 :
1965 : static rtx
1966 1139771 : extract_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
1967 : int unsignedp, rtx target, machine_mode mode,
1968 : machine_mode tmode, bool reverse, bool fallback_p,
1969 : rtx *alt_rtl)
1970 : {
1971 1139771 : rtx op0 = str_rtx;
1972 1139771 : machine_mode mode1;
1973 :
1974 1139771 : if (tmode == VOIDmode)
1975 0 : tmode = mode;
1976 :
1977 1150515 : while (GET_CODE (op0) == SUBREG)
1978 : {
1979 10744 : bitnum += SUBREG_BYTE (op0) * BITS_PER_UNIT;
1980 10744 : op0 = SUBREG_REG (op0);
1981 : }
1982 :
1983 : /* If we have an out-of-bounds access to a register, just return an
1984 : uninitialized register of the required mode. This can occur if the
1985 : source code contains an out-of-bounds access to a small array. */
1986 2094117 : if (REG_P (op0) && known_ge (bitnum, GET_MODE_BITSIZE (GET_MODE (op0))))
1987 0 : return gen_reg_rtx (tmode);
1988 :
1989 1139771 : if (REG_P (op0)
1990 954346 : && mode == GET_MODE (op0)
1991 152277 : && known_eq (bitnum, 0U)
1992 1382061 : && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0))))
1993 : {
1994 20633 : if (reverse)
1995 0 : op0 = flip_storage_order (mode, op0);
1996 : /* We're trying to extract a full register from itself. */
1997 : return op0;
1998 : }
1999 :
2000 : /* First try to check for vector from vector extractions. */
2001 1119138 : rtx res = extract_vector_from_vector (op0, target, mode, tmode, bitsize,
2002 : bitnum, alt_rtl);
2003 1119138 : if (res)
2004 : return res;
2005 :
2006 : /* Use vec_extract patterns for extracting parts of vectors whenever
2007 : available. If that fails, see whether the current modes and bitregion
2008 : give a natural subreg. */
2009 1110123 : res = extract_element_from_vector (op0, target, mode, tmode, bitsize, bitnum,
2010 : alt_rtl);
2011 1110123 : if (res)
2012 : return res;
2013 :
2014 : /* If this mode occupies multiple registers and we're extracting
2015 : a single one, we can usually get to that vector directly
2016 : without actual extraction.
2017 : This helps targets see through multi-vector modes. */
2018 1051452 : poly_uint64 bitnum_new = bitnum;
2019 1051452 : res = extract_vector_from_multireg (op0, tmode, bitsize, &bitnum_new);
2020 1051452 : if (res)
2021 : {
2022 989 : rtx vec = extract_vector_from_vector (res, target, mode, tmode, bitsize,
2023 : bitnum_new, alt_rtl);
2024 989 : if (vec)
2025 : return vec;
2026 :
2027 989 : rtx el = extract_element_from_vector (res, target, mode, tmode, bitsize,
2028 : bitnum_new, alt_rtl);
2029 989 : if (el)
2030 : return el;
2031 :
2032 4 : op0 = res;
2033 4 : bitnum = bitnum_new;
2034 : }
2035 :
2036 : /* Make sure we are playing with integral modes. Pun with subregs
2037 : if we aren't. */
2038 1050467 : opt_scalar_int_mode op0_mode = int_mode_for_mode (GET_MODE (op0));
2039 1050467 : scalar_int_mode imode;
2040 1050467 : bool need_stack_p = false;
2041 1050467 : if (!op0_mode.exists (&imode) || imode != GET_MODE (op0))
2042 : {
2043 197700 : if (MEM_P (op0))
2044 159929 : op0 = adjust_bitfield_address_size (op0, op0_mode.else_blk (),
2045 : 0, MEM_SIZE (op0));
2046 37783 : else if (op0_mode.exists (&imode)
2047 38719 : && known_ge
2048 : ((unsigned) UNITS_PER_WORD,
2049 : (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
2050 : {
2051 37752 : op0 = gen_lowpart (imode, op0);
2052 :
2053 : /* If we got a SUBREG, force it into a register since we
2054 : aren't going to be able to do another SUBREG on it. */
2055 37752 : if (GET_CODE (op0) == SUBREG)
2056 37473 : op0 = force_reg (imode, op0);
2057 : }
2058 : else
2059 : need_stack_p = true;
2060 : }
2061 852767 : else if (!MEM_P (op0)
2062 1759871 : && maybe_lt ((unsigned) UNITS_PER_WORD,
2063 827259 : (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
2064 : need_stack_p = true;
2065 :
2066 197390 : if (need_stack_p)
2067 : {
2068 62 : poly_int64 size = GET_MODE_SIZE (GET_MODE (op0));
2069 31 : rtx mem = assign_stack_temp (GET_MODE (op0), size);
2070 31 : emit_move_insn (mem, op0);
2071 31 : op0 = adjust_bitfield_address_size (mem, BLKmode, 0, size);
2072 : }
2073 :
2074 : /* ??? We currently assume TARGET is at least as big as BITSIZE.
2075 : If that's wrong, the solution is to test for it and set TARGET to 0
2076 : if needed. */
2077 :
2078 : /* Get the mode of the field to use for atomic access or subreg
2079 : conversion. */
2080 1050467 : if (!SCALAR_INT_MODE_P (tmode)
2081 1050467 : || !mode_for_size (bitsize, GET_MODE_CLASS (tmode), 0).exists (&mode1))
2082 : mode1 = mode;
2083 1050467 : gcc_assert (mode1 != BLKmode);
2084 :
2085 : /* Extraction of a full MODE1 value can be done with a subreg as long
2086 : as the least significant bit of the value is the least significant
2087 : bit of either OP0 or a word of OP0. */
2088 1050467 : if (!MEM_P (op0) && !reverse && op0_mode.exists (&imode))
2089 : {
2090 864997 : rtx sub = extract_bit_field_as_subreg (mode1, op0, imode,
2091 : bitsize, bitnum);
2092 864997 : if (sub)
2093 683825 : return convert_extracted_bit_field (sub, mode, tmode, unsignedp);
2094 : }
2095 :
2096 : /* Extraction of a full MODE1 value can be done with a load as long as
2097 : the field is on a byte boundary and is sufficiently aligned. */
2098 366642 : poly_uint64 bytenum;
2099 366642 : if (simple_mem_bitfield_p (op0, bitsize, bitnum, mode1, &bytenum))
2100 : {
2101 45358 : op0 = adjust_bitfield_address (op0, mode1, bytenum);
2102 45358 : if (reverse)
2103 55 : op0 = flip_storage_order (mode1, op0);
2104 45358 : return convert_extracted_bit_field (op0, mode, tmode, unsignedp);
2105 : }
2106 :
2107 : /* If we have a memory source and a non-constant bit offset, restrict
2108 : the memory to the referenced bytes. This is a worst-case fallback
2109 : but is useful for things like vector booleans. */
2110 321284 : if (MEM_P (op0) && !bitnum.is_constant ())
2111 : {
2112 : bytenum = bits_to_bytes_round_down (bitnum);
2113 : bitnum = num_trailing_bits (bitnum);
2114 : poly_uint64 bytesize = bits_to_bytes_round_up (bitnum + bitsize);
2115 : op0 = adjust_bitfield_address_size (op0, BLKmode, bytenum, bytesize);
2116 : op0_mode = opt_scalar_int_mode ();
2117 : }
2118 :
2119 : /* It's possible we'll need to handle other cases here for
2120 : polynomial bitnum and bitsize. */
2121 :
2122 : /* From here on we need to be looking at a fixed-size insertion. */
2123 321284 : return extract_integral_bit_field (op0, op0_mode, bitsize.to_constant (),
2124 : bitnum.to_constant (), unsignedp,
2125 321284 : target, mode, tmode, reverse, fallback_p);
2126 : }
2127 :
2128 : /* Subroutine of extract_bit_field_1, with the same arguments, except
2129 : that BITSIZE and BITNUM are constant. Handle cases specific to
2130 : integral modes. If OP0_MODE is defined, it is the mode of OP0,
2131 : otherwise OP0 is a BLKmode MEM. */
2132 :
2133 : static rtx
2134 321284 : extract_integral_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
2135 : unsigned HOST_WIDE_INT bitsize,
2136 : unsigned HOST_WIDE_INT bitnum, int unsignedp,
2137 : rtx target, machine_mode mode, machine_mode tmode,
2138 : bool reverse, bool fallback_p)
2139 : {
2140 : /* Handle fields bigger than a word. */
2141 :
2142 324553 : if (bitsize > BITS_PER_WORD)
2143 : {
2144 : /* Here we transfer the words of the field
2145 : in the order least significant first.
2146 : This is because the most significant word is the one which may
2147 : be less than full. */
2148 :
2149 1519 : const bool backwards = WORDS_BIG_ENDIAN;
2150 1519 : unsigned int nwords = (bitsize + (BITS_PER_WORD - 1)) / BITS_PER_WORD;
2151 1519 : unsigned int i;
2152 1519 : rtx_insn *last;
2153 :
2154 1519 : if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
2155 1495 : target = gen_reg_rtx (mode);
2156 :
2157 : /* In case we're about to clobber a base register or something
2158 : (see gcc.c-torture/execute/20040625-1.c). */
2159 1519 : if (reg_mentioned_p (target, op0))
2160 0 : target = gen_reg_rtx (mode);
2161 :
2162 : /* Indicate for flow that the entire target reg is being set. */
2163 1519 : emit_clobber (target);
2164 :
2165 : /* The mode must be fixed-size, since extract_bit_field_1 handles
2166 : extractions from variable-sized objects before calling this
2167 : function. */
2168 1519 : unsigned int target_size
2169 1519 : = GET_MODE_SIZE (GET_MODE (target)).to_constant ();
2170 1519 : last = get_last_insn ();
2171 4557 : for (i = 0; i < nwords; i++)
2172 : {
2173 : /* If I is 0, use the low-order word in both field and target;
2174 : if I is 1, use the next to lowest word; and so on. */
2175 : /* Word number in TARGET to use. */
2176 3038 : unsigned int wordnum
2177 : = (backwards ? target_size / UNITS_PER_WORD - i - 1 : i);
2178 : /* Offset from start of field in OP0. */
2179 6076 : unsigned int bit_offset = (backwards ^ reverse
2180 3038 : ? MAX ((int) bitsize - ((int) i + 1)
2181 : * BITS_PER_WORD,
2182 : 0)
2183 3106 : : (int) i * BITS_PER_WORD);
2184 3038 : rtx target_part = operand_subword (target, wordnum, 1, VOIDmode);
2185 3038 : rtx result_part
2186 3214 : = extract_bit_field_1 (op0, MIN (BITS_PER_WORD,
2187 : bitsize - i * BITS_PER_WORD),
2188 3038 : bitnum + bit_offset,
2189 : (unsignedp ? 1 : -1), target_part,
2190 : mode, word_mode, reverse, fallback_p, NULL);
2191 :
2192 3038 : if (!result_part)
2193 : {
2194 0 : delete_insns_since (last);
2195 0 : return NULL;
2196 : }
2197 :
2198 3038 : if (!target_part)
2199 0 : store_bit_field (target, BITS_PER_WORD, i * BITS_PER_WORD,
2200 : 0, 0, word_mode, result_part, false, i == 0);
2201 3038 : else if (result_part != target_part)
2202 2922 : emit_move_insn (target_part, result_part);
2203 : }
2204 :
2205 1519 : if (unsignedp)
2206 : {
2207 : /* Unless we've filled TARGET, the upper regs in a multi-reg value
2208 : need to be zero'd out. */
2209 1533 : if (target_size > nwords * UNITS_PER_WORD)
2210 : {
2211 0 : unsigned int i, total_words;
2212 :
2213 0 : total_words = target_size / UNITS_PER_WORD;
2214 0 : for (i = nwords; i < total_words; i++)
2215 0 : emit_move_insn
2216 0 : (operand_subword (target,
2217 0 : backwards ? total_words - i - 1 : i,
2218 : 1, VOIDmode),
2219 : const0_rtx);
2220 : }
2221 : return target;
2222 : }
2223 :
2224 : /* Signed bit field: sign-extend with two arithmetic shifts. */
2225 40 : target = expand_shift (LSHIFT_EXPR, mode, target,
2226 20 : GET_MODE_BITSIZE (mode) - bitsize, NULL_RTX, 0);
2227 40 : return expand_shift (RSHIFT_EXPR, mode, target,
2228 20 : GET_MODE_BITSIZE (mode) - bitsize, NULL_RTX, 0);
2229 : }
2230 :
2231 : /* If OP0 is a multi-word register, narrow it to the affected word.
2232 : If the region spans two words, defer to extract_split_bit_field. */
2233 503481 : if (!MEM_P (op0) && GET_MODE_SIZE (op0_mode.require ()) > UNITS_PER_WORD)
2234 : {
2235 3086 : if (bitnum % BITS_PER_WORD + bitsize > BITS_PER_WORD)
2236 : {
2237 1142 : if (!fallback_p)
2238 : return NULL_RTX;
2239 60 : target = extract_split_bit_field (op0, op0_mode, bitsize, bitnum,
2240 : unsignedp, reverse);
2241 60 : return convert_extracted_bit_field (target, mode, tmode, unsignedp);
2242 : }
2243 : /* If OP0 is a hard register, copy it to a pseudo before calling
2244 : force_subreg. */
2245 1944 : if (REG_P (op0) && HARD_REGISTER_P (op0))
2246 1 : op0 = copy_to_reg (op0);
2247 1944 : op0 = force_subreg (word_mode, op0, op0_mode.require (),
2248 2408 : bitnum / BITS_PER_WORD * UNITS_PER_WORD);
2249 1944 : op0_mode = word_mode;
2250 2176 : bitnum %= BITS_PER_WORD;
2251 : }
2252 :
2253 : /* From here on we know the desired field is smaller than a word.
2254 : If OP0 is a register, it too fits within a word. */
2255 318623 : enum extraction_pattern pattern = unsignedp ? EP_extzv : EP_extv;
2256 318623 : extraction_insn extv;
2257 318623 : if (!MEM_P (op0)
2258 180035 : && !reverse
2259 : /* ??? We could limit the structure size to the part of OP0 that
2260 : contains the field, with appropriate checks for endianness
2261 : and TARGET_TRULY_NOOP_TRUNCATION. */
2262 498644 : && get_best_reg_extraction_insn (&extv, pattern,
2263 512025 : GET_MODE_BITSIZE (op0_mode.require ()),
2264 : tmode))
2265 : {
2266 166640 : rtx result = extract_bit_field_using_extv (&extv, op0, op0_mode,
2267 : bitsize, bitnum,
2268 : unsignedp, target, mode,
2269 : tmode);
2270 166640 : if (result)
2271 : return result;
2272 : }
2273 :
2274 : /* If OP0 is a memory, try copying it to a register and seeing if a
2275 : cheap register alternative is available. */
2276 316948 : if (MEM_P (op0) & !reverse)
2277 : {
2278 138365 : if (get_best_mem_extraction_insn (&extv, pattern, bitsize, bitnum,
2279 : tmode))
2280 : {
2281 0 : rtx result = extract_bit_field_using_extv (&extv, op0, op0_mode,
2282 : bitsize, bitnum,
2283 : unsignedp, target, mode,
2284 : tmode);
2285 0 : if (result)
2286 0 : return result;
2287 : }
2288 :
2289 138365 : rtx_insn *last = get_last_insn ();
2290 :
2291 : /* Try loading part of OP0 into a register and extracting the
2292 : bitfield from that. */
2293 138365 : unsigned HOST_WIDE_INT bitpos;
2294 138365 : rtx xop0 = adjust_bit_field_mem_for_reg (pattern, op0, bitsize, bitnum,
2295 : 0, 0, tmode, &bitpos);
2296 138365 : if (xop0)
2297 : {
2298 135718 : xop0 = copy_to_reg (xop0);
2299 135718 : rtx result = extract_bit_field_1 (xop0, bitsize, bitpos,
2300 : unsignedp, target,
2301 : mode, tmode, reverse, false, NULL);
2302 135718 : if (result)
2303 : return result;
2304 135718 : delete_insns_since (last);
2305 : }
2306 : }
2307 :
2308 316948 : if (!fallback_p)
2309 : return NULL;
2310 :
2311 : /* Find a correspondingly-sized integer field, so we can apply
2312 : shifts and masks to it. */
2313 182312 : scalar_int_mode int_mode;
2314 182312 : if (!int_mode_for_mode (tmode).exists (&int_mode))
2315 : /* If this fails, we should probably push op0 out to memory and then
2316 : do a load. */
2317 0 : int_mode = int_mode_for_mode (mode).require ();
2318 :
2319 182312 : target = extract_fixed_bit_field (int_mode, op0, op0_mode, bitsize,
2320 : bitnum, target, unsignedp, reverse);
2321 :
2322 : /* Complex values must be reversed piecewise, so we need to undo the global
2323 : reversal, convert to the complex mode and reverse again. */
2324 182312 : if (reverse && COMPLEX_MODE_P (tmode))
2325 : {
2326 0 : target = flip_storage_order (int_mode, target);
2327 0 : target = convert_extracted_bit_field (target, mode, tmode, unsignedp);
2328 0 : target = flip_storage_order (tmode, target);
2329 : }
2330 : else
2331 182312 : target = convert_extracted_bit_field (target, mode, tmode, unsignedp);
2332 :
2333 : return target;
2334 : }
2335 :
2336 : /* Generate code to extract a byte-field from STR_RTX
2337 : containing BITSIZE bits, starting at BITNUM,
2338 : and put it in TARGET if possible (if TARGET is nonzero).
2339 : Regardless of TARGET, we return the rtx for where the value is placed.
2340 :
2341 : STR_RTX is the structure containing the byte (a REG or MEM).
2342 : UNSIGNEDP is nonzero if this is an unsigned bit field.
2343 : MODE is the natural mode of the field value once extracted.
2344 : TMODE is the mode the caller would like the value to have;
2345 : but the value may be returned with type MODE instead.
2346 :
2347 : If REVERSE is true, the extraction is to be done in reverse order.
2348 :
2349 : If a TARGET is specified and we can store in it at no extra cost,
2350 : we do so, and return TARGET.
2351 : Otherwise, we return a REG of mode TMODE or MODE, with TMODE preferred
2352 : if they are equally easy.
2353 :
2354 : If the result can be stored at TARGET, and ALT_RTL is non-NULL,
2355 : then *ALT_RTL is set to TARGET (before legitimziation). */
2356 :
2357 : rtx
2358 1001015 : extract_bit_field (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
2359 : int unsignedp, rtx target, machine_mode mode,
2360 : machine_mode tmode, bool reverse, rtx *alt_rtl)
2361 : {
2362 1001015 : machine_mode mode1;
2363 :
2364 : /* Handle -fstrict-volatile-bitfields in the cases where it applies. */
2365 2002030 : if (maybe_ne (GET_MODE_BITSIZE (GET_MODE (str_rtx)), 0))
2366 : mode1 = GET_MODE (str_rtx);
2367 260927 : else if (target && maybe_ne (GET_MODE_BITSIZE (GET_MODE (target)), 0))
2368 : mode1 = GET_MODE (target);
2369 : else
2370 : mode1 = tmode;
2371 :
2372 1001015 : unsigned HOST_WIDE_INT ibitsize, ibitnum;
2373 1001015 : scalar_int_mode int_mode;
2374 1001015 : if (bitsize.is_constant (&ibitsize)
2375 1001015 : && bitnum.is_constant (&ibitnum)
2376 1866597 : && is_a <scalar_int_mode> (mode1, &int_mode)
2377 865589 : && strict_volatile_bitfield_p (str_rtx, ibitsize, ibitnum,
2378 : int_mode, 0, 0))
2379 : {
2380 : /* Extraction of a full INT_MODE value can be done with a simple load.
2381 : We know here that the field can be accessed with one single
2382 : instruction. For targets that support unaligned memory,
2383 : an unaligned access may be necessary. */
2384 14 : if (ibitsize == GET_MODE_BITSIZE (int_mode))
2385 : {
2386 0 : rtx result = adjust_bitfield_address (str_rtx, int_mode,
2387 : ibitnum / BITS_PER_UNIT);
2388 0 : if (reverse)
2389 0 : result = flip_storage_order (int_mode, result);
2390 0 : gcc_assert (ibitnum % BITS_PER_UNIT == 0);
2391 0 : return convert_extracted_bit_field (result, mode, tmode, unsignedp);
2392 : }
2393 :
2394 7 : str_rtx = narrow_bit_field_mem (str_rtx, int_mode, ibitsize, ibitnum,
2395 : &ibitnum);
2396 14 : gcc_assert (ibitnum + ibitsize <= GET_MODE_BITSIZE (int_mode));
2397 7 : str_rtx = copy_to_reg (str_rtx);
2398 7 : return extract_bit_field_1 (str_rtx, ibitsize, ibitnum, unsignedp,
2399 : target, mode, tmode, reverse, true, alt_rtl);
2400 : }
2401 :
2402 1001008 : return extract_bit_field_1 (str_rtx, bitsize, bitnum, unsignedp,
2403 1001008 : target, mode, tmode, reverse, true, alt_rtl);
2404 : }
2405 :
2406 : /* Use shifts and boolean operations to extract a field of BITSIZE bits
2407 : from bit BITNUM of OP0. If OP0_MODE is defined, it is the mode of OP0,
2408 : otherwise OP0 is a BLKmode MEM.
2409 :
2410 : UNSIGNEDP is nonzero for an unsigned bit field (don't sign-extend value).
2411 : If REVERSE is true, the extraction is to be done in reverse order.
2412 :
2413 : If TARGET is nonzero, attempts to store the value there
2414 : and return TARGET, but this is not guaranteed.
2415 : If TARGET is not used, create a pseudo-reg of mode TMODE for the value. */
2416 :
2417 : static rtx
2418 200590 : extract_fixed_bit_field (machine_mode tmode, rtx op0,
2419 : opt_scalar_int_mode op0_mode,
2420 : unsigned HOST_WIDE_INT bitsize,
2421 : unsigned HOST_WIDE_INT bitnum, rtx target,
2422 : int unsignedp, bool reverse)
2423 : {
2424 200590 : scalar_int_mode mode;
2425 200590 : if (MEM_P (op0))
2426 : {
2427 152246 : if (!get_best_mode (bitsize, bitnum, 0, 0, MEM_ALIGN (op0),
2428 152246 : BITS_PER_WORD, MEM_VOLATILE_P (op0), &mode))
2429 : /* The only way this should occur is if the field spans word
2430 : boundaries. */
2431 4283 : return extract_split_bit_field (op0, op0_mode, bitsize, bitnum,
2432 4283 : unsignedp, reverse);
2433 :
2434 147963 : op0 = narrow_bit_field_mem (op0, mode, bitsize, bitnum, &bitnum);
2435 : }
2436 : else
2437 48344 : mode = op0_mode.require ();
2438 :
2439 196307 : return extract_fixed_bit_field_1 (tmode, op0, mode, bitsize, bitnum,
2440 196307 : target, unsignedp, reverse);
2441 : }
2442 :
2443 : /* Helper function for extract_fixed_bit_field, extracts
2444 : the bit field always using MODE, which is the mode of OP0.
2445 : If UNSIGNEDP is -1, the result need not be sign or zero extended.
2446 : The other arguments are as for extract_fixed_bit_field. */
2447 :
2448 : static rtx
2449 196307 : extract_fixed_bit_field_1 (machine_mode tmode, rtx op0, scalar_int_mode mode,
2450 : unsigned HOST_WIDE_INT bitsize,
2451 : unsigned HOST_WIDE_INT bitnum, rtx target,
2452 : int unsignedp, bool reverse)
2453 : {
2454 : /* Note that bitsize + bitnum can be greater than GET_MODE_BITSIZE (mode)
2455 : for invalid input, such as extract equivalent of f5 from
2456 : gcc.dg/pr48335-2.c. */
2457 :
2458 196307 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
2459 : /* BITNUM is the distance between our msb and that of OP0.
2460 : Convert it to the distance from the lsb. */
2461 478 : bitnum = GET_MODE_BITSIZE (mode) - bitsize - bitnum;
2462 :
2463 : /* Now BITNUM is always the distance between the field's lsb and that of OP0.
2464 : We have reduced the big-endian case to the little-endian case. */
2465 196307 : if (reverse)
2466 239 : op0 = flip_storage_order (mode, op0);
2467 :
2468 196307 : if (unsignedp)
2469 : {
2470 137298 : if (bitnum)
2471 : {
2472 : /* If the field does not already start at the lsb,
2473 : shift it so it does. */
2474 : /* Maybe propagate the target for the shift. */
2475 61509 : rtx subtarget = (target != 0 && REG_P (target) ? target : 0);
2476 61509 : if (tmode != mode)
2477 40908 : subtarget = 0;
2478 61509 : op0 = expand_shift (RSHIFT_EXPR, mode, op0, bitnum, subtarget, 1);
2479 : }
2480 : /* Convert the value to the desired mode. TMODE must also be a
2481 : scalar integer for this conversion to make sense, since we
2482 : shouldn't reinterpret the bits. */
2483 137298 : scalar_int_mode new_mode = as_a <scalar_int_mode> (tmode);
2484 137298 : if (mode != new_mode)
2485 60325 : op0 = convert_to_mode (new_mode, op0, 1);
2486 :
2487 : /* Unless the msb of the field used to be the msb when we shifted,
2488 : mask out the upper bits. */
2489 :
2490 137298 : if (GET_MODE_BITSIZE (mode) != bitnum + bitsize
2491 137298 : && unsignedp != -1)
2492 93686 : return expand_binop (new_mode, and_optab, op0,
2493 : mask_rtx (new_mode, 0, bitsize, 0),
2494 93686 : target, 1, OPTAB_LIB_WIDEN);
2495 : return op0;
2496 : }
2497 :
2498 : /* To extract a signed bit-field, first shift its msb to the msb of the word,
2499 : then arithmetic-shift its lsb to the lsb of the word. */
2500 59009 : op0 = force_reg (mode, op0);
2501 :
2502 : /* Find the narrowest integer mode that contains the field. */
2503 :
2504 59009 : opt_scalar_int_mode mode_iter;
2505 145987 : FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
2506 291974 : if (GET_MODE_BITSIZE (mode_iter.require ()) >= bitsize + bitnum)
2507 : break;
2508 :
2509 59009 : mode = mode_iter.require ();
2510 59009 : op0 = convert_to_mode (mode, op0, 0);
2511 :
2512 59009 : if (mode != tmode)
2513 4427 : target = 0;
2514 :
2515 118018 : if (GET_MODE_BITSIZE (mode) != (bitsize + bitnum))
2516 : {
2517 54203 : int amount = GET_MODE_BITSIZE (mode) - (bitsize + bitnum);
2518 : /* Maybe propagate the target for the shift. */
2519 54203 : rtx subtarget = (target != 0 && REG_P (target) ? target : 0);
2520 54203 : op0 = expand_shift (LSHIFT_EXPR, mode, op0, amount, subtarget, 1);
2521 : }
2522 :
2523 118018 : return expand_shift (RSHIFT_EXPR, mode, op0,
2524 59009 : GET_MODE_BITSIZE (mode) - bitsize, target, 0);
2525 : }
2526 :
2527 : /* Return a constant integer (CONST_INT or CONST_DOUBLE) rtx with the value
2528 : VALUE << BITPOS. */
2529 :
2530 : static rtx
2531 73369 : lshift_value (machine_mode mode, unsigned HOST_WIDE_INT value,
2532 : int bitpos)
2533 : {
2534 73369 : return immed_wide_int_const (wi::lshift (value, bitpos), mode);
2535 : }
2536 :
2537 : /* Extract a bit field that is split across two words
2538 : and return an RTX for the result.
2539 :
2540 : OP0 is the REG, SUBREG or MEM rtx for the first of the two words.
2541 : BITSIZE is the field width; BITPOS, position of its first bit, in the word.
2542 : UNSIGNEDP is 1 if should zero-extend the contents; else sign-extend.
2543 : If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is
2544 : a BLKmode MEM.
2545 :
2546 : If REVERSE is true, the extraction is to be done in reverse order. */
2547 :
2548 : static rtx
2549 4343 : extract_split_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
2550 : unsigned HOST_WIDE_INT bitsize,
2551 : unsigned HOST_WIDE_INT bitpos, int unsignedp,
2552 : bool reverse)
2553 : {
2554 4343 : unsigned int unit;
2555 4343 : unsigned int bitsdone = 0;
2556 4343 : rtx result = NULL_RTX;
2557 4343 : int first = 1;
2558 :
2559 : /* Make sure UNIT isn't larger than BITS_PER_WORD, we can only handle that
2560 : much at a time. */
2561 4343 : if (REG_P (op0) || GET_CODE (op0) == SUBREG)
2562 60 : unit = BITS_PER_WORD;
2563 : else
2564 6362 : unit = MIN (MEM_ALIGN (op0), BITS_PER_WORD);
2565 :
2566 18097 : while (bitsdone < bitsize)
2567 : {
2568 13754 : unsigned HOST_WIDE_INT thissize;
2569 13754 : rtx part;
2570 13754 : unsigned HOST_WIDE_INT thispos;
2571 13754 : unsigned HOST_WIDE_INT offset;
2572 :
2573 13754 : offset = (bitpos + bitsdone) / unit;
2574 13754 : thispos = (bitpos + bitsdone) % unit;
2575 :
2576 : /* THISSIZE must not overrun a word boundary. Otherwise,
2577 : extract_fixed_bit_field will call us again, and we will mutually
2578 : recurse forever. */
2579 13754 : thissize = MIN (bitsize - bitsdone, BITS_PER_WORD);
2580 13754 : thissize = MIN (thissize, unit - thispos);
2581 :
2582 : /* If OP0 is a register, then handle OFFSET here. */
2583 13754 : rtx op0_piece = op0;
2584 13754 : opt_scalar_int_mode op0_piece_mode = op0_mode;
2585 13754 : if (SUBREG_P (op0) || REG_P (op0))
2586 : {
2587 120 : op0_piece = operand_subword_force (op0, offset, op0_mode.require ());
2588 120 : op0_piece_mode = word_mode;
2589 120 : offset = 0;
2590 : }
2591 :
2592 : /* Extract the parts in bit-counting order,
2593 : whose meaning is determined by BYTES_PER_UNIT.
2594 : OFFSET is in UNITs, and UNIT is in bits. */
2595 27508 : part = extract_fixed_bit_field (word_mode, op0_piece, op0_piece_mode,
2596 13754 : thissize, offset * unit + thispos,
2597 : 0, 1, reverse);
2598 13754 : bitsdone += thissize;
2599 :
2600 : /* Shift this part into place for the result. */
2601 13754 : if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
2602 : {
2603 4 : if (bitsize != bitsdone)
2604 2 : part = expand_shift (LSHIFT_EXPR, word_mode, part,
2605 2 : bitsize - bitsdone, 0, 1);
2606 : }
2607 : else
2608 : {
2609 13750 : if (bitsdone != thissize)
2610 9409 : part = expand_shift (LSHIFT_EXPR, word_mode, part,
2611 9409 : bitsdone - thissize, 0, 1);
2612 : }
2613 :
2614 13754 : if (first)
2615 : result = part;
2616 : else
2617 : /* Combine the parts with bitwise or. This works
2618 : because we extracted each part as an unsigned bit field. */
2619 9411 : result = expand_binop (word_mode, ior_optab, part, result, NULL_RTX, 1,
2620 : OPTAB_LIB_WIDEN);
2621 :
2622 13754 : first = 0;
2623 : }
2624 :
2625 : /* Unsigned bit field: we are done. */
2626 4343 : if (unsignedp)
2627 : return result;
2628 : /* Signed bit field: sign-extend with two arithmetic shifts. */
2629 1464 : result = expand_shift (LSHIFT_EXPR, word_mode, result,
2630 1464 : BITS_PER_WORD - bitsize, NULL_RTX, 0);
2631 1464 : return expand_shift (RSHIFT_EXPR, word_mode, result,
2632 1464 : BITS_PER_WORD - bitsize, NULL_RTX, 0);
2633 : }
2634 :
2635 : /* Try to read the low bits of SRC as an rvalue of mode MODE, preserving
2636 : the bit pattern. SRC_MODE is the mode of SRC; if this is smaller than
2637 : MODE, fill the upper bits with zeros. Fail if the layout of either
2638 : mode is unknown (as for CC modes) or if the extraction would involve
2639 : unprofitable mode punning. Return the value on success, otherwise
2640 : return null.
2641 :
2642 : This is different from gen_lowpart* in these respects:
2643 :
2644 : - the returned value must always be considered an rvalue
2645 :
2646 : - when MODE is wider than SRC_MODE, the extraction involves
2647 : a zero extension
2648 :
2649 : - when MODE is smaller than SRC_MODE, the extraction involves
2650 : a truncation (and is thus subject to TARGET_TRULY_NOOP_TRUNCATION).
2651 :
2652 : In other words, this routine performs a computation, whereas the
2653 : gen_lowpart* routines are conceptually lvalue or rvalue subreg
2654 : operations. */
2655 :
2656 : rtx
2657 119680 : extract_low_bits (machine_mode mode, machine_mode src_mode, rtx src)
2658 : {
2659 119680 : scalar_int_mode int_mode, src_int_mode;
2660 :
2661 119680 : if (mode == src_mode)
2662 : return src;
2663 :
2664 81937 : if (CONSTANT_P (src))
2665 : {
2666 : /* simplify_gen_subreg can't be used here, as if simplify_subreg
2667 : fails, it will happily create (subreg (symbol_ref)) or similar
2668 : invalid SUBREGs. */
2669 19342 : poly_uint64 byte = subreg_lowpart_offset (mode, src_mode);
2670 19342 : rtx ret = simplify_subreg (mode, src, src_mode, byte);
2671 19342 : if (ret)
2672 : return ret;
2673 :
2674 22 : if (GET_MODE (src) == VOIDmode
2675 22 : || !validate_subreg (mode, src_mode, src, byte))
2676 : return NULL_RTX;
2677 :
2678 17 : src = force_reg (GET_MODE (src), src);
2679 17 : return gen_rtx_SUBREG (mode, src, byte);
2680 : }
2681 :
2682 62595 : if (GET_MODE_CLASS (mode) == MODE_CC || GET_MODE_CLASS (src_mode) == MODE_CC)
2683 : return NULL_RTX;
2684 :
2685 125190 : if (known_eq (GET_MODE_BITSIZE (mode), GET_MODE_BITSIZE (src_mode))
2686 62595 : && targetm.modes_tieable_p (mode, src_mode))
2687 : {
2688 3972 : rtx x = gen_lowpart_common (mode, src);
2689 3972 : if (x)
2690 : return x;
2691 : }
2692 :
2693 58634 : if (!int_mode_for_mode (src_mode).exists (&src_int_mode)
2694 58634 : || !int_mode_for_mode (mode).exists (&int_mode))
2695 : return NULL_RTX;
2696 :
2697 58621 : if (!targetm.modes_tieable_p (src_int_mode, src_mode))
2698 : return NULL_RTX;
2699 57531 : if (!targetm.modes_tieable_p (int_mode, mode))
2700 : return NULL_RTX;
2701 :
2702 55458 : src = gen_lowpart (src_int_mode, src);
2703 55458 : if (!validate_subreg (int_mode, src_int_mode, src,
2704 : subreg_lowpart_offset (int_mode, src_int_mode)))
2705 : return NULL_RTX;
2706 :
2707 55446 : src = convert_modes (int_mode, src_int_mode, src, true);
2708 55446 : src = gen_lowpart (mode, src);
2709 55446 : return src;
2710 : }
2711 :
2712 : /* Add INC into TARGET. */
2713 :
2714 : void
2715 1200 : expand_inc (rtx target, rtx inc)
2716 : {
2717 1200 : rtx value = expand_binop (GET_MODE (target), add_optab,
2718 : target, inc,
2719 : target, 0, OPTAB_LIB_WIDEN);
2720 1200 : if (value != target)
2721 61 : emit_move_insn (target, value);
2722 1200 : }
2723 :
2724 : /* Subtract DEC from TARGET. */
2725 :
2726 : void
2727 1235 : expand_dec (rtx target, rtx dec)
2728 : {
2729 1235 : rtx value = expand_binop (GET_MODE (target), sub_optab,
2730 : target, dec,
2731 : target, 0, OPTAB_LIB_WIDEN);
2732 1235 : if (value != target)
2733 0 : emit_move_insn (target, value);
2734 1235 : }
2735 :
2736 : /* Output a shift instruction for expression code CODE,
2737 : with SHIFTED being the rtx for the value to shift,
2738 : and AMOUNT the rtx for the amount to shift by.
2739 : Store the result in the rtx TARGET, if that is convenient.
2740 : If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2741 : Return the rtx for where the value is.
2742 : If that cannot be done, abort the compilation unless MAY_FAIL is true,
2743 : in which case 0 is returned. */
2744 :
2745 : static rtx
2746 1559195 : expand_shift_1 (enum tree_code code, machine_mode mode, rtx shifted,
2747 : rtx amount, rtx target, int unsignedp, bool may_fail = false)
2748 : {
2749 1559195 : rtx op1, temp = 0;
2750 1559195 : int left = (code == LSHIFT_EXPR || code == LROTATE_EXPR);
2751 1559195 : int rotate = (code == LROTATE_EXPR || code == RROTATE_EXPR);
2752 1559195 : optab lshift_optab = ashl_optab;
2753 1559195 : optab rshift_arith_optab = ashr_optab;
2754 1559195 : optab rshift_uns_optab = lshr_optab;
2755 1559195 : optab lrotate_optab = rotl_optab;
2756 1559195 : optab rrotate_optab = rotr_optab;
2757 1559195 : machine_mode op1_mode;
2758 1559195 : scalar_mode scalar_mode = GET_MODE_INNER (mode);
2759 1559195 : int attempt;
2760 1559195 : bool speed = optimize_insn_for_speed_p ();
2761 :
2762 1559195 : op1 = amount;
2763 1559195 : op1_mode = GET_MODE (op1);
2764 :
2765 : /* Determine whether the shift/rotate amount is a vector, or scalar. If the
2766 : shift amount is a vector, use the vector/vector shift patterns. */
2767 1559195 : if (VECTOR_MODE_P (mode) && VECTOR_MODE_P (op1_mode))
2768 : {
2769 1559195 : lshift_optab = vashl_optab;
2770 1559195 : rshift_arith_optab = vashr_optab;
2771 1559195 : rshift_uns_optab = vlshr_optab;
2772 1559195 : lrotate_optab = vrotl_optab;
2773 1559195 : rrotate_optab = vrotr_optab;
2774 : }
2775 :
2776 : /* Previously detected shift-counts computed by NEGATE_EXPR
2777 : and shifted in the other direction; but that does not work
2778 : on all machines. */
2779 :
2780 1559195 : if (SHIFT_COUNT_TRUNCATED)
2781 : {
2782 : if (CONST_INT_P (op1)
2783 : && ((unsigned HOST_WIDE_INT) INTVAL (op1) >=
2784 : (unsigned HOST_WIDE_INT) GET_MODE_BITSIZE (scalar_mode)))
2785 : op1 = gen_int_shift_amount (mode,
2786 : (unsigned HOST_WIDE_INT) INTVAL (op1)
2787 : % GET_MODE_BITSIZE (scalar_mode));
2788 : else if (GET_CODE (op1) == SUBREG
2789 : && subreg_lowpart_p (op1)
2790 : && SCALAR_INT_MODE_P (GET_MODE (SUBREG_REG (op1)))
2791 : && SCALAR_INT_MODE_P (GET_MODE (op1)))
2792 : op1 = SUBREG_REG (op1);
2793 : }
2794 :
2795 : /* Canonicalize rotates by constant amount. We may canonicalize
2796 : to reduce the immediate or if the ISA can rotate by constants
2797 : in only on direction. */
2798 1559195 : if (rotate && reverse_rotate_by_imm_p (scalar_mode, left, op1))
2799 : {
2800 3292 : op1 = gen_int_shift_amount (mode, (GET_MODE_BITSIZE (scalar_mode)
2801 3292 : - INTVAL (op1)));
2802 3292 : left = !left;
2803 3292 : code = left ? LROTATE_EXPR : RROTATE_EXPR;
2804 : }
2805 :
2806 : /* Rotation of 16bit values by 8 bits is effectively equivalent to a bswaphi.
2807 : Note that this is not the case for bigger values. For instance a rotation
2808 : of 0x01020304 by 16 bits gives 0x03040102 which is different from
2809 : 0x04030201 (bswapsi). */
2810 1559195 : if (rotate
2811 8657 : && CONST_INT_P (op1)
2812 5442 : && INTVAL (op1) == BITS_PER_UNIT
2813 1070 : && GET_MODE_SIZE (scalar_mode) == 2
2814 1560090 : && optab_handler (bswap_optab, mode) != CODE_FOR_nothing)
2815 894 : return expand_unop (mode, bswap_optab, shifted, NULL_RTX, unsignedp);
2816 :
2817 1558301 : if (op1 == const0_rtx)
2818 : return shifted;
2819 :
2820 : /* Check whether its cheaper to implement a left shift by a constant
2821 : bit count by a sequence of additions. */
2822 1508329 : if (code == LSHIFT_EXPR
2823 903018 : && CONST_INT_P (op1)
2824 872564 : && INTVAL (op1) > 0
2825 872539 : && INTVAL (op1) < GET_MODE_PRECISION (scalar_mode)
2826 872539 : && INTVAL (op1) < MAX_BITS_PER_WORD
2827 867766 : && (shift_cost (speed, mode, INTVAL (op1))
2828 867766 : > INTVAL (op1) * add_cost (speed, mode))
2829 1511378 : && shift_cost (speed, mode, INTVAL (op1)) != MAX_COST)
2830 : {
2831 : int i;
2832 6286 : for (i = 0; i < INTVAL (op1); i++)
2833 : {
2834 3237 : temp = force_reg (mode, shifted);
2835 3237 : shifted = expand_binop (mode, add_optab, temp, temp, NULL_RTX,
2836 : unsignedp, OPTAB_LIB_WIDEN);
2837 : }
2838 : return shifted;
2839 : }
2840 :
2841 3010598 : for (attempt = 0; temp == 0 && attempt < 3; attempt++)
2842 : {
2843 1505356 : enum optab_methods methods;
2844 :
2845 1505356 : if (attempt == 0)
2846 : methods = OPTAB_DIRECT;
2847 76 : else if (attempt == 1)
2848 : methods = OPTAB_WIDEN;
2849 : else
2850 38 : methods = OPTAB_LIB_WIDEN;
2851 :
2852 1505356 : if (rotate)
2853 : {
2854 : /* Widening does not work for rotation. */
2855 7839 : if (methods == OPTAB_WIDEN)
2856 38 : continue;
2857 7801 : else if (methods == OPTAB_LIB_WIDEN)
2858 : {
2859 : /* If we have been unable to open-code this by a rotation,
2860 : do it as the IOR or PLUS of two shifts. I.e., to rotate
2861 : A by N bits, compute
2862 : (A << N) | ((unsigned) A >> ((-N) & (C - 1)))
2863 : where C is the bitsize of A. If N cannot be zero,
2864 : use PLUS instead of IOR.
2865 :
2866 : It is theoretically possible that the target machine might
2867 : not be able to perform either shift and hence we would
2868 : be making two libcalls rather than just the one for the
2869 : shift (similarly if IOR could not be done). We will allow
2870 : this extremely unlikely lossage to avoid complicating the
2871 : code below. */
2872 :
2873 38 : rtx subtarget = target == shifted ? 0 : target;
2874 38 : rtx new_amount, other_amount;
2875 38 : rtx temp1;
2876 :
2877 38 : new_amount = op1;
2878 38 : if (op1 == const0_rtx)
2879 : return shifted;
2880 38 : else if (CONST_INT_P (op1))
2881 23 : other_amount = gen_int_shift_amount
2882 23 : (mode, GET_MODE_BITSIZE (scalar_mode) - INTVAL (op1));
2883 : else
2884 : {
2885 15 : other_amount
2886 30 : = simplify_gen_unary (NEG, GET_MODE (op1),
2887 15 : op1, GET_MODE (op1));
2888 15 : HOST_WIDE_INT mask = GET_MODE_PRECISION (scalar_mode) - 1;
2889 15 : other_amount
2890 15 : = simplify_gen_binary (AND, GET_MODE (op1), other_amount,
2891 15 : gen_int_mode (mask, GET_MODE (op1)));
2892 : }
2893 :
2894 38 : shifted = force_reg (mode, shifted);
2895 :
2896 51 : temp = expand_shift_1 (left ? LSHIFT_EXPR : RSHIFT_EXPR,
2897 : mode, shifted, new_amount, 0, 1);
2898 51 : temp1 = expand_shift_1 (left ? RSHIFT_EXPR : LSHIFT_EXPR,
2899 : mode, shifted, other_amount,
2900 : subtarget, 1);
2901 38 : return expand_binop (mode,
2902 38 : CONST_INT_P (op1) ? add_optab : ior_optab,
2903 38 : temp, temp1, target, unsignedp, methods);
2904 : }
2905 :
2906 11369 : temp = expand_binop (mode,
2907 : left ? lrotate_optab : rrotate_optab,
2908 : shifted, op1, target, unsignedp, methods);
2909 : }
2910 1497517 : else if (unsignedp)
2911 1246695 : temp = expand_binop (mode,
2912 : left ? lshift_optab : rshift_uns_optab,
2913 : shifted, op1, target, unsignedp, methods);
2914 :
2915 : /* Do arithmetic shifts.
2916 : Also, if we are going to widen the operand, we can just as well
2917 : use an arithmetic right-shift instead of a logical one. */
2918 1505280 : if (temp == 0 && ! rotate
2919 600894 : && (! unsignedp || (! left && methods == OPTAB_WIDEN)))
2920 : {
2921 : enum optab_methods methods1 = methods;
2922 :
2923 : /* If trying to widen a log shift to an arithmetic shift,
2924 : don't accept an arithmetic shift of the same size. */
2925 : if (unsignedp)
2926 : methods1 = OPTAB_MUST_WIDEN;
2927 :
2928 : /* Arithmetic shift */
2929 :
2930 848370 : temp = expand_binop (mode,
2931 : left ? lshift_optab : rshift_arith_optab,
2932 : shifted, op1, target, unsignedp, methods1);
2933 : }
2934 :
2935 : /* We used to try extzv here for logical right shifts, but that was
2936 : only useful for one machine, the VAX, and caused poor code
2937 : generation there for lshrdi3, so the code was deleted and a
2938 : define_expand for lshrsi3 was added to vax.md. */
2939 : }
2940 :
2941 1505242 : gcc_assert (temp != NULL_RTX || may_fail);
2942 : return temp;
2943 : }
2944 :
2945 : /* Output a shift instruction for expression code CODE,
2946 : with SHIFTED being the rtx for the value to shift,
2947 : and AMOUNT the amount to shift by.
2948 : Store the result in the rtx TARGET, if that is convenient.
2949 : If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2950 : Return the rtx for where the value is. */
2951 :
2952 : rtx
2953 1247585 : expand_shift (enum tree_code code, machine_mode mode, rtx shifted,
2954 : poly_int64 amount, rtx target, int unsignedp)
2955 : {
2956 1247585 : return expand_shift_1 (code, mode, shifted,
2957 : gen_int_shift_amount (mode, amount),
2958 1247585 : target, unsignedp);
2959 : }
2960 :
2961 : /* Likewise, but return 0 if that cannot be done. */
2962 :
2963 : rtx
2964 325 : maybe_expand_shift (enum tree_code code, machine_mode mode, rtx shifted,
2965 : int amount, rtx target, int unsignedp)
2966 : {
2967 325 : return expand_shift_1 (code, mode,
2968 325 : shifted, GEN_INT (amount), target, unsignedp, true);
2969 : }
2970 :
2971 : /* Output a shift instruction for expression code CODE,
2972 : with SHIFTED being the rtx for the value to shift,
2973 : and AMOUNT the tree for the amount to shift by.
2974 : Store the result in the rtx TARGET, if that is convenient.
2975 : If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2976 : Return the rtx for where the value is. */
2977 :
2978 : rtx
2979 311209 : expand_variable_shift (enum tree_code code, machine_mode mode, rtx shifted,
2980 : tree amount, rtx target, int unsignedp)
2981 : {
2982 311209 : return expand_shift_1 (code, mode,
2983 311209 : shifted, expand_normal (amount), target, unsignedp);
2984 : }
2985 :
2986 :
2987 : static void synth_mult (struct algorithm *, unsigned HOST_WIDE_INT,
2988 : const struct mult_cost *, machine_mode mode);
2989 : static rtx expand_mult_const (machine_mode, rtx, HOST_WIDE_INT, rtx,
2990 : const struct algorithm *, enum mult_variant);
2991 : static unsigned HOST_WIDE_INT invert_mod2n (unsigned HOST_WIDE_INT, int);
2992 : static rtx extract_high_half (scalar_int_mode, rtx);
2993 : static rtx expmed_mult_highpart (scalar_int_mode, rtx, rtx, rtx, int, int);
2994 :
2995 : /* Compute and return the best algorithm for multiplying by T.
2996 : The algorithm must cost less than cost_limit
2997 : If retval.cost >= COST_LIMIT, no algorithm was found and all
2998 : other field of the returned struct are undefined.
2999 : MODE is the machine mode of the multiplication. */
3000 :
3001 : static void
3002 38852805 : synth_mult (struct algorithm *alg_out, unsigned HOST_WIDE_INT t,
3003 : const struct mult_cost *cost_limit, machine_mode mode)
3004 : {
3005 38852805 : int m;
3006 38852805 : struct algorithm *alg_in, *best_alg;
3007 38852805 : struct mult_cost best_cost;
3008 38852805 : struct mult_cost new_limit;
3009 38852805 : int op_cost, op_latency;
3010 38852805 : unsigned HOST_WIDE_INT orig_t = t;
3011 38852805 : unsigned HOST_WIDE_INT q;
3012 38852805 : int maxm, hash_index;
3013 38852805 : bool cache_hit = false;
3014 38852805 : enum alg_code cache_alg = alg_zero;
3015 38852805 : bool speed = optimize_insn_for_speed_p ();
3016 38852805 : scalar_int_mode imode;
3017 38852805 : struct alg_hash_entry *entry_ptr;
3018 :
3019 : /* Indicate that no algorithm is yet found. If no algorithm
3020 : is found, this value will be returned and indicate failure. */
3021 38852805 : alg_out->cost.cost = cost_limit->cost + 1;
3022 38852805 : alg_out->cost.latency = cost_limit->latency + 1;
3023 :
3024 38852805 : if (cost_limit->cost < 0
3025 31675199 : || (cost_limit->cost == 0 && cost_limit->latency <= 0))
3026 31410494 : return;
3027 :
3028 : /* Be prepared for vector modes. */
3029 52495560 : imode = as_a <scalar_int_mode> (GET_MODE_INNER (mode));
3030 :
3031 77311683 : maxm = MIN (BITS_PER_WORD, GET_MODE_BITSIZE (imode));
3032 :
3033 : /* Restrict the bits of "t" to the multiplication's mode. */
3034 26247780 : t &= GET_MODE_MASK (imode);
3035 :
3036 : /* t == 1 can be done in zero cost. */
3037 26247780 : if (t == 1)
3038 : {
3039 6049887 : alg_out->ops = 1;
3040 6049887 : alg_out->cost.cost = 0;
3041 6049887 : alg_out->cost.latency = 0;
3042 6049887 : alg_out->op[0] = alg_m;
3043 6049887 : return;
3044 : }
3045 :
3046 : /* t == 0 sometimes has a cost. If it does and it exceeds our limit,
3047 : fail now. */
3048 20197893 : if (t == 0)
3049 : {
3050 535853 : if (MULT_COST_LESS (cost_limit, zero_cost (speed)))
3051 : return;
3052 : else
3053 : {
3054 535853 : alg_out->ops = 1;
3055 535853 : alg_out->cost.cost = zero_cost (speed);
3056 535853 : alg_out->cost.latency = zero_cost (speed);
3057 535853 : alg_out->op[0] = alg_zero;
3058 535853 : return;
3059 : }
3060 : }
3061 :
3062 : /* We'll be needing a couple extra algorithm structures now. */
3063 :
3064 19662040 : alg_in = XALLOCA (struct algorithm);
3065 19662040 : best_alg = XALLOCA (struct algorithm);
3066 19662040 : best_cost = *cost_limit;
3067 :
3068 : /* Compute the hash index. */
3069 19662040 : hash_index = (t ^ (unsigned int) mode ^ (speed * 256)) % NUM_ALG_HASH_ENTRIES;
3070 :
3071 : /* See if we already know what to do for T. */
3072 19662040 : entry_ptr = alg_hash_entry_ptr (hash_index);
3073 19662040 : if (entry_ptr->t == t
3074 15389564 : && entry_ptr->mode == mode
3075 15389564 : && entry_ptr->speed == speed
3076 15389564 : && entry_ptr->alg != alg_unknown)
3077 : {
3078 15389564 : cache_alg = entry_ptr->alg;
3079 :
3080 15389564 : if (cache_alg == alg_impossible)
3081 : {
3082 : /* The cache tells us that it's impossible to synthesize
3083 : multiplication by T within entry_ptr->cost. */
3084 7580459 : if (!CHEAPER_MULT_COST (&entry_ptr->cost, cost_limit))
3085 : /* COST_LIMIT is at least as restrictive as the one
3086 : recorded in the hash table, in which case we have no
3087 : hope of synthesizing a multiplication. Just
3088 : return. */
3089 : return;
3090 :
3091 : /* If we get here, COST_LIMIT is less restrictive than the
3092 : one recorded in the hash table, so we may be able to
3093 : synthesize a multiplication. Proceed as if we didn't
3094 : have the cache entry. */
3095 : }
3096 : else
3097 : {
3098 7809105 : if (CHEAPER_MULT_COST (cost_limit, &entry_ptr->cost))
3099 : /* The cached algorithm shows that this multiplication
3100 : requires more cost than COST_LIMIT. Just return. This
3101 : way, we don't clobber this cache entry with
3102 : alg_impossible but retain useful information. */
3103 : return;
3104 :
3105 7046022 : cache_hit = true;
3106 :
3107 7046022 : switch (cache_alg)
3108 : {
3109 4640749 : case alg_shift:
3110 4640749 : goto do_alg_shift;
3111 :
3112 952122 : case alg_add_t_m2:
3113 952122 : case alg_sub_t_m2:
3114 952122 : goto do_alg_addsub_t_m2;
3115 :
3116 120935 : case alg_add_factor:
3117 120935 : case alg_sub_factor:
3118 120935 : goto do_alg_addsub_factor;
3119 :
3120 1332205 : case alg_add_t2_m:
3121 1332205 : goto do_alg_add_t2_m;
3122 :
3123 11 : case alg_sub_t2_m:
3124 11 : goto do_alg_sub_t2_m;
3125 :
3126 0 : default:
3127 0 : gcc_unreachable ();
3128 : }
3129 : }
3130 : }
3131 :
3132 : /* If we have a group of zero bits at the low-order part of T, try
3133 : multiplying by the remaining bits and then doing a shift. */
3134 :
3135 5286620 : if ((t & 1) == 0)
3136 : {
3137 2715041 : do_alg_shift:
3138 7355790 : m = ctz_or_zero (t); /* m = number of low zero bits */
3139 7355790 : if (m < maxm)
3140 : {
3141 7354050 : q = t >> m;
3142 : /* The function expand_shift will choose between a shift and
3143 : a sequence of additions, so the observed cost is given as
3144 : MIN (m * add_cost(speed, mode), shift_cost(speed, mode, m)). */
3145 7354050 : op_cost = m * add_cost (speed, mode);
3146 7354050 : if (shift_cost (speed, mode, m) < op_cost)
3147 : op_cost = shift_cost (speed, mode, m);
3148 7354050 : new_limit.cost = best_cost.cost - op_cost;
3149 7354050 : new_limit.latency = best_cost.latency - op_cost;
3150 7354050 : synth_mult (alg_in, q, &new_limit, mode);
3151 :
3152 7354050 : alg_in->cost.cost += op_cost;
3153 7354050 : alg_in->cost.latency += op_cost;
3154 7354050 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3155 : {
3156 4269357 : best_cost = alg_in->cost;
3157 4269357 : std::swap (alg_in, best_alg);
3158 4269357 : best_alg->log[best_alg->ops] = m;
3159 4269357 : best_alg->op[best_alg->ops] = alg_shift;
3160 : }
3161 :
3162 : /* See if treating ORIG_T as a signed number yields a better
3163 : sequence. Try this sequence only for a negative ORIG_T
3164 : as it would be useless for a non-negative ORIG_T. */
3165 7354050 : if ((HOST_WIDE_INT) orig_t < 0)
3166 : {
3167 : /* Shift ORIG_T as follows because a right shift of a
3168 : negative-valued signed type is implementation
3169 : defined. */
3170 633145 : q = ~(~orig_t >> m);
3171 : /* The function expand_shift will choose between a shift
3172 : and a sequence of additions, so the observed cost is
3173 : given as MIN (m * add_cost(speed, mode),
3174 : shift_cost(speed, mode, m)). */
3175 633145 : op_cost = m * add_cost (speed, mode);
3176 633145 : if (shift_cost (speed, mode, m) < op_cost)
3177 : op_cost = shift_cost (speed, mode, m);
3178 633145 : new_limit.cost = best_cost.cost - op_cost;
3179 633145 : new_limit.latency = best_cost.latency - op_cost;
3180 633145 : synth_mult (alg_in, q, &new_limit, mode);
3181 :
3182 633145 : alg_in->cost.cost += op_cost;
3183 633145 : alg_in->cost.latency += op_cost;
3184 633145 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3185 : {
3186 600998 : best_cost = alg_in->cost;
3187 600998 : std::swap (alg_in, best_alg);
3188 600998 : best_alg->log[best_alg->ops] = m;
3189 600998 : best_alg->op[best_alg->ops] = alg_shift;
3190 : }
3191 : }
3192 : }
3193 1740 : else if (2 * BITS_PER_WORD <= HOST_BITS_PER_WIDE_INT
3194 1740 : && GET_MODE_BITSIZE (imode) == 2 * BITS_PER_WORD
3195 1728 : && m >= BITS_PER_WORD
3196 3468 : && imode == mode)
3197 : {
3198 1728 : q = t >> m;
3199 1728 : int op1_cost = shift_cost (speed, mode, m - BITS_PER_WORD);
3200 1728 : int op2_cost = zero_cost (speed);
3201 1728 : op_latency = MAX (op1_cost, op2_cost);
3202 1728 : op_cost = op1_cost + op2_cost;
3203 :
3204 1728 : new_limit.cost = best_cost.cost - op_cost;
3205 1728 : new_limit.latency = best_cost.latency - op_latency;
3206 1728 : synth_mult (alg_in, q, &new_limit, mode);
3207 1728 : alg_in->cost.cost += op_cost;
3208 1728 : alg_in->cost.latency += op_latency;
3209 1728 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3210 : {
3211 1662 : best_cost = alg_in->cost;
3212 1662 : std::swap (alg_in, best_alg);
3213 1662 : best_alg->log[best_alg->ops] = m;
3214 1662 : best_alg->op[best_alg->ops] = alg_shift;
3215 : }
3216 : }
3217 7355790 : if (cache_hit)
3218 4640749 : goto done;
3219 : }
3220 :
3221 : /* If we have an odd number, add or subtract one. */
3222 2715041 : if ((t & 1) != 0)
3223 : {
3224 3523701 : unsigned HOST_WIDE_INT w;
3225 :
3226 0 : do_alg_addsub_t_m2:
3227 42476792 : for (w = 1; (w & t) != 0; w <<= 1)
3228 : ;
3229 : /* If T was -1, then W will be zero after the loop. This is another
3230 : case where T ends with ...111. Handling this with (T + 1) and
3231 : subtract 1 produces slightly better code and results in algorithm
3232 : selection much faster than treating it like the ...0111 case
3233 : below. */
3234 3523701 : if (w == 0
3235 3104815 : || (w > 2
3236 : /* Reject the case where t is 3.
3237 : Thus we prefer addition in that case. */
3238 3104815 : && t != 3))
3239 : {
3240 : /* T ends with ...111. Multiply by (T + 1) and subtract T. */
3241 :
3242 1933486 : op_cost = add_cost (speed, mode);
3243 1933486 : new_limit.cost = best_cost.cost - op_cost;
3244 1933486 : new_limit.latency = best_cost.latency - op_cost;
3245 1933486 : synth_mult (alg_in, t + 1, &new_limit, mode);
3246 :
3247 1933486 : alg_in->cost.cost += op_cost;
3248 1933486 : alg_in->cost.latency += op_cost;
3249 1933486 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3250 : {
3251 720385 : best_cost = alg_in->cost;
3252 720385 : std::swap (alg_in, best_alg);
3253 720385 : best_alg->log[best_alg->ops] = 0;
3254 720385 : best_alg->op[best_alg->ops] = alg_sub_t_m2;
3255 : }
3256 : }
3257 : else
3258 : {
3259 : /* T ends with ...01 or ...011. Multiply by (T - 1) and add T. */
3260 :
3261 1590215 : op_cost = add_cost (speed, mode);
3262 1590215 : new_limit.cost = best_cost.cost - op_cost;
3263 1590215 : new_limit.latency = best_cost.latency - op_cost;
3264 1590215 : synth_mult (alg_in, t - 1, &new_limit, mode);
3265 :
3266 1590215 : alg_in->cost.cost += op_cost;
3267 1590215 : alg_in->cost.latency += op_cost;
3268 1590215 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3269 : {
3270 197738 : best_cost = alg_in->cost;
3271 197738 : std::swap (alg_in, best_alg);
3272 197738 : best_alg->log[best_alg->ops] = 0;
3273 197738 : best_alg->op[best_alg->ops] = alg_add_t_m2;
3274 : }
3275 : }
3276 :
3277 : /* We may be able to calculate a * -7, a * -15, a * -31, etc
3278 : quickly with a - a * n for some appropriate constant n. */
3279 3523701 : m = exact_log2 (-orig_t + 1);
3280 3523701 : if (m >= 0 && m < maxm)
3281 : {
3282 743633 : op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
3283 : /* If the target has a cheap shift-and-subtract insn use
3284 : that in preference to a shift insn followed by a sub insn.
3285 : Assume that the shift-and-sub is "atomic" with a latency
3286 : equal to it's cost, otherwise assume that on superscalar
3287 : hardware the shift may be executed concurrently with the
3288 : earlier steps in the algorithm. */
3289 743633 : if (shiftsub1_cost (speed, mode, m) <= op_cost)
3290 : {
3291 : op_cost = shiftsub1_cost (speed, mode, m);
3292 : op_latency = op_cost;
3293 : }
3294 : else
3295 737759 : op_latency = add_cost (speed, mode);
3296 :
3297 743633 : new_limit.cost = best_cost.cost - op_cost;
3298 743633 : new_limit.latency = best_cost.latency - op_latency;
3299 743633 : synth_mult (alg_in, (unsigned HOST_WIDE_INT) (-orig_t + 1) >> m,
3300 : &new_limit, mode);
3301 :
3302 743633 : alg_in->cost.cost += op_cost;
3303 743633 : alg_in->cost.latency += op_latency;
3304 743633 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3305 : {
3306 191358 : best_cost = alg_in->cost;
3307 191358 : std::swap (alg_in, best_alg);
3308 191358 : best_alg->log[best_alg->ops] = m;
3309 191358 : best_alg->op[best_alg->ops] = alg_sub_t_m2;
3310 : }
3311 : }
3312 :
3313 3523701 : if (cache_hit)
3314 952122 : goto done;
3315 : }
3316 :
3317 : /* Look for factors of t of the form
3318 : t = q(2**m +- 1), 2 <= m <= floor(log2(t - 1)).
3319 : If we find such a factor, we can multiply by t using an algorithm that
3320 : multiplies by q, shift the result by m and add/subtract it to itself.
3321 :
3322 : We search for large factors first and loop down, even if large factors
3323 : are less probable than small; if we find a large factor we will find a
3324 : good sequence quickly, and therefore be able to prune (by decreasing
3325 : COST_LIMIT) the search. */
3326 :
3327 2715041 : do_alg_addsub_factor:
3328 114360357 : for (m = floor_log2 (t - 1); m >= 2; m--)
3329 : {
3330 111742992 : unsigned HOST_WIDE_INT d;
3331 :
3332 111742992 : d = (HOST_WIDE_INT_1U << m) + 1;
3333 111742992 : if (t % d == 0 && t > d && m < maxm
3334 1305287 : && (!cache_hit || cache_alg == alg_add_factor))
3335 : {
3336 1305287 : op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
3337 1305287 : if (shiftadd_cost (speed, mode, m) <= op_cost)
3338 : op_cost = shiftadd_cost (speed, mode, m);
3339 :
3340 1305287 : op_latency = op_cost;
3341 :
3342 :
3343 1305287 : new_limit.cost = best_cost.cost - op_cost;
3344 1305287 : new_limit.latency = best_cost.latency - op_latency;
3345 1305287 : synth_mult (alg_in, t / d, &new_limit, mode);
3346 :
3347 1305287 : alg_in->cost.cost += op_cost;
3348 1305287 : alg_in->cost.latency += op_latency;
3349 1305287 : if (alg_in->cost.latency < op_cost)
3350 263947 : alg_in->cost.latency = op_cost;
3351 1305287 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3352 : {
3353 118733 : best_cost = alg_in->cost;
3354 118733 : std::swap (alg_in, best_alg);
3355 118733 : best_alg->log[best_alg->ops] = m;
3356 118733 : best_alg->op[best_alg->ops] = alg_add_factor;
3357 : }
3358 : /* Other factors will have been taken care of in the recursion. */
3359 : break;
3360 : }
3361 :
3362 110437705 : d = (HOST_WIDE_INT_1U << m) - 1;
3363 110437705 : if (t % d == 0 && t > d && m < maxm
3364 1484903 : && (!cache_hit || cache_alg == alg_sub_factor))
3365 : {
3366 1484903 : op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
3367 1484903 : if (shiftsub0_cost (speed, mode, m) <= op_cost)
3368 : op_cost = shiftsub0_cost (speed, mode, m);
3369 :
3370 1484903 : op_latency = op_cost;
3371 :
3372 1484903 : new_limit.cost = best_cost.cost - op_cost;
3373 1484903 : new_limit.latency = best_cost.latency - op_latency;
3374 1484903 : synth_mult (alg_in, t / d, &new_limit, mode);
3375 :
3376 1484903 : alg_in->cost.cost += op_cost;
3377 1484903 : alg_in->cost.latency += op_latency;
3378 1484903 : if (alg_in->cost.latency < op_cost)
3379 360987 : alg_in->cost.latency = op_cost;
3380 1484903 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3381 : {
3382 33754 : best_cost = alg_in->cost;
3383 33754 : std::swap (alg_in, best_alg);
3384 33754 : best_alg->log[best_alg->ops] = m;
3385 33754 : best_alg->op[best_alg->ops] = alg_sub_factor;
3386 : }
3387 : break;
3388 : }
3389 : }
3390 5407555 : if (cache_hit)
3391 120935 : goto done;
3392 :
3393 : /* Try shift-and-add (load effective address) instructions,
3394 : i.e. do a*3, a*5, a*9. */
3395 5286620 : if ((t & 1) != 0)
3396 : {
3397 2571579 : do_alg_add_t2_m:
3398 3903784 : q = t - 1;
3399 3903784 : m = ctz_hwi (q);
3400 3903784 : if (q && m < maxm)
3401 : {
3402 3903762 : op_cost = shiftadd_cost (speed, mode, m);
3403 3903762 : new_limit.cost = best_cost.cost - op_cost;
3404 3903762 : new_limit.latency = best_cost.latency - op_cost;
3405 3903762 : synth_mult (alg_in, (t - 1) >> m, &new_limit, mode);
3406 :
3407 3903762 : alg_in->cost.cost += op_cost;
3408 3903762 : alg_in->cost.latency += op_cost;
3409 3903762 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3410 : {
3411 1387034 : best_cost = alg_in->cost;
3412 1387034 : std::swap (alg_in, best_alg);
3413 1387034 : best_alg->log[best_alg->ops] = m;
3414 1387034 : best_alg->op[best_alg->ops] = alg_add_t2_m;
3415 : }
3416 : }
3417 3903784 : if (cache_hit)
3418 1332205 : goto done;
3419 :
3420 2571579 : do_alg_sub_t2_m:
3421 2571590 : q = t + 1;
3422 2571590 : m = ctz_hwi (q);
3423 2571590 : if (q && m < maxm)
3424 : {
3425 2549073 : op_cost = shiftsub0_cost (speed, mode, m);
3426 2549073 : new_limit.cost = best_cost.cost - op_cost;
3427 2549073 : new_limit.latency = best_cost.latency - op_cost;
3428 2549073 : synth_mult (alg_in, (t + 1) >> m, &new_limit, mode);
3429 :
3430 2549073 : alg_in->cost.cost += op_cost;
3431 2549073 : alg_in->cost.latency += op_cost;
3432 2549073 : if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
3433 : {
3434 95 : best_cost = alg_in->cost;
3435 95 : std::swap (alg_in, best_alg);
3436 95 : best_alg->log[best_alg->ops] = m;
3437 95 : best_alg->op[best_alg->ops] = alg_sub_t2_m;
3438 : }
3439 : }
3440 : if (cache_hit)
3441 : goto done;
3442 : }
3443 :
3444 2715041 : done:
3445 : /* If best_cost has not decreased, we have not found any algorithm. */
3446 12332642 : if (!CHEAPER_MULT_COST (&best_cost, cost_limit))
3447 : {
3448 : /* We failed to find an algorithm. Record alg_impossible for
3449 : this case (that is, <T, MODE, COST_LIMIT>) so that next time
3450 : we are asked to find an algorithm for T within the same or
3451 : lower COST_LIMIT, we can immediately return to the
3452 : caller. */
3453 4890331 : entry_ptr->t = t;
3454 4890331 : entry_ptr->mode = mode;
3455 4890331 : entry_ptr->speed = speed;
3456 4890331 : entry_ptr->alg = alg_impossible;
3457 4890331 : entry_ptr->cost = *cost_limit;
3458 4890331 : return;
3459 : }
3460 :
3461 : /* Cache the result. */
3462 7442311 : if (!cache_hit)
3463 : {
3464 722024 : entry_ptr->t = t;
3465 722024 : entry_ptr->mode = mode;
3466 722024 : entry_ptr->speed = speed;
3467 722024 : entry_ptr->alg = best_alg->op[best_alg->ops];
3468 722024 : entry_ptr->cost.cost = best_cost.cost;
3469 722024 : entry_ptr->cost.latency = best_cost.latency;
3470 : }
3471 :
3472 : /* If we are getting a too long sequence for `struct algorithm'
3473 : to record, make this search fail. */
3474 7442311 : if (best_alg->ops == MAX_BITS_PER_WORD)
3475 : return;
3476 :
3477 : /* Copy the algorithm from temporary space to the space at alg_out.
3478 : We avoid using structure assignment because the majority of
3479 : best_alg is normally undefined, and this is a critical function. */
3480 7442311 : alg_out->ops = best_alg->ops + 1;
3481 7442311 : alg_out->cost = best_cost;
3482 7442311 : memcpy (alg_out->op, best_alg->op,
3483 7442311 : alg_out->ops * sizeof *alg_out->op);
3484 7442311 : memcpy (alg_out->log, best_alg->log,
3485 : alg_out->ops * sizeof *alg_out->log);
3486 : }
3487 :
3488 : /* Find the cheapest way of multiplying a value of mode MODE by VAL.
3489 : Try three variations:
3490 :
3491 : - a shift/add sequence based on VAL itself
3492 : - a shift/add sequence based on -VAL, followed by a negation
3493 : - a shift/add sequence based on VAL - 1, followed by an addition.
3494 :
3495 : Return true if the cheapest of these cost less than MULT_COST,
3496 : describing the algorithm in *ALG and final fixup in *VARIANT. */
3497 :
3498 : bool
3499 7359174 : choose_mult_variant (machine_mode mode, HOST_WIDE_INT val,
3500 : struct algorithm *alg, enum mult_variant *variant,
3501 : int mult_cost)
3502 : {
3503 7359174 : struct algorithm alg2;
3504 7359174 : struct mult_cost limit;
3505 7359174 : int op_cost;
3506 7359174 : bool speed = optimize_insn_for_speed_p ();
3507 :
3508 : /* Fail quickly for impossible bounds. */
3509 7359174 : if (mult_cost < 0)
3510 : return false;
3511 :
3512 : /* Ensure that mult_cost provides a reasonable upper bound.
3513 : Any constant multiplication can be performed with less
3514 : than 2 * bits additions. */
3515 14715186 : op_cost = 2 * GET_MODE_UNIT_BITSIZE (mode) * add_cost (speed, mode);
3516 7357593 : if (mult_cost > op_cost)
3517 : mult_cost = op_cost;
3518 :
3519 7357593 : *variant = basic_variant;
3520 7357593 : limit.cost = mult_cost;
3521 7357593 : limit.latency = mult_cost;
3522 7357593 : synth_mult (alg, val, &limit, mode);
3523 :
3524 : /* This works only if the inverted value actually fits in an
3525 : `unsigned int' */
3526 14715186 : if (HOST_BITS_PER_INT >= GET_MODE_UNIT_BITSIZE (mode))
3527 : {
3528 2638337 : op_cost = neg_cost (speed, mode);
3529 2638337 : if (MULT_COST_LESS (&alg->cost, mult_cost))
3530 : {
3531 2530549 : limit.cost = alg->cost.cost - op_cost;
3532 2530549 : limit.latency = alg->cost.latency - op_cost;
3533 : }
3534 : else
3535 : {
3536 107788 : limit.cost = mult_cost - op_cost;
3537 107788 : limit.latency = mult_cost - op_cost;
3538 : }
3539 :
3540 2638337 : synth_mult (&alg2, -val, &limit, mode);
3541 2638337 : alg2.cost.cost += op_cost;
3542 2638337 : alg2.cost.latency += op_cost;
3543 2638337 : if (CHEAPER_MULT_COST (&alg2.cost, &alg->cost))
3544 14756 : *alg = alg2, *variant = negate_variant;
3545 : }
3546 :
3547 : /* This proves very useful for division-by-constant. */
3548 7357593 : op_cost = add_cost (speed, mode);
3549 7357593 : if (MULT_COST_LESS (&alg->cost, mult_cost))
3550 : {
3551 6502714 : limit.cost = alg->cost.cost - op_cost;
3552 6502714 : limit.latency = alg->cost.latency - op_cost;
3553 : }
3554 : else
3555 : {
3556 854879 : limit.cost = mult_cost - op_cost;
3557 854879 : limit.latency = mult_cost - op_cost;
3558 : }
3559 :
3560 7357593 : if (val != HOST_WIDE_INT_MIN
3561 7357603 : || GET_MODE_UNIT_PRECISION (mode) == HOST_BITS_PER_WIDE_INT)
3562 : {
3563 7357593 : synth_mult (&alg2, val - HOST_WIDE_INT_1U, &limit, mode);
3564 7357593 : alg2.cost.cost += op_cost;
3565 7357593 : alg2.cost.latency += op_cost;
3566 7357593 : if (CHEAPER_MULT_COST (&alg2.cost, &alg->cost))
3567 2752 : *alg = alg2, *variant = add_variant;
3568 : }
3569 :
3570 7357593 : return MULT_COST_LESS (&alg->cost, mult_cost);
3571 : }
3572 :
3573 : /* A subroutine of expand_mult, used for constant multiplications.
3574 : Multiply OP0 by VAL in mode MODE, storing the result in TARGET if
3575 : convenient. Use the shift/add sequence described by ALG and apply
3576 : the final fixup specified by VARIANT. */
3577 :
3578 : static rtx
3579 141123 : expand_mult_const (machine_mode mode, rtx op0, HOST_WIDE_INT val,
3580 : rtx target, const struct algorithm *alg,
3581 : enum mult_variant variant)
3582 : {
3583 141123 : unsigned HOST_WIDE_INT val_so_far;
3584 141123 : rtx_insn *insn;
3585 141123 : rtx accum, tem;
3586 141123 : int opno;
3587 141123 : machine_mode nmode;
3588 :
3589 : /* Avoid referencing memory over and over and invalid sharing
3590 : on SUBREGs. */
3591 141123 : op0 = force_reg (mode, op0);
3592 :
3593 : /* ACCUM starts out either as OP0 or as a zero, depending on
3594 : the first operation. */
3595 :
3596 141123 : if (alg->op[0] == alg_zero)
3597 : {
3598 5492 : accum = copy_to_mode_reg (mode, CONST0_RTX (mode));
3599 5492 : val_so_far = 0;
3600 : }
3601 135631 : else if (alg->op[0] == alg_m)
3602 : {
3603 135631 : accum = copy_to_mode_reg (mode, op0);
3604 135631 : val_so_far = 1;
3605 : }
3606 : else
3607 0 : gcc_unreachable ();
3608 :
3609 401932 : for (opno = 1; opno < alg->ops; opno++)
3610 : {
3611 260809 : int log = alg->log[opno];
3612 260809 : rtx shift_subtarget = optimize ? 0 : accum;
3613 243923 : rtx add_target
3614 141123 : = (opno == alg->ops - 1 && target != 0 && variant != add_variant
3615 42520 : && !optimize)
3616 260809 : ? target : 0;
3617 260809 : rtx accum_target = optimize ? 0 : accum;
3618 260809 : rtx accum_inner;
3619 :
3620 260809 : switch (alg->op[opno])
3621 : {
3622 113944 : case alg_shift:
3623 113944 : tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
3624 : /* REG_EQUAL note will be attached to the following insn. */
3625 113944 : emit_move_insn (accum, tem);
3626 113944 : val_so_far <<= log;
3627 113944 : break;
3628 :
3629 7073 : case alg_add_t_m2:
3630 7073 : tem = expand_shift (LSHIFT_EXPR, mode, op0, log, NULL_RTX, 0);
3631 14146 : accum = force_operand (gen_rtx_PLUS (mode, accum, tem),
3632 : add_target ? add_target : accum_target);
3633 7073 : val_so_far += HOST_WIDE_INT_1U << log;
3634 7073 : break;
3635 :
3636 20398 : case alg_sub_t_m2:
3637 20398 : tem = expand_shift (LSHIFT_EXPR, mode, op0, log, NULL_RTX, 0);
3638 40796 : accum = force_operand (gen_rtx_MINUS (mode, accum, tem),
3639 : add_target ? add_target : accum_target);
3640 20398 : val_so_far -= HOST_WIDE_INT_1U << log;
3641 20398 : break;
3642 :
3643 118155 : case alg_add_t2_m:
3644 118155 : accum = expand_shift (LSHIFT_EXPR, mode, accum,
3645 118155 : log, shift_subtarget, 0);
3646 236310 : accum = force_operand (gen_rtx_PLUS (mode, accum, op0),
3647 : add_target ? add_target : accum_target);
3648 118155 : val_so_far = (val_so_far << log) + 1;
3649 118155 : break;
3650 :
3651 0 : case alg_sub_t2_m:
3652 0 : accum = expand_shift (LSHIFT_EXPR, mode, accum,
3653 0 : log, shift_subtarget, 0);
3654 0 : accum = force_operand (gen_rtx_MINUS (mode, accum, op0),
3655 : add_target ? add_target : accum_target);
3656 0 : val_so_far = (val_so_far << log) - 1;
3657 0 : break;
3658 :
3659 1104 : case alg_add_factor:
3660 1104 : tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
3661 2208 : accum = force_operand (gen_rtx_PLUS (mode, accum, tem),
3662 : add_target ? add_target : accum_target);
3663 1104 : val_so_far += val_so_far << log;
3664 1104 : break;
3665 :
3666 135 : case alg_sub_factor:
3667 135 : tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
3668 270 : accum = force_operand (gen_rtx_MINUS (mode, tem, accum),
3669 : (add_target
3670 135 : ? add_target : (optimize ? 0 : tem)));
3671 135 : val_so_far = (val_so_far << log) - val_so_far;
3672 135 : break;
3673 :
3674 0 : default:
3675 0 : gcc_unreachable ();
3676 : }
3677 :
3678 260809 : if (SCALAR_INT_MODE_P (mode))
3679 : {
3680 : /* Write a REG_EQUAL note on the last insn so that we can cse
3681 : multiplication sequences. Note that if ACCUM is a SUBREG,
3682 : we've set the inner register and must properly indicate that. */
3683 250786 : tem = op0, nmode = mode;
3684 250786 : accum_inner = accum;
3685 250786 : if (GET_CODE (accum) == SUBREG)
3686 : {
3687 0 : accum_inner = SUBREG_REG (accum);
3688 0 : nmode = GET_MODE (accum_inner);
3689 0 : tem = gen_lowpart (nmode, op0);
3690 : }
3691 :
3692 : /* Don't add a REG_EQUAL note if tem is a paradoxical SUBREG.
3693 : In that case, only the low bits of accum would be guaranteed to
3694 : be equal to the content of the REG_EQUAL note, the upper bits
3695 : can be anything. */
3696 250786 : if (!paradoxical_subreg_p (tem))
3697 : {
3698 250786 : insn = get_last_insn ();
3699 250786 : wide_int wval_so_far
3700 250786 : = wi::uhwi (val_so_far,
3701 250786 : GET_MODE_PRECISION (as_a <scalar_mode> (nmode)));
3702 250786 : rtx c = immed_wide_int_const (wval_so_far, nmode);
3703 250786 : set_dst_reg_note (insn, REG_EQUAL, gen_rtx_MULT (nmode, tem, c),
3704 : accum_inner);
3705 250786 : }
3706 : }
3707 : }
3708 :
3709 141123 : if (variant == negate_variant)
3710 : {
3711 587 : val_so_far = -val_so_far;
3712 587 : accum = expand_unop (mode, neg_optab, accum, target, 0);
3713 : }
3714 140536 : else if (variant == add_variant)
3715 : {
3716 23 : val_so_far = val_so_far + 1;
3717 23 : accum = force_operand (gen_rtx_PLUS (mode, accum, op0), target);
3718 : }
3719 :
3720 : /* Compare only the bits of val and val_so_far that are significant
3721 : in the result mode, to avoid sign-/zero-extension confusion. */
3722 141123 : nmode = GET_MODE_INNER (mode);
3723 141123 : val &= GET_MODE_MASK (nmode);
3724 141123 : val_so_far &= GET_MODE_MASK (nmode);
3725 141123 : gcc_assert (val == (HOST_WIDE_INT) val_so_far);
3726 :
3727 141123 : return accum;
3728 : }
3729 :
3730 : /* Perform a multiplication and return an rtx for the result.
3731 : MODE is mode of value; OP0 and OP1 are what to multiply (rtx's);
3732 : TARGET is a suggestion for where to store the result (an rtx).
3733 :
3734 : We check specially for a constant integer as OP1.
3735 : If you want this check for OP0 as well, then before calling
3736 : you should swap the two operands if OP0 would be constant. */
3737 :
3738 : rtx
3739 1126807 : expand_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
3740 : int unsignedp, bool no_libcall)
3741 : {
3742 1126807 : enum mult_variant variant;
3743 1126807 : struct algorithm algorithm;
3744 1126807 : rtx scalar_op1;
3745 1126807 : int max_cost;
3746 1126807 : bool speed = optimize_insn_for_speed_p ();
3747 1126807 : bool do_trapv = flag_trapv && SCALAR_INT_MODE_P (mode) && !unsignedp;
3748 :
3749 1126807 : if (CONSTANT_P (op0))
3750 275 : std::swap (op0, op1);
3751 :
3752 : /* For vectors, there are several simplifications that can be made if
3753 : all elements of the vector constant are identical. */
3754 1126807 : scalar_op1 = unwrap_const_vec_duplicate (op1);
3755 :
3756 1126807 : if (INTEGRAL_MODE_P (mode))
3757 : {
3758 1007357 : rtx fake_reg;
3759 1007357 : HOST_WIDE_INT coeff;
3760 1007357 : bool is_neg;
3761 1007357 : int mode_bitsize;
3762 :
3763 1007357 : if (op1 == CONST0_RTX (mode))
3764 : return op1;
3765 1007357 : if (op1 == CONST1_RTX (mode))
3766 : return op0;
3767 968254 : if (op1 == CONSTM1_RTX (mode))
3768 2830 : return expand_unop (mode, do_trapv ? negv_optab : neg_optab,
3769 1415 : op0, target, 0);
3770 :
3771 966839 : if (do_trapv)
3772 34 : goto skip_synth;
3773 :
3774 : /* If mode is integer vector mode, check if the backend supports
3775 : vector lshift (by scalar or vector) at all. If not, we can't use
3776 : synthesized multiply. */
3777 966805 : if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
3778 14964 : && optab_handler (vashl_optab, mode) == CODE_FOR_nothing
3779 979352 : && optab_handler (ashl_optab, mode) == CODE_FOR_nothing)
3780 0 : goto skip_synth;
3781 :
3782 : /* These are the operations that are potentially turned into
3783 : a sequence of shifts and additions. */
3784 966805 : mode_bitsize = GET_MODE_UNIT_BITSIZE (mode);
3785 :
3786 : /* synth_mult does an `unsigned int' multiply. As long as the mode is
3787 : less than or equal in size to `unsigned int' this doesn't matter.
3788 : If the mode is larger than `unsigned int', then synth_mult works
3789 : only if the constant value exactly fits in an `unsigned int' without
3790 : any truncation. This means that multiplying by negative values does
3791 : not work; results are off by 2^32 on a 32 bit machine. */
3792 966805 : if (CONST_INT_P (scalar_op1))
3793 : {
3794 705692 : coeff = INTVAL (scalar_op1);
3795 705692 : is_neg = coeff < 0;
3796 : }
3797 : #if TARGET_SUPPORTS_WIDE_INT
3798 261113 : else if (CONST_WIDE_INT_P (scalar_op1))
3799 : #else
3800 : else if (CONST_DOUBLE_AS_INT_P (scalar_op1))
3801 : #endif
3802 : {
3803 1154 : machine_mode scalar_mode = VECTOR_MODE_P (mode)
3804 1154 : ? GET_MODE_INNER (mode) : mode;
3805 1154 : int shift = wi::exact_log2 (rtx_mode_t (scalar_op1, scalar_mode));
3806 : /* Perfect power of 2 (other than 1, which is handled above). */
3807 1154 : if (shift > 0)
3808 106 : return expand_shift (LSHIFT_EXPR, mode, op0,
3809 106 : shift, target, unsignedp);
3810 : else
3811 1048 : goto skip_synth;
3812 : }
3813 : else
3814 259959 : goto skip_synth;
3815 :
3816 : /* We used to test optimize here, on the grounds that it's better to
3817 : produce a smaller program when -O is not used. But this causes
3818 : such a terrible slowdown sometimes that it seems better to always
3819 : use synth_mult. */
3820 :
3821 : /* Special case powers of two. */
3822 705692 : if (EXACT_POWER_OF_2_OR_ZERO_P (coeff)
3823 475674 : && !(is_neg && mode_bitsize > HOST_BITS_PER_WIDE_INT))
3824 475666 : return expand_shift (LSHIFT_EXPR, mode, op0,
3825 951332 : floor_log2 (coeff), target, unsignedp);
3826 :
3827 230026 : fake_reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
3828 :
3829 : /* Attempt to handle multiplication of DImode values by negative
3830 : coefficients, by performing the multiplication by a positive
3831 : multiplier and then inverting the result. */
3832 230026 : if (is_neg && mode_bitsize > HOST_BITS_PER_WIDE_INT)
3833 : {
3834 : /* Its safe to use -coeff even for INT_MIN, as the
3835 : result is interpreted as an unsigned coefficient.
3836 : Exclude cost of op0 from max_cost to match the cost
3837 : calculation of the synth_mult. */
3838 218 : coeff = -(unsigned HOST_WIDE_INT) coeff;
3839 218 : max_cost = (set_src_cost (gen_rtx_MULT (mode, fake_reg, op1),
3840 : mode, speed)
3841 218 : - neg_cost (speed, mode));
3842 218 : if (max_cost <= 0)
3843 0 : goto skip_synth;
3844 :
3845 : /* Special case powers of two. */
3846 218 : if (EXACT_POWER_OF_2_OR_ZERO_P (coeff))
3847 : {
3848 340 : rtx temp = expand_shift (LSHIFT_EXPR, mode, op0,
3849 170 : floor_log2 (coeff), target, unsignedp);
3850 170 : return expand_unop (mode, neg_optab, temp, target, 0);
3851 : }
3852 :
3853 48 : if (choose_mult_variant (mode, coeff, &algorithm, &variant,
3854 : max_cost))
3855 : {
3856 47 : rtx temp = expand_mult_const (mode, op0, coeff, NULL_RTX,
3857 : &algorithm, variant);
3858 47 : return expand_unop (mode, neg_optab, temp, target, 0);
3859 : }
3860 1 : goto skip_synth;
3861 : }
3862 :
3863 : /* Exclude cost of op0 from max_cost to match the cost
3864 : calculation of the synth_mult. */
3865 229808 : max_cost = set_src_cost (gen_rtx_MULT (mode, fake_reg, op1), mode, speed);
3866 229808 : if (choose_mult_variant (mode, coeff, &algorithm, &variant, max_cost))
3867 139954 : return expand_mult_const (mode, op0, coeff, target,
3868 139954 : &algorithm, variant);
3869 : }
3870 89854 : skip_synth:
3871 :
3872 : /* Expand x*2.0 as x+x. */
3873 38459 : if (CONST_DOUBLE_AS_FLOAT_P (scalar_op1)
3874 508805 : && real_equal (CONST_DOUBLE_REAL_VALUE (scalar_op1), &dconst2))
3875 : {
3876 6016 : op0 = force_reg (GET_MODE (op0), op0);
3877 12032 : return expand_binop (mode, add_optab, op0, op0,
3878 : target, unsignedp,
3879 6016 : no_libcall ? OPTAB_WIDEN : OPTAB_LIB_WIDEN);
3880 : }
3881 :
3882 : /* This used to use umul_optab if unsigned, but for non-widening multiply
3883 : there is no difference between signed and unsigned. */
3884 1392956 : op0 = expand_binop (mode, do_trapv ? smulv_optab : smul_optab,
3885 : op0, op1, target, unsignedp,
3886 : no_libcall ? OPTAB_WIDEN : OPTAB_LIB_WIDEN);
3887 464330 : gcc_assert (op0 || no_libcall);
3888 : return op0;
3889 : }
3890 :
3891 : /* Return a cost estimate for multiplying a register by the given
3892 : COEFFicient in the given MODE and SPEED. */
3893 :
3894 : int
3895 6783926 : mult_by_coeff_cost (HOST_WIDE_INT coeff, machine_mode mode, bool speed)
3896 : {
3897 6783926 : int max_cost;
3898 6783926 : struct algorithm algorithm;
3899 6783926 : enum mult_variant variant;
3900 :
3901 6783926 : rtx fake_reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
3902 6783926 : max_cost = set_src_cost (gen_rtx_MULT (mode, fake_reg, fake_reg),
3903 : mode, speed);
3904 6783926 : if (choose_mult_variant (mode, coeff, &algorithm, &variant, max_cost))
3905 6022802 : return algorithm.cost.cost;
3906 : else
3907 : return max_cost;
3908 : }
3909 :
3910 : /* Perform a widening multiplication and return an rtx for the result.
3911 : MODE is mode of value; OP0 and OP1 are what to multiply (rtx's);
3912 : TARGET is a suggestion for where to store the result (an rtx).
3913 : THIS_OPTAB is the optab we should use, it must be either umul_widen_optab
3914 : or smul_widen_optab.
3915 :
3916 : We check specially for a constant integer as OP1, comparing the
3917 : cost of a widening multiply against the cost of a sequence of shifts
3918 : and adds. */
3919 :
3920 : rtx
3921 24739 : expand_widening_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
3922 : int unsignedp, optab this_optab)
3923 : {
3924 24739 : bool speed = optimize_insn_for_speed_p ();
3925 24739 : rtx cop1;
3926 :
3927 24739 : if (CONST_INT_P (op1)
3928 5738 : && GET_MODE (op0) != VOIDmode
3929 5738 : && (cop1 = convert_modes (mode, GET_MODE (op0), op1,
3930 : this_optab == umul_widen_optab))
3931 5738 : && CONST_INT_P (cop1)
3932 29929 : && (INTVAL (cop1) >= 0
3933 27634 : || HWI_COMPUTABLE_MODE_P (mode)))
3934 : {
3935 4951 : HOST_WIDE_INT coeff = INTVAL (cop1);
3936 4951 : int max_cost;
3937 4951 : enum mult_variant variant;
3938 4951 : struct algorithm algorithm;
3939 :
3940 4951 : if (coeff == 0)
3941 2056 : return CONST0_RTX (mode);
3942 :
3943 : /* Special case powers of two. */
3944 4575 : if (EXACT_POWER_OF_2_OR_ZERO_P (coeff))
3945 : {
3946 570 : op0 = convert_to_mode (mode, op0, this_optab == umul_widen_optab);
3947 570 : return expand_shift (LSHIFT_EXPR, mode, op0,
3948 570 : floor_log2 (coeff), target, unsignedp);
3949 : }
3950 :
3951 : /* Exclude cost of op0 from max_cost to match the cost
3952 : calculation of the synth_mult. */
3953 4005 : max_cost = mul_widen_cost (speed, mode);
3954 4005 : if (choose_mult_variant (mode, coeff, &algorithm, &variant,
3955 : max_cost))
3956 : {
3957 1110 : op0 = convert_to_mode (mode, op0, this_optab == umul_widen_optab);
3958 1110 : return expand_mult_const (mode, op0, coeff, target,
3959 1110 : &algorithm, variant);
3960 : }
3961 : }
3962 22683 : return expand_binop (mode, this_optab, op0, op1, target,
3963 22683 : unsignedp, OPTAB_LIB_WIDEN);
3964 : }
3965 :
3966 : /* Choose a minimal N + 1 bit approximation to 2**K / D that can be used to
3967 : replace division by D, put the least significant N bits of the result in
3968 : *MULTIPLIER_PTR, the value K - N in *POST_SHIFT_PTR, and return the most
3969 : significant bit.
3970 :
3971 : The width of operations is N (should be <= HOST_BITS_PER_WIDE_INT), the
3972 : needed precision is PRECISION (should be <= N).
3973 :
3974 : PRECISION should be as small as possible so this function can choose the
3975 : multiplier more freely. If PRECISION is <= N - 1, the most significant
3976 : bit returned by the function will be zero.
3977 :
3978 : Using this function, x / D is equal to (x*m) / 2**N >> (*POST_SHIFT_PTR),
3979 : where m is the full N + 1 bit multiplier. */
3980 :
3981 : unsigned HOST_WIDE_INT
3982 75459 : choose_multiplier (unsigned HOST_WIDE_INT d, int n, int precision,
3983 : unsigned HOST_WIDE_INT *multiplier_ptr,
3984 : int *post_shift_ptr)
3985 : {
3986 75459 : int lgup, post_shift;
3987 75459 : int pow1, pow2;
3988 :
3989 : /* lgup = ceil(log2(d)) */
3990 : /* Assuming d > 1, we have d >= 2^(lgup-1) + 1 */
3991 75459 : lgup = ceil_log2 (d);
3992 :
3993 75459 : gcc_assert (lgup <= n);
3994 75459 : gcc_assert (lgup <= precision);
3995 :
3996 75459 : pow1 = n + lgup;
3997 75459 : pow2 = n + lgup - precision;
3998 :
3999 : /* mlow = 2^(n + lgup)/d */
4000 : /* Trivially from above we have mlow < 2^(n+1) */
4001 75459 : wide_int val = wi::set_bit_in_zero (pow1, HOST_BITS_PER_DOUBLE_INT);
4002 75459 : wide_int mlow = wi::udiv_trunc (val, d);
4003 :
4004 : /* mhigh = (2^(n + lgup) + 2^(n + lgup - precision))/d */
4005 : /* From above we have mhigh < 2^(n+1) assuming lgup <= precision */
4006 : /* From precision <= n, the difference between the numerators of mhigh and
4007 : mlow is >= 2^lgup >= d. Therefore the difference of the quotients in
4008 : the Euclidean division by d is at least 1, so we have mlow < mhigh and
4009 : the exact value of 2^(n + lgup)/d lies in the interval [mlow; mhigh). */
4010 75459 : val |= wi::set_bit_in_zero (pow2, HOST_BITS_PER_DOUBLE_INT);
4011 75459 : wide_int mhigh = wi::udiv_trunc (val, d);
4012 :
4013 : /* Reduce to lowest terms. */
4014 : /* If precision <= n - 1, then the difference between the numerators of
4015 : mhigh and mlow is >= 2^(lgup + 1) >= 2 * 2^lgup >= 2 * d. Therefore
4016 : the difference of the quotients in the Euclidean division by d is at
4017 : least 2, which means that mhigh and mlow differ by at least one bit
4018 : not in the last place. The conclusion is that the first iteration of
4019 : the loop below completes and shifts mhigh and mlow by 1 bit, which in
4020 : particular means that mhigh < 2^n, that is to say, the most significant
4021 : bit in the n + 1 bit value is zero. */
4022 269445 : for (post_shift = lgup; post_shift > 0; post_shift--)
4023 : {
4024 188212 : unsigned HOST_WIDE_INT ml_lo = wi::extract_uhwi (mlow, 1,
4025 : HOST_BITS_PER_WIDE_INT);
4026 188212 : unsigned HOST_WIDE_INT mh_lo = wi::extract_uhwi (mhigh, 1,
4027 : HOST_BITS_PER_WIDE_INT);
4028 188212 : if (ml_lo >= mh_lo)
4029 : break;
4030 :
4031 118527 : mlow = wi::uhwi (ml_lo, HOST_BITS_PER_DOUBLE_INT);
4032 118527 : mhigh = wi::uhwi (mh_lo, HOST_BITS_PER_DOUBLE_INT);
4033 : }
4034 :
4035 75459 : *post_shift_ptr = post_shift;
4036 :
4037 75459 : if (n < HOST_BITS_PER_WIDE_INT)
4038 : {
4039 42493 : unsigned HOST_WIDE_INT mask = (HOST_WIDE_INT_1U << n) - 1;
4040 42493 : *multiplier_ptr = mhigh.to_uhwi () & mask;
4041 42493 : return mhigh.to_uhwi () > mask;
4042 : }
4043 : else
4044 : {
4045 32966 : *multiplier_ptr = mhigh.to_uhwi ();
4046 32966 : return wi::extract_uhwi (mhigh, HOST_BITS_PER_WIDE_INT, 1);
4047 : }
4048 75459 : }
4049 :
4050 : /* Compute the inverse of X mod 2**N, i.e., find Y such that X * Y is congruent
4051 : to 1 modulo 2**N, assuming that X is odd. Bézout's lemma guarantees that Y
4052 : exists for any given positive N. */
4053 :
4054 : static unsigned HOST_WIDE_INT
4055 46011 : invert_mod2n (unsigned HOST_WIDE_INT x, int n)
4056 : {
4057 46011 : gcc_assert ((x & 1) == 1);
4058 :
4059 : /* The algorithm notes that the choice Y = X satisfies X*Y == 1 mod 2^3,
4060 : since X is odd. Then each Newton-Raphson iteration doubles the number
4061 : of bits of significance in Y (Hensel's lemma). */
4062 :
4063 47029 : const unsigned HOST_WIDE_INT mask
4064 : = (n == HOST_BITS_PER_WIDE_INT
4065 46011 : ? HOST_WIDE_INT_M1U
4066 1018 : : (HOST_WIDE_INT_1U << n) - 1);
4067 46011 : unsigned HOST_WIDE_INT y = x;
4068 46011 : int nbit = 3;
4069 :
4070 275018 : while (nbit < n)
4071 : {
4072 229007 : y = y * (2 - x*y) & mask; /* Modulo 2^N */
4073 229007 : nbit *= 2;
4074 : }
4075 :
4076 46011 : return y;
4077 : }
4078 :
4079 : /* Emit code to adjust ADJ_OPERAND after multiplication of wrong signedness
4080 : flavor of OP0 and OP1. ADJ_OPERAND is already the high half of the
4081 : product OP0 x OP1. If UNSIGNEDP is nonzero, adjust the signed product
4082 : to become unsigned, if UNSIGNEDP is zero, adjust the unsigned product to
4083 : become signed.
4084 :
4085 : The result is put in TARGET if that is convenient.
4086 :
4087 : MODE is the mode of operation. */
4088 :
4089 : rtx
4090 0 : expand_mult_highpart_adjust (scalar_int_mode mode, rtx adj_operand, rtx op0,
4091 : rtx op1, rtx target, int unsignedp)
4092 : {
4093 0 : rtx tem;
4094 0 : enum rtx_code adj_code = unsignedp ? PLUS : MINUS;
4095 :
4096 0 : tem = expand_shift (RSHIFT_EXPR, mode, op0,
4097 0 : GET_MODE_BITSIZE (mode) - 1, NULL_RTX, 0);
4098 0 : tem = expand_and (mode, tem, op1, NULL_RTX);
4099 0 : adj_operand
4100 0 : = force_operand (gen_rtx_fmt_ee (adj_code, mode, adj_operand, tem),
4101 : adj_operand);
4102 :
4103 0 : tem = expand_shift (RSHIFT_EXPR, mode, op1,
4104 0 : GET_MODE_BITSIZE (mode) - 1, NULL_RTX, 0);
4105 0 : tem = expand_and (mode, tem, op0, NULL_RTX);
4106 0 : target = force_operand (gen_rtx_fmt_ee (adj_code, mode, adj_operand, tem),
4107 : target);
4108 :
4109 0 : return target;
4110 : }
4111 :
4112 : /* Subroutine of expmed_mult_highpart. Return the MODE high part of OP. */
4113 :
4114 : static rtx
4115 20035 : extract_high_half (scalar_int_mode mode, rtx op)
4116 : {
4117 20035 : if (mode == word_mode)
4118 0 : return gen_highpart (mode, op);
4119 :
4120 20035 : scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
4121 :
4122 40070 : op = expand_shift (RSHIFT_EXPR, wider_mode, op,
4123 20035 : GET_MODE_BITSIZE (mode), 0, 1);
4124 20035 : return convert_modes (mode, wider_mode, op, 0);
4125 : }
4126 :
4127 : /* Like expmed_mult_highpart, but only consider using multiplication optab. */
4128 :
4129 : rtx
4130 46481 : expmed_mult_highpart_optab (scalar_int_mode mode, rtx op0, rtx op1,
4131 : rtx target, int unsignedp, int max_cost)
4132 : {
4133 46481 : const scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
4134 46481 : const bool speed = optimize_insn_for_speed_p ();
4135 46481 : const int size = GET_MODE_BITSIZE (mode);
4136 46481 : optab moptab;
4137 46481 : rtx tem;
4138 :
4139 : /* Firstly, try using a multiplication insn that only generates the needed
4140 : high part of the product, and in the sign flavor of unsignedp. */
4141 46481 : if (mul_highpart_cost (speed, mode) < max_cost)
4142 : {
4143 44449 : moptab = unsignedp ? umul_highpart_optab : smul_highpart_optab;
4144 44449 : tem = expand_binop (mode, moptab, op0, op1, target, unsignedp,
4145 : OPTAB_DIRECT);
4146 44449 : if (tem)
4147 : return tem;
4148 : }
4149 :
4150 : /* Secondly, same as above, but use sign flavor opposite of unsignedp.
4151 : Need to adjust the result after the multiplication. */
4152 22233 : if (size - 1 < BITS_PER_WORD
4153 44210 : && (mul_highpart_cost (speed, mode)
4154 21977 : + 2 * shift_cost (speed, mode, size-1)
4155 21977 : + 4 * add_cost (speed, mode) < max_cost))
4156 : {
4157 5499 : moptab = unsignedp ? smul_highpart_optab : umul_highpart_optab;
4158 5499 : tem = expand_binop (mode, moptab, op0, op1, target, !unsignedp,
4159 : OPTAB_DIRECT);
4160 5499 : if (tem)
4161 : /* We used the wrong signedness. Adjust the result. */
4162 0 : return expand_mult_highpart_adjust (mode, tem, op0, op1, tem,
4163 0 : unsignedp);
4164 : }
4165 :
4166 : /* Try widening multiplication. */
4167 22233 : moptab = unsignedp ? umul_widen_optab : smul_widen_optab;
4168 22233 : if (convert_optab_handler (moptab, wider_mode, mode) != CODE_FOR_nothing
4169 22233 : && mul_widen_cost (speed, wider_mode) < max_cost)
4170 : {
4171 388 : tem = expand_binop (wider_mode, moptab, op0, op1, NULL_RTX, unsignedp,
4172 : OPTAB_WIDEN);
4173 388 : if (tem)
4174 388 : return extract_high_half (mode, tem);
4175 : }
4176 :
4177 : /* Try widening the mode and perform a non-widening multiplication. */
4178 21845 : if (optab_handler (smul_optab, wider_mode) != CODE_FOR_nothing
4179 21240 : && size - 1 < BITS_PER_WORD
4180 43082 : && (mul_cost (speed, wider_mode) + shift_cost (speed, mode, size-1)
4181 : < max_cost))
4182 : {
4183 19635 : rtx_insn *insns;
4184 19635 : rtx wop0, wop1;
4185 :
4186 : /* We need to widen the operands, for example to ensure the
4187 : constant multiplier is correctly sign or zero extended.
4188 : Use a sequence to clean-up any instructions emitted by
4189 : the conversions if things don't work out. */
4190 19635 : start_sequence ();
4191 19635 : wop0 = convert_modes (wider_mode, mode, op0, unsignedp);
4192 19635 : wop1 = convert_modes (wider_mode, mode, op1, unsignedp);
4193 19635 : tem = expand_binop (wider_mode, smul_optab, wop0, wop1, 0,
4194 : unsignedp, OPTAB_WIDEN);
4195 19635 : insns = end_sequence ();
4196 :
4197 19635 : if (tem)
4198 : {
4199 19635 : emit_insn (insns);
4200 19635 : return extract_high_half (mode, tem);
4201 : }
4202 : }
4203 :
4204 : /* Try widening multiplication of opposite signedness, and adjust. */
4205 2210 : moptab = unsignedp ? smul_widen_optab : umul_widen_optab;
4206 2210 : if (convert_optab_handler (moptab, wider_mode, mode) != CODE_FOR_nothing
4207 456 : && size - 1 < BITS_PER_WORD
4208 3004 : && (mul_widen_cost (speed, wider_mode)
4209 397 : + 2 * shift_cost (speed, mode, size-1)
4210 397 : + 4 * add_cost (speed, mode) < max_cost))
4211 : {
4212 0 : tem = expand_binop (wider_mode, moptab, op0, op1, NULL_RTX, !unsignedp,
4213 : OPTAB_WIDEN);
4214 0 : if (tem != 0)
4215 : {
4216 0 : tem = extract_high_half (mode, tem);
4217 : /* We used the wrong signedness. Adjust the result. */
4218 0 : return expand_mult_highpart_adjust (mode, tem, op0, op1, target,
4219 0 : unsignedp);
4220 : }
4221 : }
4222 :
4223 : return 0;
4224 : }
4225 :
4226 : /* Emit code to multiply OP0 and OP1 (where OP1 is an integer constant),
4227 : putting the high half of the result in TARGET if that is convenient,
4228 : and return where the result is. If the operation cannot be performed,
4229 : 0 is returned.
4230 :
4231 : MODE is the mode of operation and result.
4232 :
4233 : UNSIGNEDP nonzero means unsigned multiply.
4234 :
4235 : MAX_COST is the total allowed cost for the expanded RTL. */
4236 :
4237 : static rtx
4238 46481 : expmed_mult_highpart (scalar_int_mode mode, rtx op0, rtx op1,
4239 : rtx target, int unsignedp, int max_cost)
4240 : {
4241 46481 : const bool speed = optimize_insn_for_speed_p ();
4242 46481 : unsigned HOST_WIDE_INT cnst1;
4243 46481 : int extra_cost;
4244 46481 : bool sign_adjust = false;
4245 46481 : enum mult_variant variant;
4246 46481 : struct algorithm alg;
4247 46481 : rtx narrow_op1, tem;
4248 :
4249 : /* We can't support modes wider than HOST_BITS_PER_INT. */
4250 46481 : gcc_assert (HWI_COMPUTABLE_MODE_P (mode));
4251 :
4252 46481 : cnst1 = INTVAL (op1) & GET_MODE_MASK (mode);
4253 46481 : narrow_op1 = gen_int_mode (INTVAL (op1), mode);
4254 :
4255 : /* We can't optimize modes wider than BITS_PER_WORD.
4256 : ??? We might be able to perform double-word arithmetic if
4257 : mode == word_mode, however all the cost calculations in
4258 : synth_mult etc. assume single-word operations. */
4259 46481 : scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
4260 96079 : if (GET_MODE_BITSIZE (wider_mode) > BITS_PER_WORD)
4261 24856 : return expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
4262 24856 : unsignedp, max_cost);
4263 :
4264 43250 : extra_cost = shift_cost (speed, mode, GET_MODE_BITSIZE (mode) - 1);
4265 :
4266 : /* Check whether we try to multiply by a negative constant. */
4267 32066 : if (!unsignedp && ((cnst1 >> (GET_MODE_BITSIZE (mode) - 1)) & 1))
4268 : {
4269 2292 : sign_adjust = true;
4270 2292 : extra_cost += add_cost (speed, mode);
4271 : }
4272 :
4273 : /* See whether shift/add multiplication is cheap enough. */
4274 21625 : if (choose_mult_variant (wider_mode, cnst1, &alg, &variant,
4275 : max_cost - extra_cost))
4276 : {
4277 : /* See whether the specialized multiplication optabs are
4278 : cheaper than the shift/add version. */
4279 39910 : tem = expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
4280 : unsignedp,
4281 19955 : alg.cost.cost + extra_cost);
4282 19955 : if (tem)
4283 : return tem;
4284 :
4285 12 : tem = convert_to_mode (wider_mode, op0, unsignedp);
4286 12 : tem = expand_mult_const (wider_mode, tem, cnst1, 0, &alg, variant);
4287 12 : tem = extract_high_half (mode, tem);
4288 :
4289 : /* Adjust result for signedness. */
4290 12 : if (sign_adjust)
4291 0 : tem = force_operand (gen_rtx_MINUS (mode, tem, op0), tem);
4292 :
4293 : return tem;
4294 : }
4295 1670 : return expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
4296 1670 : unsignedp, max_cost);
4297 : }
4298 :
4299 :
4300 : /* Expand signed modulus of OP0 by a power of two D in mode MODE. */
4301 :
4302 : static rtx
4303 2547 : expand_smod_pow2 (scalar_int_mode mode, rtx op0, HOST_WIDE_INT d)
4304 : {
4305 2547 : rtx result, temp, shift;
4306 2547 : rtx_code_label *label;
4307 2547 : int logd;
4308 2547 : int prec = GET_MODE_PRECISION (mode);
4309 :
4310 2547 : logd = floor_log2 (d);
4311 2547 : result = gen_reg_rtx (mode);
4312 :
4313 : /* Avoid conditional branches when they're expensive. */
4314 2547 : if (BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2
4315 2547 : && optimize_insn_for_speed_p ())
4316 : {
4317 2546 : rtx signmask = emit_store_flag (result, LT, op0, const0_rtx,
4318 : mode, 0, -1);
4319 2546 : if (signmask)
4320 : {
4321 2546 : HOST_WIDE_INT masklow = (HOST_WIDE_INT_1 << logd) - 1;
4322 2546 : signmask = force_reg (mode, signmask);
4323 5092 : shift = gen_int_shift_amount (mode, GET_MODE_BITSIZE (mode) - logd);
4324 :
4325 : /* Use the rtx_cost of a LSHIFTRT instruction to determine
4326 : which instruction sequence to use. If logical right shifts
4327 : are expensive the use 2 XORs, 2 SUBs and an AND, otherwise
4328 : use a LSHIFTRT, 1 ADD, 1 SUB and an AND. */
4329 :
4330 2546 : temp = gen_rtx_LSHIFTRT (mode, result, shift);
4331 2546 : if (optab_handler (lshr_optab, mode) == CODE_FOR_nothing
4332 2546 : || (set_src_cost (temp, mode, optimize_insn_for_speed_p ())
4333 : > COSTS_N_INSNS (2)))
4334 : {
4335 109 : temp = expand_binop (mode, xor_optab, op0, signmask,
4336 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4337 109 : temp = expand_binop (mode, sub_optab, temp, signmask,
4338 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4339 109 : temp = expand_binop (mode, and_optab, temp,
4340 109 : gen_int_mode (masklow, mode),
4341 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4342 109 : temp = expand_binop (mode, xor_optab, temp, signmask,
4343 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4344 109 : temp = expand_binop (mode, sub_optab, temp, signmask,
4345 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4346 : }
4347 : else
4348 : {
4349 2437 : signmask = expand_binop (mode, lshr_optab, signmask, shift,
4350 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4351 2437 : signmask = force_reg (mode, signmask);
4352 :
4353 2437 : temp = expand_binop (mode, add_optab, op0, signmask,
4354 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4355 2437 : temp = expand_binop (mode, and_optab, temp,
4356 2437 : gen_int_mode (masklow, mode),
4357 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4358 2437 : temp = expand_binop (mode, sub_optab, temp, signmask,
4359 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4360 : }
4361 : return temp;
4362 : }
4363 : }
4364 :
4365 : /* Mask contains the mode's signbit and the significant bits of the
4366 : modulus. By including the signbit in the operation, many targets
4367 : can avoid an explicit compare operation in the following comparison
4368 : against zero. */
4369 1 : wide_int mask = wi::mask (logd, false, prec);
4370 1 : mask = wi::set_bit (mask, prec - 1);
4371 :
4372 2 : temp = expand_binop (mode, and_optab, op0,
4373 1 : immed_wide_int_const (mask, mode),
4374 : result, 1, OPTAB_LIB_WIDEN);
4375 1 : if (temp != result)
4376 0 : emit_move_insn (result, temp);
4377 :
4378 1 : label = gen_label_rtx ();
4379 1 : do_cmp_and_jump (result, const0_rtx, GE, mode, label);
4380 :
4381 1 : temp = expand_binop (mode, sub_optab, result, const1_rtx, result,
4382 : 0, OPTAB_LIB_WIDEN);
4383 :
4384 1 : mask = wi::mask (logd, true, prec);
4385 2 : temp = expand_binop (mode, ior_optab, temp,
4386 1 : immed_wide_int_const (mask, mode),
4387 : result, 1, OPTAB_LIB_WIDEN);
4388 1 : temp = expand_binop (mode, add_optab, temp, const1_rtx, result,
4389 : 0, OPTAB_LIB_WIDEN);
4390 1 : if (temp != result)
4391 0 : emit_move_insn (result, temp);
4392 1 : emit_label (label);
4393 1 : return result;
4394 1 : }
4395 :
4396 : /* Expand signed division of OP0 by a power of two D in mode MODE.
4397 : This routine is only called for positive values of D. */
4398 :
4399 : static rtx
4400 10710 : expand_sdiv_pow2 (scalar_int_mode mode, rtx op0, HOST_WIDE_INT d)
4401 : {
4402 10710 : rtx temp;
4403 10710 : rtx_code_label *label;
4404 10710 : int logd;
4405 :
4406 10710 : logd = floor_log2 (d);
4407 :
4408 10710 : if (d == 2
4409 10710 : && BRANCH_COST (optimize_insn_for_speed_p (),
4410 : false) >= 1)
4411 : {
4412 6982 : temp = gen_reg_rtx (mode);
4413 6982 : temp = emit_store_flag (temp, LT, op0, const0_rtx, mode, 0, 1);
4414 6982 : if (temp != NULL_RTX)
4415 : {
4416 6982 : temp = expand_binop (mode, add_optab, temp, op0, NULL_RTX,
4417 : 0, OPTAB_LIB_WIDEN);
4418 6982 : return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
4419 : }
4420 : }
4421 :
4422 7455 : if (HAVE_conditional_move
4423 3728 : && BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2)
4424 : {
4425 3728 : rtx temp2;
4426 :
4427 3728 : start_sequence ();
4428 3728 : temp2 = copy_to_mode_reg (mode, op0);
4429 3728 : temp = expand_binop (mode, add_optab, temp2, gen_int_mode (d - 1, mode),
4430 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
4431 3728 : temp = force_reg (mode, temp);
4432 :
4433 : /* Construct "temp2 = (temp2 < 0) ? temp : temp2". */
4434 3728 : temp2 = emit_conditional_move (temp2, { LT, temp2, const0_rtx, mode },
4435 : temp, temp2, mode, 0);
4436 3728 : if (temp2)
4437 : {
4438 3663 : rtx_insn *seq = end_sequence ();
4439 3663 : emit_insn (seq);
4440 3663 : return expand_shift (RSHIFT_EXPR, mode, temp2, logd, NULL_RTX, 0);
4441 : }
4442 65 : end_sequence ();
4443 : }
4444 :
4445 65 : if (BRANCH_COST (optimize_insn_for_speed_p (),
4446 : false) >= 2)
4447 : {
4448 65 : int ushift = GET_MODE_BITSIZE (mode) - logd;
4449 :
4450 65 : temp = gen_reg_rtx (mode);
4451 65 : temp = emit_store_flag (temp, LT, op0, const0_rtx, mode, 0, -1);
4452 65 : if (temp != NULL_RTX)
4453 : {
4454 130 : if (GET_MODE_BITSIZE (mode) >= BITS_PER_WORD
4455 65 : || shift_cost (optimize_insn_for_speed_p (), mode, ushift)
4456 : > COSTS_N_INSNS (1))
4457 65 : temp = expand_binop (mode, and_optab, temp,
4458 65 : gen_int_mode (d - 1, mode),
4459 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
4460 : else
4461 0 : temp = expand_shift (RSHIFT_EXPR, mode, temp,
4462 0 : ushift, NULL_RTX, 1);
4463 65 : temp = expand_binop (mode, add_optab, temp, op0, NULL_RTX,
4464 : 0, OPTAB_LIB_WIDEN);
4465 65 : return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
4466 : }
4467 : }
4468 :
4469 0 : label = gen_label_rtx ();
4470 0 : temp = copy_to_mode_reg (mode, op0);
4471 0 : do_cmp_and_jump (temp, const0_rtx, GE, mode, label);
4472 0 : expand_inc (temp, gen_int_mode (d - 1, mode));
4473 0 : emit_label (label);
4474 0 : return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
4475 : }
4476 :
4477 : /* Emit the code to divide OP0 by OP1, putting the result in TARGET
4478 : if that is convenient, and returning where the result is.
4479 : You may request either the quotient or the remainder as the result;
4480 : specify REM_FLAG nonzero to get the remainder.
4481 :
4482 : CODE is the expression code for which kind of division this is;
4483 : it controls how rounding is done. MODE is the machine mode to use.
4484 : UNSIGNEDP nonzero means do unsigned division. */
4485 :
4486 : /* ??? For CEIL_MOD_EXPR, can compute incorrect remainder with ANDI
4487 : and then correct it by or'ing in missing high bits
4488 : if result of ANDI is nonzero.
4489 : For ROUND_MOD_EXPR, can use ANDI and then sign-extend the result.
4490 : This could optimize to a bfexts instruction.
4491 : But C doesn't use these operations, so their optimizations are
4492 : left for later. */
4493 : /* ??? For modulo, we don't actually need the highpart of the first product,
4494 : the low part will do nicely. And for small divisors, the second multiply
4495 : can also be a low-part only multiply or even be completely left out.
4496 : E.g. to calculate the remainder of a division by 3 with a 32 bit
4497 : multiply, multiply with 0x55555556 and extract the upper two bits;
4498 : the result is exact for inputs up to 0x1fffffff.
4499 : The input range can be reduced by using cross-sum rules.
4500 : For odd divisors >= 3, the following table gives right shift counts
4501 : so that if a number is shifted by an integer multiple of the given
4502 : amount, the remainder stays the same:
4503 : 2, 4, 3, 6, 10, 12, 4, 8, 18, 6, 11, 20, 18, 0, 5, 10, 12, 0, 12, 20,
4504 : 14, 12, 23, 21, 8, 0, 20, 18, 0, 0, 6, 12, 0, 22, 0, 18, 20, 30, 0, 0,
4505 : 0, 8, 0, 11, 12, 10, 36, 0, 30, 0, 0, 12, 0, 0, 0, 0, 44, 12, 24, 0,
4506 : 20, 0, 7, 14, 0, 18, 36, 0, 0, 46, 60, 0, 42, 0, 15, 24, 20, 0, 0, 33,
4507 : 0, 20, 0, 0, 18, 0, 60, 0, 0, 0, 0, 0, 40, 18, 0, 0, 12
4508 :
4509 : Cross-sum rules for even numbers can be derived by leaving as many bits
4510 : to the right alone as the divisor has zeros to the right.
4511 : E.g. if x is an unsigned 32 bit number:
4512 : (x mod 12) == (((x & 1023) + ((x >> 8) & ~3)) * 0x15555558 >> 2 * 3) >> 28
4513 : */
4514 :
4515 : /* Helper for expand_divmod's unsigned constant division. For OP0 in
4516 : INT_MODE divided by a constant needing a (SIZE+1)-bit multiplier ML
4517 : with right shift POST_SHIFT (the mh != 0 case), try to obtain
4518 : the quotient from the high part of a single multiply in a mode twice
4519 : as wide as INT_MODE. Return the quotient in INT_MODE, having emitted
4520 : the insns, or NULL_RTX when the transformation is unavailable or not
4521 : cheaper than the classic sub/shift/add sequence. EXTRA_COST is the
4522 : cost of that sequence's follow-up ops, MAX_COST bounds the multiply
4523 : and SPEED selects the cost model.
4524 :
4525 : The magic constant occupies at most 2*SIZE bits and so must fit in a
4526 : HOST_WIDE_INT (always 64 bits today; checked below). A wider INT_MODE
4527 : such as DImode -- which would need a 128-bit magic and a single-word
4528 : high-part multiply in a 2x-wide mode that common targets lack -- is
4529 : therefore excluded. */
4530 :
4531 : static rtx
4532 2340 : expand_wide_mulh_udiv (scalar_int_mode int_mode, rtx op0,
4533 : unsigned HOST_WIDE_INT ml, int size, int post_shift,
4534 : int extra_cost, int max_cost, bool speed)
4535 : {
4536 2340 : scalar_int_mode wide_mode;
4537 :
4538 : /* We need POST_SHIFT >= 1, a wider integer mode that still fits in a
4539 : word, and the pre-shifted magic constant to fit in a HOST_WIDE_INT. */
4540 2340 : if (post_shift < 1
4541 2340 : || !GET_MODE_2XWIDER_MODE (int_mode).exists (&wide_mode)
4542 2433 : || GET_MODE_BITSIZE (wide_mode) > BITS_PER_WORD
4543 3384 : || GET_MODE_BITSIZE (wide_mode) > HOST_BITS_PER_WIDE_INT)
4544 : return NULL_RTX;
4545 :
4546 : /* The caller obtained ML and POST_SHIFT from choose_multiplier, which
4547 : guarantees POST_SHIFT <= ceil (log2 (d)) <= SIZE for a SIZE-bit
4548 : divisor d, so the shift count below is non-negative. */
4549 1044 : gcc_checking_assert (post_shift <= size);
4550 :
4551 : /* Pre-shift the (SIZE+1)-bit magic constant (2^SIZE + ML) by
4552 : (SIZE - POST_SHIFT) so that the quotient ends up in the high part
4553 : of the widened product. Since ML < 2^SIZE and POST_SHIFT >= 1, the
4554 : result is below 2^(2*SIZE) and thus fits in both WIDE_MODE and an
4555 : unsigned HOST_WIDE_INT (2*SIZE <= HOST_BITS_PER_WIDE_INT was
4556 : checked above). */
4557 1044 : unsigned HOST_WIDE_INT magic
4558 1044 : = ((HOST_WIDE_INT_1U << size) + ml) << (size - post_shift);
4559 :
4560 1044 : start_sequence ();
4561 1044 : rtx x_wide = convert_to_mode (wide_mode, op0, 1);
4562 1044 : rtx hi = expmed_mult_highpart (wide_mode, x_wide,
4563 1044 : gen_int_mode (magic, wide_mode),
4564 : NULL_RTX, 1, max_cost);
4565 1044 : rtx quotient = hi ? convert_to_mode (int_mode, hi, 1) : NULL_RTX;
4566 1044 : rtx_insn *insns = end_sequence ();
4567 :
4568 : /* Use the widened multiply only when it is no more expensive than
4569 : the classic sub/shift/add sequence. */
4570 1044 : unsigned classic_cost = mul_highpart_cost (speed, int_mode) + extra_cost;
4571 1044 : if (quotient == NULL_RTX || seq_cost (insns, speed) > classic_cost)
4572 : return NULL_RTX;
4573 :
4574 967 : emit_insn (insns);
4575 967 : return quotient;
4576 : }
4577 :
4578 : rtx
4579 242366 : expand_divmod (int rem_flag, enum tree_code code, machine_mode mode,
4580 : rtx op0, rtx op1, rtx target, int unsignedp,
4581 : enum optab_methods methods)
4582 : {
4583 242366 : machine_mode compute_mode;
4584 242366 : rtx tquotient;
4585 242366 : rtx quotient = 0, remainder = 0;
4586 242366 : rtx_insn *last;
4587 242366 : rtx_insn *insn;
4588 242366 : optab optab1, optab2;
4589 242366 : int op1_is_constant, op1_is_pow2 = 0;
4590 242366 : int max_cost, extra_cost;
4591 242366 : bool speed = optimize_insn_for_speed_p ();
4592 :
4593 242366 : op1_is_constant = CONST_INT_P (op1);
4594 242366 : if (op1_is_constant)
4595 : {
4596 138637 : wide_int ext_op1 = rtx_mode_t (op1, mode);
4597 169071 : op1_is_pow2 = (wi::popcount (ext_op1) == 1
4598 277274 : || (! unsignedp
4599 169071 : && wi::popcount (wi::neg (ext_op1)) == 1));
4600 138637 : }
4601 :
4602 : /*
4603 : This is the structure of expand_divmod:
4604 :
4605 : First comes code to fix up the operands so we can perform the operations
4606 : correctly and efficiently.
4607 :
4608 : Second comes a switch statement with code specific for each rounding mode.
4609 : For some special operands this code emits all RTL for the desired
4610 : operation, for other cases, it generates only a quotient and stores it in
4611 : QUOTIENT. The case for trunc division/remainder might leave quotient = 0,
4612 : to indicate that it has not done anything.
4613 :
4614 : Last comes code that finishes the operation. If QUOTIENT is set and
4615 : REM_FLAG is set, the remainder is computed as OP0 - QUOTIENT * OP1. If
4616 : QUOTIENT is not set, it is computed using trunc rounding.
4617 :
4618 : We try to generate special code for division and remainder when OP1 is a
4619 : constant. If |OP1| = 2**n we can use shifts and some other fast
4620 : operations. For other values of OP1, we compute a carefully selected
4621 : fixed-point approximation m = 1/OP1, and generate code that multiplies OP0
4622 : by m.
4623 :
4624 : In all cases but EXACT_DIV_EXPR, this multiplication requires the upper
4625 : half of the product. Different strategies for generating the product are
4626 : implemented in expmed_mult_highpart.
4627 :
4628 : If what we actually want is the remainder, we generate that by another
4629 : by-constant multiplication and a subtraction. */
4630 :
4631 : /* We shouldn't be called with OP1 == const1_rtx, but some of the
4632 : code below will malfunction if we are, so check here and handle
4633 : the special case if so. */
4634 242366 : if (op1 == const1_rtx)
4635 15 : return rem_flag ? const0_rtx : op0;
4636 :
4637 : /* When dividing by -1, we could get an overflow.
4638 : negv_optab can handle overflows. */
4639 242351 : if (! unsignedp && op1 == constm1_rtx)
4640 : {
4641 0 : if (rem_flag)
4642 0 : return const0_rtx;
4643 0 : return expand_unop (mode, flag_trapv && GET_MODE_CLASS (mode) == MODE_INT
4644 0 : ? negv_optab : neg_optab, op0, target, 0);
4645 : }
4646 :
4647 242351 : if (target
4648 : /* Don't use the function value register as a target
4649 : since we have to read it as well as write it,
4650 : and function-inlining gets confused by this. */
4651 242351 : && ((REG_P (target) && REG_FUNCTION_VALUE_P (target))
4652 : /* Don't clobber an operand while doing a multi-step calculation. */
4653 95289 : || ((rem_flag || op1_is_constant)
4654 74704 : && (reg_mentioned_p (target, op0)
4655 72266 : || (MEM_P (op0) && MEM_P (target))))
4656 92121 : || reg_mentioned_p (target, op1)
4657 92036 : || (MEM_P (op1) && MEM_P (target))))
4658 : target = 0;
4659 :
4660 : /* Get the mode in which to perform this computation. Normally it will
4661 : be MODE, but sometimes we can't do the desired operation in MODE.
4662 : If so, pick a wider mode in which we can do the operation. Convert
4663 : to that mode at the start to avoid repeated conversions.
4664 :
4665 : First see what operations we need. These depend on the expression
4666 : we are evaluating. (We assume that divxx3 insns exist under the
4667 : same conditions that modxx3 insns and that these insns don't normally
4668 : fail. If these assumptions are not correct, we may generate less
4669 : efficient code in some cases.)
4670 :
4671 : Then see if we find a mode in which we can open-code that operation
4672 : (either a division, modulus, or shift). Finally, check for the smallest
4673 : mode for which we can do the operation with a library call. */
4674 :
4675 : /* We might want to refine this now that we have division-by-constant
4676 : optimization. Since expmed_mult_highpart tries so many variants, it is
4677 : not straightforward to generalize this. Maybe we should make an array
4678 : of possible modes in init_expmed? Save this for GCC 2.7. */
4679 :
4680 136440 : optab1 = (op1_is_pow2
4681 242351 : ? (unsignedp ? lshr_optab : ashr_optab)
4682 161066 : : (unsignedp ? udiv_optab : sdiv_optab));
4683 324961 : optab2 = (op1_is_pow2 ? optab1
4684 161066 : : (unsignedp ? udivmod_optab : sdivmod_optab));
4685 :
4686 242351 : if (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN)
4687 : {
4688 254124 : FOR_EACH_MODE_FROM (compute_mode, mode)
4689 249889 : if (optab_handler (optab1, compute_mode) != CODE_FOR_nothing
4690 249889 : || optab_handler (optab2, compute_mode) != CODE_FOR_nothing)
4691 : break;
4692 :
4693 240558 : if (compute_mode == VOIDmode && methods == OPTAB_LIB_WIDEN)
4694 4235 : FOR_EACH_MODE_FROM (compute_mode, mode)
4695 4235 : if (optab_libfunc (optab1, compute_mode)
4696 4235 : || optab_libfunc (optab2, compute_mode))
4697 : break;
4698 : }
4699 : else
4700 : compute_mode = mode;
4701 :
4702 : /* If we still couldn't find a mode, use MODE, but expand_binop will
4703 : probably die. */
4704 6028 : if (compute_mode == VOIDmode)
4705 0 : compute_mode = mode;
4706 :
4707 242351 : if (target && GET_MODE (target) == compute_mode)
4708 : tquotient = target;
4709 : else
4710 150491 : tquotient = gen_reg_rtx (compute_mode);
4711 :
4712 : #if 0
4713 : /* It should be possible to restrict the precision to GET_MODE_BITSIZE
4714 : (mode), and thereby get better code when OP1 is a constant. Do that
4715 : later. It will require going over all usages of SIZE below. */
4716 : size = GET_MODE_BITSIZE (mode);
4717 : #endif
4718 :
4719 : /* Only deduct something for a REM if the last divide done was
4720 : for a different constant. Then set the constant of the last
4721 : divide. */
4722 242351 : max_cost = (unsignedp
4723 242351 : ? udiv_cost (speed, compute_mode)
4724 136440 : : sdiv_cost (speed, compute_mode));
4725 242351 : if (rem_flag && ! (last_div_const != 0 && op1_is_constant
4726 6289 : && INTVAL (op1) == last_div_const))
4727 63126 : max_cost -= (mul_cost (speed, compute_mode)
4728 63126 : + add_cost (speed, compute_mode));
4729 :
4730 242351 : last_div_const = ! rem_flag && op1_is_constant ? INTVAL (op1) : 0;
4731 :
4732 : /* Now convert to the best mode to use. */
4733 242351 : if (compute_mode != mode)
4734 : {
4735 0 : op0 = convert_modes (compute_mode, mode, op0, unsignedp);
4736 0 : op1 = convert_modes (compute_mode, mode, op1, unsignedp);
4737 :
4738 : /* convert_modes may have placed op1 into a register, so we
4739 : must recompute the following. */
4740 0 : op1_is_constant = CONST_INT_P (op1);
4741 0 : if (op1_is_constant)
4742 : {
4743 0 : wide_int ext_op1 = rtx_mode_t (op1, compute_mode);
4744 0 : op1_is_pow2 = (wi::popcount (ext_op1) == 1
4745 0 : || (! unsignedp
4746 0 : && wi::popcount (wi::neg (ext_op1)) == 1));
4747 0 : }
4748 : else
4749 : op1_is_pow2 = 0;
4750 : }
4751 :
4752 : /* If one of the operands is a volatile MEM, copy it into a register. */
4753 :
4754 242351 : if (MEM_P (op0) && MEM_VOLATILE_P (op0))
4755 0 : op0 = force_reg (compute_mode, op0);
4756 242351 : if (MEM_P (op1) && MEM_VOLATILE_P (op1))
4757 0 : op1 = force_reg (compute_mode, op1);
4758 :
4759 : /* If we need the remainder or if OP1 is constant, we need to
4760 : put OP0 in a register in case it has any queued subexpressions. */
4761 242351 : if (rem_flag || op1_is_constant)
4762 178453 : op0 = force_reg (compute_mode, op0);
4763 :
4764 242351 : last = get_last_insn ();
4765 :
4766 : /* Promote floor rounding to trunc rounding for unsigned operations. */
4767 242351 : if (unsignedp)
4768 : {
4769 105911 : if (code == FLOOR_DIV_EXPR)
4770 : code = TRUNC_DIV_EXPR;
4771 105858 : if (code == FLOOR_MOD_EXPR)
4772 148 : code = TRUNC_MOD_EXPR;
4773 105911 : if (code == EXACT_DIV_EXPR && op1_is_pow2)
4774 94 : code = TRUNC_DIV_EXPR;
4775 : }
4776 :
4777 242351 : if (op1 != const0_rtx)
4778 241984 : switch (code)
4779 : {
4780 193795 : case TRUNC_MOD_EXPR:
4781 193795 : case TRUNC_DIV_EXPR:
4782 193795 : if (op1_is_constant)
4783 : {
4784 91190 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
4785 91190 : int size = GET_MODE_BITSIZE (int_mode);
4786 91190 : if (unsignedp)
4787 : {
4788 53301 : unsigned HOST_WIDE_INT mh, ml;
4789 53301 : int pre_shift, post_shift;
4790 53301 : wide_int wd = rtx_mode_t (op1, int_mode);
4791 53301 : unsigned HOST_WIDE_INT d = wd.to_uhwi ();
4792 :
4793 53301 : if (wi::popcount (wd) == 1)
4794 : {
4795 27455 : pre_shift = floor_log2 (d);
4796 27455 : if (rem_flag)
4797 : {
4798 277 : unsigned HOST_WIDE_INT mask
4799 277 : = (HOST_WIDE_INT_1U << pre_shift) - 1;
4800 277 : remainder
4801 277 : = expand_binop (int_mode, and_optab, op0,
4802 277 : gen_int_mode (mask, int_mode),
4803 : remainder, 1, methods);
4804 277 : if (remainder)
4805 277 : return gen_lowpart (mode, remainder);
4806 : }
4807 27178 : quotient = expand_shift (RSHIFT_EXPR, int_mode, op0,
4808 27178 : pre_shift, tquotient, 1);
4809 : }
4810 25846 : else if (size <= HOST_BITS_PER_WIDE_INT)
4811 : {
4812 24234 : if (d >= (HOST_WIDE_INT_1U << (size - 1)))
4813 : {
4814 : /* Most significant bit of divisor is set; emit an scc
4815 : insn. */
4816 161 : quotient = emit_store_flag_force (tquotient, GEU, op0, op1,
4817 : int_mode, 1, 1);
4818 : }
4819 : else
4820 : {
4821 : /* Find a suitable multiplier and right shift count
4822 : instead of directly dividing by D. */
4823 24073 : mh = choose_multiplier (d, size, size,
4824 : &ml, &post_shift);
4825 :
4826 : /* If the suggested multiplier is more than SIZE bits,
4827 : we can do better for even divisors, using an
4828 : initial right shift. */
4829 24073 : if (mh != 0 && (d & 1) == 0)
4830 : {
4831 2380 : pre_shift = ctz_or_zero (d);
4832 2380 : mh = choose_multiplier (d >> pre_shift, size,
4833 : size - pre_shift,
4834 : &ml, &post_shift);
4835 2380 : gcc_assert (!mh);
4836 : }
4837 : else
4838 : pre_shift = 0;
4839 :
4840 2340 : if (mh != 0)
4841 : {
4842 2340 : rtx t1, t2, t3, t4;
4843 :
4844 2433 : if (post_shift - 1 >= BITS_PER_WORD)
4845 0 : goto fail1;
4846 :
4847 2340 : extra_cost
4848 2340 : = (shift_cost (speed, int_mode, post_shift - 1)
4849 2340 : + shift_cost (speed, int_mode, 1)
4850 2340 : + 2 * add_cost (speed, int_mode));
4851 :
4852 : /* Try a single widened multiply first; use it when
4853 : it is no more expensive. */
4854 2340 : quotient
4855 2340 : = expand_wide_mulh_udiv (int_mode, op0, ml, size,
4856 : post_shift, extra_cost,
4857 : max_cost, speed);
4858 2340 : if (quotient == NULL_RTX)
4859 : {
4860 1373 : t1 = expmed_mult_highpart
4861 1373 : (int_mode, op0, gen_int_mode (ml, int_mode),
4862 : NULL_RTX, 1, max_cost - extra_cost);
4863 1373 : if (t1 == 0)
4864 107 : goto fail1;
4865 1266 : t2 = force_operand (gen_rtx_MINUS (int_mode,
4866 : op0, t1),
4867 : NULL_RTX);
4868 1266 : t3 = expand_shift (RSHIFT_EXPR, int_mode,
4869 : t2, 1, NULL_RTX, 1);
4870 1266 : t4 = force_operand (gen_rtx_PLUS (int_mode,
4871 : t1, t3),
4872 : NULL_RTX);
4873 1266 : quotient = expand_shift
4874 1266 : (RSHIFT_EXPR, int_mode, t4,
4875 1266 : post_shift - 1, tquotient, 1);
4876 : }
4877 : }
4878 : else
4879 : {
4880 21733 : rtx t1, t2;
4881 :
4882 23556 : if (pre_shift >= BITS_PER_WORD
4883 21733 : || post_shift >= BITS_PER_WORD)
4884 3 : goto fail1;
4885 :
4886 21730 : t1 = expand_shift
4887 43460 : (RSHIFT_EXPR, int_mode, op0,
4888 21730 : pre_shift, NULL_RTX, 1);
4889 21730 : extra_cost
4890 21730 : = (shift_cost (speed, int_mode, pre_shift)
4891 21730 : + shift_cost (speed, int_mode, post_shift));
4892 21730 : t2 = expmed_mult_highpart
4893 21730 : (int_mode, t1,
4894 21730 : gen_int_mode (ml, int_mode),
4895 : NULL_RTX, 1, max_cost - extra_cost);
4896 21730 : if (t2 == 0)
4897 974 : goto fail1;
4898 20756 : quotient = expand_shift
4899 20756 : (RSHIFT_EXPR, int_mode, t2,
4900 20756 : post_shift, tquotient, 1);
4901 : }
4902 : }
4903 : }
4904 : else /* Too wide mode to use tricky code */
4905 : break;
4906 :
4907 50328 : insn = get_last_insn ();
4908 50328 : if (insn != last)
4909 50328 : set_dst_reg_note (insn, REG_EQUAL,
4910 : gen_rtx_UDIV (int_mode, op0, op1),
4911 : quotient);
4912 51689 : }
4913 : else /* TRUNC_DIV, signed */
4914 : {
4915 37889 : unsigned HOST_WIDE_INT ml;
4916 37889 : int post_shift;
4917 37889 : rtx mlr;
4918 37889 : HOST_WIDE_INT d = INTVAL (op1);
4919 37889 : unsigned HOST_WIDE_INT abs_d;
4920 :
4921 : /* Not prepared to handle division/remainder by
4922 : 0xffffffffffffffff8000000000000000 etc. */
4923 37889 : if (d == HOST_WIDE_INT_MIN && size > HOST_BITS_PER_WIDE_INT)
4924 : break;
4925 :
4926 : /* Since d might be INT_MIN, we have to cast to
4927 : unsigned HOST_WIDE_INT before negating to avoid
4928 : undefined signed overflow. */
4929 37889 : abs_d = (d >= 0
4930 37889 : ? (unsigned HOST_WIDE_INT) d
4931 : : - (unsigned HOST_WIDE_INT) d);
4932 :
4933 : /* n rem d = n rem -d */
4934 37889 : if (rem_flag && d < 0)
4935 : {
4936 195 : d = abs_d;
4937 195 : op1 = gen_int_mode (abs_d, int_mode);
4938 : }
4939 :
4940 37889 : if (d == 1)
4941 : quotient = op0;
4942 37889 : else if (d == -1)
4943 0 : quotient = expand_unop (int_mode, neg_optab, op0,
4944 : tquotient, 0);
4945 37889 : else if (size <= HOST_BITS_PER_WIDE_INT
4946 36488 : && abs_d == HOST_WIDE_INT_1U << (size - 1))
4947 : {
4948 : /* This case is not handled correctly below. */
4949 241 : quotient = emit_store_flag (tquotient, EQ, op0, op1,
4950 : int_mode, 1, 1);
4951 241 : if (quotient == 0)
4952 1302 : goto fail1;
4953 : }
4954 37648 : else if (EXACT_POWER_OF_2_OR_ZERO_P (d)
4955 13252 : && (size <= HOST_BITS_PER_WIDE_INT || d >= 0)
4956 2681 : && (rem_flag
4957 2681 : ? smod_pow2_cheap (speed, int_mode)
4958 10571 : : sdiv_pow2_cheap (speed, int_mode))
4959 : /* We assume that cheap metric is true if the
4960 : optab has an expander for this mode. */
4961 52161 : && ((optab_handler ((rem_flag ? smod_optab
4962 : : sdiv_optab),
4963 : int_mode)
4964 : != CODE_FOR_nothing)
4965 698 : || (optab_handler (sdivmod_optab, int_mode)
4966 : != CODE_FOR_nothing)))
4967 : ;
4968 36953 : else if (EXACT_POWER_OF_2_OR_ZERO_P (abs_d))
4969 : {
4970 13289 : if (rem_flag)
4971 : {
4972 2547 : remainder = expand_smod_pow2 (int_mode, op0, d);
4973 2547 : if (remainder)
4974 2547 : return gen_lowpart (mode, remainder);
4975 : }
4976 :
4977 10742 : if (sdiv_pow2_cheap (speed, int_mode)
4978 10742 : && ((optab_handler (sdiv_optab, int_mode)
4979 : != CODE_FOR_nothing)
4980 35 : || (optab_handler (sdivmod_optab, int_mode)
4981 : != CODE_FOR_nothing)))
4982 32 : quotient = expand_divmod (0, TRUNC_DIV_EXPR,
4983 : int_mode, op0,
4984 32 : gen_int_mode (abs_d,
4985 : int_mode),
4986 : NULL_RTX, 0);
4987 : else
4988 10710 : quotient = expand_sdiv_pow2 (int_mode, op0, abs_d);
4989 :
4990 : /* We have computed OP0 / abs(OP1). If OP1 is negative,
4991 : negate the quotient. */
4992 10742 : if (d < 0)
4993 : {
4994 732 : insn = get_last_insn ();
4995 732 : if (insn != last
4996 732 : && abs_d < (HOST_WIDE_INT_1U
4997 : << (HOST_BITS_PER_WIDE_INT - 1)))
4998 732 : set_dst_reg_note (insn, REG_EQUAL,
4999 732 : gen_rtx_DIV (int_mode, op0,
5000 : gen_int_mode
5001 : (abs_d,
5002 : int_mode)),
5003 : quotient);
5004 :
5005 732 : quotient = expand_unop (int_mode, neg_optab,
5006 : quotient, quotient, 0);
5007 : }
5008 : }
5009 23664 : else if (size <= HOST_BITS_PER_WIDE_INT)
5010 : {
5011 22325 : choose_multiplier (abs_d, size, size - 1,
5012 : &ml, &post_shift);
5013 22325 : if (ml < HOST_WIDE_INT_1U << (size - 1))
5014 : {
5015 16863 : rtx t1, t2, t3;
5016 :
5017 17978 : if (post_shift >= BITS_PER_WORD
5018 16863 : || size - 1 >= BITS_PER_WORD)
5019 250 : goto fail1;
5020 :
5021 16613 : extra_cost = (shift_cost (speed, int_mode, post_shift)
5022 16613 : + shift_cost (speed, int_mode, size - 1)
5023 16613 : + add_cost (speed, int_mode));
5024 16613 : t1 = expmed_mult_highpart
5025 16613 : (int_mode, op0, gen_int_mode (ml, int_mode),
5026 : NULL_RTX, 0, max_cost - extra_cost);
5027 16613 : if (t1 == 0)
5028 791 : goto fail1;
5029 15822 : t2 = expand_shift
5030 31644 : (RSHIFT_EXPR, int_mode, t1,
5031 15822 : post_shift, NULL_RTX, 0);
5032 15822 : t3 = expand_shift
5033 15822 : (RSHIFT_EXPR, int_mode, op0,
5034 15822 : size - 1, NULL_RTX, 0);
5035 15822 : if (d < 0)
5036 198 : quotient
5037 198 : = force_operand (gen_rtx_MINUS (int_mode, t3, t2),
5038 : tquotient);
5039 : else
5040 15624 : quotient
5041 15624 : = force_operand (gen_rtx_MINUS (int_mode, t2, t3),
5042 : tquotient);
5043 : }
5044 : else
5045 : {
5046 5462 : rtx t1, t2, t3, t4;
5047 :
5048 5822 : if (post_shift >= BITS_PER_WORD
5049 5457 : || size - 1 >= BITS_PER_WORD)
5050 43 : goto fail1;
5051 :
5052 5419 : ml |= HOST_WIDE_INT_M1U << (size - 1);
5053 5419 : mlr = gen_int_mode (ml, int_mode);
5054 5419 : extra_cost = (shift_cost (speed, int_mode, post_shift)
5055 5419 : + shift_cost (speed, int_mode, size - 1)
5056 5419 : + 2 * add_cost (speed, int_mode));
5057 5419 : t1 = expmed_mult_highpart (int_mode, op0, mlr,
5058 : NULL_RTX, 0,
5059 : max_cost - extra_cost);
5060 5419 : if (t1 == 0)
5061 218 : goto fail1;
5062 5201 : t2 = force_operand (gen_rtx_PLUS (int_mode, t1, op0),
5063 : NULL_RTX);
5064 5201 : t3 = expand_shift
5065 10402 : (RSHIFT_EXPR, int_mode, t2,
5066 5201 : post_shift, NULL_RTX, 0);
5067 5201 : t4 = expand_shift
5068 5201 : (RSHIFT_EXPR, int_mode, op0,
5069 5201 : size - 1, NULL_RTX, 0);
5070 5201 : if (d < 0)
5071 52 : quotient
5072 52 : = force_operand (gen_rtx_MINUS (int_mode, t4, t3),
5073 : tquotient);
5074 : else
5075 5149 : quotient
5076 5149 : = force_operand (gen_rtx_MINUS (int_mode, t3, t4),
5077 : tquotient);
5078 : }
5079 : }
5080 : else /* Too wide mode to use tricky code */
5081 : break;
5082 :
5083 32701 : insn = get_last_insn ();
5084 32701 : if (insn != last)
5085 32006 : set_dst_reg_note (insn, REG_EQUAL,
5086 : gen_rtx_DIV (int_mode, op0, op1),
5087 : quotient);
5088 : }
5089 : break;
5090 : }
5091 102605 : fail1:
5092 104991 : delete_insns_since (last);
5093 104991 : break;
5094 :
5095 1765 : case FLOOR_DIV_EXPR:
5096 1765 : case FLOOR_MOD_EXPR:
5097 : /* We will come here only for signed operations. */
5098 1765 : if (op1_is_constant && HWI_COMPUTABLE_MODE_P (compute_mode))
5099 : {
5100 971 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
5101 971 : int size = GET_MODE_BITSIZE (int_mode);
5102 971 : unsigned HOST_WIDE_INT mh, ml;
5103 971 : int pre_shift, post_shift;
5104 971 : HOST_WIDE_INT d = INTVAL (op1);
5105 :
5106 971 : if (d > 0)
5107 : {
5108 : /* We could just as easily deal with negative constants here,
5109 : but it does not seem worth the trouble for GCC 2.6. */
5110 945 : if (EXACT_POWER_OF_2_OR_ZERO_P (d))
5111 : {
5112 641 : pre_shift = floor_log2 (d);
5113 641 : if (rem_flag)
5114 : {
5115 64 : unsigned HOST_WIDE_INT mask
5116 64 : = (HOST_WIDE_INT_1U << pre_shift) - 1;
5117 64 : remainder = expand_binop
5118 64 : (int_mode, and_optab, op0,
5119 64 : gen_int_mode (mask, int_mode),
5120 : remainder, 0, methods);
5121 64 : if (remainder)
5122 64 : return gen_lowpart (mode, remainder);
5123 : }
5124 577 : quotient = expand_shift
5125 577 : (RSHIFT_EXPR, int_mode, op0,
5126 577 : pre_shift, tquotient, 0);
5127 : }
5128 : else
5129 : {
5130 304 : rtx t1, t2, t3, t4;
5131 :
5132 304 : mh = choose_multiplier (d, size, size - 1,
5133 : &ml, &post_shift);
5134 304 : gcc_assert (!mh);
5135 :
5136 328 : if (post_shift < BITS_PER_WORD
5137 304 : && size - 1 < BITS_PER_WORD)
5138 : {
5139 302 : t1 = expand_shift
5140 302 : (RSHIFT_EXPR, int_mode, op0,
5141 302 : size - 1, NULL_RTX, 0);
5142 302 : t2 = expand_binop (int_mode, xor_optab, op0, t1,
5143 : NULL_RTX, 0, OPTAB_WIDEN);
5144 302 : extra_cost = (shift_cost (speed, int_mode, post_shift)
5145 302 : + shift_cost (speed, int_mode, size - 1)
5146 302 : + 2 * add_cost (speed, int_mode));
5147 302 : t3 = expmed_mult_highpart
5148 302 : (int_mode, t2, gen_int_mode (ml, int_mode),
5149 : NULL_RTX, 1, max_cost - extra_cost);
5150 302 : if (t3 != 0)
5151 : {
5152 271 : t4 = expand_shift
5153 542 : (RSHIFT_EXPR, int_mode, t3,
5154 271 : post_shift, NULL_RTX, 1);
5155 271 : quotient = expand_binop (int_mode, xor_optab,
5156 : t4, t1, tquotient, 0,
5157 : OPTAB_WIDEN);
5158 : }
5159 : }
5160 : }
5161 : }
5162 : else
5163 : {
5164 26 : rtx nsign, t1, t2, t3, t4;
5165 26 : t1 = force_operand (gen_rtx_PLUS (int_mode,
5166 : op0, constm1_rtx), NULL_RTX);
5167 26 : t2 = expand_binop (int_mode, ior_optab, op0, t1, NULL_RTX,
5168 : 0, OPTAB_WIDEN);
5169 52 : nsign = expand_shift (RSHIFT_EXPR, int_mode, t2,
5170 26 : size - 1, NULL_RTX, 0);
5171 26 : t3 = force_operand (gen_rtx_MINUS (int_mode, t1, nsign),
5172 : NULL_RTX);
5173 26 : t4 = expand_divmod (0, TRUNC_DIV_EXPR, int_mode, t3, op1,
5174 : NULL_RTX, 0);
5175 26 : if (t4)
5176 : {
5177 26 : rtx t5;
5178 26 : t5 = expand_unop (int_mode, one_cmpl_optab, nsign,
5179 : NULL_RTX, 0);
5180 26 : quotient = force_operand (gen_rtx_PLUS (int_mode, t4, t5),
5181 : tquotient);
5182 : }
5183 : }
5184 : }
5185 :
5186 907 : if (quotient != 0)
5187 : break;
5188 827 : delete_insns_since (last);
5189 :
5190 : /* Try using an instruction that produces both the quotient and
5191 : remainder, using truncation. We can easily compensate the quotient
5192 : or remainder to get floor rounding, once we have the remainder.
5193 : Notice that we compute also the final remainder value here,
5194 : and return the result right away. */
5195 827 : if (target == 0 || GET_MODE (target) != compute_mode)
5196 126 : target = gen_reg_rtx (compute_mode);
5197 :
5198 827 : if (rem_flag)
5199 : {
5200 329 : remainder
5201 329 : = REG_P (target) ? target : gen_reg_rtx (compute_mode);
5202 329 : quotient = gen_reg_rtx (compute_mode);
5203 : }
5204 : else
5205 : {
5206 498 : quotient
5207 498 : = REG_P (target) ? target : gen_reg_rtx (compute_mode);
5208 498 : remainder = gen_reg_rtx (compute_mode);
5209 : }
5210 :
5211 827 : if (expand_twoval_binop (sdivmod_optab, op0, op1,
5212 : quotient, remainder, 0))
5213 : {
5214 : /* This could be computed with a branch-less sequence.
5215 : Save that for later. */
5216 792 : rtx tem;
5217 792 : rtx_code_label *label = gen_label_rtx ();
5218 792 : do_cmp_and_jump (remainder, const0_rtx, EQ, compute_mode, label);
5219 792 : tem = expand_binop (compute_mode, xor_optab, op0, op1,
5220 : NULL_RTX, 0, OPTAB_WIDEN);
5221 792 : do_cmp_and_jump (tem, const0_rtx, GE, compute_mode, label);
5222 792 : expand_dec (quotient, const1_rtx);
5223 792 : expand_inc (remainder, op1);
5224 792 : emit_label (label);
5225 1281 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5226 : }
5227 :
5228 : /* No luck with division elimination or divmod. Have to do it
5229 : by conditionally adjusting op0 *and* the result. */
5230 35 : {
5231 35 : rtx_code_label *label1, *label2, *label3, *label4, *label5;
5232 35 : rtx adjusted_op0;
5233 35 : rtx tem;
5234 :
5235 35 : quotient = gen_reg_rtx (compute_mode);
5236 35 : adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
5237 35 : label1 = gen_label_rtx ();
5238 35 : label2 = gen_label_rtx ();
5239 35 : label3 = gen_label_rtx ();
5240 35 : label4 = gen_label_rtx ();
5241 35 : label5 = gen_label_rtx ();
5242 35 : do_cmp_and_jump (op1, const0_rtx, LT, compute_mode, label2);
5243 35 : do_cmp_and_jump (adjusted_op0, const0_rtx, LT, compute_mode, label1);
5244 35 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5245 : quotient, 0, methods);
5246 35 : if (tem != quotient)
5247 35 : emit_move_insn (quotient, tem);
5248 35 : emit_jump_insn (targetm.gen_jump (label5));
5249 35 : emit_barrier ();
5250 35 : emit_label (label1);
5251 35 : expand_inc (adjusted_op0, const1_rtx);
5252 35 : emit_jump_insn (targetm.gen_jump (label4));
5253 35 : emit_barrier ();
5254 35 : emit_label (label2);
5255 35 : do_cmp_and_jump (adjusted_op0, const0_rtx, GT, compute_mode, label3);
5256 35 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5257 : quotient, 0, methods);
5258 35 : if (tem != quotient)
5259 35 : emit_move_insn (quotient, tem);
5260 35 : emit_jump_insn (targetm.gen_jump (label5));
5261 35 : emit_barrier ();
5262 35 : emit_label (label3);
5263 35 : expand_dec (adjusted_op0, const1_rtx);
5264 35 : emit_label (label4);
5265 35 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5266 : quotient, 0, methods);
5267 35 : if (tem != quotient)
5268 35 : emit_move_insn (quotient, tem);
5269 35 : expand_dec (quotient, const1_rtx);
5270 35 : emit_label (label5);
5271 : }
5272 35 : break;
5273 :
5274 402 : case CEIL_DIV_EXPR:
5275 402 : case CEIL_MOD_EXPR:
5276 402 : if (unsignedp)
5277 : {
5278 0 : if (op1_is_constant
5279 0 : && EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1))
5280 0 : && (HWI_COMPUTABLE_MODE_P (compute_mode)
5281 0 : || INTVAL (op1) >= 0))
5282 : {
5283 0 : scalar_int_mode int_mode
5284 0 : = as_a <scalar_int_mode> (compute_mode);
5285 0 : rtx t1, t2, t3;
5286 0 : unsigned HOST_WIDE_INT d = INTVAL (op1);
5287 0 : t1 = expand_shift (RSHIFT_EXPR, int_mode, op0,
5288 0 : floor_log2 (d), tquotient, 1);
5289 0 : t2 = expand_binop (int_mode, and_optab, op0,
5290 0 : gen_int_mode (d - 1, int_mode),
5291 : NULL_RTX, 1, methods);
5292 0 : t3 = gen_reg_rtx (int_mode);
5293 0 : t3 = emit_store_flag (t3, NE, t2, const0_rtx, int_mode, 1, 1);
5294 0 : if (t3 == 0)
5295 : {
5296 0 : rtx_code_label *lab;
5297 0 : lab = gen_label_rtx ();
5298 0 : do_cmp_and_jump (t2, const0_rtx, EQ, int_mode, lab);
5299 0 : expand_inc (t1, const1_rtx);
5300 0 : emit_label (lab);
5301 0 : quotient = t1;
5302 : }
5303 : else
5304 0 : quotient = force_operand (gen_rtx_PLUS (int_mode, t1, t3),
5305 : tquotient);
5306 : break;
5307 : }
5308 :
5309 : /* Try using an instruction that produces both the quotient and
5310 : remainder, using truncation. We can easily compensate the
5311 : quotient or remainder to get ceiling rounding, once we have the
5312 : remainder. Notice that we compute also the final remainder
5313 : value here, and return the result right away. */
5314 0 : if (target == 0 || GET_MODE (target) != compute_mode)
5315 0 : target = gen_reg_rtx (compute_mode);
5316 :
5317 0 : if (rem_flag)
5318 : {
5319 0 : remainder = (REG_P (target)
5320 0 : ? target : gen_reg_rtx (compute_mode));
5321 0 : quotient = gen_reg_rtx (compute_mode);
5322 : }
5323 : else
5324 : {
5325 0 : quotient = (REG_P (target)
5326 0 : ? target : gen_reg_rtx (compute_mode));
5327 0 : remainder = gen_reg_rtx (compute_mode);
5328 : }
5329 :
5330 0 : if (expand_twoval_binop (udivmod_optab, op0, op1, quotient,
5331 : remainder, 1))
5332 : {
5333 : /* This could be computed with a branch-less sequence.
5334 : Save that for later. */
5335 0 : rtx_code_label *label = gen_label_rtx ();
5336 0 : do_cmp_and_jump (remainder, const0_rtx, EQ,
5337 : compute_mode, label);
5338 0 : expand_inc (quotient, const1_rtx);
5339 0 : expand_dec (remainder, op1);
5340 0 : emit_label (label);
5341 0 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5342 : }
5343 :
5344 : /* No luck with division elimination or divmod. Have to do it
5345 : by conditionally adjusting op0 *and* the result. */
5346 0 : {
5347 0 : rtx_code_label *label1, *label2;
5348 0 : rtx adjusted_op0, tem;
5349 :
5350 0 : quotient = gen_reg_rtx (compute_mode);
5351 0 : adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
5352 0 : label1 = gen_label_rtx ();
5353 0 : label2 = gen_label_rtx ();
5354 0 : do_cmp_and_jump (adjusted_op0, const0_rtx, NE,
5355 : compute_mode, label1);
5356 0 : emit_move_insn (quotient, const0_rtx);
5357 0 : emit_jump_insn (targetm.gen_jump (label2));
5358 0 : emit_barrier ();
5359 0 : emit_label (label1);
5360 0 : expand_dec (adjusted_op0, const1_rtx);
5361 0 : tem = expand_binop (compute_mode, udiv_optab, adjusted_op0, op1,
5362 : quotient, 1, methods);
5363 0 : if (tem != quotient)
5364 0 : emit_move_insn (quotient, tem);
5365 0 : expand_inc (quotient, const1_rtx);
5366 0 : emit_label (label2);
5367 : }
5368 : }
5369 : else /* signed */
5370 : {
5371 402 : if (op1_is_constant && EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1))
5372 31 : && INTVAL (op1) >= 0)
5373 : {
5374 : /* This is extremely similar to the code for the unsigned case
5375 : above. For 2.7 we should merge these variants, but for
5376 : 2.6.1 I don't want to touch the code for unsigned since that
5377 : get used in C. The signed case will only be used by other
5378 : languages (Ada). */
5379 :
5380 31 : rtx t1, t2, t3;
5381 31 : unsigned HOST_WIDE_INT d = INTVAL (op1);
5382 62 : t1 = expand_shift (RSHIFT_EXPR, compute_mode, op0,
5383 31 : floor_log2 (d), tquotient, 0);
5384 31 : t2 = expand_binop (compute_mode, and_optab, op0,
5385 31 : gen_int_mode (d - 1, compute_mode),
5386 : NULL_RTX, 1, methods);
5387 31 : t3 = gen_reg_rtx (compute_mode);
5388 31 : t3 = emit_store_flag (t3, NE, t2, const0_rtx,
5389 : compute_mode, 1, 1);
5390 31 : if (t3 == 0)
5391 : {
5392 0 : rtx_code_label *lab;
5393 0 : lab = gen_label_rtx ();
5394 0 : do_cmp_and_jump (t2, const0_rtx, EQ, compute_mode, lab);
5395 0 : expand_inc (t1, const1_rtx);
5396 0 : emit_label (lab);
5397 0 : quotient = t1;
5398 : }
5399 : else
5400 31 : quotient = force_operand (gen_rtx_PLUS (compute_mode,
5401 : t1, t3),
5402 : tquotient);
5403 : break;
5404 : }
5405 :
5406 : /* Try using an instruction that produces both the quotient and
5407 : remainder, using truncation. We can easily compensate the
5408 : quotient or remainder to get ceiling rounding, once we have the
5409 : remainder. Notice that we compute also the final remainder
5410 : value here, and return the result right away. */
5411 371 : if (target == 0 || GET_MODE (target) != compute_mode)
5412 15 : target = gen_reg_rtx (compute_mode);
5413 371 : if (rem_flag)
5414 : {
5415 149 : remainder= (REG_P (target)
5416 149 : ? target : gen_reg_rtx (compute_mode));
5417 149 : quotient = gen_reg_rtx (compute_mode);
5418 : }
5419 : else
5420 : {
5421 222 : quotient = (REG_P (target)
5422 222 : ? target : gen_reg_rtx (compute_mode));
5423 222 : remainder = gen_reg_rtx (compute_mode);
5424 : }
5425 :
5426 371 : if (expand_twoval_binop (sdivmod_optab, op0, op1, quotient,
5427 : remainder, 0))
5428 : {
5429 : /* This could be computed with a branch-less sequence.
5430 : Save that for later. */
5431 371 : rtx tem;
5432 371 : rtx_code_label *label = gen_label_rtx ();
5433 371 : do_cmp_and_jump (remainder, const0_rtx, EQ,
5434 : compute_mode, label);
5435 371 : tem = expand_binop (compute_mode, xor_optab, op0, op1,
5436 : NULL_RTX, 0, OPTAB_WIDEN);
5437 371 : do_cmp_and_jump (tem, const0_rtx, LT, compute_mode, label);
5438 371 : expand_inc (quotient, const1_rtx);
5439 371 : expand_dec (remainder, op1);
5440 371 : emit_label (label);
5441 593 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5442 : }
5443 :
5444 : /* No luck with division elimination or divmod. Have to do it
5445 : by conditionally adjusting op0 *and* the result. */
5446 0 : {
5447 0 : rtx_code_label *label1, *label2, *label3, *label4, *label5;
5448 0 : rtx adjusted_op0;
5449 0 : rtx tem;
5450 :
5451 0 : quotient = gen_reg_rtx (compute_mode);
5452 0 : adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
5453 0 : label1 = gen_label_rtx ();
5454 0 : label2 = gen_label_rtx ();
5455 0 : label3 = gen_label_rtx ();
5456 0 : label4 = gen_label_rtx ();
5457 0 : label5 = gen_label_rtx ();
5458 0 : do_cmp_and_jump (op1, const0_rtx, LT, compute_mode, label2);
5459 0 : do_cmp_and_jump (adjusted_op0, const0_rtx, GT,
5460 : compute_mode, label1);
5461 0 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5462 : quotient, 0, methods);
5463 0 : if (tem != quotient)
5464 0 : emit_move_insn (quotient, tem);
5465 0 : emit_jump_insn (targetm.gen_jump (label5));
5466 0 : emit_barrier ();
5467 0 : emit_label (label1);
5468 0 : expand_dec (adjusted_op0, const1_rtx);
5469 0 : emit_jump_insn (targetm.gen_jump (label4));
5470 0 : emit_barrier ();
5471 0 : emit_label (label2);
5472 0 : do_cmp_and_jump (adjusted_op0, const0_rtx, LT,
5473 : compute_mode, label3);
5474 0 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5475 : quotient, 0, methods);
5476 0 : if (tem != quotient)
5477 0 : emit_move_insn (quotient, tem);
5478 0 : emit_jump_insn (targetm.gen_jump (label5));
5479 0 : emit_barrier ();
5480 0 : emit_label (label3);
5481 0 : expand_inc (adjusted_op0, const1_rtx);
5482 0 : emit_label (label4);
5483 0 : tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
5484 : quotient, 0, methods);
5485 0 : if (tem != quotient)
5486 0 : emit_move_insn (quotient, tem);
5487 0 : expand_inc (quotient, const1_rtx);
5488 0 : emit_label (label5);
5489 : }
5490 : }
5491 : break;
5492 :
5493 46020 : case EXACT_DIV_EXPR:
5494 46020 : if (op1_is_constant && HWI_COMPUTABLE_MODE_P (compute_mode))
5495 : {
5496 46011 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
5497 46011 : int size = GET_MODE_BITSIZE (int_mode);
5498 46011 : HOST_WIDE_INT d = INTVAL (op1);
5499 46011 : unsigned HOST_WIDE_INT ml;
5500 46011 : int pre_shift;
5501 46011 : rtx t1;
5502 :
5503 46011 : pre_shift = ctz_or_zero (d);
5504 46011 : ml = invert_mod2n (d >> pre_shift, size);
5505 46011 : t1 = expand_shift (RSHIFT_EXPR, int_mode, op0,
5506 46011 : pre_shift, NULL_RTX, unsignedp);
5507 46011 : quotient = expand_mult (int_mode, t1, gen_int_mode (ml, int_mode),
5508 : NULL_RTX, 1);
5509 :
5510 46011 : insn = get_last_insn ();
5511 92022 : set_dst_reg_note (insn, REG_EQUAL,
5512 : gen_rtx_fmt_ee (unsignedp ? UDIV : DIV,
5513 : int_mode, op0, op1),
5514 : quotient);
5515 : }
5516 : break;
5517 :
5518 2 : case ROUND_DIV_EXPR:
5519 2 : case ROUND_MOD_EXPR:
5520 2 : if (unsignedp)
5521 : {
5522 0 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
5523 0 : rtx tem;
5524 0 : rtx_code_label *label;
5525 0 : label = gen_label_rtx ();
5526 0 : quotient = gen_reg_rtx (int_mode);
5527 0 : remainder = gen_reg_rtx (int_mode);
5528 0 : if (expand_twoval_binop (udivmod_optab, op0, op1, quotient, remainder, 1) == 0)
5529 : {
5530 0 : rtx tem;
5531 0 : quotient = expand_binop (int_mode, udiv_optab, op0, op1,
5532 : quotient, 1, methods);
5533 0 : tem = expand_mult (int_mode, quotient, op1, NULL_RTX, 1);
5534 0 : remainder = expand_binop (int_mode, sub_optab, op0, tem,
5535 : remainder, 1, methods);
5536 : }
5537 0 : tem = plus_constant (int_mode, op1, -1);
5538 0 : tem = expand_shift (RSHIFT_EXPR, int_mode, tem, 1, NULL_RTX, 1);
5539 0 : do_cmp_and_jump (remainder, tem, LEU, int_mode, label);
5540 0 : expand_inc (quotient, const1_rtx);
5541 0 : expand_dec (remainder, op1);
5542 0 : emit_label (label);
5543 : }
5544 : else
5545 : {
5546 2 : scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
5547 2 : int size = GET_MODE_BITSIZE (int_mode);
5548 2 : rtx abs_rem, abs_op1, tem, mask;
5549 2 : rtx_code_label *label;
5550 2 : label = gen_label_rtx ();
5551 2 : quotient = gen_reg_rtx (int_mode);
5552 2 : remainder = gen_reg_rtx (int_mode);
5553 2 : if (expand_twoval_binop (sdivmod_optab, op0, op1, quotient, remainder, 0) == 0)
5554 : {
5555 0 : rtx tem;
5556 0 : quotient = expand_binop (int_mode, sdiv_optab, op0, op1,
5557 : quotient, 0, methods);
5558 0 : tem = expand_mult (int_mode, quotient, op1, NULL_RTX, 0);
5559 0 : remainder = expand_binop (int_mode, sub_optab, op0, tem,
5560 : remainder, 0, methods);
5561 : }
5562 2 : abs_rem = expand_abs (int_mode, remainder, NULL_RTX, 1, 0);
5563 2 : abs_op1 = expand_abs (int_mode, op1, NULL_RTX, 1, 0);
5564 2 : tem = expand_shift (LSHIFT_EXPR, int_mode, abs_rem,
5565 : 1, NULL_RTX, 1);
5566 2 : do_cmp_and_jump (tem, abs_op1, LTU, int_mode, label);
5567 2 : tem = expand_binop (int_mode, xor_optab, op0, op1,
5568 : NULL_RTX, 0, OPTAB_WIDEN);
5569 4 : mask = expand_shift (RSHIFT_EXPR, int_mode, tem,
5570 2 : size - 1, NULL_RTX, 0);
5571 2 : tem = expand_binop (int_mode, xor_optab, mask, const1_rtx,
5572 : NULL_RTX, 0, OPTAB_WIDEN);
5573 2 : tem = expand_binop (int_mode, sub_optab, tem, mask,
5574 : NULL_RTX, 0, OPTAB_WIDEN);
5575 2 : expand_inc (quotient, tem);
5576 2 : tem = expand_binop (int_mode, xor_optab, mask, op1,
5577 : NULL_RTX, 0, OPTAB_WIDEN);
5578 2 : tem = expand_binop (int_mode, sub_optab, tem, mask,
5579 : NULL_RTX, 0, OPTAB_WIDEN);
5580 2 : expand_dec (remainder, tem);
5581 2 : emit_label (label);
5582 : }
5583 3 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5584 :
5585 0 : default:
5586 0 : gcc_unreachable ();
5587 : }
5588 :
5589 237048 : if (quotient == 0)
5590 : {
5591 109013 : if (target && GET_MODE (target) != compute_mode)
5592 65577 : target = 0;
5593 :
5594 109013 : if (rem_flag)
5595 : {
5596 : /* Try to produce the remainder without producing the quotient.
5597 : If we seem to have a divmod pattern that does not require widening,
5598 : don't try widening here. We should really have a WIDEN argument
5599 : to expand_twoval_binop, since what we'd really like to do here is
5600 : 1) try a mod insn in compute_mode
5601 : 2) try a divmod insn in compute_mode
5602 : 3) try a div insn in compute_mode and multiply-subtract to get
5603 : remainder
5604 : 4) try the same things with widening allowed. */
5605 41936 : remainder
5606 43430 : = sign_expand_binop (compute_mode, umod_optab, smod_optab,
5607 : op0, op1, target,
5608 : unsignedp,
5609 41936 : ((optab_handler (optab2, compute_mode)
5610 : != CODE_FOR_nothing)
5611 : ? OPTAB_DIRECT : OPTAB_WIDEN));
5612 41936 : if (remainder == 0)
5613 : {
5614 : /* No luck there. Can we do remainder and divide at once
5615 : without a library call? */
5616 41729 : remainder = gen_reg_rtx (compute_mode);
5617 66257 : if (! expand_twoval_binop ((unsignedp
5618 : ? udivmod_optab
5619 : : sdivmod_optab),
5620 : op0, op1,
5621 : NULL_RTX, remainder, unsignedp))
5622 : remainder = 0;
5623 : }
5624 :
5625 40442 : if (remainder)
5626 40649 : return gen_lowpart (mode, remainder);
5627 : }
5628 :
5629 : /* Produce the quotient. Try a quotient insn, but not a library call.
5630 : If we have a divmod in this mode, use it in preference to widening
5631 : the div (for this test we assume it will not fail). Note that optab2
5632 : is set to the one of the two optabs that the call below will use. */
5633 68364 : quotient
5634 73646 : = sign_expand_binop (compute_mode, udiv_optab, sdiv_optab,
5635 : op0, op1, rem_flag ? NULL_RTX : target,
5636 : unsignedp,
5637 68364 : ((optab_handler (optab2, compute_mode)
5638 : != CODE_FOR_nothing)
5639 : ? OPTAB_DIRECT : OPTAB_WIDEN));
5640 :
5641 68364 : if (quotient == 0)
5642 : {
5643 : /* No luck there. Try a quotient-and-remainder insn,
5644 : keeping the quotient alone. */
5645 67929 : quotient = gen_reg_rtx (compute_mode);
5646 99196 : if (! expand_twoval_binop (unsignedp ? udivmod_optab : sdivmod_optab,
5647 : op0, op1,
5648 : quotient, NULL_RTX, unsignedp))
5649 : {
5650 3561 : quotient = 0;
5651 3561 : if (! rem_flag)
5652 : /* Still no luck. If we are not computing the remainder,
5653 : use a library call for the quotient. */
5654 2312 : quotient = sign_expand_binop (compute_mode,
5655 : udiv_optab, sdiv_optab,
5656 : op0, op1, target,
5657 : unsignedp, methods);
5658 : }
5659 : }
5660 : }
5661 :
5662 196400 : if (rem_flag)
5663 : {
5664 22235 : if (target && GET_MODE (target) != compute_mode)
5665 15138 : target = 0;
5666 :
5667 22235 : if (quotient == 0)
5668 : {
5669 : /* No divide instruction either. Use library for remainder. */
5670 1249 : remainder = sign_expand_binop (compute_mode, umod_optab, smod_optab,
5671 : op0, op1, target,
5672 : unsignedp, methods);
5673 : /* No remainder function. Try a quotient-and-remainder
5674 : function, keeping the remainder. */
5675 1249 : if (!remainder
5676 0 : && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
5677 : {
5678 0 : remainder = gen_reg_rtx (compute_mode);
5679 0 : if (!expand_twoval_binop_libfunc
5680 0 : (unsignedp ? udivmod_optab : sdivmod_optab,
5681 : op0, op1,
5682 : NULL_RTX, remainder,
5683 : unsignedp ? UMOD : MOD))
5684 0 : remainder = NULL_RTX;
5685 : }
5686 : }
5687 : else
5688 : {
5689 : /* We divided. Now finish doing X - Y * (X / Y). */
5690 20986 : remainder = expand_mult (compute_mode, quotient, op1,
5691 : NULL_RTX, unsignedp);
5692 20986 : remainder = expand_binop (compute_mode, sub_optab, op0,
5693 : remainder, target, unsignedp,
5694 : methods);
5695 : }
5696 : }
5697 :
5698 197649 : if (methods != OPTAB_LIB_WIDEN
5699 1793 : && (rem_flag ? remainder : quotient) == NULL_RTX)
5700 : return NULL_RTX;
5701 :
5702 373063 : return gen_lowpart (mode, rem_flag ? remainder : quotient);
5703 : }
5704 :
5705 : /* Return a tree node with data type TYPE, describing the value of X.
5706 : Usually this is an VAR_DECL, if there is no obvious better choice.
5707 : X may be an expression, however we only support those expressions
5708 : generated by loop.c. */
5709 :
5710 : tree
5711 686906 : make_tree (tree type, rtx x)
5712 : {
5713 686906 : tree t;
5714 :
5715 686906 : switch (GET_CODE (x))
5716 : {
5717 22976 : case CONST_INT:
5718 22976 : case CONST_WIDE_INT:
5719 22976 : t = wide_int_to_tree (type, rtx_mode_t (x, TYPE_MODE (type)));
5720 22976 : return t;
5721 :
5722 0 : case CONST_POLY_INT:
5723 0 : return wide_int_to_tree (type, const_poly_int_value (x));
5724 :
5725 0 : case CONST_DOUBLE:
5726 0 : STATIC_ASSERT (HOST_BITS_PER_WIDE_INT * 2 <= MAX_BITSIZE_MODE_ANY_INT);
5727 0 : if (TARGET_SUPPORTS_WIDE_INT == 0 && GET_MODE (x) == VOIDmode)
5728 : t = wide_int_to_tree (type,
5729 : wide_int::from_array (&CONST_DOUBLE_LOW (x), 2,
5730 : HOST_BITS_PER_WIDE_INT * 2));
5731 : else
5732 0 : t = build_real (type, *CONST_DOUBLE_REAL_VALUE (x));
5733 :
5734 0 : return t;
5735 :
5736 0 : case CONST_VECTOR:
5737 0 : {
5738 0 : unsigned int npatterns = CONST_VECTOR_NPATTERNS (x);
5739 0 : unsigned int nelts_per_pattern = CONST_VECTOR_NELTS_PER_PATTERN (x);
5740 0 : tree itype = TREE_TYPE (type);
5741 :
5742 : /* Build a tree with vector elements. */
5743 0 : tree_vector_builder elts (type, npatterns, nelts_per_pattern);
5744 0 : unsigned int count = elts.encoded_nelts ();
5745 0 : for (unsigned int i = 0; i < count; ++i)
5746 : {
5747 0 : rtx elt = CONST_VECTOR_ELT (x, i);
5748 0 : elts.quick_push (make_tree (itype, elt));
5749 : }
5750 :
5751 0 : return elts.build ();
5752 0 : }
5753 :
5754 0 : case PLUS:
5755 0 : return fold_build2 (PLUS_EXPR, type, make_tree (type, XEXP (x, 0)),
5756 : make_tree (type, XEXP (x, 1)));
5757 :
5758 0 : case MINUS:
5759 0 : return fold_build2 (MINUS_EXPR, type, make_tree (type, XEXP (x, 0)),
5760 : make_tree (type, XEXP (x, 1)));
5761 :
5762 0 : case NEG:
5763 0 : return fold_build1 (NEGATE_EXPR, type, make_tree (type, XEXP (x, 0)));
5764 :
5765 0 : case MULT:
5766 0 : return fold_build2 (MULT_EXPR, type, make_tree (type, XEXP (x, 0)),
5767 : make_tree (type, XEXP (x, 1)));
5768 :
5769 0 : case ASHIFT:
5770 0 : return fold_build2 (LSHIFT_EXPR, type, make_tree (type, XEXP (x, 0)),
5771 : make_tree (type, XEXP (x, 1)));
5772 :
5773 0 : case LSHIFTRT:
5774 0 : t = unsigned_type_for (type);
5775 0 : return fold_convert (type, build2 (RSHIFT_EXPR, t,
5776 : make_tree (t, XEXP (x, 0)),
5777 : make_tree (type, XEXP (x, 1))));
5778 :
5779 0 : case ASHIFTRT:
5780 0 : t = signed_type_for (type);
5781 0 : return fold_convert (type, build2 (RSHIFT_EXPR, t,
5782 : make_tree (t, XEXP (x, 0)),
5783 : make_tree (type, XEXP (x, 1))));
5784 :
5785 0 : case DIV:
5786 0 : if (TREE_CODE (type) != REAL_TYPE)
5787 0 : t = signed_type_for (type);
5788 : else
5789 : t = type;
5790 :
5791 0 : return fold_convert (type, build2 (TRUNC_DIV_EXPR, t,
5792 : make_tree (t, XEXP (x, 0)),
5793 : make_tree (t, XEXP (x, 1))));
5794 0 : case UDIV:
5795 0 : t = unsigned_type_for (type);
5796 0 : return fold_convert (type, build2 (TRUNC_DIV_EXPR, t,
5797 : make_tree (t, XEXP (x, 0)),
5798 : make_tree (t, XEXP (x, 1))));
5799 :
5800 0 : case SIGN_EXTEND:
5801 0 : case ZERO_EXTEND:
5802 0 : t = lang_hooks.types.type_for_mode (GET_MODE (XEXP (x, 0)),
5803 : GET_CODE (x) == ZERO_EXTEND);
5804 0 : return fold_convert (type, make_tree (t, XEXP (x, 0)));
5805 :
5806 0 : case CONST:
5807 0 : return make_tree (type, XEXP (x, 0));
5808 :
5809 0 : case SYMBOL_REF:
5810 0 : t = SYMBOL_REF_DECL (x);
5811 0 : if (t)
5812 0 : return fold_convert (type, build_fold_addr_expr (t));
5813 : /* fall through. */
5814 :
5815 663930 : default:
5816 663930 : t = build_decl (RTL_LOCATION (x), VAR_DECL, NULL_TREE, type);
5817 :
5818 : /* If TYPE is a POINTER_TYPE, we might need to convert X from
5819 : address mode to pointer mode. */
5820 663930 : if (POINTER_TYPE_P (type))
5821 828064 : x = convert_memory_address_addr_space
5822 414032 : (SCALAR_INT_TYPE_MODE (type), x, TYPE_ADDR_SPACE (TREE_TYPE (type)));
5823 :
5824 : /* Note that we do *not* use SET_DECL_RTL here, because we do not
5825 : want set_decl_rtl to go adjusting REG_ATTRS for this temporary. */
5826 663930 : t->decl_with_rtl.rtl = x;
5827 :
5828 663930 : return t;
5829 : }
5830 : }
5831 :
5832 : /* Compute the logical-and of OP0 and OP1, storing it in TARGET
5833 : and returning TARGET.
5834 :
5835 : If TARGET is 0, a pseudo-register or constant is returned. */
5836 :
5837 : rtx
5838 73933 : expand_and (machine_mode mode, rtx op0, rtx op1, rtx target)
5839 : {
5840 73933 : rtx tem = 0;
5841 :
5842 73933 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
5843 80 : tem = simplify_binary_operation (AND, mode, op0, op1);
5844 80 : if (tem == 0)
5845 73853 : tem = expand_binop (mode, and_optab, op0, op1, target, 0, OPTAB_LIB_WIDEN);
5846 :
5847 73933 : if (target == 0)
5848 : target = tem;
5849 45141 : else if (tem != target)
5850 13 : emit_move_insn (target, tem);
5851 73933 : return target;
5852 : }
5853 :
5854 : /* Helper function for emit_store_flag. */
5855 : rtx
5856 753849 : emit_cstore (rtx target, enum insn_code icode, enum rtx_code code,
5857 : machine_mode mode, machine_mode compare_mode,
5858 : int unsignedp, rtx x, rtx y, int normalizep,
5859 : machine_mode target_mode)
5860 : {
5861 753849 : class expand_operand ops[4];
5862 753849 : rtx op0, comparison, subtarget;
5863 753849 : rtx_insn *last;
5864 753849 : scalar_int_mode result_mode = targetm.cstore_mode (icode);
5865 753849 : scalar_int_mode int_target_mode;
5866 :
5867 753849 : last = get_last_insn ();
5868 753849 : x = prepare_operand (icode, x, 2, mode, compare_mode, unsignedp);
5869 753849 : y = prepare_operand (icode, y, 3, mode, compare_mode, unsignedp);
5870 753849 : if (!x || !y)
5871 : {
5872 284 : delete_insns_since (last);
5873 284 : return NULL_RTX;
5874 : }
5875 :
5876 753565 : if (target_mode == VOIDmode)
5877 : int_target_mode = result_mode;
5878 : else
5879 753549 : int_target_mode = as_a <scalar_int_mode> (target_mode);
5880 753565 : if (!target)
5881 69616 : target = gen_reg_rtx (int_target_mode);
5882 :
5883 753565 : comparison = gen_rtx_fmt_ee (code, result_mode, x, y);
5884 :
5885 753565 : create_output_operand (&ops[0], optimize ? NULL_RTX : target, result_mode);
5886 753565 : create_fixed_operand (&ops[1], comparison);
5887 753565 : create_fixed_operand (&ops[2], x);
5888 753565 : create_fixed_operand (&ops[3], y);
5889 753565 : if (!maybe_expand_insn (icode, 4, ops))
5890 : {
5891 155075 : delete_insns_since (last);
5892 155075 : return NULL_RTX;
5893 : }
5894 598490 : subtarget = ops[0].value;
5895 :
5896 : /* If we are converting to a wider mode, first convert to
5897 : INT_TARGET_MODE, then normalize. This produces better combining
5898 : opportunities on machines that have a SIGN_EXTRACT when we are
5899 : testing a single bit. This mostly benefits the 68k.
5900 :
5901 : If STORE_FLAG_VALUE does not have the sign bit set when
5902 : interpreted in MODE, we can do this conversion as unsigned, which
5903 : is usually more efficient. */
5904 598490 : if (GET_MODE_PRECISION (int_target_mode) > GET_MODE_PRECISION (result_mode))
5905 : {
5906 115728 : gcc_assert (GET_MODE_PRECISION (result_mode) != 1
5907 : || STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1);
5908 :
5909 115728 : bool unsignedp = (STORE_FLAG_VALUE >= 0);
5910 115728 : convert_move (target, subtarget, unsignedp);
5911 :
5912 115728 : op0 = target;
5913 115728 : result_mode = int_target_mode;
5914 : }
5915 : else
5916 : op0 = subtarget;
5917 :
5918 : /* If we want to keep subexpressions around, don't reuse our last
5919 : target. */
5920 598490 : if (optimize)
5921 479834 : subtarget = 0;
5922 :
5923 : /* Now normalize to the proper value in MODE. Sometimes we don't
5924 : have to do anything. */
5925 598490 : if (normalizep == 0 || normalizep == STORE_FLAG_VALUE)
5926 : ;
5927 : /* STORE_FLAG_VALUE might be the most negative number, so write
5928 : the comparison this way to avoid a compiler-time warning. */
5929 354 : else if (- normalizep == STORE_FLAG_VALUE)
5930 354 : op0 = expand_unop (result_mode, neg_optab, op0, subtarget, 0);
5931 :
5932 : /* We don't want to use STORE_FLAG_VALUE < 0 below since this makes
5933 : it hard to use a value of just the sign bit due to ANSI integer
5934 : constant typing rules. */
5935 0 : else if (val_signbit_known_set_p (result_mode, STORE_FLAG_VALUE))
5936 0 : op0 = expand_shift (RSHIFT_EXPR, result_mode, op0,
5937 0 : GET_MODE_BITSIZE (result_mode) - 1, subtarget,
5938 : normalizep == 1);
5939 : else
5940 : {
5941 0 : gcc_assert (STORE_FLAG_VALUE & 1);
5942 :
5943 0 : op0 = expand_and (result_mode, op0, const1_rtx, subtarget);
5944 0 : if (normalizep == -1)
5945 : op0 = expand_unop (result_mode, neg_optab, op0, op0, 0);
5946 : }
5947 :
5948 : /* If we were converting to a smaller mode, do the conversion now. */
5949 598490 : if (int_target_mode != result_mode)
5950 : {
5951 0 : convert_move (target, op0, 0);
5952 0 : return target;
5953 : }
5954 : else
5955 : return op0;
5956 : }
5957 :
5958 :
5959 : /* A subroutine of emit_store_flag only including "tricks" that do not
5960 : need a recursive call. These are kept separate to avoid infinite
5961 : loops. */
5962 :
5963 : static rtx
5964 721559 : emit_store_flag_1 (rtx target, enum rtx_code code, rtx op0, rtx op1,
5965 : machine_mode mode, int unsignedp, int normalizep,
5966 : machine_mode target_mode)
5967 : {
5968 721559 : rtx subtarget;
5969 721559 : enum insn_code icode;
5970 721559 : machine_mode compare_mode;
5971 721559 : enum mode_class mclass;
5972 :
5973 721559 : if (unsignedp)
5974 166763 : code = unsigned_condition (code);
5975 :
5976 : /* If one operand is constant, make it the second one. Only do this
5977 : if the other operand is not constant as well. */
5978 :
5979 721559 : if (swap_commutative_operands_p (op0, op1))
5980 : {
5981 5355 : std::swap (op0, op1);
5982 5355 : code = swap_condition (code);
5983 : }
5984 :
5985 721559 : if (mode == VOIDmode)
5986 42449 : mode = GET_MODE (op0);
5987 :
5988 721559 : if (CONST_SCALAR_INT_P (op1))
5989 335062 : canonicalize_comparison (mode, &code, &op1);
5990 :
5991 : /* For some comparisons with 1 and -1, we can convert this to
5992 : comparisons with zero. This will often produce more opportunities for
5993 : store-flag insns. */
5994 :
5995 721559 : switch (code)
5996 : {
5997 37707 : case LT:
5998 37707 : if (op1 == const1_rtx)
5999 59 : op1 = const0_rtx, code = LE;
6000 : break;
6001 27139 : case LE:
6002 27139 : if (op1 == constm1_rtx)
6003 0 : op1 = const0_rtx, code = LT;
6004 : break;
6005 39857 : case GE:
6006 39857 : if (op1 == const1_rtx)
6007 0 : op1 = const0_rtx, code = GT;
6008 : break;
6009 31213 : case GT:
6010 31213 : if (op1 == constm1_rtx)
6011 145 : op1 = const0_rtx, code = GE;
6012 : break;
6013 6310 : case GEU:
6014 6310 : if (op1 == const1_rtx)
6015 0 : op1 = const0_rtx, code = NE;
6016 : break;
6017 10664 : case LTU:
6018 10664 : if (op1 == const1_rtx)
6019 10 : op1 = const0_rtx, code = EQ;
6020 : break;
6021 : default:
6022 : break;
6023 : }
6024 :
6025 : /* If this is A < 0 or A >= 0, we can do this by taking the ones
6026 : complement of A (for GE) and shifting the sign bit to the low bit. */
6027 721559 : scalar_int_mode int_mode;
6028 200871 : if (op1 == const0_rtx && (code == LT || code == GE)
6029 721559 : && is_int_mode (mode, &int_mode)
6030 721559 : && (normalizep || STORE_FLAG_VALUE == 1
6031 : || val_signbit_p (int_mode, STORE_FLAG_VALUE)))
6032 : {
6033 41231 : scalar_int_mode int_target_mode;
6034 41231 : subtarget = target;
6035 :
6036 41231 : if (!target)
6037 : int_target_mode = int_mode;
6038 : else
6039 : {
6040 : /* If the result is to be wider than OP0, it is best to convert it
6041 : first. If it is to be narrower, it is *incorrect* to convert it
6042 : first. */
6043 41231 : int_target_mode = as_a <scalar_int_mode> (target_mode);
6044 123693 : if (GET_MODE_SIZE (int_target_mode) > GET_MODE_SIZE (int_mode))
6045 : {
6046 461 : op0 = convert_modes (int_target_mode, int_mode, op0, 0);
6047 461 : int_mode = int_target_mode;
6048 : }
6049 : }
6050 :
6051 41231 : if (int_target_mode != int_mode)
6052 27135 : subtarget = 0;
6053 :
6054 41231 : if (code == GE)
6055 20789 : op0 = expand_unop (int_mode, one_cmpl_optab, op0,
6056 : ((STORE_FLAG_VALUE == 1 || normalizep)
6057 : ? 0 : subtarget), 0);
6058 :
6059 41231 : if (STORE_FLAG_VALUE == 1 || normalizep)
6060 : /* If we are supposed to produce a 0/1 value, we want to do
6061 : a logical shift from the sign bit to the low-order bit; for
6062 : a -1/0 value, we do an arithmetic shift. */
6063 82462 : op0 = expand_shift (RSHIFT_EXPR, int_mode, op0,
6064 41231 : GET_MODE_BITSIZE (int_mode) - 1,
6065 : subtarget, normalizep != -1);
6066 :
6067 41231 : if (int_mode != int_target_mode)
6068 27135 : op0 = convert_modes (int_target_mode, int_mode, op0, 0);
6069 :
6070 : return op0;
6071 : }
6072 :
6073 : /* Next try expanding this via the backend's cstore<mode>4. */
6074 680328 : mclass = GET_MODE_CLASS (mode);
6075 692329 : FOR_EACH_WIDER_MODE_FROM (compare_mode, mode)
6076 : {
6077 685046 : machine_mode optab_mode = mclass == MODE_CC ? CCmode : compare_mode;
6078 685046 : icode = optab_handler (cstore_optab, optab_mode);
6079 685046 : if (icode != CODE_FOR_nothing)
6080 : {
6081 673045 : do_pending_stack_adjust ();
6082 673045 : rtx tem = emit_cstore (target, icode, code, mode, compare_mode,
6083 : unsignedp, op0, op1, normalizep, target_mode);
6084 673045 : if (tem)
6085 : return tem;
6086 :
6087 89966 : if (GET_MODE_CLASS (mode) == MODE_FLOAT)
6088 : {
6089 80784 : enum rtx_code scode = swap_condition (code);
6090 :
6091 80784 : tem = emit_cstore (target, icode, scode, mode, compare_mode,
6092 : unsignedp, op1, op0, normalizep, target_mode);
6093 80784 : if (tem)
6094 : return tem;
6095 : }
6096 : break;
6097 : }
6098 : }
6099 :
6100 : /* If we are comparing a double-word integer with zero or -1, we can
6101 : convert the comparison into one involving a single word. */
6102 81858 : if (is_int_mode (mode, &int_mode)
6103 16052 : && GET_MODE_BITSIZE (int_mode) == BITS_PER_WORD * 2
6104 8898 : && (!MEM_P (op0) || ! MEM_VOLATILE_P (op0)))
6105 : {
6106 8898 : rtx tem;
6107 8898 : if ((code == EQ || code == NE)
6108 0 : && (op1 == const0_rtx || op1 == constm1_rtx))
6109 : {
6110 0 : rtx op00, op01;
6111 :
6112 : /* Do a logical OR or AND of the two words and compare the
6113 : result. */
6114 0 : op00 = force_subreg (word_mode, op0, int_mode, 0);
6115 0 : op01 = force_subreg (word_mode, op0, int_mode, UNITS_PER_WORD);
6116 0 : tem = expand_binop (word_mode,
6117 0 : op1 == const0_rtx ? ior_optab : and_optab,
6118 : op00, op01, NULL_RTX, unsignedp,
6119 : OPTAB_DIRECT);
6120 :
6121 0 : if (tem != 0)
6122 0 : tem = emit_store_flag (NULL_RTX, code, tem, op1, word_mode,
6123 : unsignedp, normalizep);
6124 : }
6125 8898 : else if ((code == LT || code == GE) && op1 == const0_rtx)
6126 : {
6127 0 : rtx op0h;
6128 :
6129 : /* If testing the sign bit, can just test on high word. */
6130 0 : op0h = force_highpart_subreg (word_mode, op0, int_mode);
6131 0 : tem = emit_store_flag (NULL_RTX, code, op0h, op1, word_mode,
6132 : unsignedp, normalizep);
6133 0 : }
6134 : else
6135 : tem = NULL_RTX;
6136 :
6137 0 : if (tem)
6138 : {
6139 0 : if (target_mode == VOIDmode || GET_MODE (tem) == target_mode)
6140 : return tem;
6141 0 : if (!target)
6142 0 : target = gen_reg_rtx (target_mode);
6143 :
6144 0 : convert_move (target, tem,
6145 0 : !val_signbit_known_set_p (word_mode,
6146 : (normalizep ? normalizep
6147 : : STORE_FLAG_VALUE)));
6148 0 : return target;
6149 : }
6150 : }
6151 :
6152 : return 0;
6153 : }
6154 :
6155 : /* Subroutine of emit_store_flag that handles cases in which the operands
6156 : are scalar integers. SUBTARGET is the target to use for temporary
6157 : operations and TRUEVAL is the value to store when the condition is
6158 : true. All other arguments are as for emit_store_flag. */
6159 :
6160 : rtx
6161 4591 : emit_store_flag_int (rtx target, rtx subtarget, enum rtx_code code, rtx op0,
6162 : rtx op1, scalar_int_mode mode, int unsignedp,
6163 : int normalizep, rtx trueval)
6164 : {
6165 4591 : machine_mode target_mode = target ? GET_MODE (target) : VOIDmode;
6166 4591 : rtx_insn *last = get_last_insn ();
6167 :
6168 : /* If this is an equality comparison of integers, we can try to exclusive-or
6169 : (or subtract) the two operands and use a recursive call to try the
6170 : comparison with zero. Don't do any of these cases if branches are
6171 : very cheap. */
6172 :
6173 4591 : if ((code == EQ || code == NE) && op1 != const0_rtx)
6174 : {
6175 0 : rtx tem = expand_binop (mode, xor_optab, op0, op1, subtarget, 1,
6176 : OPTAB_WIDEN);
6177 :
6178 0 : if (tem == 0)
6179 0 : tem = expand_binop (mode, sub_optab, op0, op1, subtarget, 1,
6180 : OPTAB_WIDEN);
6181 0 : if (tem != 0)
6182 0 : tem = emit_store_flag (target, code, tem, const0_rtx,
6183 : mode, unsignedp, normalizep);
6184 0 : if (tem != 0)
6185 : return tem;
6186 :
6187 0 : delete_insns_since (last);
6188 : }
6189 :
6190 : /* For integer comparisons, try the reverse comparison. However, for
6191 : small X and if we'd have anyway to extend, implementing "X != 0"
6192 : as "-(int)X >> 31" is still cheaper than inverting "(int)X == 0". */
6193 4591 : rtx_code rcode = reverse_condition (code);
6194 4591 : if (can_compare_p (rcode, mode, ccp_store_flag)
6195 4591 : && ! (optab_handler (cstore_optab, mode) == CODE_FOR_nothing
6196 0 : && code == NE
6197 0 : && GET_MODE_SIZE (mode) < UNITS_PER_WORD
6198 0 : && op1 == const0_rtx))
6199 : {
6200 4591 : int want_add = ((STORE_FLAG_VALUE == 1 && normalizep == -1)
6201 : || (STORE_FLAG_VALUE == -1 && normalizep == 1));
6202 :
6203 : /* Again, for the reverse comparison, use either an addition or a XOR. */
6204 4591 : if (want_add
6205 4591 : && rtx_cost (GEN_INT (normalizep), mode, PLUS, 1,
6206 : optimize_insn_for_speed_p ()) == 0)
6207 : {
6208 0 : rtx tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6209 : STORE_FLAG_VALUE, target_mode);
6210 0 : if (tem != 0)
6211 0 : tem = expand_binop (target_mode, add_optab, tem,
6212 0 : gen_int_mode (normalizep, target_mode),
6213 : target, 0, OPTAB_WIDEN);
6214 0 : if (tem != 0)
6215 : return tem;
6216 : }
6217 4591 : else if (!want_add
6218 4591 : && rtx_cost (trueval, mode, XOR, 1,
6219 : optimize_insn_for_speed_p ()) == 0)
6220 : {
6221 4591 : rtx tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6222 : normalizep, target_mode);
6223 4591 : if (tem != 0)
6224 0 : tem = expand_binop (target_mode, xor_optab, tem, trueval, target,
6225 0 : INTVAL (trueval) >= 0, OPTAB_WIDEN);
6226 0 : if (tem != 0)
6227 : return tem;
6228 : }
6229 :
6230 4591 : delete_insns_since (last);
6231 : }
6232 :
6233 : /* Some other cases we can do are EQ, NE, LE, and GT comparisons with
6234 : the constant zero. Reject all other comparisons at this point. Only
6235 : do LE and GT if branches are expensive since they are expensive on
6236 : 2-operand machines. */
6237 :
6238 4591 : if (op1 != const0_rtx
6239 4591 : || (code != EQ && code != NE
6240 91 : && (BRANCH_COST (optimize_insn_for_speed_p (),
6241 91 : false) <= 1 || (code != LE && code != GT))))
6242 : return 0;
6243 :
6244 : /* Try to put the result of the comparison in the sign bit. Assume we can't
6245 : do the necessary operation below. */
6246 :
6247 233 : rtx tem = 0;
6248 :
6249 : /* To see if A <= 0, compute (A | (A - 1)). A <= 0 iff that result has
6250 : the sign bit set. */
6251 :
6252 233 : if (code == LE)
6253 : {
6254 : /* This is destructive, so SUBTARGET can't be OP0. */
6255 56 : if (rtx_equal_p (subtarget, op0))
6256 0 : subtarget = 0;
6257 :
6258 56 : tem = expand_binop (mode, sub_optab, op0, const1_rtx, subtarget, 0,
6259 : OPTAB_WIDEN);
6260 56 : if (tem)
6261 56 : tem = expand_binop (mode, ior_optab, op0, tem, subtarget, 0,
6262 : OPTAB_WIDEN);
6263 : }
6264 :
6265 : /* To see if A > 0, compute (((signed) A) << BITS) - A, where BITS is the
6266 : number of bits in the mode of OP0, minus one. */
6267 :
6268 233 : if (code == GT)
6269 : {
6270 35 : if (rtx_equal_p (subtarget, op0))
6271 0 : subtarget = 0;
6272 :
6273 35 : tem = maybe_expand_shift (RSHIFT_EXPR, mode, op0,
6274 35 : GET_MODE_BITSIZE (mode) - 1,
6275 : subtarget, 0);
6276 35 : if (tem)
6277 35 : tem = expand_binop (mode, sub_optab, tem, op0, subtarget, 0,
6278 : OPTAB_WIDEN);
6279 : }
6280 :
6281 233 : if (code == EQ || code == NE)
6282 : {
6283 : /* For EQ or NE, one way to do the comparison is to apply an operation
6284 : that converts the operand into a positive number if it is nonzero
6285 : or zero if it was originally zero. Then, for EQ, we subtract 1 and
6286 : for NE we negate. This puts the result in the sign bit. Then we
6287 : normalize with a shift, if needed.
6288 :
6289 : Two operations that can do the above actions are ABS and FFS, so try
6290 : them. If that doesn't work, and MODE is smaller than a full word,
6291 : we can use zero-extension to the wider mode (an unsigned conversion)
6292 : as the operation. */
6293 :
6294 : /* Note that ABS doesn't yield a positive number for INT_MIN, but
6295 : that is compensated by the subsequent overflow when subtracting
6296 : one / negating. */
6297 :
6298 142 : if (optab_handler (abs_optab, mode) != CODE_FOR_nothing)
6299 142 : tem = expand_unop (mode, abs_optab, op0, subtarget, 1);
6300 0 : else if (optab_handler (ffs_optab, mode) != CODE_FOR_nothing)
6301 0 : tem = expand_unop (mode, ffs_optab, op0, subtarget, 1);
6302 0 : else if (GET_MODE_SIZE (mode) < UNITS_PER_WORD)
6303 : {
6304 0 : tem = convert_modes (word_mode, mode, op0, 1);
6305 0 : mode = word_mode;
6306 : }
6307 :
6308 142 : if (tem != 0)
6309 : {
6310 142 : if (code == EQ)
6311 0 : tem = expand_binop (mode, sub_optab, tem, const1_rtx, subtarget,
6312 : 0, OPTAB_WIDEN);
6313 : else
6314 142 : tem = expand_unop (mode, neg_optab, tem, subtarget, 0);
6315 : }
6316 :
6317 : /* If we couldn't do it that way, for NE we can "or" the two's complement
6318 : of the value with itself. For EQ, we take the one's complement of
6319 : that "or", which is an extra insn, so we only handle EQ if branches
6320 : are expensive. */
6321 :
6322 142 : if (tem == 0
6323 142 : && (code == NE
6324 0 : || BRANCH_COST (optimize_insn_for_speed_p (),
6325 : false) > 1))
6326 : {
6327 0 : if (rtx_equal_p (subtarget, op0))
6328 0 : subtarget = 0;
6329 :
6330 0 : tem = expand_unop (mode, neg_optab, op0, subtarget, 0);
6331 0 : tem = expand_binop (mode, ior_optab, tem, op0, subtarget, 0,
6332 : OPTAB_WIDEN);
6333 :
6334 0 : if (tem && code == EQ)
6335 0 : tem = expand_unop (mode, one_cmpl_optab, tem, subtarget, 0);
6336 : }
6337 : }
6338 :
6339 233 : if (tem && normalizep)
6340 233 : tem = maybe_expand_shift (RSHIFT_EXPR, mode, tem,
6341 233 : GET_MODE_BITSIZE (mode) - 1,
6342 : subtarget, normalizep == 1);
6343 :
6344 233 : if (tem)
6345 : {
6346 233 : if (!target)
6347 : ;
6348 233 : else if (GET_MODE (tem) != target_mode)
6349 : {
6350 91 : convert_move (target, tem, 0);
6351 91 : tem = target;
6352 : }
6353 142 : else if (!subtarget)
6354 : {
6355 73 : emit_move_insn (target, tem);
6356 73 : tem = target;
6357 : }
6358 : }
6359 : else
6360 0 : delete_insns_since (last);
6361 :
6362 : return tem;
6363 : }
6364 :
6365 : /* Emit a store-flags instruction for comparison CODE on OP0 and OP1
6366 : and storing in TARGET. Normally return TARGET.
6367 : Return 0 if that cannot be done.
6368 :
6369 : MODE is the mode to use for OP0 and OP1 should they be CONST_INTs. If
6370 : it is VOIDmode, they cannot both be CONST_INT.
6371 :
6372 : UNSIGNEDP is for the case where we have to widen the operands
6373 : to perform the operation. It says to use zero-extension.
6374 :
6375 : NORMALIZEP is 1 if we should convert the result to be either zero
6376 : or one. Normalize is -1 if we should convert the result to be
6377 : either zero or -1. If NORMALIZEP is zero, the result will be left
6378 : "raw" out of the scc insn. */
6379 :
6380 : rtx
6381 651364 : emit_store_flag (rtx target, enum rtx_code code, rtx op0, rtx op1,
6382 : machine_mode mode, int unsignedp, int normalizep)
6383 : {
6384 651364 : machine_mode target_mode = target ? GET_MODE (target) : VOIDmode;
6385 651364 : enum rtx_code rcode;
6386 651364 : rtx subtarget;
6387 651364 : rtx tem, trueval;
6388 651364 : rtx_insn *last;
6389 :
6390 : /* If we compare constants, we shouldn't use a store-flag operation,
6391 : but a constant load. We can get there via the vanilla route that
6392 : usually generates a compare-branch sequence, but will in this case
6393 : fold the comparison to a constant, and thus elide the branch. */
6394 651364 : if (CONSTANT_P (op0) && CONSTANT_P (op1))
6395 : return NULL_RTX;
6396 :
6397 651048 : tem = emit_store_flag_1 (target, code, op0, op1, mode, unsignedp, normalizep,
6398 : target_mode);
6399 651048 : if (tem)
6400 : return tem;
6401 :
6402 : /* If we reached here, we can't do this with a scc insn, however there
6403 : are some comparisons that can be done in other ways. Don't do any
6404 : of these cases if branches are very cheap. */
6405 76740 : if (BRANCH_COST (optimize_insn_for_speed_p (), false) == 0)
6406 : return 0;
6407 :
6408 : /* See what we need to return. We can only return a 1, -1, or the
6409 : sign bit. */
6410 :
6411 76740 : if (normalizep == 0)
6412 : {
6413 0 : if (STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1)
6414 0 : normalizep = STORE_FLAG_VALUE;
6415 :
6416 : else if (val_signbit_p (mode, STORE_FLAG_VALUE))
6417 : ;
6418 : else
6419 : return 0;
6420 : }
6421 :
6422 76740 : last = get_last_insn ();
6423 :
6424 : /* If optimizing, use different pseudo registers for each insn, instead
6425 : of reusing the same pseudo. This leads to better CSE, but slows
6426 : down the compiler, since there are more pseudos. */
6427 76671 : subtarget = (!optimize
6428 76740 : && (target_mode == mode)) ? target : NULL_RTX;
6429 76740 : trueval = GEN_INT (normalizep ? normalizep : STORE_FLAG_VALUE);
6430 :
6431 : /* For floating-point comparisons, try the reverse comparison or try
6432 : changing the "orderedness" of the comparison. */
6433 76740 : if (GET_MODE_CLASS (mode) == MODE_FLOAT)
6434 : {
6435 68634 : enum rtx_code first_code;
6436 68634 : bool and_them;
6437 :
6438 68634 : rcode = reverse_condition_maybe_unordered (code);
6439 68634 : if (can_compare_p (rcode, mode, ccp_store_flag)
6440 68634 : && (code == ORDERED || code == UNORDERED
6441 0 : || (! HONOR_NANS (mode) && (code == LTGT || code == UNEQ))
6442 0 : || (! HONOR_SNANS (mode) && (code == EQ || code == NE))))
6443 : {
6444 0 : int want_add = ((STORE_FLAG_VALUE == 1 && normalizep == -1)
6445 : || (STORE_FLAG_VALUE == -1 && normalizep == 1));
6446 :
6447 : /* For the reverse comparison, use either an addition or a XOR. */
6448 0 : if (want_add
6449 0 : && rtx_cost (GEN_INT (normalizep), mode, PLUS, 1,
6450 : optimize_insn_for_speed_p ()) == 0)
6451 : {
6452 0 : tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6453 : STORE_FLAG_VALUE, target_mode);
6454 0 : if (tem)
6455 0 : return expand_binop (target_mode, add_optab, tem,
6456 0 : gen_int_mode (normalizep, target_mode),
6457 : target, 0, OPTAB_WIDEN);
6458 : }
6459 0 : else if (!want_add
6460 0 : && rtx_cost (trueval, mode, XOR, 1,
6461 : optimize_insn_for_speed_p ()) == 0)
6462 : {
6463 0 : tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
6464 : normalizep, target_mode);
6465 0 : if (tem)
6466 0 : return expand_binop (target_mode, xor_optab, tem, trueval,
6467 0 : target, INTVAL (trueval) >= 0,
6468 0 : OPTAB_WIDEN);
6469 : }
6470 : }
6471 :
6472 68634 : delete_insns_since (last);
6473 :
6474 : /* Cannot split ORDERED and UNORDERED, only try the above trick. */
6475 68634 : if (code == ORDERED || code == UNORDERED)
6476 : return 0;
6477 :
6478 68492 : and_them = split_comparison (code, mode, &first_code, &code);
6479 :
6480 : /* If there are no NaNs, the first comparison should always fall through.
6481 : Effectively change the comparison to the other one. */
6482 68492 : if (!HONOR_NANS (mode))
6483 : {
6484 404 : gcc_assert (first_code == (and_them ? ORDERED : UNORDERED));
6485 243 : return emit_store_flag_1 (target, code, op0, op1, mode, 0, normalizep,
6486 243 : target_mode);
6487 : }
6488 :
6489 68249 : if (!HAVE_conditional_move)
6490 : return 0;
6491 :
6492 : /* x != y is split into x unord y || x ltgt y, but the conditional move
6493 : below evaluates its comparison unconditionally and LTGT is signaling.
6494 : Use the quiet x unord y || !(x uneq y) instead. */
6495 68249 : bool invert = false;
6496 68249 : if (code == LTGT && !and_them)
6497 : {
6498 62616 : code = UNEQ;
6499 62616 : invert = true;
6500 : }
6501 :
6502 : /* Do not change whether the comparison traps: only the quiet
6503 : EQ, NE and UNEQ remainders come from quiet comparisons. */
6504 68249 : if ((code != EQ && code != NE && code != UNEQ)
6505 2572 : && flag_trapping_math)
6506 : return 0;
6507 :
6508 : /* Try using a setcc instruction for ORDERED/UNORDERED, followed by a
6509 : conditional move. */
6510 65677 : tem = emit_store_flag_1 (subtarget, first_code, op0, op1, mode, 0,
6511 : normalizep, target_mode);
6512 65677 : if (tem == 0)
6513 : return 0;
6514 :
6515 65151 : if (and_them)
6516 2774 : tem = emit_conditional_move (target, { code, op0, op1, mode },
6517 2774 : tem, const0_rtx, GET_MODE (tem), 0);
6518 62377 : else if (invert)
6519 62377 : tem = emit_conditional_move (target, { code, op0, op1, mode },
6520 62377 : tem, trueval, GET_MODE (tem), 0);
6521 : else
6522 0 : tem = emit_conditional_move (target, { code, op0, op1, mode },
6523 0 : trueval, tem, GET_MODE (tem), 0);
6524 :
6525 65151 : if (tem == 0)
6526 725 : delete_insns_since (last);
6527 : return tem;
6528 : }
6529 :
6530 : /* The remaining tricks only apply to integer comparisons. */
6531 :
6532 8106 : scalar_int_mode int_mode;
6533 8106 : if (is_int_mode (mode, &int_mode))
6534 4591 : return emit_store_flag_int (target, subtarget, code, op0, op1, int_mode,
6535 4591 : unsignedp, normalizep, trueval);
6536 :
6537 : return 0;
6538 : }
6539 :
6540 : /* Like emit_store_flag, but always succeeds. */
6541 :
6542 : rtx
6543 599034 : emit_store_flag_force (rtx target, enum rtx_code code, rtx op0, rtx op1,
6544 : machine_mode mode, int unsignedp, int normalizep)
6545 : {
6546 599034 : rtx tem;
6547 599034 : rtx_code_label *label;
6548 599034 : rtx trueval, falseval;
6549 :
6550 : /* First see if emit_store_flag can do the job. */
6551 599034 : tem = emit_store_flag (target, code, op0, op1, mode, unsignedp, normalizep);
6552 599034 : if (tem != 0)
6553 : return tem;
6554 :
6555 : /* If one operand is constant, make it the second one. Only do this
6556 : if the other operand is not constant as well. */
6557 12155 : if (swap_commutative_operands_p (op0, op1))
6558 : {
6559 8 : std::swap (op0, op1);
6560 8 : code = swap_condition (code);
6561 : }
6562 :
6563 12155 : if (mode == VOIDmode)
6564 0 : mode = GET_MODE (op0);
6565 :
6566 12155 : if (!target)
6567 0 : target = gen_reg_rtx (word_mode);
6568 :
6569 : /* If this failed, we have to do this with set/compare/jump/set code.
6570 : For foo != 0, if foo is in OP0, just replace it with 1 if nonzero. */
6571 12155 : trueval = normalizep ? GEN_INT (normalizep) : const1_rtx;
6572 12155 : if (code == NE
6573 1746 : && GET_MODE_CLASS (mode) == MODE_INT
6574 41 : && REG_P (target)
6575 41 : && op0 == target
6576 0 : && op1 == const0_rtx)
6577 : {
6578 0 : label = gen_label_rtx ();
6579 0 : do_compare_rtx_and_jump (target, const0_rtx, EQ, unsignedp, mode,
6580 : NULL_RTX, NULL, label,
6581 : profile_probability::uninitialized ());
6582 0 : emit_move_insn (target, trueval);
6583 0 : emit_label (label);
6584 0 : return target;
6585 : }
6586 :
6587 12155 : if (!REG_P (target)
6588 12155 : || reg_mentioned_p (target, op0) || reg_mentioned_p (target, op1))
6589 5 : target = gen_reg_rtx (GET_MODE (target));
6590 :
6591 : /* Jump in the right direction if the target cannot implement CODE
6592 : but can jump on its reverse condition. */
6593 12155 : falseval = const0_rtx;
6594 12155 : if (! can_compare_p (code, mode, ccp_jump)
6595 12155 : && (! FLOAT_MODE_P (mode)
6596 7450 : || code == ORDERED || code == UNORDERED
6597 7263 : || (! HONOR_NANS (mode) && (code == LTGT || code == UNEQ))
6598 7263 : || (! HONOR_SNANS (mode) && (code == EQ || code == NE))))
6599 : {
6600 2787 : enum rtx_code rcode;
6601 2787 : if (FLOAT_MODE_P (mode))
6602 2787 : rcode = reverse_condition_maybe_unordered (code);
6603 : else
6604 0 : rcode = reverse_condition (code);
6605 :
6606 : /* Canonicalize to UNORDERED for the libcall. */
6607 2787 : if (can_compare_p (rcode, mode, ccp_jump)
6608 2787 : || (code == ORDERED && ! can_compare_p (ORDERED, mode, ccp_jump)))
6609 : {
6610 155 : falseval = trueval;
6611 155 : trueval = const0_rtx;
6612 155 : code = rcode;
6613 : }
6614 : }
6615 :
6616 12155 : emit_move_insn (target, trueval);
6617 12155 : label = gen_label_rtx ();
6618 12155 : do_compare_rtx_and_jump (op0, op1, code, unsignedp, mode, NULL_RTX, NULL,
6619 : label, profile_probability::uninitialized ());
6620 :
6621 12155 : emit_move_insn (target, falseval);
6622 12155 : emit_label (label);
6623 :
6624 12155 : return target;
6625 : }
6626 :
6627 : /* Expand a vector (left) rotate of MODE of X by an immediate AMT as a vector
6628 : permute operation. Emit code to put the result in DST if successful and
6629 : return it. Otherwise return NULL. This is intended to implement vector
6630 : rotates by byte amounts using vector permutes when the target does not offer
6631 : native vector rotate operations. */
6632 : rtx
6633 0 : expand_rotate_as_vec_perm (machine_mode mode, rtx dst, rtx x, rtx amt)
6634 : {
6635 0 : rtx amt_unwrap = unwrap_const_vec_duplicate (amt);
6636 : /* For now handle only rotate by the same integer constant in all lanes.
6637 : In principle rotates by any constant vector are representable through
6638 : permutes as long as the individual rotate amounts are multiples of
6639 : BITS_PER_UNIT. */
6640 0 : if (!CONST_INT_P (amt_unwrap))
6641 : return NULL_RTX;
6642 :
6643 0 : int rotamnt = INTVAL (amt_unwrap);
6644 0 : if (rotamnt % BITS_PER_UNIT != 0)
6645 : return NULL_RTX;
6646 0 : machine_mode qimode;
6647 0 : if (!qimode_for_vec_perm (mode).exists (&qimode))
6648 0 : return NULL_RTX;
6649 :
6650 0 : vec_perm_builder builder;
6651 0 : unsigned nunits = GET_MODE_SIZE (GET_MODE_INNER (mode));
6652 0 : poly_uint64 total_units = GET_MODE_SIZE (mode);
6653 0 : builder.new_vector (total_units, nunits, 3);
6654 0 : unsigned rot_bytes = rotamnt / BITS_PER_UNIT;
6655 0 : unsigned rot_to_perm = BYTES_BIG_ENDIAN ? rot_bytes : nunits - rot_bytes;
6656 0 : for (unsigned j = 0; j < 3 * nunits; j += nunits)
6657 0 : for (unsigned i = 0; i < nunits; i++)
6658 0 : builder.quick_push ((rot_to_perm + i) % nunits + j);
6659 :
6660 0 : rtx perm_src = lowpart_subreg (qimode, x, mode);
6661 0 : rtx perm_dst = lowpart_subreg (qimode, dst, mode);
6662 0 : rtx res
6663 0 : = expand_vec_perm_const (qimode, perm_src, perm_src, builder,
6664 : qimode, perm_dst);
6665 0 : if (!res)
6666 : return NULL_RTX;
6667 0 : if (!rtx_equal_p (res, perm_dst))
6668 0 : emit_move_insn (dst, lowpart_subreg (mode, res, qimode));
6669 : return dst;
6670 0 : }
6671 :
6672 : /* Helper function for canonicalize_cmp_for_target. Swap between inclusive
6673 : and exclusive ranges in order to create an equivalent comparison. See
6674 : canonicalize_cmp_for_target for the possible cases. */
6675 :
6676 : static enum rtx_code
6677 47 : equivalent_cmp_code (enum rtx_code code)
6678 : {
6679 47 : switch (code)
6680 : {
6681 : case GT:
6682 : return GE;
6683 0 : case GE:
6684 0 : return GT;
6685 0 : case LT:
6686 0 : return LE;
6687 0 : case LE:
6688 0 : return LT;
6689 2 : case GTU:
6690 2 : return GEU;
6691 0 : case GEU:
6692 0 : return GTU;
6693 1 : case LTU:
6694 1 : return LEU;
6695 2 : case LEU:
6696 2 : return LTU;
6697 :
6698 0 : default:
6699 0 : return code;
6700 : }
6701 : }
6702 :
6703 : /* Choose the more appropriate immediate in scalar integer comparisons. The
6704 : purpose of this is to end up with an immediate which can be loaded into a
6705 : register in fewer moves, if possible.
6706 :
6707 : For each integer comparison there exists an equivalent choice:
6708 : i) a > b or a >= b + 1
6709 : ii) a <= b or a < b + 1
6710 : iii) a >= b or a > b - 1
6711 : iv) a < b or a <= b - 1
6712 :
6713 : MODE is the mode of the first operand.
6714 : CODE points to the comparison code.
6715 : IMM points to the rtx containing the immediate. *IMM must satisfy
6716 : CONST_SCALAR_INT_P on entry and continues to satisfy CONST_SCALAR_INT_P
6717 : on exit. */
6718 :
6719 : void
6720 4841115 : canonicalize_comparison (machine_mode mode, enum rtx_code *code, rtx *imm)
6721 : {
6722 4841115 : if (!SCALAR_INT_MODE_P (mode))
6723 3986147 : return;
6724 :
6725 4837249 : int to_add = 0;
6726 4837249 : enum signop sgn = unsigned_condition_p (*code) ? UNSIGNED : SIGNED;
6727 :
6728 : /* Extract the immediate value from the rtx. */
6729 4837249 : wide_int imm_val = rtx_mode_t (*imm, mode);
6730 :
6731 4837249 : if (*code == GT || *code == GTU || *code == LE || *code == LEU)
6732 : to_add = 1;
6733 : else if (*code == GE || *code == GEU || *code == LT || *code == LTU)
6734 : to_add = -1;
6735 : else
6736 : return;
6737 :
6738 : /* Check for overflow/underflow in the case of signed values and
6739 : wrapping around in the case of unsigned values. If any occur
6740 : cancel the optimization. */
6741 855112 : wi::overflow_type overflow = wi::OVF_NONE;
6742 855112 : wide_int imm_modif;
6743 :
6744 855112 : if (to_add == 1)
6745 616383 : imm_modif = wi::add (imm_val, 1, sgn, &overflow);
6746 : else
6747 238729 : imm_modif = wi::sub (imm_val, 1, sgn, &overflow);
6748 :
6749 855112 : if (overflow)
6750 144 : return;
6751 :
6752 854968 : rtx new_imm = immed_wide_int_const (imm_modif, mode);
6753 :
6754 854968 : int old_cost = rtx_cost (*imm, mode, COMPARE, 0, true);
6755 854968 : int new_cost = rtx_cost (new_imm, mode, COMPARE, 0, true);
6756 :
6757 854968 : if (dump_file && (dump_flags & TDF_DETAILS))
6758 : {
6759 7 : fprintf (dump_file, ";; cmp: %s, old cst: ",
6760 7 : GET_RTX_NAME (*code));
6761 7 : print_rtl (dump_file, *imm);
6762 7 : fprintf (dump_file, " new cst: ");
6763 7 : print_rtl (dump_file, new_imm);
6764 7 : fprintf (dump_file, "\n");
6765 7 : fprintf (dump_file, ";; old cst cost: %d, new cst cost: %d\n",
6766 : old_cost, new_cost);
6767 : }
6768 :
6769 : /* Update the immediate and the code. */
6770 854968 : if (old_cost > new_cost)
6771 : {
6772 47 : *code = equivalent_cmp_code (*code);
6773 47 : *imm = new_imm;
6774 : }
6775 4837393 : }
6776 :
6777 :
6778 :
6779 : /* Perform possibly multi-word comparison and conditional jump to LABEL
6780 : if ARG1 OP ARG2 true where ARG1 and ARG2 are of mode MODE. This is
6781 : now a thin wrapper around do_compare_rtx_and_jump. */
6782 :
6783 : static void
6784 2434 : do_cmp_and_jump (rtx arg1, rtx arg2, enum rtx_code op, machine_mode mode,
6785 : rtx_code_label *label)
6786 : {
6787 2434 : int unsignedp = (op == LTU || op == LEU || op == GTU || op == GEU);
6788 2434 : do_compare_rtx_and_jump (arg1, arg2, op, unsignedp, mode, NULL_RTX,
6789 : NULL, label, profile_probability::uninitialized ());
6790 2434 : }
|