Line data Source code
1 : /* Lower _BitInt(N) operations to scalar operations.
2 : Copyright (C) 2023-2026 Free Software Foundation, Inc.
3 : Contributed by Jakub Jelinek <jakub@redhat.com>.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it
8 : under the terms of the GNU General Public License as published by the
9 : Free Software Foundation; either version 3, or (at your option) any
10 : later version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT
13 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "cfghooks.h"
29 : #include "tree-pass.h"
30 : #include "ssa.h"
31 : #include "fold-const.h"
32 : #include "gimplify.h"
33 : #include "gimple-iterator.h"
34 : #include "tree-cfg.h"
35 : #include "tree-dfa.h"
36 : #include "cfgloop.h"
37 : #include "cfganal.h"
38 : #include "target.h"
39 : #include "tree-ssa-live.h"
40 : #include "tree-ssa-coalesce.h"
41 : #include "domwalk.h"
42 : #include "memmodel.h"
43 : #include "optabs.h"
44 : #include "varasm.h"
45 : #include "gimple-range.h"
46 : #include "value-range.h"
47 : #include "langhooks.h"
48 : #include "gimplify-me.h"
49 : #include "diagnostic-core.h"
50 : #include "tree-eh.h"
51 : #include "tree-pretty-print.h"
52 : #include "alloc-pool.h"
53 : #include "tree-into-ssa.h"
54 : #include "tree-cfgcleanup.h"
55 : #include "tree-switch-conversion.h"
56 : #include "ubsan.h"
57 : #include "stor-layout.h"
58 : #include "gimple-lower-bitint.h"
59 : #include "attribs.h"
60 : #include "asan.h"
61 :
62 : /* Split BITINT_TYPE precisions in 4 categories. Small _BitInt, where
63 : target hook says it is a single limb, middle _BitInt which per ABI
64 : does not, but there is some INTEGER_TYPE in which arithmetics can be
65 : performed (operations on such _BitInt are lowered to casts to that
66 : arithmetic type and cast back; e.g. on x86_64 limb is DImode, but
67 : target supports TImode, so _BitInt(65) to _BitInt(128) are middle
68 : ones), large _BitInt which should by straight line code and
69 : finally huge _BitInt which should be handled by loops over the limbs. */
70 :
71 : enum bitint_prec_kind {
72 : bitint_prec_small,
73 : bitint_prec_middle,
74 : bitint_prec_large,
75 : bitint_prec_huge
76 : };
77 :
78 : /* Caches to speed up bitint_precision_kind. */
79 :
80 : static int small_max_prec, mid_min_prec, large_min_prec, huge_min_prec;
81 : static int limb_prec, abi_limb_prec;
82 : static bool bitint_big_endian;
83 : static enum bitint_ext bitint_extended;
84 :
85 : /* Categorize _BitInt(PREC) as small, middle, large or huge. */
86 :
87 : static bitint_prec_kind
88 488562 : bitint_precision_kind (int prec)
89 : {
90 488562 : if (prec <= small_max_prec)
91 : return bitint_prec_small;
92 470280 : if (huge_min_prec && prec >= huge_min_prec)
93 : return bitint_prec_huge;
94 265046 : if (large_min_prec && prec >= large_min_prec)
95 : return bitint_prec_large;
96 52483 : if (mid_min_prec && prec >= mid_min_prec)
97 : return bitint_prec_middle;
98 :
99 10031 : struct bitint_info info;
100 10031 : bool ok = targetm.c.bitint_type_info (prec, &info);
101 10031 : gcc_assert (ok);
102 10031 : scalar_int_mode limb_mode = as_a <scalar_int_mode> (info.limb_mode);
103 10031 : if (prec <= GET_MODE_PRECISION (limb_mode))
104 : {
105 2595 : small_max_prec = prec;
106 2595 : return bitint_prec_small;
107 : }
108 7436 : bitint_big_endian = info.big_endian;
109 7436 : bitint_extended = info.extended;
110 7436 : if (info.limb_mode == info.abi_limb_mode && bitint_extended == bitint_ext_full)
111 0 : bitint_extended = bitint_ext_partial;
112 7436 : if (!large_min_prec
113 14782 : && GET_MODE_PRECISION (limb_mode) <= MAX_FIXED_MODE_SIZE)
114 14692 : large_min_prec = MAX_FIXED_MODE_SIZE + 1;
115 7436 : if (!limb_prec)
116 7346 : limb_prec = GET_MODE_PRECISION (limb_mode);
117 7436 : if (!abi_limb_prec)
118 7346 : abi_limb_prec
119 7346 : = GET_MODE_PRECISION (as_a <scalar_int_mode> (info.abi_limb_mode));
120 : /* For bitint_ext_full with different limb_mode from abi_limb_mode we
121 : currently only support only abi_limb_mode twice the precision of
122 : limb_mode, and don't support big endian in that case either. */
123 7436 : gcc_assert (bitint_extended != bitint_ext_full
124 : || (abi_limb_prec == 2 * limb_prec
125 : && !bitint_big_endian));
126 7436 : if (!huge_min_prec)
127 : {
128 14692 : if (4 * limb_prec >= MAX_FIXED_MODE_SIZE)
129 7346 : huge_min_prec = 4 * limb_prec;
130 : else
131 0 : huge_min_prec = MAX_FIXED_MODE_SIZE + 1;
132 : }
133 14872 : if (prec <= MAX_FIXED_MODE_SIZE)
134 : {
135 1609 : if (!mid_min_prec || prec < mid_min_prec)
136 1609 : mid_min_prec = prec;
137 1609 : return bitint_prec_middle;
138 : }
139 5827 : if (huge_min_prec && prec >= huge_min_prec)
140 : return bitint_prec_huge;
141 : return bitint_prec_large;
142 : }
143 :
144 : /* Same for a TYPE. */
145 :
146 : static bitint_prec_kind
147 483096 : bitint_precision_kind (tree type)
148 : {
149 483096 : return bitint_precision_kind (TYPE_PRECISION (type));
150 : }
151 :
152 : /* Return minimum precision needed to describe INTEGER_CST
153 : CST. All bits above that precision up to precision of
154 : TREE_TYPE (CST) are cleared if EXT is set to 0, or set
155 : if EXT is set to -1. */
156 :
157 : static unsigned
158 5395 : bitint_min_cst_precision (tree cst, int &ext)
159 : {
160 5395 : ext = tree_int_cst_sgn (cst) < 0 ? -1 : 0;
161 5395 : wide_int w = wi::to_wide (cst);
162 5395 : unsigned min_prec = wi::min_precision (w, TYPE_SIGN (TREE_TYPE (cst)));
163 : /* For signed values, we don't need to count the sign bit,
164 : we'll use constant 0 or -1 for the upper bits. */
165 5395 : if (!TYPE_UNSIGNED (TREE_TYPE (cst)))
166 3279 : --min_prec;
167 : else
168 : {
169 : /* For unsigned values, also try signed min_precision
170 : in case the constant has lots of most significant bits set. */
171 2116 : unsigned min_prec2 = wi::min_precision (w, SIGNED) - 1;
172 2116 : if (min_prec2 < min_prec)
173 : {
174 1001 : ext = -1;
175 1001 : return min_prec2;
176 : }
177 : }
178 : return min_prec;
179 5395 : }
180 :
181 : namespace {
182 :
183 : /* If OP is middle _BitInt, cast it to corresponding INTEGER_TYPE
184 : cached in TYPE and return it. */
185 :
186 : tree
187 7768 : maybe_cast_middle_bitint (gimple_stmt_iterator *gsi, tree op, tree &type)
188 : {
189 7768 : if (op == NULL_TREE
190 7740 : || !BITINT_TYPE_P (TREE_TYPE (op))
191 14631 : || bitint_precision_kind (TREE_TYPE (op)) != bitint_prec_middle)
192 908 : return op;
193 :
194 6860 : int prec = TYPE_PRECISION (TREE_TYPE (op));
195 6860 : int uns = TYPE_UNSIGNED (TREE_TYPE (op));
196 6860 : if (type == NULL_TREE
197 2538 : || TYPE_PRECISION (type) != prec
198 9398 : || TYPE_UNSIGNED (type) != uns)
199 4322 : type = build_nonstandard_integer_type (prec, uns);
200 :
201 6860 : if (TREE_CODE (op) != SSA_NAME)
202 : {
203 2351 : tree nop = fold_convert (type, op);
204 2351 : if (is_gimple_val (nop))
205 : return nop;
206 : }
207 :
208 4509 : tree nop = make_ssa_name (type);
209 4509 : gimple *g = gimple_build_assign (nop, NOP_EXPR, op);
210 4509 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
211 4509 : return nop;
212 : }
213 :
214 : /* Return true if STMT can be handled in a loop from least to most
215 : significant limb together with its dependencies. */
216 :
217 : bool
218 48045 : mergeable_op (gimple *stmt)
219 : {
220 48045 : if (!is_gimple_assign (stmt))
221 : return false;
222 39325 : switch (gimple_assign_rhs_code (stmt))
223 : {
224 : case PLUS_EXPR:
225 : case MINUS_EXPR:
226 : case NEGATE_EXPR:
227 : case BIT_AND_EXPR:
228 : case BIT_IOR_EXPR:
229 : case BIT_XOR_EXPR:
230 : case BIT_NOT_EXPR:
231 : case SSA_NAME:
232 : case INTEGER_CST:
233 : case BIT_FIELD_REF:
234 : return true;
235 410 : case LSHIFT_EXPR:
236 410 : {
237 410 : tree cnt = gimple_assign_rhs2 (stmt);
238 410 : if (tree_fits_uhwi_p (cnt)
239 198 : && tree_to_uhwi (cnt) < (unsigned HOST_WIDE_INT) limb_prec)
240 : return true;
241 : }
242 : break;
243 6331 : CASE_CONVERT:
244 6331 : case VIEW_CONVERT_EXPR:
245 6331 : {
246 6331 : tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt));
247 6331 : tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (stmt));
248 6331 : if (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
249 6289 : && BITINT_TYPE_P (lhs_type)
250 3715 : && BITINT_TYPE_P (rhs_type)
251 3360 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
252 3324 : && bitint_precision_kind (rhs_type) >= bitint_prec_large
253 9494 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
254 3163 : == CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
255 : {
256 2195 : if (TYPE_PRECISION (rhs_type) >= TYPE_PRECISION (lhs_type))
257 : return true;
258 183 : if ((unsigned) TYPE_PRECISION (lhs_type) % (2 * limb_prec) != 0)
259 : return true;
260 17 : if (bitint_precision_kind (lhs_type) == bitint_prec_large)
261 : return true;
262 : }
263 : break;
264 : }
265 : default:
266 : break;
267 : }
268 : return false;
269 : }
270 :
271 : /* Return non-zero if stmt is .{ADD,SUB,MUL}_OVERFLOW call with
272 : _Complex large/huge _BitInt lhs which has at most two immediate uses,
273 : at most one use in REALPART_EXPR stmt in the same bb and exactly one
274 : IMAGPART_EXPR use in the same bb with a single use which casts it to
275 : non-BITINT_TYPE integral type. If there is a REALPART_EXPR use,
276 : return 2. Such cases (most common uses of those builtins) can be
277 : optimized by marking their lhs and lhs of IMAGPART_EXPR and maybe lhs
278 : of REALPART_EXPR as not needed to be backed up by a stack variable.
279 : For .UBSAN_CHECK_{ADD,SUB,MUL} return 3. */
280 :
281 : int
282 21452 : optimizable_arith_overflow (gimple *stmt)
283 : {
284 21452 : bool is_ubsan = false;
285 21452 : if (!is_gimple_call (stmt) || !gimple_call_internal_p (stmt))
286 : return false;
287 5236 : switch (gimple_call_internal_fn (stmt))
288 : {
289 : case IFN_ADD_OVERFLOW:
290 : case IFN_SUB_OVERFLOW:
291 : case IFN_MUL_OVERFLOW:
292 : break;
293 63 : case IFN_UBSAN_CHECK_ADD:
294 63 : case IFN_UBSAN_CHECK_SUB:
295 63 : case IFN_UBSAN_CHECK_MUL:
296 63 : is_ubsan = true;
297 63 : break;
298 : default:
299 : return 0;
300 : }
301 5210 : tree lhs = gimple_call_lhs (stmt);
302 5210 : if (!lhs)
303 : return 0;
304 5210 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
305 : return 0;
306 5195 : tree type = is_ubsan ? TREE_TYPE (lhs) : TREE_TYPE (TREE_TYPE (lhs));
307 0 : if (!BITINT_TYPE_P (type)
308 5195 : || bitint_precision_kind (type) < bitint_prec_large)
309 0 : return 0;
310 :
311 5195 : if (is_ubsan)
312 : {
313 48 : use_operand_p use_p;
314 48 : gimple *use_stmt;
315 48 : if (!single_imm_use (lhs, &use_p, &use_stmt)
316 48 : || gimple_bb (use_stmt) != gimple_bb (stmt)
317 48 : || !gimple_store_p (use_stmt)
318 48 : || !is_gimple_assign (use_stmt)
319 48 : || gimple_has_volatile_ops (use_stmt)
320 96 : || stmt_ends_bb_p (use_stmt))
321 0 : return 0;
322 : return 3;
323 : }
324 :
325 5147 : imm_use_iterator ui;
326 5147 : use_operand_p use_p;
327 5147 : int seen = 0;
328 5147 : gimple *realpart = NULL, *cast = NULL;
329 19887 : FOR_EACH_IMM_USE_FAST (use_p, ui, lhs)
330 : {
331 9807 : gimple *g = USE_STMT (use_p);
332 9807 : if (is_gimple_debug (g))
333 0 : continue;
334 9807 : if (!is_gimple_assign (g) || gimple_bb (g) != gimple_bb (stmt))
335 : return 0;
336 9807 : if (gimple_assign_rhs_code (g) == REALPART_EXPR)
337 : {
338 4660 : if ((seen & 1) != 0)
339 : return 0;
340 4660 : seen |= 1;
341 4660 : realpart = g;
342 : }
343 5147 : else if (gimple_assign_rhs_code (g) == IMAGPART_EXPR)
344 : {
345 5147 : if ((seen & 2) != 0)
346 214 : return 0;
347 5147 : seen |= 2;
348 :
349 5147 : use_operand_p use2_p;
350 5147 : gimple *use_stmt;
351 5147 : tree lhs2 = gimple_assign_lhs (g);
352 5147 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs2))
353 : return 0;
354 5147 : if (!single_imm_use (lhs2, &use2_p, &use_stmt)
355 5147 : || gimple_bb (use_stmt) != gimple_bb (stmt)
356 10294 : || !gimple_assign_cast_p (use_stmt))
357 : return 0;
358 :
359 4937 : lhs2 = gimple_assign_lhs (use_stmt);
360 9874 : if (!INTEGRAL_TYPE_P (TREE_TYPE (lhs2))
361 9874 : || BITINT_TYPE_P (TREE_TYPE (lhs2)))
362 : return 0;
363 4933 : cast = use_stmt;
364 : }
365 : else
366 : return 0;
367 214 : }
368 4933 : if ((seen & 2) == 0)
369 : return 0;
370 4933 : if (seen == 3)
371 : {
372 : /* Punt if the cast stmt appears before realpart stmt, because
373 : if both appear, the lowering wants to emit all the code
374 : at the location of realpart stmt. */
375 4660 : gimple_stmt_iterator gsi = gsi_for_stmt (realpart);
376 4660 : unsigned int cnt = 0;
377 4663 : do
378 : {
379 4663 : gsi_prev_nondebug (&gsi);
380 4663 : if (gsi_end_p (gsi) || gsi_stmt (gsi) == cast)
381 : return 0;
382 4660 : if (gsi_stmt (gsi) == stmt)
383 : return 2;
384 : /* If realpart is too far from stmt, punt as well.
385 : Usually it will appear right after it. */
386 3 : if (++cnt == 32)
387 : return 0;
388 : }
389 : while (1);
390 : }
391 : return 1;
392 : }
393 :
394 : /* If STMT is some kind of comparison (GIMPLE_COND, comparison assignment)
395 : comparing large/huge _BitInt types, return the comparison code and if
396 : non-NULL fill in the comparison operands to *POP1 and *POP2. */
397 :
398 : tree_code
399 36002 : comparison_op (gimple *stmt, tree *pop1, tree *pop2)
400 : {
401 36002 : tree op1 = NULL_TREE, op2 = NULL_TREE;
402 36002 : tree_code code = ERROR_MARK;
403 36002 : if (gimple_code (stmt) == GIMPLE_COND)
404 : {
405 6552 : code = gimple_cond_code (stmt);
406 6552 : op1 = gimple_cond_lhs (stmt);
407 6552 : op2 = gimple_cond_rhs (stmt);
408 : }
409 29450 : else if (is_gimple_assign (stmt))
410 : {
411 29423 : code = gimple_assign_rhs_code (stmt);
412 29423 : op1 = gimple_assign_rhs1 (stmt);
413 29423 : if (TREE_CODE_CLASS (code) == tcc_comparison
414 29423 : || TREE_CODE_CLASS (code) == tcc_binary)
415 2205 : op2 = gimple_assign_rhs2 (stmt);
416 : }
417 36002 : if (TREE_CODE_CLASS (code) != tcc_comparison)
418 : return ERROR_MARK;
419 7334 : tree type = TREE_TYPE (op1);
420 10 : if (!BITINT_TYPE_P (type)
421 7344 : || bitint_precision_kind (type) < bitint_prec_large)
422 0 : return ERROR_MARK;
423 7334 : if (pop1)
424 : {
425 7272 : *pop1 = op1;
426 7272 : *pop2 = op2;
427 : }
428 : return code;
429 : }
430 :
431 : /* Class used during large/huge _BitInt lowering containing all the
432 : state for the methods. */
433 :
434 : struct bitint_large_huge
435 : {
436 7344 : bitint_large_huge ()
437 7344 : : m_names (NULL), m_loads (NULL), m_preserved (NULL),
438 7344 : m_single_use_names (NULL), m_map (NULL), m_vars (NULL),
439 7344 : m_limb_type (NULL_TREE), m_data (vNULL),
440 7344 : m_returns_twice_calls (vNULL) {}
441 :
442 : ~bitint_large_huge ();
443 :
444 : void insert_before (gimple *);
445 : tree limb_access_type (tree, tree);
446 : tree limb_access (tree, tree, tree, bool, bool = false);
447 : tree build_bit_field_ref (tree, tree, unsigned HOST_WIDE_INT,
448 : unsigned HOST_WIDE_INT);
449 : void if_then (gimple *, profile_probability, edge &, edge &);
450 : void if_then_else (gimple *, profile_probability, edge &, edge &);
451 : void if_then_if_then_else (gimple *g, gimple *,
452 : profile_probability, profile_probability,
453 : edge &, edge &, edge &);
454 : tree handle_operand (tree, tree);
455 : tree prepare_data_in_out (tree, tree, tree *, tree = NULL_TREE);
456 : tree add_cast (tree, tree);
457 : tree handle_plus_minus (tree_code, tree, tree, tree);
458 : tree handle_lshift (tree, tree, tree);
459 : tree handle_cast (tree, tree, tree);
460 : tree handle_bit_field_ref (tree, tree);
461 : tree handle_load (gimple *, tree);
462 : tree handle_stmt (gimple *, tree);
463 : tree handle_operand_addr (tree, gimple *, int *, int *);
464 : tree create_loop (tree, tree *);
465 : tree lower_mergeable_stmt (gimple *, tree_code &, tree, tree);
466 : tree lower_comparison_stmt (gimple *, tree_code &, tree, tree);
467 : void lower_shift_stmt (tree, gimple *);
468 : void lower_muldiv_stmt (tree, gimple *);
469 : void lower_float_conv_stmt (tree, gimple *);
470 : tree arith_overflow_extract_bits (unsigned int, unsigned int, tree,
471 : unsigned int, bool);
472 : void finish_arith_overflow (tree, tree, tree, tree, tree, tree, gimple *,
473 : unsigned, tree_code);
474 : void lower_addsub_overflow (tree, gimple *);
475 : void lower_mul_overflow (tree, gimple *);
476 : void lower_cplxpart_stmt (tree, gimple *);
477 : void lower_complexexpr_stmt (gimple *);
478 : void lower_bit_query (gimple *);
479 : void lower_bswap_bitreverse (tree, gimple *);
480 : void lower_call (tree, gimple *);
481 : void lower_asm (gimple *);
482 : void lower_stmt (gimple *);
483 :
484 : /* Bitmap of large/huge _BitInt SSA_NAMEs except those can be
485 : merged with their uses. */
486 : bitmap m_names;
487 : /* Subset of those for lhs of load statements. These will be
488 : cleared in m_names if the loads will be mergeable with all
489 : their uses. */
490 : bitmap m_loads;
491 : /* Bitmap of large/huge _BitInt SSA_NAMEs that should survive
492 : to later passes (arguments or return values of calls). */
493 : bitmap m_preserved;
494 : /* Subset of m_names which have a single use. As the lowering
495 : can replace various original statements with their lowered
496 : form even before it is done iterating over all basic blocks,
497 : testing has_single_use for the purpose of emitting clobbers
498 : doesn't work properly. */
499 : bitmap m_single_use_names;
500 : /* Used for coalescing/partitioning of large/huge _BitInt SSA_NAMEs
501 : set in m_names. */
502 : var_map m_map;
503 : /* Mapping of the partitions to corresponding decls. */
504 : tree *m_vars;
505 : /* Unsigned integer type with limb precision. */
506 : tree m_limb_type;
507 : /* Its TYPE_SIZE_UNIT. */
508 : unsigned HOST_WIDE_INT m_limb_size;
509 : /* Location of a gimple stmt which is being currently lowered. */
510 : location_t m_loc;
511 : /* Current stmt iterator where code is being lowered currently. */
512 : gimple_stmt_iterator m_gsi;
513 : /* Statement after which any clobbers should be added if non-NULL. */
514 : gimple *m_after_stmt;
515 : /* Set when creating loops to the loop header bb and its preheader. */
516 : basic_block m_bb, m_preheader_bb;
517 : /* Stmt iterator after which initialization statements should be emitted. */
518 : gimple_stmt_iterator m_init_gsi;
519 : /* Decl into which a mergeable statement stores result. */
520 : tree m_lhs;
521 : /* handle_operand/handle_stmt can be invoked in various ways.
522 :
523 : lower_mergeable_stmt for large _BitInt calls those with constant
524 : idx only, expanding to straight line code, for huge _BitInt
525 : emits a loop from least significant limb upwards, where each loop
526 : iteration handles 2 limbs, plus there can be up to one full limb
527 : and one partial limb processed after the loop, where handle_operand
528 : and/or handle_stmt are called with constant idx. m_upwards_2limb
529 : is set for this case, false otherwise. m_upwards is true if it
530 : is either large or huge _BitInt handled by lower_mergeable_stmt,
531 : i.e. indexes always increase.
532 :
533 : Another way is used by lower_comparison_stmt, which walks limbs
534 : from most significant to least significant, partial limb if any
535 : processed first with constant idx and then loop processing a single
536 : limb per iteration with non-constant idx.
537 :
538 : Another way is used in lower_shift_stmt, where for LSHIFT_EXPR
539 : destination limbs are processed from most significant to least
540 : significant or for RSHIFT_EXPR the other way around, in loops or
541 : straight line code, but idx usually is non-constant (so from
542 : handle_operand/handle_stmt POV random access). The LSHIFT_EXPR
543 : handling there can access even partial limbs using non-constant
544 : idx (then m_var_msb should be true, for all the other cases
545 : including lower_mergeable_stmt/lower_comparison_stmt that is
546 : not the case and so m_var_msb should be false.
547 :
548 : m_first should be set the first time handle_operand/handle_stmt
549 : is called and clear when it is called for some other limb with
550 : the same argument. If the lowering of an operand (e.g. INTEGER_CST)
551 : or statement (e.g. +/-/<< with < limb_prec constant) needs some
552 : state between the different calls, when m_first is true it should
553 : push some trees to m_data vector and also make sure m_data_cnt is
554 : incremented by how many trees were pushed, and when m_first is
555 : false, it can use the m_data[m_data_cnt] etc. data or update them,
556 : just needs to bump m_data_cnt by the same amount as when it was
557 : called with m_first set. The toplevel calls to
558 : handle_operand/handle_stmt should set m_data_cnt to 0 and truncate
559 : m_data vector when setting m_first to true.
560 :
561 : m_cast_conditional and m_bitfld_load are used when handling a
562 : bit-field load inside of a widening cast. handle_cast sometimes
563 : needs to do runtime comparisons and handle_operand only conditionally
564 : or even in two separate conditional blocks for one idx (once with
565 : constant index after comparing the runtime one for equality with the
566 : constant). In these cases, m_cast_conditional is set to true and
567 : the bit-field load then communicates its m_data_cnt to handle_cast
568 : using m_bitfld_load. */
569 : bool m_first;
570 : bool m_var_msb;
571 : unsigned m_upwards_2limb;
572 : bool m_upwards;
573 : bool m_cast_conditional;
574 : unsigned m_bitfld_load;
575 : vec<tree> m_data;
576 : unsigned int m_data_cnt;
577 : vec<gimple *> m_returns_twice_calls;
578 : };
579 :
580 7344 : bitint_large_huge::~bitint_large_huge ()
581 : {
582 7344 : BITMAP_FREE (m_names);
583 7344 : BITMAP_FREE (m_loads);
584 7344 : BITMAP_FREE (m_preserved);
585 7344 : BITMAP_FREE (m_single_use_names);
586 7344 : if (m_map)
587 5678 : delete_var_map (m_map);
588 7344 : XDELETEVEC (m_vars);
589 7344 : m_data.release ();
590 7344 : m_returns_twice_calls.release ();
591 7344 : }
592 :
593 : /* Insert gimple statement G before current location
594 : and set its gimple_location. */
595 :
596 : void
597 363999 : bitint_large_huge::insert_before (gimple *g)
598 : {
599 363999 : gimple_set_location (g, m_loc);
600 363999 : gsi_insert_before (&m_gsi, g, GSI_SAME_STMT);
601 363999 : }
602 :
603 : /* Return type for accessing limb IDX of BITINT_TYPE TYPE.
604 : This is normally m_limb_type, except for a partial most
605 : significant limb if any. */
606 :
607 : tree
608 130811 : bitint_large_huge::limb_access_type (tree type, tree idx)
609 : {
610 130811 : if (type == NULL_TREE)
611 5913 : return m_limb_type;
612 124898 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
613 124898 : unsigned int prec = TYPE_PRECISION (type);
614 124898 : gcc_assert (i * limb_prec < prec
615 : || (bitint_extended == bitint_ext_full
616 : && abi_limb_prec > limb_prec
617 : && i * limb_prec
618 : < CEIL (prec, abi_limb_prec) * abi_limb_prec));
619 249796 : if (bitint_big_endian
620 124898 : ? (i != 0 || (prec % limb_prec) == 0)
621 124898 : : (i + 1) * limb_prec <= prec)
622 81506 : return m_limb_type;
623 : else
624 86784 : return build_nonstandard_integer_type (prec % limb_prec,
625 43392 : TYPE_UNSIGNED (type));
626 : }
627 :
628 : /* Return a tree how to access limb IDX of VAR corresponding to BITINT_TYPE
629 : TYPE. If WRITE_P is true, it will be a store, otherwise a read. */
630 :
631 : tree
632 156125 : bitint_large_huge::limb_access (tree type, tree var, tree idx, bool write_p,
633 : bool abi_load_p)
634 : {
635 156125 : tree atype = (tree_fits_uhwi_p (idx)
636 156125 : ? limb_access_type (type, idx) : m_limb_type);
637 :
638 156125 : tree ltype = (bitint_extended && abi_load_p) ? atype : m_limb_type;
639 :
640 156125 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (var));
641 156125 : tree ret;
642 156125 : if (DECL_P (var) && tree_fits_uhwi_p (idx))
643 : {
644 95985 : if (as != TYPE_ADDR_SPACE (ltype))
645 0 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
646 0 : | ENCODE_QUAL_ADDR_SPACE (as));
647 95985 : tree ptype = build_pointer_type (strip_array_types (TREE_TYPE (var)));
648 95985 : unsigned HOST_WIDE_INT off = tree_to_uhwi (idx) * m_limb_size;
649 95985 : if (bitint_big_endian)
650 0 : off += m_limb_size - tree_to_uhwi (TYPE_SIZE_UNIT (ltype));
651 95985 : ret = build2 (MEM_REF, ltype,
652 : build_fold_addr_expr (var),
653 95985 : build_int_cst (ptype, off));
654 95985 : TREE_THIS_VOLATILE (ret) = TREE_THIS_VOLATILE (var);
655 95985 : TREE_SIDE_EFFECTS (ret) = TREE_SIDE_EFFECTS (var);
656 95985 : }
657 60140 : else if (TREE_CODE (var) == MEM_REF && tree_fits_uhwi_p (idx))
658 : {
659 4302 : if (as != TYPE_ADDR_SPACE (ltype))
660 0 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
661 0 : | ENCODE_QUAL_ADDR_SPACE (as));
662 4302 : unsigned HOST_WIDE_INT off = tree_to_uhwi (idx) * m_limb_size;
663 4302 : if (bitint_big_endian)
664 0 : off += m_limb_size - tree_to_uhwi (TYPE_SIZE_UNIT (ltype));
665 4302 : ret
666 8604 : = build2 (MEM_REF, ltype, unshare_expr (TREE_OPERAND (var, 0)),
667 8604 : size_binop (PLUS_EXPR, TREE_OPERAND (var, 1),
668 : build_int_cst (TREE_TYPE (TREE_OPERAND (var, 1)),
669 : off)));
670 4302 : TREE_THIS_VOLATILE (ret) = TREE_THIS_VOLATILE (var);
671 4302 : TREE_SIDE_EFFECTS (ret) = TREE_SIDE_EFFECTS (var);
672 4302 : TREE_THIS_NOTRAP (ret) = TREE_THIS_NOTRAP (var);
673 4302 : }
674 : else
675 : {
676 55838 : ltype = m_limb_type;
677 55838 : if (as != TYPE_ADDR_SPACE (ltype))
678 17 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
679 17 : | ENCODE_QUAL_ADDR_SPACE (as));
680 55838 : var = unshare_expr (var);
681 55838 : if (TREE_CODE (TREE_TYPE (var)) != ARRAY_TYPE
682 83846 : || !useless_type_conversion_p (m_limb_type,
683 28008 : TREE_TYPE (TREE_TYPE (var))))
684 : {
685 28772 : unsigned HOST_WIDE_INT nelts
686 28772 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var))), limb_prec);
687 28772 : tree atype = build_array_type_nelts (ltype, nelts);
688 28772 : var = build1 (VIEW_CONVERT_EXPR, atype, var);
689 : }
690 55838 : ret = build4 (ARRAY_REF, ltype, var, idx, NULL_TREE, NULL_TREE);
691 : }
692 156125 : if (!write_p && !useless_type_conversion_p (atype, ltype))
693 : {
694 19031 : gimple *g = gimple_build_assign (make_ssa_name (m_limb_type), ret);
695 19031 : insert_before (g);
696 19031 : ret = gimple_assign_lhs (g);
697 19031 : ret = build1 (NOP_EXPR, atype, ret);
698 : }
699 156125 : return ret;
700 : }
701 :
702 : /* Build a BIT_FIELD_REF to access BITSIZE bits with FTYPE type at
703 : offset BITPOS inside of OBJ. */
704 :
705 : tree
706 273 : bitint_large_huge::build_bit_field_ref (tree ftype, tree obj,
707 : unsigned HOST_WIDE_INT bitsize,
708 : unsigned HOST_WIDE_INT bitpos)
709 : {
710 546 : if (INTEGRAL_TYPE_P (TREE_TYPE (obj))
711 282 : && !type_has_mode_precision_p (TREE_TYPE (obj)))
712 : {
713 9 : unsigned HOST_WIDE_INT nelts
714 9 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))), limb_prec);
715 9 : tree ltype = m_limb_type;
716 9 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (obj));
717 9 : if (as != TYPE_ADDR_SPACE (ltype))
718 0 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
719 0 : | ENCODE_QUAL_ADDR_SPACE (as));
720 9 : tree atype = build_array_type_nelts (ltype, nelts);
721 9 : obj = build1 (VIEW_CONVERT_EXPR, atype, obj);
722 : }
723 273 : return build3 (BIT_FIELD_REF, ftype, obj, bitsize_int (bitsize),
724 273 : bitsize_int (bitpos));
725 : }
726 :
727 : /* Emit a half diamond,
728 : if (COND)
729 : |\
730 : | \
731 : | \
732 : | new_bb1
733 : | /
734 : | /
735 : |/
736 : or if (COND) new_bb1;
737 : PROB is the probability that the condition is true.
738 : Updates m_gsi to start of new_bb1.
739 : Sets EDGE_TRUE to edge from new_bb1 to successor and
740 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND) bb. */
741 :
742 : void
743 4318 : bitint_large_huge::if_then (gimple *cond, profile_probability prob,
744 : edge &edge_true, edge &edge_false)
745 : {
746 4318 : insert_before (cond);
747 4318 : edge e1 = split_block (gsi_bb (m_gsi), cond);
748 4318 : edge e2 = split_block (e1->dest, (gimple *) NULL);
749 4318 : edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
750 4318 : e1->flags = EDGE_TRUE_VALUE;
751 4318 : e1->probability = prob;
752 4318 : e3->probability = prob.invert ();
753 4318 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
754 4318 : edge_true = e2;
755 4318 : edge_false = e3;
756 4318 : m_gsi = gsi_after_labels (e1->dest);
757 4318 : }
758 :
759 : /* Emit a full diamond,
760 : if (COND)
761 : /\
762 : / \
763 : / \
764 : new_bb1 new_bb2
765 : \ /
766 : \ /
767 : \/
768 : or if (COND) new_bb2; else new_bb1;
769 : PROB is the probability that the condition is true.
770 : Updates m_gsi to start of new_bb2.
771 : Sets EDGE_TRUE to edge from new_bb1 to successor and
772 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND) bb. */
773 :
774 : void
775 112 : bitint_large_huge::if_then_else (gimple *cond, profile_probability prob,
776 : edge &edge_true, edge &edge_false)
777 : {
778 112 : insert_before (cond);
779 112 : edge e1 = split_block (gsi_bb (m_gsi), cond);
780 112 : edge e2 = split_block (e1->dest, (gimple *) NULL);
781 112 : basic_block bb = create_empty_bb (e1->dest);
782 112 : add_bb_to_loop (bb, e1->dest->loop_father);
783 112 : edge e3 = make_edge (e1->src, bb, EDGE_TRUE_VALUE);
784 112 : e1->flags = EDGE_FALSE_VALUE;
785 112 : e3->probability = prob;
786 112 : e1->probability = prob.invert ();
787 112 : bb->count = e1->src->count.apply_probability (prob);
788 112 : set_immediate_dominator (CDI_DOMINATORS, bb, e1->src);
789 112 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
790 112 : edge_true = make_single_succ_edge (bb, e2->dest, EDGE_FALLTHRU);
791 112 : edge_false = e2;
792 112 : m_gsi = gsi_after_labels (bb);
793 112 : }
794 :
795 : /* Emit a half diamond with full diamond in it
796 : if (COND1)
797 : |\
798 : | \
799 : | \
800 : | if (COND2)
801 : | / \
802 : | / \
803 : |new_bb1 new_bb2
804 : | | /
805 : \ | /
806 : \ | /
807 : \ | /
808 : \|/
809 : or if (COND1) { if (COND2) new_bb2; else new_bb1; }
810 : PROB1 is the probability that the condition 1 is true.
811 : PROB2 is the probability that the condition 2 is true.
812 : Updates m_gsi to start of new_bb1.
813 : Sets EDGE_TRUE_TRUE to edge from new_bb2 to successor,
814 : EDGE_TRUE_FALSE to edge from new_bb1 to successor and
815 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND1) bb.
816 : If COND2 is NULL, this is equivalent to
817 : if_then (COND1, PROB1, EDGE_TRUE_FALSE, EDGE_FALSE);
818 : EDGE_TRUE_TRUE = NULL; */
819 :
820 : void
821 2098 : bitint_large_huge::if_then_if_then_else (gimple *cond1, gimple *cond2,
822 : profile_probability prob1,
823 : profile_probability prob2,
824 : edge &edge_true_true,
825 : edge &edge_true_false,
826 : edge &edge_false)
827 : {
828 2098 : edge e2, e3, e4 = NULL;
829 2098 : if_then (cond1, prob1, e2, e3);
830 2098 : if (cond2 == NULL)
831 : {
832 1257 : edge_true_true = NULL;
833 1257 : edge_true_false = e2;
834 1257 : edge_false = e3;
835 1257 : return;
836 : }
837 841 : insert_before (cond2);
838 841 : e2 = split_block (gsi_bb (m_gsi), cond2);
839 841 : basic_block bb = create_empty_bb (e2->dest);
840 841 : add_bb_to_loop (bb, e2->dest->loop_father);
841 841 : e4 = make_edge (e2->src, bb, EDGE_TRUE_VALUE);
842 841 : set_immediate_dominator (CDI_DOMINATORS, bb, e2->src);
843 841 : e4->probability = prob2;
844 841 : e2->flags = EDGE_FALSE_VALUE;
845 841 : e2->probability = prob2.invert ();
846 841 : bb->count = e2->src->count.apply_probability (prob2);
847 841 : e4 = make_single_succ_edge (bb, e3->dest, EDGE_FALLTHRU);
848 841 : e2 = find_edge (e2->dest, e3->dest);
849 841 : edge_true_true = e4;
850 841 : edge_true_false = e2;
851 841 : edge_false = e3;
852 841 : m_gsi = gsi_after_labels (e2->src);
853 : }
854 :
855 : /* Emit code to access limb IDX from OP. */
856 :
857 : tree
858 111474 : bitint_large_huge::handle_operand (tree op, tree idx)
859 : {
860 111474 : switch (TREE_CODE (op))
861 : {
862 77040 : case SSA_NAME:
863 77040 : if (m_names == NULL
864 77040 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (op)))
865 : {
866 14507 : if (SSA_NAME_IS_DEFAULT_DEF (op))
867 : {
868 47 : if (m_first)
869 : {
870 16 : tree v = create_tmp_reg (m_limb_type);
871 16 : if (SSA_NAME_VAR (op) && VAR_P (SSA_NAME_VAR (op)))
872 : {
873 16 : DECL_NAME (v) = DECL_NAME (SSA_NAME_VAR (op));
874 16 : DECL_SOURCE_LOCATION (v)
875 16 : = DECL_SOURCE_LOCATION (SSA_NAME_VAR (op));
876 : }
877 16 : v = get_or_create_ssa_default_def (cfun, v);
878 16 : m_data.safe_push (v);
879 : }
880 47 : tree ret = m_data[m_data_cnt];
881 47 : m_data_cnt++;
882 47 : if (tree_fits_uhwi_p (idx))
883 : {
884 45 : tree type = limb_access_type (TREE_TYPE (op), idx);
885 45 : ret = add_cast (type, ret);
886 : }
887 47 : return ret;
888 : }
889 14460 : location_t loc_save = m_loc;
890 14460 : m_loc = gimple_location (SSA_NAME_DEF_STMT (op));
891 14460 : tree ret = handle_stmt (SSA_NAME_DEF_STMT (op), idx);
892 14460 : m_loc = loc_save;
893 14460 : return ret;
894 : }
895 62533 : int p;
896 62533 : gimple *g;
897 62533 : tree t;
898 62533 : p = var_to_partition (m_map, op);
899 62533 : gcc_assert (m_vars[p] != NULL_TREE);
900 62533 : t = limb_access (TREE_TYPE (op), m_vars[p], idx, false);
901 62533 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t);
902 62533 : insert_before (g);
903 62533 : t = gimple_assign_lhs (g);
904 62533 : if (m_first
905 22639 : && m_single_use_names
906 21767 : && m_vars[p] != m_lhs
907 21667 : && m_after_stmt
908 71294 : && bitmap_bit_p (m_single_use_names, SSA_NAME_VERSION (op)))
909 : {
910 8445 : tree clobber = build_clobber (TREE_TYPE (m_vars[p]),
911 : CLOBBER_STORAGE_END);
912 8445 : g = gimple_build_assign (m_vars[p], clobber);
913 8445 : gimple_stmt_iterator gsi = gsi_for_stmt (m_after_stmt);
914 8445 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
915 : }
916 : return t;
917 34434 : case INTEGER_CST:
918 34434 : if (tree_fits_uhwi_p (idx))
919 : {
920 23839 : tree c, type = limb_access_type (TREE_TYPE (op), idx);
921 23839 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
922 23839 : if (m_first)
923 : {
924 6214 : m_data.safe_push (NULL_TREE);
925 6214 : m_data.safe_push (NULL_TREE);
926 : }
927 23839 : if (bitint_big_endian)
928 0 : i = CEIL (TYPE_PRECISION (TREE_TYPE (op)), limb_prec) - 1 - i;
929 23839 : if (limb_prec != HOST_BITS_PER_WIDE_INT)
930 : {
931 0 : wide_int w = wi::rshift (wi::to_wide (op), i * limb_prec,
932 0 : TYPE_SIGN (TREE_TYPE (op)));
933 0 : c = wide_int_to_tree (type,
934 0 : wide_int::from (w, TYPE_PRECISION (type),
935 : UNSIGNED));
936 0 : }
937 23839 : else if (i >= TREE_INT_CST_EXT_NUNITS (op))
938 7511 : c = build_int_cst (type,
939 13334 : tree_int_cst_sgn (op) < 0 ? -1 : 0);
940 : else
941 16328 : c = build_int_cst (type, TREE_INT_CST_ELT (op, i));
942 23839 : m_data_cnt += 2;
943 23839 : return c;
944 : }
945 10595 : if (m_first
946 10595 : || (m_data[m_data_cnt] == NULL_TREE
947 159 : && m_data[m_data_cnt + 1] == NULL_TREE))
948 : {
949 5343 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
950 5343 : unsigned int rem = prec % ((m_upwards_2limb ? 2 : 1) * limb_prec);
951 5343 : int ext;
952 5343 : unsigned min_prec = bitint_min_cst_precision (op, ext);
953 5343 : if (m_first)
954 : {
955 5184 : m_data.safe_push (NULL_TREE);
956 5184 : m_data.safe_push (NULL_TREE);
957 : }
958 5343 : if (integer_zerop (op))
959 : {
960 837 : tree c = build_zero_cst (m_limb_type);
961 837 : m_data[m_data_cnt] = c;
962 837 : m_data[m_data_cnt + 1] = c;
963 : }
964 4506 : else if (integer_all_onesp (op))
965 : {
966 678 : tree c = build_all_ones_cst (m_limb_type);
967 678 : m_data[m_data_cnt] = c;
968 678 : m_data[m_data_cnt + 1] = c;
969 : }
970 3828 : else if (m_upwards_2limb && min_prec <= (unsigned) limb_prec)
971 : {
972 : /* Single limb constant. Use a phi with that limb from
973 : the preheader edge and 0 or -1 constant from the other edge
974 : and for the second limb in the loop. */
975 894 : tree out;
976 894 : gcc_assert (m_first);
977 894 : m_data.pop ();
978 894 : m_data.pop ();
979 894 : prepare_data_in_out (fold_convert (m_limb_type, op), idx, &out,
980 894 : build_int_cst (m_limb_type, ext));
981 894 : }
982 2934 : else if (min_prec > prec - rem - 2 * limb_prec)
983 : {
984 : /* Constant which has enough significant bits that it isn't
985 : worth trying to save .rodata space by extending from smaller
986 : number. */
987 2415 : tree type;
988 2415 : if (m_var_msb)
989 25 : type = TREE_TYPE (op);
990 : else
991 : /* If we have a guarantee the most significant partial limb
992 : (if any) will be only accessed through handle_operand
993 : with INTEGER_CST idx, we don't need to include the partial
994 : limb in .rodata. */
995 2390 : type = build_bitint_type (prec - rem, 1);
996 2415 : tree c = tree_output_constant_def (fold_convert (type, op));
997 2415 : m_data[m_data_cnt] = c;
998 2415 : m_data[m_data_cnt + 1] = NULL_TREE;
999 : }
1000 519 : else if (m_upwards_2limb)
1001 : {
1002 : /* Constant with smaller number of bits. Trade conditional
1003 : code for .rodata space by extending from smaller number. */
1004 444 : min_prec = CEIL (min_prec, 2 * limb_prec) * (2 * limb_prec);
1005 444 : tree type = build_bitint_type (min_prec, 1);
1006 444 : tree c = tree_output_constant_def (fold_convert (type, op));
1007 444 : tree ridx = idx;
1008 444 : if (bitint_big_endian)
1009 : {
1010 0 : ridx = make_ssa_name (sizetype);
1011 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1012 0 : size_int (min_prec / limb_prec
1013 : - ((HOST_WIDE_INT)
1014 : CEIL (prec,
1015 : limb_prec))));
1016 0 : insert_before (g);
1017 : }
1018 444 : tree ridx2 = make_ssa_name (sizetype);
1019 444 : g = gimple_build_assign (ridx2, PLUS_EXPR, ridx,
1020 : bitint_big_endian
1021 0 : ? size_int (-1) : size_one_node);
1022 444 : insert_before (g);
1023 444 : if (bitint_big_endian)
1024 0 : g = gimple_build_cond (GE_EXPR, idx,
1025 0 : size_int (CEIL (prec, limb_prec)
1026 : - min_prec / limb_prec),
1027 : NULL_TREE, NULL_TREE);
1028 : else
1029 444 : g = gimple_build_cond (LT_EXPR, idx,
1030 444 : size_int (min_prec / limb_prec),
1031 : NULL_TREE, NULL_TREE);
1032 444 : edge edge_true, edge_false;
1033 888 : if_then (g, (min_prec >= (prec - rem) / 2
1034 312 : ? profile_probability::likely ()
1035 132 : : profile_probability::unlikely ()),
1036 : edge_true, edge_false);
1037 444 : tree c1 = limb_access (TREE_TYPE (op), c, ridx, false);
1038 444 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c1)), c1);
1039 444 : insert_before (g);
1040 444 : c1 = gimple_assign_lhs (g);
1041 444 : tree c2 = limb_access (TREE_TYPE (op), c, ridx2, false);
1042 444 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c2)), c2);
1043 444 : insert_before (g);
1044 444 : c2 = gimple_assign_lhs (g);
1045 444 : tree c3 = build_int_cst (m_limb_type, ext);
1046 444 : m_gsi = gsi_after_labels (edge_true->dest);
1047 444 : m_data[m_data_cnt] = make_ssa_name (m_limb_type);
1048 444 : m_data[m_data_cnt + 1] = make_ssa_name (m_limb_type);
1049 444 : gphi *phi = create_phi_node (m_data[m_data_cnt],
1050 : edge_true->dest);
1051 444 : add_phi_arg (phi, c1, edge_true, UNKNOWN_LOCATION);
1052 444 : add_phi_arg (phi, c3, edge_false, UNKNOWN_LOCATION);
1053 444 : phi = create_phi_node (m_data[m_data_cnt + 1], edge_true->dest);
1054 444 : add_phi_arg (phi, c2, edge_true, UNKNOWN_LOCATION);
1055 444 : add_phi_arg (phi, c3, edge_false, UNKNOWN_LOCATION);
1056 : }
1057 : else
1058 : {
1059 : /* Constant with smaller number of bits. Trade conditional
1060 : code for .rodata space by extending from smaller number.
1061 : Version for loops with random access to the limbs or
1062 : downwards loops. */
1063 75 : min_prec = CEIL (min_prec, limb_prec) * limb_prec;
1064 75 : tree c;
1065 75 : if (min_prec <= (unsigned) limb_prec)
1066 21 : c = fold_convert (m_limb_type, op);
1067 : else
1068 : {
1069 54 : tree type = build_bitint_type (min_prec, 1);
1070 54 : c = tree_output_constant_def (fold_convert (type, op));
1071 : }
1072 75 : m_data[m_data_cnt] = c;
1073 75 : m_data[m_data_cnt + 1] = integer_type_node;
1074 : }
1075 5343 : t = m_data[m_data_cnt];
1076 : }
1077 : else
1078 5252 : t = m_data[m_data_cnt + 1];
1079 10595 : if (m_data[m_data_cnt + 1] == NULL_TREE)
1080 : {
1081 4774 : tree ridx = idx;
1082 4774 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
1083 4774 : tree c = m_data[m_data_cnt];
1084 4774 : unsigned int min_prec = TYPE_PRECISION (TREE_TYPE (c));
1085 4774 : if (bitint_big_endian
1086 0 : && CEIL (min_prec, limb_prec) != CEIL (prec, limb_prec))
1087 : {
1088 0 : ridx = make_ssa_name (sizetype);
1089 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1090 0 : size_int (CEIL (min_prec, limb_prec)
1091 : - ((HOST_WIDE_INT)
1092 : CEIL (prec, limb_prec))));
1093 0 : insert_before (g);
1094 : }
1095 4774 : t = limb_access (TREE_TYPE (op), c, ridx, false);
1096 4774 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t);
1097 4774 : insert_before (g);
1098 4774 : t = gimple_assign_lhs (g);
1099 : }
1100 5821 : else if (m_data[m_data_cnt + 1] == integer_type_node)
1101 : {
1102 115 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
1103 115 : unsigned rem = prec % ((m_upwards_2limb ? 2 : 1) * limb_prec);
1104 115 : int ext = wi::neg_p (wi::to_wide (op)) ? -1 : 0;
1105 115 : tree c = m_data[m_data_cnt];
1106 115 : unsigned min_prec = TYPE_PRECISION (TREE_TYPE (c));
1107 115 : if (bitint_big_endian)
1108 0 : g = gimple_build_cond (GE_EXPR, idx,
1109 0 : size_int (CEIL (prec, limb_prec)
1110 : - min_prec / limb_prec),
1111 : NULL_TREE, NULL_TREE);
1112 : else
1113 115 : g = gimple_build_cond (LT_EXPR, idx,
1114 115 : size_int (min_prec / limb_prec),
1115 : NULL_TREE, NULL_TREE);
1116 115 : edge edge_true, edge_false;
1117 230 : if_then (g, (min_prec >= (prec - rem) / 2
1118 29 : ? profile_probability::likely ()
1119 86 : : profile_probability::unlikely ()),
1120 : edge_true, edge_false);
1121 115 : if (min_prec > (unsigned) limb_prec)
1122 : {
1123 70 : tree ridx = idx;
1124 70 : if (bitint_big_endian)
1125 : {
1126 0 : ridx = make_ssa_name (sizetype);
1127 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1128 0 : size_int (min_prec / limb_prec
1129 : - ((HOST_WIDE_INT)
1130 : CEIL (prec,
1131 : limb_prec))));
1132 0 : insert_before (g);
1133 : }
1134 70 : c = limb_access (TREE_TYPE (op), c, ridx, false);
1135 70 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c)), c);
1136 70 : insert_before (g);
1137 70 : c = gimple_assign_lhs (g);
1138 : }
1139 115 : tree c2 = build_int_cst (m_limb_type, ext);
1140 115 : m_gsi = gsi_after_labels (edge_true->dest);
1141 115 : t = make_ssa_name (m_limb_type);
1142 115 : gphi *phi = create_phi_node (t, edge_true->dest);
1143 115 : add_phi_arg (phi, c, edge_true, UNKNOWN_LOCATION);
1144 115 : add_phi_arg (phi, c2, edge_false, UNKNOWN_LOCATION);
1145 : }
1146 10595 : m_data_cnt += 2;
1147 10595 : return t;
1148 0 : default:
1149 0 : gcc_unreachable ();
1150 : }
1151 : }
1152 :
1153 : /* Helper method, add a PHI node with VAL from preheader edge if
1154 : inside of a loop and m_first. Keep state in a pair of m_data
1155 : elements. If VAL_OUT is non-NULL, use that as PHI argument from
1156 : the latch edge, otherwise create a new SSA_NAME for it and let
1157 : caller initialize it. */
1158 :
1159 : tree
1160 15851 : bitint_large_huge::prepare_data_in_out (tree val, tree idx, tree *data_out,
1161 : tree val_out)
1162 : {
1163 15851 : if (!m_first)
1164 : {
1165 9425 : *data_out = tree_fits_uhwi_p (idx) ? NULL_TREE : m_data[m_data_cnt + 1];
1166 9425 : return m_data[m_data_cnt];
1167 : }
1168 :
1169 6426 : *data_out = NULL_TREE;
1170 6426 : if (tree_fits_uhwi_p (idx))
1171 : {
1172 2004 : m_data.safe_push (val);
1173 2004 : m_data.safe_push (NULL_TREE);
1174 2004 : return val;
1175 : }
1176 :
1177 4422 : tree in = make_ssa_name (TREE_TYPE (val));
1178 4422 : gphi *phi = create_phi_node (in, m_bb);
1179 4422 : edge e1 = find_edge (m_preheader_bb, m_bb);
1180 4422 : edge e2 = EDGE_PRED (m_bb, 0);
1181 4422 : if (e1 == e2)
1182 4422 : e2 = EDGE_PRED (m_bb, 1);
1183 4422 : add_phi_arg (phi, val, e1, UNKNOWN_LOCATION);
1184 4422 : tree out = val_out ? val_out : make_ssa_name (TREE_TYPE (val));
1185 4422 : add_phi_arg (phi, out, e2, UNKNOWN_LOCATION);
1186 4422 : m_data.safe_push (in);
1187 4422 : m_data.safe_push (out);
1188 4422 : return in;
1189 : }
1190 :
1191 : /* Return VAL cast to TYPE. If VAL is INTEGER_CST, just
1192 : convert it without emitting any code, otherwise emit
1193 : the conversion statement before the current location. */
1194 :
1195 : tree
1196 38729 : bitint_large_huge::add_cast (tree type, tree val)
1197 : {
1198 38729 : if (TREE_CODE (val) == INTEGER_CST)
1199 4577 : return fold_convert (type, val);
1200 :
1201 34152 : tree lhs = make_ssa_name (type);
1202 34152 : gimple *g = gimple_build_assign (lhs, NOP_EXPR, val);
1203 34152 : insert_before (g);
1204 34152 : return lhs;
1205 : }
1206 :
1207 : /* Helper of handle_stmt method, handle PLUS_EXPR or MINUS_EXPR. */
1208 :
1209 : tree
1210 13353 : bitint_large_huge::handle_plus_minus (tree_code code, tree rhs1, tree rhs2,
1211 : tree idx)
1212 : {
1213 13353 : tree lhs, data_out, ctype;
1214 13353 : tree rhs1_type = TREE_TYPE (rhs1);
1215 13353 : gimple *g;
1216 13353 : tree data_in = prepare_data_in_out (build_zero_cst (m_limb_type), idx,
1217 : &data_out);
1218 :
1219 19139 : if (optab_handler (code == PLUS_EXPR ? uaddc5_optab : usubc5_optab,
1220 13353 : TYPE_MODE (m_limb_type)) != CODE_FOR_nothing)
1221 : {
1222 13353 : ctype = build_complex_type (m_limb_type);
1223 13353 : if (!types_compatible_p (rhs1_type, m_limb_type))
1224 : {
1225 1071 : if (!TYPE_UNSIGNED (rhs1_type))
1226 : {
1227 360 : tree type = unsigned_type_for (rhs1_type);
1228 360 : rhs1 = add_cast (type, rhs1);
1229 360 : rhs2 = add_cast (type, rhs2);
1230 : }
1231 1071 : rhs1 = add_cast (m_limb_type, rhs1);
1232 1071 : rhs2 = add_cast (m_limb_type, rhs2);
1233 : }
1234 13353 : lhs = make_ssa_name (ctype);
1235 19139 : g = gimple_build_call_internal (code == PLUS_EXPR
1236 : ? IFN_UADDC : IFN_USUBC,
1237 : 3, rhs1, rhs2, data_in);
1238 13353 : gimple_call_set_lhs (g, lhs);
1239 13353 : insert_before (g);
1240 13353 : if (data_out == NULL_TREE)
1241 11134 : data_out = make_ssa_name (m_limb_type);
1242 13353 : g = gimple_build_assign (data_out, IMAGPART_EXPR,
1243 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1244 13353 : insert_before (g);
1245 : }
1246 0 : else if (types_compatible_p (rhs1_type, m_limb_type))
1247 : {
1248 0 : ctype = build_complex_type (m_limb_type);
1249 0 : lhs = make_ssa_name (ctype);
1250 0 : g = gimple_build_call_internal (code == PLUS_EXPR
1251 : ? IFN_ADD_OVERFLOW : IFN_SUB_OVERFLOW,
1252 : 2, rhs1, rhs2);
1253 0 : gimple_call_set_lhs (g, lhs);
1254 0 : insert_before (g);
1255 0 : if (data_out == NULL_TREE)
1256 0 : data_out = make_ssa_name (m_limb_type);
1257 0 : if (!integer_zerop (data_in))
1258 : {
1259 0 : rhs1 = make_ssa_name (m_limb_type);
1260 0 : g = gimple_build_assign (rhs1, REALPART_EXPR,
1261 : build1 (REALPART_EXPR, m_limb_type, lhs));
1262 0 : insert_before (g);
1263 0 : rhs2 = make_ssa_name (m_limb_type);
1264 0 : g = gimple_build_assign (rhs2, IMAGPART_EXPR,
1265 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1266 0 : insert_before (g);
1267 0 : lhs = make_ssa_name (ctype);
1268 0 : g = gimple_build_call_internal (code == PLUS_EXPR
1269 : ? IFN_ADD_OVERFLOW
1270 : : IFN_SUB_OVERFLOW,
1271 : 2, rhs1, data_in);
1272 0 : gimple_call_set_lhs (g, lhs);
1273 0 : insert_before (g);
1274 0 : data_in = make_ssa_name (m_limb_type);
1275 0 : g = gimple_build_assign (data_in, IMAGPART_EXPR,
1276 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1277 0 : insert_before (g);
1278 0 : g = gimple_build_assign (data_out, PLUS_EXPR, rhs2, data_in);
1279 0 : insert_before (g);
1280 : }
1281 : else
1282 : {
1283 0 : g = gimple_build_assign (data_out, IMAGPART_EXPR,
1284 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1285 0 : insert_before (g);
1286 : }
1287 : }
1288 : else
1289 : {
1290 : /* Always perform the two additions or subtractions in
1291 : unsigned type, avoid introducing a temporary UB. While
1292 : the end result of the 2 additions or 2 subtractions in
1293 : a valid program should not overflow, temporarily it can.
1294 : See PR126503. */
1295 0 : tree utype = unsigned_type_for (rhs1_type);
1296 0 : tree in = add_cast (utype, data_in);
1297 0 : lhs = make_ssa_name (utype);
1298 0 : if (utype != rhs1_type)
1299 : {
1300 0 : rhs1 = add_cast (utype, rhs1);
1301 0 : rhs2 = add_cast (utype, rhs2);
1302 : }
1303 0 : g = gimple_build_assign (lhs, code, rhs1, rhs2);
1304 0 : insert_before (g);
1305 0 : rhs1 = make_ssa_name (utype);
1306 0 : g = gimple_build_assign (rhs1, code, lhs, in);
1307 0 : insert_before (g);
1308 0 : m_data[m_data_cnt] = NULL_TREE;
1309 0 : m_data_cnt += 2;
1310 0 : return utype != rhs1_type ? add_cast (rhs1_type, rhs1) : rhs1;
1311 : }
1312 13353 : rhs1 = make_ssa_name (m_limb_type);
1313 13353 : g = gimple_build_assign (rhs1, REALPART_EXPR,
1314 : build1 (REALPART_EXPR, m_limb_type, lhs));
1315 13353 : insert_before (g);
1316 13353 : if (!types_compatible_p (rhs1_type, m_limb_type))
1317 1071 : rhs1 = add_cast (rhs1_type, rhs1);
1318 13353 : m_data[m_data_cnt] = data_out;
1319 13353 : m_data_cnt += 2;
1320 13353 : return rhs1;
1321 : }
1322 :
1323 : /* Helper function for handle_stmt method, handle LSHIFT_EXPR by
1324 : count in [0, limb_prec - 1] range. */
1325 :
1326 : tree
1327 152 : bitint_large_huge::handle_lshift (tree rhs1, tree rhs2, tree idx)
1328 : {
1329 152 : unsigned HOST_WIDE_INT cnt = tree_to_uhwi (rhs2);
1330 152 : gcc_checking_assert (cnt < (unsigned) limb_prec);
1331 152 : if (cnt == 0)
1332 : return rhs1;
1333 :
1334 152 : tree lhs, data_out, rhs1_type = TREE_TYPE (rhs1);
1335 152 : gimple *g;
1336 152 : tree data_in = prepare_data_in_out (build_zero_cst (m_limb_type), idx,
1337 : &data_out);
1338 :
1339 152 : if (!integer_zerop (data_in))
1340 : {
1341 136 : lhs = make_ssa_name (m_limb_type);
1342 136 : g = gimple_build_assign (lhs, RSHIFT_EXPR, data_in,
1343 : build_int_cst (unsigned_type_node,
1344 136 : limb_prec - cnt));
1345 136 : insert_before (g);
1346 136 : if (!types_compatible_p (rhs1_type, m_limb_type))
1347 35 : lhs = add_cast (rhs1_type, lhs);
1348 : data_in = lhs;
1349 : }
1350 152 : if (types_compatible_p (rhs1_type, m_limb_type))
1351 : {
1352 117 : if (data_out == NULL_TREE)
1353 82 : data_out = make_ssa_name (m_limb_type);
1354 117 : g = gimple_build_assign (data_out, rhs1);
1355 117 : insert_before (g);
1356 : }
1357 152 : if (cnt < (unsigned) TYPE_PRECISION (rhs1_type))
1358 : {
1359 137 : lhs = make_ssa_name (rhs1_type);
1360 137 : g = gimple_build_assign (lhs, LSHIFT_EXPR, rhs1, rhs2);
1361 137 : insert_before (g);
1362 137 : if (!integer_zerop (data_in))
1363 : {
1364 121 : rhs1 = lhs;
1365 121 : lhs = make_ssa_name (rhs1_type);
1366 121 : g = gimple_build_assign (lhs, BIT_IOR_EXPR, rhs1, data_in);
1367 121 : insert_before (g);
1368 : }
1369 : }
1370 : else
1371 : lhs = data_in;
1372 152 : m_data[m_data_cnt] = data_out;
1373 152 : m_data_cnt += 2;
1374 152 : return lhs;
1375 : }
1376 :
1377 : /* Helper function for handle_stmt method, handle an integral
1378 : to integral conversion. */
1379 :
1380 : tree
1381 7558 : bitint_large_huge::handle_cast (tree lhs_type, tree rhs1, tree idx)
1382 : {
1383 7558 : tree rhs_type = TREE_TYPE (rhs1);
1384 7558 : gimple *g;
1385 7558 : if ((TREE_CODE (rhs1) == SSA_NAME || TREE_CODE (rhs1) == INTEGER_CST)
1386 7558 : && BITINT_TYPE_P (lhs_type)
1387 7558 : && BITINT_TYPE_P (rhs_type)
1388 6565 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
1389 14123 : && bitint_precision_kind (rhs_type) >= bitint_prec_large)
1390 : {
1391 5926 : if (TYPE_PRECISION (rhs_type) >= TYPE_PRECISION (lhs_type)
1392 : /* If lhs has bigger precision than rhs, we can use
1393 : the simple case only if there is a guarantee that
1394 : the most significant limb is handled in straight
1395 : line code. If m_var_msb (on left shifts) or
1396 : if m_upwards_2limb * limb_prec is equal to
1397 : lhs precision or if not m_upwards_2limb and lhs_type
1398 : has precision which is multiple of limb_prec that is
1399 : not the case. */
1400 5926 : || (!m_var_msb
1401 1524 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1402 1524 : == CEIL (TYPE_PRECISION (rhs_type), limb_prec))
1403 370 : && ((!m_upwards_2limb
1404 182 : && (TYPE_PRECISION (lhs_type) % limb_prec != 0))
1405 267 : || (m_upwards_2limb
1406 376 : && (m_upwards_2limb * limb_prec
1407 188 : < TYPE_PRECISION (lhs_type))))))
1408 : {
1409 4667 : tree ridx = idx;
1410 4667 : if (bitint_big_endian
1411 4667 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1412 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1413 : {
1414 0 : HOST_WIDE_INT diff = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1415 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1416 0 : if (tree_fits_uhwi_p (idx))
1417 0 : ridx = size_int (tree_to_uhwi (idx) + diff);
1418 : else
1419 : {
1420 0 : tree t = make_ssa_name (sizetype);
1421 0 : g = gimple_build_assign (t, PLUS_EXPR, idx, size_int (diff));
1422 0 : insert_before (g);
1423 0 : ridx = t;
1424 : }
1425 : }
1426 4667 : rhs1 = handle_operand (rhs1, ridx);
1427 4667 : if (tree_fits_uhwi_p (idx))
1428 : {
1429 2386 : tree type = limb_access_type (lhs_type, idx);
1430 2386 : if (!types_compatible_p (type, TREE_TYPE (rhs1)))
1431 1249 : rhs1 = add_cast (type, rhs1);
1432 : }
1433 4667 : return rhs1;
1434 : }
1435 1259 : tree t;
1436 : /* Indexes lower than this don't need any special processing. */
1437 1259 : unsigned low = ((unsigned) TYPE_PRECISION (rhs_type)
1438 1259 : - !TYPE_UNSIGNED (rhs_type)) / limb_prec;
1439 : /* Indexes >= than this always contain an extension. */
1440 1259 : unsigned high = CEIL ((unsigned) TYPE_PRECISION (rhs_type), limb_prec);
1441 1259 : unsigned lcnt = CEIL ((unsigned) TYPE_PRECISION (lhs_type), limb_prec);
1442 1259 : unsigned lowe = bitint_big_endian ? lcnt - 1 - low : low;
1443 1259 : bool save_first = m_first;
1444 1259 : if (m_first)
1445 : {
1446 414 : m_data.safe_push (NULL_TREE);
1447 414 : m_data.safe_push (NULL_TREE);
1448 414 : m_data.safe_push (NULL_TREE);
1449 414 : if (TYPE_UNSIGNED (rhs_type))
1450 : /* No need to keep state between iterations. */
1451 : ;
1452 193 : else if (m_upwards && !m_upwards_2limb)
1453 : /* We need to keep state between iterations, but
1454 : not within any loop, everything is straight line
1455 : code with only increasing indexes. */
1456 : ;
1457 153 : else if (!m_upwards_2limb)
1458 : {
1459 3 : unsigned save_data_cnt = m_data_cnt;
1460 3 : gimple_stmt_iterator save_gsi = m_gsi;
1461 3 : m_gsi = m_init_gsi;
1462 3 : if (gsi_end_p (m_gsi))
1463 0 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
1464 : else
1465 3 : gsi_next (&m_gsi);
1466 3 : m_data_cnt = save_data_cnt + 3;
1467 3 : t = handle_operand (rhs1, size_int (bitint_big_endian
1468 : ? high - 1 - low : low));
1469 3 : m_first = false;
1470 3 : m_data[save_data_cnt + 2]
1471 3 : = build_int_cst (NULL_TREE, m_data_cnt);
1472 3 : m_data_cnt = save_data_cnt;
1473 3 : t = add_cast (signed_type_for (m_limb_type), t);
1474 3 : tree lpm1 = build_int_cst (unsigned_type_node, limb_prec - 1);
1475 3 : tree n = make_ssa_name (TREE_TYPE (t));
1476 3 : g = gimple_build_assign (n, RSHIFT_EXPR, t, lpm1);
1477 3 : insert_before (g);
1478 3 : m_data[save_data_cnt + 1] = add_cast (m_limb_type, n);
1479 3 : m_init_gsi = m_gsi;
1480 3 : if (gsi_end_p (m_init_gsi))
1481 0 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
1482 : else
1483 3 : gsi_prev (&m_init_gsi);
1484 3 : m_gsi = save_gsi;
1485 : }
1486 150 : else if (m_upwards_2limb * limb_prec < TYPE_PRECISION (rhs_type))
1487 : /* We need to keep state between iterations, but
1488 : fortunately not within the loop, only afterwards. */
1489 : ;
1490 : else
1491 : {
1492 146 : tree out;
1493 146 : m_data.truncate (m_data_cnt);
1494 146 : prepare_data_in_out (build_zero_cst (m_limb_type), idx, &out);
1495 146 : m_data.safe_push (NULL_TREE);
1496 : }
1497 : }
1498 :
1499 1259 : unsigned save_data_cnt = m_data_cnt;
1500 1259 : m_data_cnt += 3;
1501 1259 : if (!tree_fits_uhwi_p (idx))
1502 : {
1503 688 : if (m_upwards_2limb
1504 668 : && low >= m_upwards_2limb - m_first)
1505 : {
1506 158 : if (bitint_big_endian
1507 158 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1508 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1509 : {
1510 0 : HOST_WIDE_INT diff
1511 0 : = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1512 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1513 0 : tree t = make_ssa_name (sizetype);
1514 0 : g = gimple_build_assign (t, PLUS_EXPR, idx, size_int (diff));
1515 0 : insert_before (g);
1516 0 : idx = t;
1517 : }
1518 158 : rhs1 = handle_operand (rhs1, idx);
1519 158 : if (m_first)
1520 131 : m_data[save_data_cnt + 2]
1521 262 : = build_int_cst (NULL_TREE, m_data_cnt);
1522 158 : m_first = save_first;
1523 158 : return rhs1;
1524 : }
1525 1326 : bool single_comparison
1526 530 : = low == high || (m_upwards_2limb && (low & 1) == m_first);
1527 266 : tree idxc = idx;
1528 266 : if (!single_comparison
1529 266 : && m_upwards_2limb
1530 246 : && !m_first
1531 112 : && low + 1 == m_upwards_2limb)
1532 : /* In this case we know that idx <= low always,
1533 : so effectively we just needs a single comparison,
1534 : idx < low or idx == low, but we'd need to emit different
1535 : code for the 2 branches than single_comparison normally
1536 : emits. So, instead of special-casing that, emit a
1537 : low <= low comparison which cfg cleanup will clean up
1538 : at the end of the pass. */
1539 89 : idxc = size_int (lowe);
1540 530 : if (bitint_big_endian)
1541 0 : g = gimple_build_cond (single_comparison ? GT_EXPR : GE_EXPR,
1542 0 : idxc, size_int (lowe),
1543 : NULL_TREE, NULL_TREE);
1544 : else
1545 796 : g = gimple_build_cond (single_comparison ? LT_EXPR : LE_EXPR,
1546 530 : idxc, size_int (low), NULL_TREE, NULL_TREE);
1547 530 : edge edge_true_true, edge_true_false, edge_false;
1548 796 : if_then_if_then_else (g, (single_comparison ? NULL
1549 266 : : gimple_build_cond (EQ_EXPR, idx,
1550 266 : size_int (lowe),
1551 : NULL_TREE,
1552 : NULL_TREE)),
1553 : profile_probability::likely (),
1554 : profile_probability::unlikely (),
1555 : edge_true_true, edge_true_false, edge_false);
1556 530 : bool save_cast_conditional = m_cast_conditional;
1557 530 : m_cast_conditional = true;
1558 530 : m_bitfld_load = 0;
1559 530 : tree t1 = idx, t2 = NULL_TREE;
1560 530 : if (bitint_big_endian
1561 530 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1562 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1563 : {
1564 0 : HOST_WIDE_INT diff = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1565 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1566 0 : t1 = make_ssa_name (sizetype);
1567 0 : g = gimple_build_assign (t1, PLUS_EXPR, idx, size_int (diff));
1568 0 : insert_before (g);
1569 : }
1570 530 : t1 = handle_operand (rhs1, t1);
1571 530 : if (m_first)
1572 208 : m_data[save_data_cnt + 2]
1573 416 : = build_int_cst (NULL_TREE, m_data_cnt);
1574 530 : tree ext = NULL_TREE;
1575 530 : tree bitfld = NULL_TREE;
1576 530 : if (!single_comparison)
1577 : {
1578 266 : m_gsi = gsi_after_labels (edge_true_true->src);
1579 266 : m_first = false;
1580 266 : m_data_cnt = save_data_cnt + 3;
1581 266 : if (m_bitfld_load)
1582 : {
1583 4 : bitfld = m_data[m_bitfld_load];
1584 4 : m_data[m_bitfld_load] = m_data[m_bitfld_load + 2];
1585 4 : m_bitfld_load = 0;
1586 : }
1587 266 : t2 = handle_operand (rhs1, size_int (bitint_big_endian
1588 : ? high - 1 - low : low));
1589 266 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (t2)))
1590 221 : t2 = add_cast (m_limb_type, t2);
1591 266 : if (!TYPE_UNSIGNED (rhs_type) && m_upwards_2limb)
1592 : {
1593 146 : ext = add_cast (signed_type_for (m_limb_type), t2);
1594 292 : tree lpm1 = build_int_cst (unsigned_type_node,
1595 146 : limb_prec - 1);
1596 146 : tree n = make_ssa_name (TREE_TYPE (ext));
1597 146 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
1598 146 : insert_before (g);
1599 146 : ext = add_cast (m_limb_type, n);
1600 : }
1601 : }
1602 530 : tree t3;
1603 530 : if (TYPE_UNSIGNED (rhs_type))
1604 278 : t3 = build_zero_cst (m_limb_type);
1605 252 : else if (m_upwards_2limb && (save_first || ext != NULL_TREE))
1606 168 : t3 = m_data[save_data_cnt];
1607 : else
1608 84 : t3 = m_data[save_data_cnt + 1];
1609 530 : m_gsi = gsi_after_labels (edge_true_false->dest);
1610 530 : t = make_ssa_name (m_limb_type);
1611 530 : gphi *phi = create_phi_node (t, edge_true_false->dest);
1612 530 : add_phi_arg (phi, t1, edge_true_false, UNKNOWN_LOCATION);
1613 530 : add_phi_arg (phi, t3, edge_false, UNKNOWN_LOCATION);
1614 530 : if (edge_true_true)
1615 266 : add_phi_arg (phi, t2, edge_true_true, UNKNOWN_LOCATION);
1616 530 : if (ext)
1617 : {
1618 146 : tree t4 = make_ssa_name (m_limb_type);
1619 146 : phi = create_phi_node (t4, edge_true_false->dest);
1620 146 : add_phi_arg (phi, build_zero_cst (m_limb_type), edge_true_false,
1621 : UNKNOWN_LOCATION);
1622 146 : add_phi_arg (phi, m_data[save_data_cnt], edge_false,
1623 : UNKNOWN_LOCATION);
1624 146 : add_phi_arg (phi, ext, edge_true_true, UNKNOWN_LOCATION);
1625 146 : if (!save_cast_conditional)
1626 : {
1627 136 : g = gimple_build_assign (m_data[save_data_cnt + 1], t4);
1628 136 : insert_before (g);
1629 : }
1630 : else
1631 10 : for (basic_block bb = gsi_bb (m_gsi);;)
1632 : {
1633 10 : edge e1 = single_succ_edge (bb);
1634 10 : edge e2 = find_edge (e1->dest, m_bb), e3;
1635 10 : tree t5 = (e2 ? m_data[save_data_cnt + 1]
1636 10 : : make_ssa_name (m_limb_type));
1637 10 : phi = create_phi_node (t5, e1->dest);
1638 10 : edge_iterator ei;
1639 30 : FOR_EACH_EDGE (e3, ei, e1->dest->preds)
1640 30 : add_phi_arg (phi, (e3 == e1 ? t4
1641 10 : : build_zero_cst (m_limb_type)),
1642 : e3, UNKNOWN_LOCATION);
1643 10 : if (e2)
1644 : break;
1645 0 : t4 = t5;
1646 0 : bb = e1->dest;
1647 0 : }
1648 : }
1649 530 : if (m_bitfld_load)
1650 : {
1651 8 : tree t4;
1652 8 : if (!save_first && !save_cast_conditional)
1653 2 : t4 = m_data[m_bitfld_load + 1];
1654 : else
1655 6 : t4 = make_ssa_name (m_limb_type);
1656 8 : phi = create_phi_node (t4, edge_true_false->dest);
1657 12 : add_phi_arg (phi,
1658 4 : edge_true_true ? bitfld : m_data[m_bitfld_load],
1659 : edge_true_false, UNKNOWN_LOCATION);
1660 8 : add_phi_arg (phi, m_data[m_bitfld_load + 2],
1661 : edge_false, UNKNOWN_LOCATION);
1662 8 : if (edge_true_true)
1663 4 : add_phi_arg (phi, m_data[m_bitfld_load], edge_true_true,
1664 : UNKNOWN_LOCATION);
1665 8 : if (save_cast_conditional)
1666 4 : for (basic_block bb = gsi_bb (m_gsi);;)
1667 : {
1668 4 : edge e1 = single_succ_edge (bb);
1669 4 : edge e2 = find_edge (e1->dest, m_bb), e3;
1670 4 : tree t5 = ((e2 && !save_first) ? m_data[m_bitfld_load + 1]
1671 4 : : make_ssa_name (m_limb_type));
1672 4 : phi = create_phi_node (t5, e1->dest);
1673 4 : edge_iterator ei;
1674 14 : FOR_EACH_EDGE (e3, ei, e1->dest->preds)
1675 16 : add_phi_arg (phi, (e3 == e1 ? t4
1676 6 : : build_zero_cst (m_limb_type)),
1677 : e3, UNKNOWN_LOCATION);
1678 4 : t4 = t5;
1679 4 : if (e2)
1680 : break;
1681 0 : bb = e1->dest;
1682 0 : }
1683 8 : m_data[m_bitfld_load] = t4;
1684 8 : m_data[m_bitfld_load + 2] = t4;
1685 8 : m_bitfld_load = 0;
1686 : }
1687 530 : m_cast_conditional = save_cast_conditional;
1688 530 : m_first = save_first;
1689 530 : return t;
1690 : }
1691 : else
1692 : {
1693 571 : unsigned tidx = tree_to_uhwi (idx);
1694 571 : if (bitint_big_endian)
1695 0 : tidx = lcnt - 1 - tidx;
1696 571 : if (tidx < low)
1697 : {
1698 152 : t = handle_operand (rhs1, (bitint_big_endian
1699 0 : ? size_int (high - 1 - tidx) : idx));
1700 152 : if (m_first)
1701 71 : m_data[save_data_cnt + 2]
1702 142 : = build_int_cst (NULL_TREE, m_data_cnt);
1703 : }
1704 419 : else if (tidx < high)
1705 : {
1706 68 : t = handle_operand (rhs1, size_int (bitint_big_endian
1707 : ? high - 1 - low : low));
1708 68 : if (m_first)
1709 1 : m_data[save_data_cnt + 2]
1710 2 : = build_int_cst (NULL_TREE, m_data_cnt);
1711 68 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (t)))
1712 60 : t = add_cast (m_limb_type, t);
1713 68 : tree ext = NULL_TREE;
1714 68 : if (!TYPE_UNSIGNED (rhs_type) && m_upwards)
1715 : {
1716 44 : ext = add_cast (signed_type_for (m_limb_type), t);
1717 88 : tree lpm1 = build_int_cst (unsigned_type_node,
1718 44 : limb_prec - 1);
1719 44 : tree n = make_ssa_name (TREE_TYPE (ext));
1720 44 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
1721 44 : insert_before (g);
1722 44 : ext = add_cast (m_limb_type, n);
1723 44 : m_data[save_data_cnt + 1] = ext;
1724 : }
1725 : }
1726 : else
1727 : {
1728 351 : if (TYPE_UNSIGNED (rhs_type) && m_first)
1729 : {
1730 0 : handle_operand (rhs1, (bitint_big_endian
1731 0 : ? size_int (high - 1)
1732 : : size_zero_node));
1733 0 : m_data[save_data_cnt + 2]
1734 0 : = build_int_cst (NULL_TREE, m_data_cnt);
1735 : }
1736 : else
1737 351 : m_data_cnt = tree_to_uhwi (m_data[save_data_cnt + 2]);
1738 351 : if (TYPE_UNSIGNED (rhs_type))
1739 184 : t = build_zero_cst (m_limb_type);
1740 167 : else if (m_bb
1741 16 : && m_data[save_data_cnt]
1742 180 : && ((tidx & 1) == 0 || tidx != low + 1))
1743 : t = m_data[save_data_cnt];
1744 : else
1745 161 : t = m_data[save_data_cnt + 1];
1746 : }
1747 571 : tree type = limb_access_type (lhs_type, idx);
1748 571 : if (!useless_type_conversion_p (type, m_limb_type))
1749 292 : t = add_cast (type, t);
1750 571 : m_first = save_first;
1751 571 : return t;
1752 : }
1753 : }
1754 0 : else if (BITINT_TYPE_P (lhs_type)
1755 1632 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
1756 3264 : && INTEGRAL_TYPE_P (rhs_type))
1757 : {
1758 : /* Add support for 3 or more limbs filled in from normal integral
1759 : type if this assert fails. If no target chooses limb mode smaller
1760 : than half of largest supported normal integral type, this will not
1761 : be needed. */
1762 1632 : gcc_assert (TYPE_PRECISION (rhs_type) <= 2 * limb_prec);
1763 1632 : tree r1 = NULL_TREE, r2 = NULL_TREE, rext = NULL_TREE;
1764 1632 : if (m_first)
1765 : {
1766 594 : gimple_stmt_iterator save_gsi = m_gsi;
1767 594 : m_gsi = m_init_gsi;
1768 594 : if (gsi_end_p (m_gsi))
1769 58 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
1770 : else
1771 536 : gsi_next (&m_gsi);
1772 375 : if (BITINT_TYPE_P (rhs_type)
1773 594 : && bitint_precision_kind (rhs_type) == bitint_prec_middle)
1774 : {
1775 63 : tree type = NULL_TREE;
1776 63 : rhs1 = maybe_cast_middle_bitint (&m_gsi, rhs1, type);
1777 63 : rhs_type = TREE_TYPE (rhs1);
1778 : }
1779 594 : r1 = rhs1;
1780 594 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
1781 526 : r1 = add_cast (m_limb_type, rhs1);
1782 594 : if (TYPE_PRECISION (rhs_type) > limb_prec)
1783 : {
1784 109 : g = gimple_build_assign (make_ssa_name (rhs_type),
1785 : RSHIFT_EXPR, rhs1,
1786 : build_int_cst (unsigned_type_node,
1787 109 : limb_prec));
1788 109 : insert_before (g);
1789 109 : r2 = add_cast (m_limb_type, gimple_assign_lhs (g));
1790 : }
1791 594 : if (TYPE_UNSIGNED (rhs_type))
1792 287 : rext = build_zero_cst (m_limb_type);
1793 : else
1794 : {
1795 307 : rext = add_cast (signed_type_for (m_limb_type), r2 ? r2 : r1);
1796 307 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rext)),
1797 : RSHIFT_EXPR, rext,
1798 : build_int_cst (unsigned_type_node,
1799 307 : limb_prec - 1));
1800 307 : insert_before (g);
1801 307 : rext = add_cast (m_limb_type, gimple_assign_lhs (g));
1802 : }
1803 594 : m_init_gsi = m_gsi;
1804 594 : if (gsi_end_p (m_init_gsi))
1805 598 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
1806 : else
1807 295 : gsi_prev (&m_init_gsi);
1808 594 : m_gsi = save_gsi;
1809 : }
1810 1632 : tree t;
1811 1632 : if (m_upwards_2limb)
1812 : {
1813 776 : if (m_first)
1814 : {
1815 297 : tree out1, out2;
1816 297 : prepare_data_in_out (r1, idx, &out1, rext);
1817 297 : if (TYPE_PRECISION (rhs_type) > limb_prec)
1818 : {
1819 70 : prepare_data_in_out (r2, idx, &out2, rext);
1820 70 : m_data.pop ();
1821 70 : t = m_data.pop ();
1822 70 : m_data[m_data_cnt + 1] = t;
1823 : }
1824 : else
1825 227 : m_data[m_data_cnt + 1] = rext;
1826 297 : m_data.safe_push (rext);
1827 297 : t = m_data[m_data_cnt];
1828 : }
1829 479 : else if (!tree_fits_uhwi_p (idx))
1830 297 : t = m_data[m_data_cnt + 1];
1831 : else
1832 : {
1833 182 : tree type = limb_access_type (lhs_type, idx);
1834 182 : t = m_data[m_data_cnt + 2];
1835 182 : if (!useless_type_conversion_p (type, m_limb_type))
1836 153 : t = add_cast (type, t);
1837 : }
1838 776 : m_data_cnt += 3;
1839 776 : return t;
1840 : }
1841 856 : else if (m_first)
1842 : {
1843 297 : m_data.safe_push (r1);
1844 297 : m_data.safe_push (r2);
1845 297 : m_data.safe_push (rext);
1846 : }
1847 856 : unsigned lcnt = CEIL ((unsigned) TYPE_PRECISION (lhs_type), limb_prec);
1848 856 : if (tree_fits_uhwi_p (idx))
1849 : {
1850 812 : tree type = limb_access_type (lhs_type, idx);
1851 812 : if (bitint_big_endian
1852 812 : ? tree_to_uhwi (idx) == lcnt - 1 : integer_zerop (idx))
1853 269 : t = m_data[m_data_cnt];
1854 543 : else if (TYPE_PRECISION (rhs_type) > limb_prec
1855 543 : && (bitint_big_endian
1856 72 : ? tree_to_uhwi (idx) == lcnt - 2
1857 72 : : integer_onep (idx)))
1858 33 : t = m_data[m_data_cnt + 1];
1859 : else
1860 510 : t = m_data[m_data_cnt + 2];
1861 812 : if (!useless_type_conversion_p (type, m_limb_type))
1862 250 : t = add_cast (type, t);
1863 812 : m_data_cnt += 3;
1864 812 : return t;
1865 : }
1866 44 : g = gimple_build_cond (NE_EXPR, idx,
1867 : bitint_big_endian
1868 0 : ? size_int (lcnt - 1) : size_zero_node,
1869 : NULL_TREE, NULL_TREE);
1870 44 : edge e2, e3, e4 = NULL;
1871 44 : if_then (g, profile_probability::likely (), e2, e3);
1872 44 : if (m_data[m_data_cnt + 1])
1873 : {
1874 14 : g = gimple_build_cond (EQ_EXPR, idx,
1875 : bitint_big_endian
1876 0 : ? size_int (lcnt - 2) : size_one_node,
1877 : NULL_TREE, NULL_TREE);
1878 14 : insert_before (g);
1879 14 : edge e5 = split_block (gsi_bb (m_gsi), g);
1880 14 : e4 = make_edge (e5->src, e2->dest, EDGE_TRUE_VALUE);
1881 14 : e2 = find_edge (e5->dest, e2->dest);
1882 14 : e4->probability = profile_probability::unlikely ();
1883 14 : e5->flags = EDGE_FALSE_VALUE;
1884 14 : e5->probability = e4->probability.invert ();
1885 : }
1886 44 : m_gsi = gsi_after_labels (e2->dest);
1887 44 : t = make_ssa_name (m_limb_type);
1888 44 : gphi *phi = create_phi_node (t, e2->dest);
1889 44 : add_phi_arg (phi, m_data[m_data_cnt + 2], e2, UNKNOWN_LOCATION);
1890 44 : add_phi_arg (phi, m_data[m_data_cnt], e3, UNKNOWN_LOCATION);
1891 44 : if (e4)
1892 14 : add_phi_arg (phi, m_data[m_data_cnt + 1], e4, UNKNOWN_LOCATION);
1893 44 : m_data_cnt += 3;
1894 44 : return t;
1895 : }
1896 : return NULL_TREE;
1897 : }
1898 :
1899 : /* Helper function for handle_stmt method, handle a BIT_FIELD_REF. */
1900 :
1901 : tree
1902 23 : bitint_large_huge::handle_bit_field_ref (tree op, tree idx)
1903 : {
1904 23 : if (tree_fits_uhwi_p (idx))
1905 : {
1906 13 : if (m_first)
1907 4 : m_data.safe_push (NULL);
1908 13 : ++m_data_cnt;
1909 13 : unsigned HOST_WIDE_INT sz = tree_to_uhwi (TYPE_SIZE (m_limb_type));
1910 13 : unsigned i = tree_to_uhwi (idx);
1911 13 : if (bitint_big_endian)
1912 0 : i = CEIL (TYPE_PRECISION (TREE_TYPE (op)), limb_prec) - 1 - i;
1913 26 : tree bfr = build3 (BIT_FIELD_REF, m_limb_type,
1914 13 : TREE_OPERAND (op, 0),
1915 13 : TYPE_SIZE (m_limb_type),
1916 13 : size_binop (PLUS_EXPR, TREE_OPERAND (op, 2),
1917 : bitsize_int (i * sz)));
1918 13 : tree r = make_ssa_name (m_limb_type);
1919 13 : gimple *g = gimple_build_assign (r, bfr);
1920 13 : insert_before (g);
1921 13 : tree type = limb_access_type (TREE_TYPE (op), idx);
1922 13 : if (!useless_type_conversion_p (type, m_limb_type))
1923 0 : r = add_cast (type, r);
1924 13 : return r;
1925 : }
1926 10 : tree var;
1927 10 : if (m_first)
1928 : {
1929 5 : unsigned HOST_WIDE_INT sz = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (op)));
1930 5 : machine_mode mode;
1931 5 : tree type, bfr;
1932 5 : if (bitwise_mode_for_size (sz).exists (&mode)
1933 0 : && known_eq (GET_MODE_BITSIZE (mode), sz))
1934 0 : type = bitwise_type_for_mode (mode);
1935 : else
1936 : {
1937 5 : mode = VOIDmode;
1938 5 : type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_OPERAND (op, 0)));
1939 : }
1940 5 : if (TYPE_ALIGN (type) < TYPE_ALIGN (TREE_TYPE (op)))
1941 0 : type = build_aligned_type (type, TYPE_ALIGN (TREE_TYPE (op)));
1942 5 : var = create_tmp_var (type);
1943 5 : TREE_ADDRESSABLE (var) = 1;
1944 5 : gimple *g;
1945 5 : if (mode != VOIDmode)
1946 : {
1947 0 : bfr = build3 (BIT_FIELD_REF, type, TREE_OPERAND (op, 0),
1948 0 : TYPE_SIZE (type), TREE_OPERAND (op, 2));
1949 0 : g = gimple_build_assign (make_ssa_name (type),
1950 : BIT_FIELD_REF, bfr);
1951 0 : gimple_set_location (g, m_loc);
1952 0 : gsi_insert_after (&m_init_gsi, g, GSI_NEW_STMT);
1953 0 : bfr = gimple_assign_lhs (g);
1954 : }
1955 : else
1956 5 : bfr = TREE_OPERAND (op, 0);
1957 5 : g = gimple_build_assign (var, bfr);
1958 5 : gimple_set_location (g, m_loc);
1959 5 : gsi_insert_after (&m_init_gsi, g, GSI_NEW_STMT);
1960 5 : if (mode == VOIDmode)
1961 : {
1962 5 : unsigned HOST_WIDE_INT nelts
1963 5 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (op))), limb_prec);
1964 5 : tree atype = build_array_type_nelts (m_limb_type, nelts);
1965 5 : var = build2 (MEM_REF, atype, build_fold_addr_expr (var),
1966 : build_int_cst (build_pointer_type (type),
1967 5 : tree_to_uhwi (TREE_OPERAND (op, 2))
1968 5 : / BITS_PER_UNIT));
1969 : }
1970 5 : m_data.safe_push (var);
1971 : }
1972 : else
1973 5 : var = unshare_expr (m_data[m_data_cnt]);
1974 10 : ++m_data_cnt;
1975 10 : var = limb_access (TREE_TYPE (op), var, idx, false);
1976 10 : tree r = make_ssa_name (m_limb_type);
1977 10 : gimple *g = gimple_build_assign (r, var);
1978 10 : insert_before (g);
1979 10 : return r;
1980 : }
1981 :
1982 : /* Add a new EH edge from SRC to EH_EDGE->dest, where EH_EDGE
1983 : is an older EH edge, and except for virtual PHIs duplicate the
1984 : PHI argument from the EH_EDGE to the new EH edge. */
1985 :
1986 : static void
1987 20 : add_eh_edge (basic_block src, edge eh_edge)
1988 : {
1989 20 : edge e = make_edge (src, eh_edge->dest, EDGE_EH);
1990 20 : e->probability = profile_probability::very_unlikely ();
1991 20 : for (gphi_iterator gsi = gsi_start_phis (eh_edge->dest);
1992 27 : !gsi_end_p (gsi); gsi_next (&gsi))
1993 : {
1994 7 : gphi *phi = gsi.phi ();
1995 7 : tree lhs = gimple_phi_result (phi);
1996 14 : if (virtual_operand_p (lhs))
1997 4 : continue;
1998 3 : const phi_arg_d *arg = gimple_phi_arg (phi, eh_edge->dest_idx);
1999 3 : add_phi_arg (phi, arg->def, e, arg->locus);
2000 : }
2001 20 : }
2002 :
2003 : /* Helper function for handle_stmt method, handle a load from memory. */
2004 :
2005 : tree
2006 21344 : bitint_large_huge::handle_load (gimple *stmt, tree idx)
2007 : {
2008 21344 : tree rhs1 = gimple_assign_rhs1 (stmt);
2009 21344 : tree rhs_type = TREE_TYPE (rhs1);
2010 21344 : bool eh = stmt_ends_bb_p (stmt);
2011 21344 : bool load_bitfield_p = false;
2012 21344 : edge eh_edge = NULL;
2013 21344 : gimple *g;
2014 :
2015 21344 : if (TREE_CODE (rhs1) == BIT_FIELD_REF
2016 21344 : && integer_zerop (TREE_OPERAND (rhs1, 2)))
2017 2 : rhs1 = TREE_OPERAND (rhs1, 0);
2018 :
2019 21344 : if (eh)
2020 : {
2021 10 : edge_iterator ei;
2022 10 : basic_block bb = gimple_bb (stmt);
2023 :
2024 10 : FOR_EACH_EDGE (eh_edge, ei, bb->succs)
2025 10 : if (eh_edge->flags & EDGE_EH)
2026 : break;
2027 : }
2028 :
2029 21344 : if (TREE_CODE (rhs1) == COMPONENT_REF
2030 21344 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (rhs1, 1)))
2031 : {
2032 1225 : tree fld = TREE_OPERAND (rhs1, 1);
2033 : /* For little-endian, we can allow as inputs bit-fields
2034 : which start at a limb boundary. */
2035 1225 : gcc_assert (tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld)));
2036 1225 : if (!bitint_big_endian
2037 1225 : && DECL_OFFSET_ALIGN (fld) >= TYPE_ALIGN (TREE_TYPE (rhs1))
2038 2450 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % limb_prec) == 0)
2039 : {
2040 739 : load_bitfield_p = true;
2041 751 : goto normal_load;
2042 : }
2043 : /* Even if DECL_FIELD_BIT_OFFSET (fld) is a multiple of BITS_PER_UNIT,
2044 : handle it normally for now. */
2045 486 : if (!bitint_big_endian
2046 486 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % BITS_PER_UNIT) == 0)
2047 : {
2048 12 : load_bitfield_p = true;
2049 12 : goto normal_load;
2050 : }
2051 474 : tree repr = DECL_BIT_FIELD_REPRESENTATIVE (fld);
2052 474 : poly_int64 bitoffset;
2053 474 : poly_uint64 field_offset, repr_offset;
2054 474 : bool var_field_off = false;
2055 474 : if (poly_int_tree_p (DECL_FIELD_OFFSET (fld), &field_offset)
2056 948 : && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
2057 474 : bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
2058 : else
2059 : {
2060 : bitoffset = 0;
2061 : var_field_off = true;
2062 : }
2063 474 : bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2064 474 : - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
2065 948 : tree nrhs1 = build3 (COMPONENT_REF, TREE_TYPE (repr),
2066 474 : TREE_OPERAND (rhs1, 0), repr,
2067 0 : var_field_off ? TREE_OPERAND (rhs1, 2) : NULL_TREE);
2068 474 : HOST_WIDE_INT bo = bitoffset.to_constant ();
2069 474 : unsigned bo_idx = (unsigned HOST_WIDE_INT) bo / limb_prec;
2070 474 : unsigned bo_bit = (unsigned HOST_WIDE_INT) bo % limb_prec;
2071 474 : unsigned bo_last = 0;
2072 474 : unsigned bo_shift = bo_bit;
2073 474 : unsigned nelts = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
2074 474 : if (bitint_big_endian)
2075 : {
2076 0 : bo_last = CEIL (TYPE_PRECISION (rhs_type) + bo_bit, limb_prec) - 1;
2077 0 : bo_shift = (TYPE_PRECISION (rhs_type) + bo_bit) % limb_prec;
2078 0 : if (bo_shift)
2079 0 : bo_shift = limb_prec - bo_shift;
2080 : }
2081 474 : if (m_first)
2082 : {
2083 141 : if (m_upwards && bo_shift)
2084 : {
2085 138 : gimple_stmt_iterator save_gsi = m_gsi;
2086 138 : m_gsi = m_init_gsi;
2087 138 : if (gsi_end_p (m_gsi))
2088 57 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
2089 : else
2090 81 : gsi_next (&m_gsi);
2091 138 : tree t = limb_access (NULL_TREE, nrhs1,
2092 138 : size_int (bo_idx + bo_last), true);
2093 138 : tree iv = make_ssa_name (m_limb_type);
2094 138 : g = gimple_build_assign (iv, t);
2095 138 : insert_before (g);
2096 138 : if (eh)
2097 : {
2098 2 : maybe_duplicate_eh_stmt (g, stmt);
2099 2 : if (eh_edge)
2100 : {
2101 2 : edge e = split_block (gsi_bb (m_gsi), g);
2102 2 : add_eh_edge (e->src, eh_edge);
2103 2 : m_gsi = gsi_after_labels (e->dest);
2104 2 : if (gsi_bb (save_gsi) == e->src)
2105 : {
2106 1 : if (gsi_end_p (save_gsi))
2107 0 : save_gsi = gsi_end_bb (e->dest);
2108 : else
2109 1 : save_gsi = gsi_for_stmt (gsi_stmt (save_gsi));
2110 : }
2111 2 : if (m_preheader_bb == e->src)
2112 1 : m_preheader_bb = e->dest;
2113 : }
2114 : }
2115 138 : m_init_gsi = m_gsi;
2116 138 : if (gsi_end_p (m_init_gsi))
2117 222 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
2118 : else
2119 27 : gsi_prev (&m_init_gsi);
2120 138 : m_gsi = save_gsi;
2121 138 : tree out;
2122 138 : prepare_data_in_out (iv, idx, &out);
2123 138 : out = m_data[m_data_cnt];
2124 138 : m_data.safe_push (out);
2125 138 : }
2126 : else
2127 : {
2128 3 : m_data.safe_push (NULL_TREE);
2129 3 : m_data.safe_push (NULL_TREE);
2130 3 : m_data.safe_push (NULL_TREE);
2131 : }
2132 : }
2133 :
2134 474 : tree nidx0 = NULL_TREE, nidx1 = NULL_TREE;
2135 474 : tree iv = m_data[m_data_cnt];
2136 474 : if (m_cast_conditional && iv)
2137 : {
2138 12 : gcc_assert (!m_bitfld_load);
2139 12 : m_bitfld_load = m_data_cnt;
2140 : }
2141 474 : if (tree_fits_uhwi_p (idx))
2142 : {
2143 272 : unsigned prec = TYPE_PRECISION (rhs_type);
2144 272 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
2145 272 : if (bitint_big_endian)
2146 0 : i = nelts - 1 - i;
2147 272 : gcc_assert (i * limb_prec < prec);
2148 272 : if (bo_shift)
2149 272 : nidx1 = size_int (bo_idx + (bitint_big_endian
2150 : ? bo_last - i - 1 : i + 1));
2151 272 : if ((i + 1) * limb_prec > prec)
2152 : {
2153 100 : prec %= limb_prec;
2154 100 : if (prec + bo_bit <= (unsigned) limb_prec)
2155 272 : nidx1 = NULL_TREE;
2156 : }
2157 272 : if (!iv)
2158 4 : nidx0 = size_int (bo_idx + (bitint_big_endian ? bo_last - i : i));
2159 : }
2160 : else
2161 : {
2162 202 : HOST_WIDE_INT adj = bo_idx;
2163 202 : if (bitint_big_endian)
2164 0 : adj += (HOST_WIDE_INT) bo_last + 1 - nelts;
2165 202 : if (!iv)
2166 : {
2167 4 : if (adj == 0)
2168 : nidx0 = idx;
2169 : else
2170 : {
2171 0 : nidx0 = make_ssa_name (sizetype);
2172 0 : g = gimple_build_assign (nidx0, PLUS_EXPR, idx,
2173 0 : size_int (adj));
2174 0 : insert_before (g);
2175 : }
2176 : }
2177 202 : if (bo_shift)
2178 : {
2179 202 : if (bitint_big_endian && adj == 1)
2180 : nidx1 = idx;
2181 : else
2182 : {
2183 202 : nidx1 = make_ssa_name (sizetype);
2184 202 : g = gimple_build_assign (nidx1, PLUS_EXPR, idx,
2185 404 : size_int (adj + (bitint_big_endian
2186 : ? -1 : 1)));
2187 202 : insert_before (g);
2188 : }
2189 : }
2190 : }
2191 :
2192 680 : tree iv2 = NULL_TREE;
2193 206 : if (nidx0)
2194 : {
2195 8 : tree t = limb_access (NULL_TREE, nrhs1, nidx0, true);
2196 8 : iv = make_ssa_name (m_limb_type);
2197 8 : g = gimple_build_assign (iv, t);
2198 8 : insert_before (g);
2199 8 : if (eh)
2200 : {
2201 0 : maybe_duplicate_eh_stmt (g, stmt);
2202 0 : if (eh_edge)
2203 : {
2204 0 : edge e = split_block (gsi_bb (m_gsi), g);
2205 0 : m_gsi = gsi_after_labels (e->dest);
2206 0 : add_eh_edge (e->src, eh_edge);
2207 : }
2208 : }
2209 : }
2210 474 : if (nidx1)
2211 : {
2212 385 : bool conditional = m_var_msb && !tree_fits_uhwi_p (idx);
2213 385 : unsigned prec = TYPE_PRECISION (rhs_type);
2214 385 : if (conditional)
2215 : {
2216 3 : if ((prec % limb_prec) == 0
2217 3 : || ((prec % limb_prec) + bo_bit > (unsigned) limb_prec))
2218 382 : conditional = false;
2219 : }
2220 385 : edge edge_true = NULL, edge_false = NULL;
2221 385 : if (conditional)
2222 : {
2223 6 : g = gimple_build_cond (NE_EXPR, idx,
2224 : bitint_big_endian
2225 : ? size_zero_node
2226 3 : : size_int (prec / limb_prec),
2227 : NULL_TREE, NULL_TREE);
2228 3 : if_then (g, profile_probability::likely (),
2229 : edge_true, edge_false);
2230 : }
2231 385 : tree t = limb_access (NULL_TREE, nrhs1, nidx1, true);
2232 385 : if (m_upwards_2limb
2233 283 : && !m_first
2234 184 : && !m_bitfld_load
2235 178 : && !tree_fits_uhwi_p (idx))
2236 95 : iv2 = m_data[m_data_cnt + 1];
2237 : else
2238 290 : iv2 = make_ssa_name (m_limb_type);
2239 385 : g = gimple_build_assign (iv2, t);
2240 385 : insert_before (g);
2241 385 : if (eh)
2242 : {
2243 5 : maybe_duplicate_eh_stmt (g, stmt);
2244 5 : if (eh_edge)
2245 : {
2246 5 : edge e = split_block (gsi_bb (m_gsi), g);
2247 5 : m_gsi = gsi_after_labels (e->dest);
2248 5 : add_eh_edge (e->src, eh_edge);
2249 : }
2250 : }
2251 385 : if (conditional)
2252 : {
2253 3 : tree iv3 = make_ssa_name (m_limb_type);
2254 3 : if (eh)
2255 0 : edge_true = find_edge (gsi_bb (m_gsi), edge_false->dest);
2256 3 : gphi *phi = create_phi_node (iv3, edge_true->dest);
2257 3 : add_phi_arg (phi, iv2, edge_true, UNKNOWN_LOCATION);
2258 3 : add_phi_arg (phi, build_zero_cst (m_limb_type),
2259 : edge_false, UNKNOWN_LOCATION);
2260 3 : m_gsi = gsi_after_labels (edge_true->dest);
2261 3 : iv2 = iv3;
2262 : }
2263 : }
2264 474 : if (bo_shift)
2265 : {
2266 474 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
2267 : iv, build_int_cst (unsigned_type_node,
2268 474 : bo_shift));
2269 474 : insert_before (g);
2270 474 : iv = gimple_assign_lhs (g);
2271 : }
2272 474 : if (iv2)
2273 : {
2274 385 : g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
2275 : iv2, build_int_cst (unsigned_type_node,
2276 385 : limb_prec - bo_shift));
2277 385 : insert_before (g);
2278 385 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
2279 : gimple_assign_lhs (g), iv);
2280 385 : insert_before (g);
2281 385 : iv = gimple_assign_lhs (g);
2282 385 : if (m_data[m_data_cnt])
2283 379 : m_data[m_data_cnt] = iv2;
2284 : }
2285 474 : if (tree_fits_uhwi_p (idx))
2286 : {
2287 272 : tree atype = limb_access_type (rhs_type, idx);
2288 272 : if (!useless_type_conversion_p (atype, TREE_TYPE (iv)))
2289 100 : iv = add_cast (atype, iv);
2290 : }
2291 474 : m_data_cnt += 3;
2292 474 : return iv;
2293 : }
2294 :
2295 20870 : normal_load:
2296 : /* Use write_p = true for loads with EH edges to make
2297 : sure limb_access doesn't add a cast as separate
2298 : statement after it. */
2299 20870 : rhs1 = limb_access (rhs_type, rhs1, idx, eh, !load_bitfield_p);
2300 20870 : tree ret = make_ssa_name (TREE_TYPE (rhs1));
2301 20870 : g = gimple_build_assign (ret, rhs1);
2302 20870 : insert_before (g);
2303 20870 : if (eh)
2304 : {
2305 3 : maybe_duplicate_eh_stmt (g, stmt);
2306 3 : if (eh_edge)
2307 : {
2308 3 : edge e = split_block (gsi_bb (m_gsi), g);
2309 3 : m_gsi = gsi_after_labels (e->dest);
2310 3 : add_eh_edge (e->src, eh_edge);
2311 : }
2312 3 : if (tree_fits_uhwi_p (idx))
2313 : {
2314 1 : tree atype = limb_access_type (rhs_type, idx);
2315 1 : if (!useless_type_conversion_p (atype, TREE_TYPE (rhs1)))
2316 1 : ret = add_cast (atype, ret);
2317 : }
2318 : }
2319 : return ret;
2320 : }
2321 :
2322 : /* Return a limb IDX from a mergeable statement STMT. */
2323 :
2324 : tree
2325 39172 : bitint_large_huge::handle_stmt (gimple *stmt, tree idx)
2326 : {
2327 39172 : tree lhs, rhs1, rhs2 = NULL_TREE;
2328 39172 : gimple *g;
2329 39172 : switch (gimple_code (stmt))
2330 : {
2331 39172 : case GIMPLE_ASSIGN:
2332 39172 : if (gimple_assign_load_p (stmt))
2333 21344 : return handle_load (stmt, idx);
2334 17828 : switch (gimple_assign_rhs_code (stmt))
2335 : {
2336 990 : case BIT_AND_EXPR:
2337 990 : case BIT_IOR_EXPR:
2338 990 : case BIT_XOR_EXPR:
2339 990 : rhs2 = handle_operand (gimple_assign_rhs2 (stmt), idx);
2340 : /* FALLTHRU */
2341 1410 : case BIT_NOT_EXPR:
2342 1410 : rhs1 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2343 1410 : lhs = make_ssa_name (TREE_TYPE (rhs1));
2344 1410 : g = gimple_build_assign (lhs, gimple_assign_rhs_code (stmt),
2345 : rhs1, rhs2);
2346 1410 : insert_before (g);
2347 1410 : return lhs;
2348 4019 : case PLUS_EXPR:
2349 4019 : case MINUS_EXPR:
2350 4019 : rhs1 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2351 4019 : rhs2 = handle_operand (gimple_assign_rhs2 (stmt), idx);
2352 4019 : return handle_plus_minus (gimple_assign_rhs_code (stmt),
2353 4019 : rhs1, rhs2, idx);
2354 163 : case NEGATE_EXPR:
2355 163 : rhs2 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2356 163 : rhs1 = build_zero_cst (TREE_TYPE (rhs2));
2357 163 : return handle_plus_minus (MINUS_EXPR, rhs1, rhs2, idx);
2358 152 : case LSHIFT_EXPR:
2359 152 : return handle_lshift (handle_operand (gimple_assign_rhs1 (stmt),
2360 : idx),
2361 152 : gimple_assign_rhs2 (stmt), idx);
2362 5087 : case SSA_NAME:
2363 5087 : case PAREN_EXPR:
2364 5087 : case INTEGER_CST:
2365 5087 : return handle_operand (gimple_assign_rhs1 (stmt), idx);
2366 6966 : CASE_CONVERT:
2367 6966 : return handle_cast (TREE_TYPE (gimple_assign_lhs (stmt)),
2368 6966 : gimple_assign_rhs1 (stmt), idx);
2369 8 : case VIEW_CONVERT_EXPR:
2370 8 : return handle_cast (TREE_TYPE (gimple_assign_lhs (stmt)),
2371 8 : TREE_OPERAND (gimple_assign_rhs1 (stmt), 0),
2372 8 : idx);
2373 23 : case BIT_FIELD_REF:
2374 23 : return handle_bit_field_ref (gimple_assign_rhs1 (stmt), idx);
2375 : default:
2376 : break;
2377 : }
2378 : break;
2379 : default:
2380 : break;
2381 : }
2382 0 : gcc_unreachable ();
2383 : }
2384 :
2385 : /* Return minimum precision of OP at STMT.
2386 : Positive value is minimum precision above which all bits
2387 : are zero, negative means all bits above negation of the
2388 : value are copies of the sign bit. */
2389 :
2390 : static int
2391 8429 : range_to_prec (tree op, gimple *stmt)
2392 : {
2393 8429 : int_range_max r;
2394 8429 : wide_int w;
2395 8429 : tree type = TREE_TYPE (op);
2396 8429 : unsigned int prec = TYPE_PRECISION (type);
2397 :
2398 8429 : if (!optimize
2399 7480 : || !get_range_query (cfun)->range_of_expr (r, op, stmt)
2400 12169 : || r.undefined_p ())
2401 : {
2402 4690 : if (TYPE_UNSIGNED (type))
2403 1900 : return prec;
2404 : else
2405 2790 : return MIN ((int) -prec, -2);
2406 : }
2407 :
2408 3739 : if (!TYPE_UNSIGNED (TREE_TYPE (op)))
2409 : {
2410 2338 : w = r.lower_bound ();
2411 2338 : if (wi::neg_p (w))
2412 : {
2413 1933 : int min_prec1 = wi::min_precision (w, SIGNED);
2414 1933 : w = r.upper_bound ();
2415 1933 : int min_prec2 = wi::min_precision (w, SIGNED);
2416 1933 : int min_prec = MAX (min_prec1, min_prec2);
2417 1933 : return MIN (-min_prec, -2);
2418 : }
2419 : }
2420 :
2421 1806 : w = r.upper_bound ();
2422 1806 : int min_prec = wi::min_precision (w, UNSIGNED);
2423 1806 : return MAX (min_prec, 1);
2424 8429 : }
2425 :
2426 : /* Return address of the first limb of OP and write into *PREC
2427 : its precision. If positive, the operand is zero extended
2428 : from that precision, if it is negative, the operand is sign-extended
2429 : from -*PREC. If PREC_STORED is NULL, it is the toplevel call,
2430 : otherwise *PREC_STORED is prec from the innermost call without
2431 : range optimizations (0 for uninitialized SSA_NAME). */
2432 :
2433 : tree
2434 3838 : bitint_large_huge::handle_operand_addr (tree op, gimple *stmt,
2435 : int *prec_stored, int *prec)
2436 : {
2437 3838 : wide_int w;
2438 3838 : location_t loc_save = m_loc;
2439 3838 : tree ret = NULL_TREE;
2440 3838 : int precs = 0;
2441 3873 : if ((!BITINT_TYPE_P (TREE_TYPE (op))
2442 3809 : || bitint_precision_kind (TREE_TYPE (op)) < bitint_prec_large)
2443 3962 : && TREE_CODE (op) != INTEGER_CST)
2444 : {
2445 124 : do_int:
2446 124 : *prec = range_to_prec (op, stmt);
2447 124 : bitint_prec_kind kind = bitint_prec_small;
2448 124 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (op)));
2449 124 : if (BITINT_TYPE_P (TREE_TYPE (op)))
2450 104 : kind = bitint_precision_kind (TREE_TYPE (op));
2451 104 : if (kind == bitint_prec_middle)
2452 : {
2453 12 : tree type = NULL_TREE;
2454 12 : op = maybe_cast_middle_bitint (&m_gsi, op, type);
2455 : }
2456 124 : tree op_type = TREE_TYPE (op);
2457 124 : unsigned HOST_WIDE_INT nelts
2458 124 : = CEIL (TYPE_PRECISION (op_type), limb_prec);
2459 : /* Add support for 3 or more limbs filled in from normal
2460 : integral type if this assert fails. If no target chooses
2461 : limb mode smaller than half of largest supported normal
2462 : integral type, this will not be needed. */
2463 124 : gcc_assert (nelts <= 2);
2464 124 : precs = (TYPE_UNSIGNED (op_type)
2465 124 : ? TYPE_PRECISION (op_type) : -TYPE_PRECISION (op_type));
2466 124 : if (*prec <= limb_prec && *prec >= -limb_prec)
2467 : {
2468 111 : nelts = 1;
2469 111 : if (TYPE_UNSIGNED (op_type))
2470 : {
2471 26 : if (precs > limb_prec)
2472 124 : precs = limb_prec;
2473 : }
2474 85 : else if (precs < -limb_prec)
2475 124 : precs = -limb_prec;
2476 : }
2477 124 : if (prec_stored)
2478 0 : *prec_stored = precs;
2479 124 : tree atype = build_array_type_nelts (m_limb_type, nelts);
2480 124 : tree var = create_tmp_var (atype);
2481 124 : tree t1 = op;
2482 124 : if (!useless_type_conversion_p (m_limb_type, op_type))
2483 124 : t1 = add_cast (m_limb_type, t1);
2484 124 : tree v = build4 (ARRAY_REF, m_limb_type, var,
2485 0 : bitint_big_endian && nelts > 1
2486 : ? size_one_node : size_zero_node,
2487 : NULL_TREE, NULL_TREE);
2488 124 : gimple *g = gimple_build_assign (v, t1);
2489 124 : insert_before (g);
2490 124 : if (nelts > 1)
2491 : {
2492 13 : tree lp = build_int_cst (unsigned_type_node, limb_prec);
2493 13 : g = gimple_build_assign (make_ssa_name (op_type),
2494 : RSHIFT_EXPR, op, lp);
2495 13 : insert_before (g);
2496 13 : tree t2 = gimple_assign_lhs (g);
2497 13 : t2 = add_cast (m_limb_type, t2);
2498 13 : v = build4 (ARRAY_REF, m_limb_type, var,
2499 : bitint_big_endian ? size_zero_node : size_one_node,
2500 : NULL_TREE, NULL_TREE);
2501 13 : g = gimple_build_assign (v, t2);
2502 13 : insert_before (g);
2503 : }
2504 124 : ret = build_fold_addr_expr (var);
2505 124 : if (!stmt_ends_bb_p (gsi_stmt (m_gsi)))
2506 : {
2507 123 : tree clobber = build_clobber (atype, CLOBBER_STORAGE_END);
2508 123 : g = gimple_build_assign (var, clobber);
2509 123 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
2510 : }
2511 124 : m_loc = loc_save;
2512 124 : goto do_ret;
2513 : }
2514 3762 : switch (TREE_CODE (op))
2515 : {
2516 2905 : case SSA_NAME:
2517 2905 : if (m_names == NULL
2518 2905 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (op)))
2519 : {
2520 102 : gimple *g = SSA_NAME_DEF_STMT (op);
2521 102 : m_loc = gimple_location (g);
2522 102 : if (gimple_assign_load_p (g))
2523 : {
2524 36 : *prec = range_to_prec (op, NULL);
2525 36 : precs = (TYPE_UNSIGNED (TREE_TYPE (op))
2526 36 : ? TYPE_PRECISION (TREE_TYPE (op))
2527 25 : : -TYPE_PRECISION (TREE_TYPE (op)));
2528 36 : if (prec_stored)
2529 6 : *prec_stored = precs;
2530 36 : ret = build_fold_addr_expr (gimple_assign_rhs1 (g));
2531 36 : ret = force_gimple_operand_gsi (&m_gsi, ret, true,
2532 : NULL_TREE, true, GSI_SAME_STMT);
2533 : }
2534 66 : else if (gimple_code (g) == GIMPLE_NOP)
2535 : {
2536 2 : *prec = TYPE_UNSIGNED (TREE_TYPE (op)) ? limb_prec : -limb_prec;
2537 2 : precs = *prec;
2538 2 : if (prec_stored)
2539 1 : *prec_stored = 0;
2540 2 : tree var = create_tmp_var (m_limb_type);
2541 2 : TREE_ADDRESSABLE (var) = 1;
2542 2 : ret = build_fold_addr_expr (var);
2543 2 : if (!stmt_ends_bb_p (gsi_stmt (m_gsi)))
2544 : {
2545 2 : tree clobber = build_clobber (m_limb_type,
2546 : CLOBBER_STORAGE_END);
2547 2 : g = gimple_build_assign (var, clobber);
2548 2 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
2549 : }
2550 : }
2551 : else
2552 : {
2553 64 : gcc_assert (gimple_assign_cast_p (g));
2554 64 : tree rhs1 = gimple_assign_rhs1 (g);
2555 64 : bitint_prec_kind kind = bitint_prec_small;
2556 64 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
2557 1 : rhs1 = TREE_OPERAND (rhs1, 0);
2558 64 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (rhs1)));
2559 64 : if (BITINT_TYPE_P (TREE_TYPE (rhs1)))
2560 49 : kind = bitint_precision_kind (TREE_TYPE (rhs1));
2561 49 : if (kind >= bitint_prec_large)
2562 : {
2563 16 : tree lhs_type = TREE_TYPE (op);
2564 16 : tree rhs_type = TREE_TYPE (rhs1);
2565 16 : int prec_stored_val = 0;
2566 16 : ret = handle_operand_addr (rhs1, g, &prec_stored_val, prec);
2567 16 : precs = prec_stored_val;
2568 16 : if (prec_stored)
2569 0 : *prec_stored = prec_stored_val;
2570 16 : if (precs == 0)
2571 : {
2572 1 : gcc_assert (*prec == limb_prec || *prec == -limb_prec);
2573 : precs = *prec;
2574 : }
2575 16 : if (TYPE_PRECISION (lhs_type) > TYPE_PRECISION (rhs_type))
2576 : {
2577 4 : if (TYPE_UNSIGNED (lhs_type)
2578 4 : && !TYPE_UNSIGNED (rhs_type))
2579 1 : gcc_assert (*prec >= 0 || prec_stored == NULL);
2580 : }
2581 : else
2582 : {
2583 12 : if (prec_stored_val == 0)
2584 : /* Non-widening cast of uninitialized value. */;
2585 11 : else if (*prec > 0 && *prec < TYPE_PRECISION (lhs_type))
2586 : ;
2587 11 : else if (TYPE_UNSIGNED (lhs_type))
2588 : {
2589 8 : gcc_assert (*prec > 0
2590 : || prec_stored_val > 0
2591 : || (-prec_stored_val
2592 : >= TYPE_PRECISION (lhs_type)));
2593 8 : *prec = TYPE_PRECISION (lhs_type);
2594 : }
2595 3 : else if (*prec < 0 && -*prec < TYPE_PRECISION (lhs_type))
2596 : ;
2597 : else
2598 3 : *prec = -TYPE_PRECISION (lhs_type);
2599 : }
2600 : }
2601 : else
2602 : {
2603 48 : op = rhs1;
2604 48 : stmt = g;
2605 48 : goto do_int;
2606 : }
2607 : }
2608 54 : m_loc = loc_save;
2609 54 : goto do_ret;
2610 : }
2611 : else
2612 : {
2613 2803 : int p = var_to_partition (m_map, op);
2614 2803 : gcc_assert (m_vars[p] != NULL_TREE);
2615 2803 : *prec = range_to_prec (op, stmt);
2616 2803 : precs = (TYPE_UNSIGNED (TREE_TYPE (op))
2617 2803 : ? TYPE_PRECISION (TREE_TYPE (op))
2618 1668 : : -TYPE_PRECISION (TREE_TYPE (op)));
2619 2803 : if (prec_stored)
2620 9 : *prec_stored = precs;
2621 2803 : ret = build_fold_addr_expr (m_vars[p]);
2622 2803 : goto do_ret;
2623 : }
2624 857 : case INTEGER_CST:
2625 857 : unsigned int min_prec, mp;
2626 857 : tree type;
2627 857 : w = wi::to_wide (op);
2628 857 : if (tree_int_cst_sgn (op) >= 0)
2629 : {
2630 642 : min_prec = wi::min_precision (w, UNSIGNED);
2631 642 : *prec = MAX (min_prec, 1);
2632 : }
2633 : else
2634 : {
2635 215 : min_prec = wi::min_precision (w, SIGNED);
2636 215 : *prec = MIN ((int) -min_prec, -2);
2637 : }
2638 857 : mp = CEIL (min_prec, limb_prec) * limb_prec;
2639 857 : if (mp == 0)
2640 : mp = 1;
2641 857 : if (mp >= (unsigned) TYPE_PRECISION (TREE_TYPE (op))
2642 857 : && (BITINT_TYPE_P (TREE_TYPE (op))
2643 24 : || TYPE_PRECISION (TREE_TYPE (op)) <= limb_prec))
2644 308 : type = TREE_TYPE (op);
2645 : else
2646 549 : type = build_bitint_type (mp, 1);
2647 20 : if (!BITINT_TYPE_P (type)
2648 857 : || bitint_precision_kind (type) == bitint_prec_small)
2649 : {
2650 586 : if (TYPE_PRECISION (type) <= limb_prec)
2651 586 : type = m_limb_type;
2652 : else
2653 : {
2654 0 : while (bitint_precision_kind (mp) == bitint_prec_small)
2655 0 : mp += limb_prec;
2656 : /* This case is for targets which e.g. have 64-bit
2657 : limb but categorize up to 128-bits _BitInts as
2658 : small. We could use type of m_limb_type[2] and
2659 : similar instead to save space. */
2660 0 : type = build_bitint_type (mp, 1);
2661 : }
2662 : }
2663 857 : if (tree_int_cst_sgn (op) >= 0)
2664 642 : precs = MAX (TYPE_PRECISION (type), 1);
2665 : else
2666 215 : precs = MIN ((int) -TYPE_PRECISION (type), -2);
2667 857 : if (prec_stored)
2668 0 : *prec_stored = precs;
2669 857 : op = tree_output_constant_def (fold_convert (type, op));
2670 857 : ret = build_fold_addr_expr (op);
2671 857 : goto do_ret;
2672 0 : default:
2673 0 : gcc_unreachable ();
2674 : }
2675 3838 : do_ret:
2676 3838 : if (bitint_big_endian && prec_stored == NULL)
2677 : {
2678 0 : int p1 = *prec < 0 ? -*prec : *prec;
2679 0 : int p2 = precs < 0 ? -precs : precs;
2680 0 : int c1 = CEIL (p1, limb_prec);
2681 0 : int c2 = CEIL (p2, limb_prec);
2682 0 : gcc_assert (c1 <= c2);
2683 0 : if (c1 != c2)
2684 : {
2685 0 : gimple *g
2686 0 : = gimple_build_assign (make_ssa_name (TREE_TYPE (ret)),
2687 : POINTER_PLUS_EXPR, ret,
2688 0 : size_int ((c2 - c1) * m_limb_size));
2689 0 : insert_before (g);
2690 0 : ret = gimple_assign_lhs (g);
2691 : }
2692 : }
2693 3838 : return ret;
2694 3838 : }
2695 :
2696 : /* Helper function, create a loop before the current location,
2697 : start with sizetype INIT value from the preheader edge. Return
2698 : a PHI result and set *IDX_NEXT to SSA_NAME it creates and uses
2699 : from the latch edge. */
2700 :
2701 : tree
2702 14669 : bitint_large_huge::create_loop (tree init, tree *idx_next)
2703 : {
2704 14669 : if (!gsi_end_p (m_gsi))
2705 12501 : gsi_prev (&m_gsi);
2706 : else
2707 4336 : m_gsi = gsi_last_bb (gsi_bb (m_gsi));
2708 14669 : edge e1 = split_block (gsi_bb (m_gsi), gsi_stmt (m_gsi));
2709 14669 : edge e2 = split_block (e1->dest, (gimple *) NULL);
2710 14669 : edge e3 = make_edge (e1->dest, e1->dest, EDGE_TRUE_VALUE);
2711 14669 : e3->probability = profile_probability::very_unlikely ();
2712 14669 : e2->flags = EDGE_FALSE_VALUE;
2713 14669 : e2->probability = e3->probability.invert ();
2714 14669 : tree idx = make_ssa_name (sizetype);
2715 14669 : gphi *phi = create_phi_node (idx, e1->dest);
2716 14669 : add_phi_arg (phi, init, e1, UNKNOWN_LOCATION);
2717 14669 : *idx_next = make_ssa_name (sizetype);
2718 14669 : add_phi_arg (phi, *idx_next, e3, UNKNOWN_LOCATION);
2719 14669 : m_gsi = gsi_after_labels (e1->dest);
2720 14669 : m_bb = e1->dest;
2721 14669 : m_preheader_bb = e1->src;
2722 14669 : class loop *loop = alloc_loop ();
2723 14669 : loop->header = e1->dest;
2724 14669 : add_loop (loop, e1->src->loop_father);
2725 14669 : return idx;
2726 : }
2727 :
2728 : /* Lower large/huge _BitInt statement mergeable or similar STMT which can be
2729 : lowered using iteration from the least significant limb up to the most
2730 : significant limb. For large _BitInt it is emitted as straight line code
2731 : before current location, for huge _BitInt as a loop handling two limbs
2732 : at once, followed by handling up to limbs in straight line code (at most
2733 : one full and one partial limb). It can also handle EQ_EXPR/NE_EXPR
2734 : comparisons, in that case CMP_CODE should be the comparison code and
2735 : CMP_OP1/CMP_OP2 the comparison operands. */
2736 :
2737 : tree
2738 23376 : bitint_large_huge::lower_mergeable_stmt (gimple *stmt, tree_code &cmp_code,
2739 : tree cmp_op1, tree cmp_op2)
2740 : {
2741 23376 : bool eq_p = cmp_code != ERROR_MARK;
2742 23376 : tree type;
2743 23376 : if (eq_p)
2744 6550 : type = TREE_TYPE (cmp_op1);
2745 : else
2746 16826 : type = TREE_TYPE (gimple_assign_lhs (stmt));
2747 23376 : gcc_assert (BITINT_TYPE_P (type));
2748 23376 : bitint_prec_kind kind = bitint_precision_kind (type);
2749 23376 : gcc_assert (kind >= bitint_prec_large);
2750 23376 : gimple *g;
2751 23376 : tree lhs = gimple_get_lhs (stmt);
2752 23376 : tree rhs1, lhs_type = lhs ? TREE_TYPE (lhs) : NULL_TREE;
2753 17266 : if (lhs
2754 17266 : && TREE_CODE (lhs) == SSA_NAME
2755 8745 : && BITINT_TYPE_P (TREE_TYPE (lhs))
2756 8305 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large)
2757 : {
2758 8305 : int p = var_to_partition (m_map, lhs);
2759 8305 : gcc_assert (m_vars[p] != NULL_TREE);
2760 8305 : m_lhs = lhs = m_vars[p];
2761 : }
2762 23376 : unsigned cnt, rem = 0, end = 0, prec = TYPE_PRECISION (type);
2763 23376 : bool sext = false;
2764 23376 : tree ext = NULL_TREE, store_operand = NULL_TREE;
2765 23376 : bool eh = false;
2766 23376 : basic_block eh_pad = NULL;
2767 23376 : tree nlhs = NULL_TREE;
2768 23376 : unsigned HOST_WIDE_INT bo_idx = 0;
2769 23376 : unsigned HOST_WIDE_INT bo_bit = 0;
2770 23376 : unsigned bo_shift = 0;
2771 23376 : unsigned bo_last = 0;
2772 23376 : bool bo_be_p = false;
2773 23376 : tree bf_cur = NULL_TREE, bf_next = NULL_TREE;
2774 23376 : if (gimple_store_p (stmt))
2775 : {
2776 8521 : store_operand = gimple_assign_rhs1 (stmt);
2777 8521 : eh = stmt_ends_bb_p (stmt);
2778 8521 : if (eh)
2779 : {
2780 2 : edge e;
2781 2 : edge_iterator ei;
2782 2 : basic_block bb = gimple_bb (stmt);
2783 :
2784 2 : FOR_EACH_EDGE (e, ei, bb->succs)
2785 2 : if (e->flags & EDGE_EH)
2786 : {
2787 2 : eh_pad = e->dest;
2788 2 : break;
2789 : }
2790 : }
2791 8521 : if (TREE_CODE (lhs) == COMPONENT_REF
2792 8521 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
2793 : {
2794 162 : tree fld = TREE_OPERAND (lhs, 1);
2795 162 : gcc_assert (tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld)));
2796 162 : tree repr = DECL_BIT_FIELD_REPRESENTATIVE (fld);
2797 162 : poly_int64 bitoffset;
2798 162 : poly_uint64 field_offset, repr_offset;
2799 162 : if (!bitint_big_endian
2800 162 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2801 162 : % BITS_PER_UNIT) == 0)
2802 : nlhs = lhs;
2803 : else
2804 : {
2805 143 : bool var_field_off = false;
2806 143 : if (poly_int_tree_p (DECL_FIELD_OFFSET (fld), &field_offset)
2807 286 : && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
2808 143 : bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
2809 : else
2810 : {
2811 : bitoffset = 0;
2812 : var_field_off = true;
2813 : }
2814 143 : bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2815 143 : - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
2816 286 : nlhs = build3 (COMPONENT_REF, TREE_TYPE (repr),
2817 143 : TREE_OPERAND (lhs, 0), repr,
2818 : var_field_off
2819 0 : ? TREE_OPERAND (lhs, 2) : NULL_TREE);
2820 143 : HOST_WIDE_INT bo = bitoffset.to_constant ();
2821 143 : bo_idx = (unsigned HOST_WIDE_INT) bo / limb_prec;
2822 143 : bo_bit = (unsigned HOST_WIDE_INT) bo % limb_prec;
2823 143 : bo_shift = bo_bit;
2824 143 : if (bitint_big_endian)
2825 : {
2826 0 : bo_last = CEIL (prec + bo_bit, limb_prec) - 1;
2827 0 : bo_shift = (prec + bo_bit) % limb_prec;
2828 0 : bo_be_p = true;
2829 0 : if (bo_shift)
2830 0 : bo_shift = limb_prec - bo_shift;
2831 : }
2832 : }
2833 : }
2834 : }
2835 8521 : if ((store_operand
2836 8521 : && TREE_CODE (store_operand) == SSA_NAME
2837 7543 : && (m_names == NULL
2838 7513 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (store_operand)))
2839 1154 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (store_operand)))
2840 31343 : || gimple_assign_cast_p (stmt))
2841 : {
2842 2155 : rhs1 = gimple_assign_rhs1 (store_operand
2843 554 : ? SSA_NAME_DEF_STMT (store_operand)
2844 : : stmt);
2845 1601 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
2846 2 : rhs1 = TREE_OPERAND (rhs1, 0);
2847 : /* Optimize mergeable ops ending with widening cast to _BitInt
2848 : (or followed by store). We can lower just the limbs of the
2849 : cast operand and widen afterwards. */
2850 1601 : if (TREE_CODE (rhs1) == SSA_NAME
2851 1601 : && (m_names == NULL
2852 1595 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))
2853 617 : && BITINT_TYPE_P (TREE_TYPE (rhs1))
2854 478 : && bitint_precision_kind (TREE_TYPE (rhs1)) >= bitint_prec_large
2855 2030 : && (CEIL ((unsigned) TYPE_PRECISION (TREE_TYPE (rhs1)),
2856 429 : limb_prec) < CEIL (prec, limb_prec)
2857 306 : || (kind == bitint_prec_huge
2858 262 : && TYPE_PRECISION (TREE_TYPE (rhs1)) < prec)))
2859 : {
2860 129 : store_operand = rhs1;
2861 129 : prec = TYPE_PRECISION (TREE_TYPE (rhs1));
2862 129 : kind = bitint_precision_kind (TREE_TYPE (rhs1));
2863 129 : if (!TYPE_UNSIGNED (TREE_TYPE (rhs1)))
2864 53 : sext = true;
2865 : }
2866 : }
2867 23376 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
2868 23376 : if (kind == bitint_prec_large)
2869 12201 : cnt = CEIL (prec, limb_prec);
2870 : else
2871 : {
2872 11175 : rem = (prec % (2 * limb_prec));
2873 11175 : end = (prec - rem) / limb_prec;
2874 11175 : cnt = 2 + CEIL (rem, limb_prec);
2875 11175 : idx = idx_first = create_loop (bitint_big_endian
2876 11175 : ? size_int (cnt - 2 + end - 1)
2877 : : size_zero_node, &idx_next);
2878 : }
2879 :
2880 23376 : basic_block edge_bb = NULL;
2881 23376 : if (eq_p)
2882 : {
2883 6550 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
2884 6550 : gsi_prev (&gsi);
2885 6550 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
2886 6550 : edge_bb = e->src;
2887 6550 : if (kind == bitint_prec_large)
2888 7186 : m_gsi = gsi_end_bb (edge_bb);
2889 : }
2890 : else
2891 16826 : m_after_stmt = stmt;
2892 23376 : if (kind != bitint_prec_large)
2893 11175 : m_upwards_2limb = end;
2894 23376 : m_upwards = true;
2895 :
2896 23376 : bool separate_ext
2897 23376 : = (prec != (unsigned) TYPE_PRECISION (type)
2898 23376 : && (CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)
2899 129 : > CEIL (prec, limb_prec)));
2900 23376 : bool zero_ms_limb = false;
2901 23376 : if (bitint_extended == bitint_ext_full
2902 0 : && !eq_p
2903 0 : && !nlhs
2904 0 : && abi_limb_prec > limb_prec
2905 23376 : && ((CEIL ((unsigned) TYPE_PRECISION (type), abi_limb_prec)
2906 0 : * abi_limb_prec / limb_prec)
2907 0 : > CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)))
2908 : {
2909 0 : if (prec == (unsigned) TYPE_PRECISION (type))
2910 : {
2911 0 : sext = !TYPE_UNSIGNED (type);
2912 0 : separate_ext = true;
2913 : }
2914 0 : else if (TYPE_UNSIGNED (type) && sext)
2915 : zero_ms_limb = true;
2916 : else
2917 : separate_ext = true;
2918 : }
2919 23499 : unsigned dst_idx_off = 0;
2920 23376 : if (separate_ext && bitint_big_endian)
2921 0 : dst_idx_off = (CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)
2922 0 : - CEIL (prec, limb_prec));
2923 :
2924 93721 : for (unsigned i = 0; i < cnt; i++)
2925 : {
2926 70345 : m_data_cnt = 0;
2927 70345 : if (kind == bitint_prec_large)
2928 37214 : idx = size_int (bitint_big_endian ? cnt - 1 - i : i);
2929 33131 : else if (i >= 2)
2930 10781 : idx = size_int (bitint_big_endian ? cnt - 1 - i : end + (i > 2));
2931 70345 : if (eq_p)
2932 : {
2933 19841 : rhs1 = handle_operand (cmp_op1, idx);
2934 19841 : tree rhs2 = handle_operand (cmp_op2, idx);
2935 19841 : g = gimple_build_cond (NE_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
2936 19841 : insert_before (g);
2937 19841 : edge e1 = split_block (gsi_bb (m_gsi), g);
2938 19841 : e1->flags = EDGE_FALSE_VALUE;
2939 19841 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
2940 19841 : e1->probability = profile_probability::unlikely ();
2941 19841 : e2->probability = e1->probability.invert ();
2942 19841 : if (i == 0)
2943 6550 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
2944 19841 : m_gsi = gsi_after_labels (e1->dest);
2945 : }
2946 : else
2947 : {
2948 50504 : if (store_operand)
2949 25792 : rhs1 = handle_operand (store_operand, idx);
2950 : else
2951 24712 : rhs1 = handle_stmt (stmt, idx);
2952 50504 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
2953 12049 : rhs1 = add_cast (m_limb_type, rhs1);
2954 50504 : if (sext && i == cnt - 1)
2955 50504 : ext = rhs1;
2956 50504 : tree nidx = idx;
2957 50504 : HOST_WIDE_INT adj = bo_idx;
2958 50504 : if (bo_be_p)
2959 0 : adj += bo_last - (CEIL (prec, limb_prec) - 1);
2960 : else
2961 50504 : adj += dst_idx_off;
2962 50504 : if (adj)
2963 : {
2964 220 : if (tree_fits_uhwi_p (idx))
2965 116 : nidx = size_int (tree_to_uhwi (idx) + adj);
2966 : else
2967 : {
2968 104 : nidx = make_ssa_name (sizetype);
2969 104 : g = gimple_build_assign (nidx, PLUS_EXPR, idx,
2970 104 : size_int (adj));
2971 104 : insert_before (g);
2972 : }
2973 : }
2974 50504 : bool done = false;
2975 50504 : basic_block new_bb = NULL;
2976 : /* Handle stores into bit-fields. */
2977 50504 : if (bo_shift)
2978 : {
2979 455 : if (i == 0)
2980 : {
2981 143 : edge e2 = NULL;
2982 143 : if (kind != bitint_prec_large)
2983 : {
2984 112 : prepare_data_in_out (build_zero_cst (m_limb_type),
2985 : idx, &bf_next);
2986 112 : bf_next = m_data.pop ();
2987 112 : bf_cur = m_data.pop ();
2988 112 : g = gimple_build_cond (EQ_EXPR, idx,
2989 : bitint_big_endian
2990 0 : ? size_int (CEIL (prec,
2991 : limb_prec) - 1)
2992 : : size_zero_node,
2993 : NULL_TREE, NULL_TREE);
2994 112 : edge edge_true;
2995 112 : if_then_else (g, profile_probability::unlikely (),
2996 : edge_true, e2);
2997 112 : new_bb = e2->dest;
2998 : }
2999 143 : tree ftype
3000 143 : = build_nonstandard_integer_type (limb_prec - bo_shift, 1);
3001 286 : tree bfr = build_bit_field_ref (ftype, unshare_expr (nlhs),
3002 143 : limb_prec - bo_shift,
3003 : bitint_big_endian
3004 0 : ? (bo_idx + bo_last)
3005 0 : * limb_prec
3006 143 : : bo_idx * limb_prec
3007 : + bo_bit);
3008 143 : tree t = add_cast (ftype, rhs1);
3009 143 : g = gimple_build_assign (bfr, t);
3010 143 : insert_before (g);
3011 143 : if (eh)
3012 : {
3013 0 : maybe_duplicate_eh_stmt (g, stmt);
3014 0 : if (eh_pad)
3015 : {
3016 0 : edge e = split_block (gsi_bb (m_gsi), g);
3017 0 : m_gsi = gsi_after_labels (e->dest);
3018 0 : add_eh_edge (e->src,
3019 : find_edge (gimple_bb (stmt), eh_pad));
3020 : }
3021 : }
3022 143 : if (kind == bitint_prec_large)
3023 : {
3024 : bf_cur = rhs1;
3025 : done = true;
3026 : }
3027 112 : else if (e2)
3028 112 : m_gsi = gsi_after_labels (e2->src);
3029 : }
3030 112 : if (!done)
3031 : {
3032 424 : tree t1 = make_ssa_name (m_limb_type);
3033 424 : tree t2 = make_ssa_name (m_limb_type);
3034 424 : tree t3 = make_ssa_name (m_limb_type);
3035 424 : g = gimple_build_assign (t1, RSHIFT_EXPR, bf_cur,
3036 : build_int_cst (unsigned_type_node,
3037 424 : limb_prec
3038 424 : - bo_shift));
3039 424 : insert_before (g);
3040 424 : g = gimple_build_assign (t2, LSHIFT_EXPR, rhs1,
3041 : build_int_cst (unsigned_type_node,
3042 424 : bo_shift));
3043 424 : insert_before (g);
3044 424 : bf_cur = rhs1;
3045 424 : g = gimple_build_assign (t3, BIT_IOR_EXPR, t1, t2);
3046 424 : insert_before (g);
3047 424 : rhs1 = t3;
3048 424 : if (bf_next && i == 1)
3049 : {
3050 112 : g = gimple_build_assign (bf_next, bf_cur);
3051 112 : insert_before (g);
3052 : }
3053 : }
3054 : }
3055 31 : if (!done)
3056 : {
3057 : /* Handle bit-field access to partial last limb if needed. */
3058 50473 : if (nlhs
3059 480 : && i == cnt - 1
3060 162 : && !separate_ext
3061 138 : && tree_fits_uhwi_p (idx))
3062 : {
3063 123 : unsigned int tprec = TYPE_PRECISION (type);
3064 123 : unsigned int rprec = (tprec - 1) % limb_prec + 1;
3065 123 : if (rprec + bo_shift < (unsigned) limb_prec)
3066 : {
3067 46 : tree ftype
3068 46 : = build_nonstandard_integer_type (rprec + bo_shift, 1);
3069 46 : tree bfr
3070 92 : = build_bit_field_ref (ftype, unshare_expr (nlhs),
3071 : rprec + bo_shift,
3072 : bitint_big_endian
3073 0 : ? bo_idx * limb_prec + bo_bit
3074 46 : : (bo_idx + tprec / limb_prec)
3075 46 : * limb_prec);
3076 46 : tree t = add_cast (ftype, rhs1);
3077 46 : g = gimple_build_assign (bfr, t);
3078 46 : done = true;
3079 46 : bf_cur = NULL_TREE;
3080 : }
3081 77 : else if (rprec + bo_shift == (unsigned) limb_prec)
3082 50427 : bf_cur = NULL_TREE;
3083 : }
3084 : /* Otherwise, stores to any other lhs. */
3085 46 : if (!done)
3086 : {
3087 100420 : tree l = limb_access (nlhs ? NULL_TREE : lhs_type,
3088 : nlhs ? nlhs : lhs, nidx, true);
3089 50427 : g = gimple_build_assign (l, rhs1);
3090 : }
3091 50473 : insert_before (g);
3092 50473 : if (eh)
3093 : {
3094 6 : maybe_duplicate_eh_stmt (g, stmt);
3095 6 : if (eh_pad)
3096 : {
3097 6 : edge e = split_block (gsi_bb (m_gsi), g);
3098 6 : m_gsi = gsi_after_labels (e->dest);
3099 6 : add_eh_edge (e->src,
3100 : find_edge (gimple_bb (stmt), eh_pad));
3101 : }
3102 : }
3103 50473 : if (new_bb)
3104 112 : m_gsi = gsi_after_labels (new_bb);
3105 : }
3106 : }
3107 70345 : m_first = false;
3108 70345 : if (kind == bitint_prec_huge && i <= 1)
3109 : {
3110 22350 : if (i == 0)
3111 : {
3112 11175 : idx = make_ssa_name (sizetype);
3113 11175 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
3114 : bitint_big_endian
3115 0 : ? size_int (-1) : size_one_node);
3116 11175 : insert_before (g);
3117 : }
3118 : else
3119 : {
3120 11175 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
3121 22350 : size_int (bitint_big_endian ? -2 : 2));
3122 11175 : insert_before (g);
3123 11175 : if (bitint_big_endian)
3124 0 : g = gimple_build_cond (NE_EXPR, idx_first, size_int (cnt - 1),
3125 : NULL_TREE, NULL_TREE);
3126 : else
3127 11175 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (end),
3128 : NULL_TREE, NULL_TREE);
3129 11175 : insert_before (g);
3130 11175 : if (eq_p)
3131 2957 : m_gsi = gsi_after_labels (edge_bb);
3132 : else
3133 8218 : m_gsi = gsi_for_stmt (stmt);
3134 11175 : m_bb = NULL;
3135 : }
3136 : }
3137 : }
3138 :
3139 23376 : if (separate_ext)
3140 : {
3141 123 : if (sext)
3142 : {
3143 50 : ext = add_cast (signed_type_for (m_limb_type), ext);
3144 100 : tree lpm1 = build_int_cst (unsigned_type_node,
3145 50 : limb_prec - 1);
3146 50 : tree n = make_ssa_name (TREE_TYPE (ext));
3147 50 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
3148 50 : insert_before (g);
3149 50 : ext = add_cast (m_limb_type, n);
3150 : }
3151 : else
3152 73 : ext = build_zero_cst (m_limb_type);
3153 123 : kind = bitint_precision_kind (type);
3154 123 : unsigned start = CEIL (prec, limb_prec);
3155 123 : prec = TYPE_PRECISION (type);
3156 123 : if (bitint_extended == bitint_ext_full
3157 0 : && !nlhs
3158 0 : && !zero_ms_limb
3159 0 : && abi_limb_prec > limb_prec)
3160 : {
3161 0 : prec = CEIL (prec, abi_limb_prec) * abi_limb_prec;
3162 0 : kind = bitint_precision_kind (prec);
3163 : }
3164 123 : unsigned total = CEIL (prec, limb_prec);
3165 123 : idx = idx_first = idx_next = NULL_TREE;
3166 123 : if (prec <= (start + 2 + (bo_shift != 0)) * limb_prec)
3167 : kind = bitint_prec_large;
3168 78 : if (kind == bitint_prec_large)
3169 45 : cnt = total - start;
3170 : else
3171 : {
3172 78 : rem = prec % limb_prec;
3173 78 : end = (prec - rem) / limb_prec;
3174 148 : cnt = (bo_shift != 0) + 1 + (rem != 0);
3175 : }
3176 123 : if (bitint_big_endian && bo_shift != 0 && (prec % limb_prec) == 0)
3177 0 : ++total;
3178 317 : for (unsigned i = 0; i < cnt; i++)
3179 : {
3180 194 : if (kind == bitint_prec_large || (i == 0 && bo_shift != 0))
3181 62 : idx = size_int (bo_idx
3182 : + (bitint_big_endian
3183 : ? total - 1 - start - i : start + i));
3184 132 : else if (i == cnt - 1 && rem != 0)
3185 108 : idx = size_int (bo_idx + (bitint_big_endian ? 0 : end));
3186 78 : else if (i == (bo_shift != 0))
3187 78 : idx = create_loop (size_int (bo_idx
3188 : + (bitint_big_endian
3189 : ? total - 1 - start - i
3190 : : start + i)), &idx_next);
3191 194 : rhs1 = ext;
3192 194 : if (bf_cur != NULL_TREE && bf_cur != ext)
3193 : {
3194 20 : tree t1 = make_ssa_name (m_limb_type);
3195 20 : g = gimple_build_assign (t1, RSHIFT_EXPR, bf_cur,
3196 : build_int_cst (unsigned_type_node,
3197 20 : limb_prec - bo_shift));
3198 20 : insert_before (g);
3199 20 : if (integer_zerop (ext))
3200 : rhs1 = t1;
3201 : else
3202 : {
3203 10 : tree t2 = make_ssa_name (m_limb_type);
3204 10 : rhs1 = make_ssa_name (m_limb_type);
3205 10 : g = gimple_build_assign (t2, LSHIFT_EXPR, ext,
3206 : build_int_cst (unsigned_type_node,
3207 10 : bo_shift));
3208 10 : insert_before (g);
3209 10 : g = gimple_build_assign (rhs1, BIT_IOR_EXPR, t1, t2);
3210 10 : insert_before (g);
3211 : }
3212 : bf_cur = ext;
3213 : }
3214 194 : bool done = false;
3215 : /* Handle bit-field access to partial last limb if needed. */
3216 194 : if (nlhs && i == cnt - 1)
3217 : {
3218 24 : unsigned int tprec = TYPE_PRECISION (type);
3219 24 : unsigned int rprec = (tprec - 1) % limb_prec + 1;
3220 24 : if (rprec + bo_shift < (unsigned) limb_prec)
3221 : {
3222 12 : tree ftype
3223 12 : = build_nonstandard_integer_type (rprec + bo_shift, 1);
3224 12 : tree bfr
3225 24 : = build_bit_field_ref (ftype, unshare_expr (nlhs),
3226 : rprec + bo_shift,
3227 : bitint_big_endian
3228 0 : ? bo_idx * limb_prec + bo_bit
3229 12 : : (bo_idx + tprec / limb_prec)
3230 12 : * limb_prec);
3231 12 : tree t = add_cast (ftype, rhs1);
3232 12 : g = gimple_build_assign (bfr, t);
3233 12 : done = true;
3234 12 : bf_cur = NULL_TREE;
3235 : }
3236 12 : else if (rprec + bo_shift == (unsigned) limb_prec)
3237 : bf_cur = NULL_TREE;
3238 : }
3239 : /* Otherwise, stores to any other lhs. */
3240 12 : if (!done)
3241 : {
3242 334 : tree l = limb_access (nlhs ? NULL_TREE : lhs_type,
3243 : nlhs ? nlhs : lhs, idx, true);
3244 :
3245 182 : if (bitint_extended
3246 0 : && sext
3247 0 : && TYPE_UNSIGNED (lhs_type)
3248 0 : && tree_fits_uhwi_p (idx)
3249 182 : && !nlhs)
3250 : {
3251 0 : rhs1 = add_cast (limb_access_type (lhs_type, idx), rhs1);
3252 0 : rhs1 = add_cast (TREE_TYPE (l), rhs1);
3253 : }
3254 :
3255 182 : g = gimple_build_assign (l, rhs1);
3256 : }
3257 194 : insert_before (g);
3258 194 : if (eh)
3259 : {
3260 0 : maybe_duplicate_eh_stmt (g, stmt);
3261 0 : if (eh_pad)
3262 : {
3263 0 : edge e = split_block (gsi_bb (m_gsi), g);
3264 0 : m_gsi = gsi_after_labels (e->dest);
3265 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3266 : }
3267 : }
3268 194 : if (kind == bitint_prec_huge && i == (bo_shift != 0))
3269 : {
3270 78 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3271 : bitint_big_endian
3272 0 : ? size_int (-1) : size_one_node);
3273 78 : insert_before (g);
3274 78 : if (bitint_big_endian && rem != 0)
3275 0 : g = gimple_build_cond (NE_EXPR, idx,
3276 0 : size_int (bo_idx + 1),
3277 : NULL_TREE, NULL_TREE);
3278 : else
3279 78 : g = gimple_build_cond (NE_EXPR, idx_next,
3280 156 : size_int (bo_idx
3281 : + (bitint_big_endian
3282 : ? 0 : end)),
3283 : NULL_TREE, NULL_TREE);
3284 78 : insert_before (g);
3285 78 : m_gsi = gsi_for_stmt (stmt);
3286 78 : m_bb = NULL;
3287 : }
3288 : }
3289 : }
3290 23376 : if (bf_cur != NULL_TREE)
3291 : {
3292 72 : unsigned int tprec = TYPE_PRECISION (type);
3293 72 : unsigned int rprec = (tprec + bo_shift) % limb_prec;
3294 72 : tree ftype = build_nonstandard_integer_type (rprec, 1);
3295 144 : tree bfr = build_bit_field_ref (ftype, unshare_expr (nlhs),
3296 : rprec,
3297 : bitint_big_endian
3298 0 : ? bo_idx * limb_prec + bo_bit
3299 72 : : (bo_idx + (tprec + bo_bit) / limb_prec)
3300 : * limb_prec);
3301 72 : rhs1 = bf_cur;
3302 72 : if (bf_cur != ext)
3303 : {
3304 64 : rhs1 = make_ssa_name (TREE_TYPE (rhs1));
3305 64 : g = gimple_build_assign (rhs1, RSHIFT_EXPR, bf_cur,
3306 : build_int_cst (unsigned_type_node,
3307 64 : limb_prec - bo_shift));
3308 64 : insert_before (g);
3309 : }
3310 72 : rhs1 = add_cast (ftype, rhs1);
3311 72 : g = gimple_build_assign (bfr, rhs1);
3312 72 : insert_before (g);
3313 72 : if (eh)
3314 : {
3315 0 : maybe_duplicate_eh_stmt (g, stmt);
3316 0 : if (eh_pad)
3317 : {
3318 0 : edge e = split_block (gsi_bb (m_gsi), g);
3319 0 : m_gsi = gsi_after_labels (e->dest);
3320 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3321 : }
3322 : }
3323 : }
3324 23376 : if (zero_ms_limb)
3325 : {
3326 0 : tree p2 = build_int_cst (sizetype,
3327 0 : CEIL ((unsigned) TYPE_PRECISION (type),
3328 : abi_limb_prec)
3329 0 : * abi_limb_prec / limb_prec - 1);
3330 0 : tree l = limb_access (lhs_type, lhs, p2, true);
3331 0 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
3332 0 : insert_before (g);
3333 0 : if (eh)
3334 : {
3335 0 : maybe_duplicate_eh_stmt (g, stmt);
3336 0 : if (eh_pad)
3337 : {
3338 0 : edge e = split_block (gsi_bb (m_gsi), g);
3339 0 : m_gsi = gsi_after_labels (e->dest);
3340 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3341 : }
3342 : }
3343 : }
3344 :
3345 23376 : if (gimple_store_p (stmt))
3346 : {
3347 8521 : unlink_stmt_vdef (stmt);
3348 17042 : release_ssa_name (gimple_vdef (stmt));
3349 8521 : gsi_remove (&m_gsi, true);
3350 : }
3351 23376 : if (eq_p)
3352 : {
3353 6550 : lhs = make_ssa_name (boolean_type_node);
3354 6550 : basic_block bb = gimple_bb (stmt);
3355 6550 : gphi *phi = create_phi_node (lhs, bb);
3356 6550 : edge e = find_edge (gsi_bb (m_gsi), bb);
3357 6550 : unsigned int n = EDGE_COUNT (bb->preds);
3358 32941 : for (unsigned int i = 0; i < n; i++)
3359 : {
3360 26391 : edge e2 = EDGE_PRED (bb, i);
3361 26391 : add_phi_arg (phi, e == e2 ? boolean_true_node : boolean_false_node,
3362 : e2, UNKNOWN_LOCATION);
3363 : }
3364 6550 : cmp_code = cmp_code == EQ_EXPR ? NE_EXPR : EQ_EXPR;
3365 6550 : return lhs;
3366 : }
3367 : else
3368 : return NULL_TREE;
3369 : }
3370 :
3371 : /* Handle a large/huge _BitInt comparison statement STMT other than
3372 : EQ_EXPR/NE_EXPR. CMP_CODE, CMP_OP1 and CMP_OP2 meaning is like in
3373 : lower_mergeable_stmt. The {GT,GE,LT,LE}_EXPR comparisons are
3374 : lowered by iteration from the most significant limb downwards to
3375 : the least significant one, for large _BitInt in straight line code,
3376 : otherwise with most significant limb handled in
3377 : straight line code followed by a loop handling one limb at a time.
3378 : Comparisons with unsigned huge _BitInt with precisions which are
3379 : multiples of limb precision can use just the loop and don't need to
3380 : handle most significant limb before the loop. The loop or straight
3381 : line code jumps to final basic block if a particular pair of limbs
3382 : is not equal. */
3383 :
3384 : tree
3385 722 : bitint_large_huge::lower_comparison_stmt (gimple *stmt, tree_code &cmp_code,
3386 : tree cmp_op1, tree cmp_op2)
3387 : {
3388 722 : tree type = TREE_TYPE (cmp_op1);
3389 722 : gcc_assert (BITINT_TYPE_P (type));
3390 722 : bitint_prec_kind kind = bitint_precision_kind (type);
3391 722 : gcc_assert (kind >= bitint_prec_large);
3392 722 : gimple *g;
3393 722 : if (!TYPE_UNSIGNED (type)
3394 441 : && integer_zerop (cmp_op2)
3395 750 : && (cmp_code == GE_EXPR || cmp_code == LT_EXPR))
3396 : {
3397 28 : unsigned end = CEIL ((unsigned) TYPE_PRECISION (type), limb_prec) - 1;
3398 56 : tree idx = size_int (bitint_big_endian ? 0 : end);
3399 28 : m_data_cnt = 0;
3400 28 : tree rhs1 = handle_operand (cmp_op1, idx);
3401 28 : if (TYPE_UNSIGNED (TREE_TYPE (rhs1)))
3402 : {
3403 24 : tree stype = signed_type_for (TREE_TYPE (rhs1));
3404 24 : rhs1 = add_cast (stype, rhs1);
3405 : }
3406 28 : tree lhs = make_ssa_name (boolean_type_node);
3407 28 : g = gimple_build_assign (lhs, cmp_code, rhs1,
3408 28 : build_zero_cst (TREE_TYPE (rhs1)));
3409 28 : insert_before (g);
3410 28 : cmp_code = NE_EXPR;
3411 28 : return lhs;
3412 : }
3413 :
3414 694 : unsigned cnt, rem = 0, end = 0;
3415 694 : tree idx = NULL_TREE, idx_next = NULL_TREE;
3416 694 : if (kind == bitint_prec_large)
3417 377 : cnt = CEIL ((unsigned) TYPE_PRECISION (type), limb_prec);
3418 : else
3419 : {
3420 317 : rem = ((unsigned) TYPE_PRECISION (type) % limb_prec);
3421 317 : if (rem == 0 && !TYPE_UNSIGNED (type))
3422 : rem = limb_prec;
3423 317 : end = ((unsigned) TYPE_PRECISION (type) - rem) / limb_prec;
3424 317 : cnt = 1 + (rem != 0);
3425 : }
3426 :
3427 694 : basic_block edge_bb = NULL;
3428 694 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
3429 694 : gsi_prev (&gsi);
3430 694 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
3431 694 : edge_bb = e->src;
3432 694 : m_gsi = gsi_end_bb (edge_bb);
3433 :
3434 694 : edge *edges = XALLOCAVEC (edge, cnt * 2);
3435 2519 : for (unsigned i = 0; i < cnt; i++)
3436 : {
3437 1825 : m_data_cnt = 0;
3438 1825 : if (kind == bitint_prec_large)
3439 1246 : idx = size_int (bitint_big_endian ? i : cnt - i - 1);
3440 579 : else if (i == cnt - 1)
3441 317 : idx = create_loop (size_int (bitint_big_endian ? cnt - 1 : end - 1),
3442 : &idx_next);
3443 : else
3444 524 : idx = size_int (bitint_big_endian ? 0 : end);
3445 1825 : tree rhs1 = handle_operand (cmp_op1, idx);
3446 1825 : tree rhs2 = handle_operand (cmp_op2, idx);
3447 1825 : if (i == 0
3448 694 : && !TYPE_UNSIGNED (type)
3449 2238 : && TYPE_UNSIGNED (TREE_TYPE (rhs1)))
3450 : {
3451 112 : tree stype = signed_type_for (TREE_TYPE (rhs1));
3452 112 : rhs1 = add_cast (stype, rhs1);
3453 112 : rhs2 = add_cast (stype, rhs2);
3454 : }
3455 1825 : g = gimple_build_cond (GT_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
3456 1825 : insert_before (g);
3457 1825 : edge e1 = split_block (gsi_bb (m_gsi), g);
3458 1825 : e1->flags = EDGE_FALSE_VALUE;
3459 1825 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
3460 1825 : e1->probability = profile_probability::likely ();
3461 1825 : e2->probability = e1->probability.invert ();
3462 1825 : if (i == 0)
3463 694 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
3464 1825 : m_gsi = gsi_after_labels (e1->dest);
3465 1825 : edges[2 * i] = e2;
3466 1825 : g = gimple_build_cond (LT_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
3467 1825 : insert_before (g);
3468 1825 : e1 = split_block (gsi_bb (m_gsi), g);
3469 1825 : e1->flags = EDGE_FALSE_VALUE;
3470 1825 : e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
3471 1825 : e1->probability = profile_probability::unlikely ();
3472 1825 : e2->probability = e1->probability.invert ();
3473 1825 : m_gsi = gsi_after_labels (e1->dest);
3474 1825 : edges[2 * i + 1] = e2;
3475 1825 : m_first = false;
3476 1825 : if (kind == bitint_prec_huge && i == cnt - 1)
3477 : {
3478 634 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3479 : bitint_big_endian ? size_one_node
3480 317 : : size_int (-1));
3481 317 : insert_before (g);
3482 317 : g = gimple_build_cond (NE_EXPR, idx,
3483 : bitint_big_endian
3484 0 : ? size_int (end - 1 + (cnt != 1))
3485 : : size_zero_node,
3486 : NULL_TREE, NULL_TREE);
3487 317 : insert_before (g);
3488 317 : edge true_edge, false_edge;
3489 317 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
3490 : &true_edge, &false_edge);
3491 317 : m_gsi = gsi_after_labels (false_edge->dest);
3492 317 : m_bb = NULL;
3493 : }
3494 : }
3495 :
3496 694 : tree lhs = make_ssa_name (boolean_type_node);
3497 694 : basic_block bb = gimple_bb (stmt);
3498 694 : gphi *phi = create_phi_node (lhs, bb);
3499 4344 : for (unsigned int i = 0; i < cnt * 2; i++)
3500 : {
3501 3650 : tree val = ((cmp_code == GT_EXPR || cmp_code == GE_EXPR)
3502 3650 : ^ (i & 1)) ? boolean_true_node : boolean_false_node;
3503 3650 : add_phi_arg (phi, val, edges[i], UNKNOWN_LOCATION);
3504 : }
3505 694 : add_phi_arg (phi, (cmp_code == GE_EXPR || cmp_code == LE_EXPR)
3506 : ? boolean_true_node : boolean_false_node,
3507 : find_edge (gsi_bb (m_gsi), bb), UNKNOWN_LOCATION);
3508 694 : cmp_code = NE_EXPR;
3509 694 : return lhs;
3510 : }
3511 :
3512 : /* Lower large/huge _BitInt left and right shift except for left
3513 : shift by < limb_prec constant. */
3514 :
3515 : void
3516 584 : bitint_large_huge::lower_shift_stmt (tree obj, gimple *stmt)
3517 : {
3518 584 : tree rhs1 = gimple_assign_rhs1 (stmt);
3519 584 : tree lhs = gimple_assign_lhs (stmt);
3520 584 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
3521 584 : tree type = TREE_TYPE (rhs1);
3522 584 : gimple *final_stmt = gsi_stmt (m_gsi);
3523 584 : gcc_assert (BITINT_TYPE_P (type)
3524 : && bitint_precision_kind (type) >= bitint_prec_large);
3525 584 : int prec = TYPE_PRECISION (type);
3526 584 : tree n = gimple_assign_rhs2 (stmt), n1, n2, n3, n4;
3527 584 : gimple *g;
3528 584 : if (obj == NULL_TREE)
3529 : {
3530 465 : int part = var_to_partition (m_map, lhs);
3531 465 : gcc_assert (m_vars[part] != NULL_TREE);
3532 : obj = m_vars[part];
3533 : }
3534 : /* Preparation code common for both left and right shifts.
3535 : unsigned n1 = n % limb_prec;
3536 : size_t n2 = n / limb_prec;
3537 : size_t n3 = n1 != 0;
3538 : unsigned n4 = (limb_prec - n1) % limb_prec;
3539 : (for power of 2 limb_prec n4 can be -n1 & limb_prec). */
3540 584 : if (TREE_CODE (n) == INTEGER_CST)
3541 : {
3542 249 : tree lp = build_int_cst (TREE_TYPE (n), limb_prec);
3543 249 : n1 = int_const_binop (TRUNC_MOD_EXPR, n, lp);
3544 249 : n2 = fold_convert (sizetype, int_const_binop (TRUNC_DIV_EXPR, n, lp));
3545 249 : n3 = size_int (!integer_zerop (n1));
3546 249 : n4 = int_const_binop (TRUNC_MOD_EXPR,
3547 249 : int_const_binop (MINUS_EXPR, lp, n1), lp);
3548 : }
3549 : else
3550 : {
3551 335 : n1 = make_ssa_name (TREE_TYPE (n));
3552 335 : n2 = make_ssa_name (sizetype);
3553 335 : n3 = make_ssa_name (sizetype);
3554 335 : n4 = make_ssa_name (TREE_TYPE (n));
3555 335 : if (pow2p_hwi (limb_prec))
3556 : {
3557 335 : tree lpm1 = build_int_cst (TREE_TYPE (n), limb_prec - 1);
3558 335 : g = gimple_build_assign (n1, BIT_AND_EXPR, n, lpm1);
3559 335 : insert_before (g);
3560 989 : g = gimple_build_assign (useless_type_conversion_p (sizetype,
3561 335 : TREE_TYPE (n))
3562 319 : ? n2 : make_ssa_name (TREE_TYPE (n)),
3563 : RSHIFT_EXPR, n,
3564 335 : build_int_cst (TREE_TYPE (n),
3565 670 : exact_log2 (limb_prec)));
3566 335 : insert_before (g);
3567 335 : if (gimple_assign_lhs (g) != n2)
3568 : {
3569 319 : g = gimple_build_assign (n2, NOP_EXPR, gimple_assign_lhs (g));
3570 319 : insert_before (g);
3571 : }
3572 335 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (n)),
3573 : NEGATE_EXPR, n1);
3574 335 : insert_before (g);
3575 335 : g = gimple_build_assign (n4, BIT_AND_EXPR, gimple_assign_lhs (g),
3576 : lpm1);
3577 335 : insert_before (g);
3578 : }
3579 : else
3580 : {
3581 0 : tree lp = build_int_cst (TREE_TYPE (n), limb_prec);
3582 0 : g = gimple_build_assign (n1, TRUNC_MOD_EXPR, n, lp);
3583 0 : insert_before (g);
3584 0 : g = gimple_build_assign (useless_type_conversion_p (sizetype,
3585 0 : TREE_TYPE (n))
3586 0 : ? n2 : make_ssa_name (TREE_TYPE (n)),
3587 : TRUNC_DIV_EXPR, n, lp);
3588 0 : insert_before (g);
3589 0 : if (gimple_assign_lhs (g) != n2)
3590 : {
3591 0 : g = gimple_build_assign (n2, NOP_EXPR, gimple_assign_lhs (g));
3592 0 : insert_before (g);
3593 : }
3594 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (n)),
3595 : MINUS_EXPR, lp, n1);
3596 0 : insert_before (g);
3597 0 : g = gimple_build_assign (n4, TRUNC_MOD_EXPR, gimple_assign_lhs (g),
3598 : lp);
3599 0 : insert_before (g);
3600 : }
3601 335 : g = gimple_build_assign (make_ssa_name (boolean_type_node), NE_EXPR, n1,
3602 335 : build_zero_cst (TREE_TYPE (n)));
3603 335 : insert_before (g);
3604 335 : g = gimple_build_assign (n3, NOP_EXPR, gimple_assign_lhs (g));
3605 335 : insert_before (g);
3606 : }
3607 1168 : tree p = build_int_cst (sizetype,
3608 584 : prec / limb_prec - (prec % limb_prec == 0));
3609 584 : if (rhs_code == RSHIFT_EXPR)
3610 : {
3611 : /* Lower
3612 : dst = src >> n;
3613 : as
3614 : unsigned n1 = n % limb_prec;
3615 : size_t n2 = n / limb_prec;
3616 : size_t n3 = n1 != 0;
3617 : unsigned n4 = (limb_prec - n1) % limb_prec;
3618 : size_t idx;
3619 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
3620 : int signed_p = (typeof (src) -1) < 0;
3621 : for (idx = n2; idx < ((!signed_p && (prec % limb_prec == 0))
3622 : ? p : p - n3); ++idx)
3623 : dst[idx - n2] = (src[idx] >> n1) | (src[idx + n3] << n4);
3624 : limb_type ext;
3625 : if (prec % limb_prec == 0)
3626 : ext = src[p];
3627 : else if (signed_p)
3628 : ext = ((signed limb_type) (src[p] << (limb_prec
3629 : - (prec % limb_prec))))
3630 : >> (limb_prec - (prec % limb_prec));
3631 : else
3632 : ext = src[p] & (((limb_type) 1 << (prec % limb_prec)) - 1);
3633 : if (!signed_p && (prec % limb_prec == 0))
3634 : ;
3635 : else if (idx < prec / 64)
3636 : {
3637 : dst[idx - n2] = (src[idx] >> n1) | (ext << n4);
3638 : ++idx;
3639 : }
3640 : idx -= n2;
3641 : if (signed_p)
3642 : {
3643 : dst[idx] = ((signed limb_type) ext) >> n1;
3644 : ext = ((signed limb_type) ext) >> (limb_prec - 1);
3645 : }
3646 : else
3647 : {
3648 : dst[idx] = ext >> n1;
3649 : ext = 0;
3650 : }
3651 : for (++idx; idx <= p; ++idx)
3652 : dst[idx] = ext; */
3653 387 : tree pmn3;
3654 387 : if (TYPE_UNSIGNED (type) && prec % limb_prec == 0)
3655 100 : pmn3 = bitint_big_endian ? size_zero_node : p;
3656 287 : else if (bitint_big_endian)
3657 : pmn3 = n3;
3658 287 : else if (TREE_CODE (n3) == INTEGER_CST)
3659 114 : pmn3 = int_const_binop (MINUS_EXPR, p, n3);
3660 : else
3661 : {
3662 173 : pmn3 = make_ssa_name (sizetype);
3663 173 : g = gimple_build_assign (pmn3, MINUS_EXPR, p, n3);
3664 173 : insert_before (g);
3665 : }
3666 387 : tree pmn2 = NULL_TREE;
3667 387 : if (bitint_big_endian)
3668 : {
3669 0 : if (TREE_CODE (n2) == INTEGER_CST)
3670 0 : pmn2 = int_const_binop (MINUS_EXPR, p, n2);
3671 : else
3672 : {
3673 0 : pmn2 = make_ssa_name (sizetype);
3674 0 : g = gimple_build_assign (pmn2, MINUS_EXPR, p, n2);
3675 0 : insert_before (g);
3676 : }
3677 0 : g = gimple_build_cond (GT_EXPR, pmn2, pmn3, NULL_TREE, NULL_TREE);
3678 : }
3679 : else
3680 387 : g = gimple_build_cond (LT_EXPR, n2, pmn3, NULL_TREE, NULL_TREE);
3681 387 : edge edge_true, edge_false;
3682 387 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3683 387 : tree idx_next;
3684 774 : tree idx = create_loop (bitint_big_endian ? pmn2 : n2, &idx_next);
3685 387 : tree idxmn2 = make_ssa_name (sizetype);
3686 387 : tree idxpn3 = make_ssa_name (sizetype);
3687 774 : g = gimple_build_assign (idxmn2,
3688 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3689 : idx, n2);
3690 387 : insert_before (g);
3691 774 : g = gimple_build_assign (idxpn3,
3692 : bitint_big_endian ? MINUS_EXPR : PLUS_EXPR,
3693 : idx, n3);
3694 387 : insert_before (g);
3695 387 : m_data_cnt = 0;
3696 387 : tree t1 = handle_operand (rhs1, idx);
3697 387 : m_first = false;
3698 387 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3699 : RSHIFT_EXPR, t1, n1);
3700 387 : insert_before (g);
3701 387 : t1 = gimple_assign_lhs (g);
3702 387 : if (!integer_zerop (n3))
3703 : {
3704 299 : m_data_cnt = 0;
3705 299 : tree t2 = handle_operand (rhs1, idxpn3);
3706 299 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3707 : LSHIFT_EXPR, t2, n4);
3708 299 : insert_before (g);
3709 299 : t2 = gimple_assign_lhs (g);
3710 299 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3711 : BIT_IOR_EXPR, t1, t2);
3712 299 : insert_before (g);
3713 299 : t1 = gimple_assign_lhs (g);
3714 : }
3715 387 : tree l = limb_access (TREE_TYPE (lhs), obj, idxmn2, true);
3716 387 : g = gimple_build_assign (l, t1);
3717 387 : insert_before (g);
3718 387 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3719 0 : bitint_big_endian ? size_int (-1)
3720 : : size_one_node);
3721 387 : insert_before (g);
3722 774 : g = gimple_build_cond (bitint_big_endian ? GT_EXPR : LT_EXPR,
3723 : idx_next, pmn3, NULL_TREE, NULL_TREE);
3724 387 : insert_before (g);
3725 387 : idx = make_ssa_name (sizetype);
3726 387 : m_gsi = gsi_for_stmt (final_stmt);
3727 387 : gphi *phi = create_phi_node (idx, gsi_bb (m_gsi));
3728 387 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3729 387 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3730 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3731 774 : add_phi_arg (phi, bitint_big_endian ? pmn2 : n2, edge_false,
3732 : UNKNOWN_LOCATION);
3733 387 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3734 387 : m_data_cnt = 0;
3735 387 : tree ms = handle_operand (rhs1, bitint_big_endian ? size_zero_node : p);
3736 387 : tree ext = ms;
3737 387 : if (!types_compatible_p (TREE_TYPE (ms), m_limb_type))
3738 230 : ext = add_cast (m_limb_type, ms);
3739 587 : if (!(TYPE_UNSIGNED (type) && prec % limb_prec == 0)
3740 487 : && !integer_zerop (n3))
3741 : {
3742 249 : if (bitint_big_endian)
3743 0 : g = gimple_build_cond (GT_EXPR, idx, size_zero_node,
3744 : NULL_TREE, NULL_TREE);
3745 : else
3746 249 : g = gimple_build_cond (LT_EXPR, idx, p, NULL_TREE, NULL_TREE);
3747 249 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3748 249 : m_data_cnt = 0;
3749 249 : t1 = handle_operand (rhs1, idx);
3750 249 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3751 : RSHIFT_EXPR, t1, n1);
3752 249 : insert_before (g);
3753 249 : t1 = gimple_assign_lhs (g);
3754 249 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3755 : LSHIFT_EXPR, ext, n4);
3756 249 : insert_before (g);
3757 249 : tree t2 = gimple_assign_lhs (g);
3758 249 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3759 : BIT_IOR_EXPR, t1, t2);
3760 249 : insert_before (g);
3761 249 : t1 = gimple_assign_lhs (g);
3762 249 : idxmn2 = make_ssa_name (sizetype);
3763 498 : g = gimple_build_assign (idxmn2, bitint_big_endian
3764 : ? PLUS_EXPR : MINUS_EXPR, idx, n2);
3765 249 : insert_before (g);
3766 249 : l = limb_access (TREE_TYPE (lhs), obj, idxmn2, true);
3767 249 : g = gimple_build_assign (l, t1);
3768 249 : insert_before (g);
3769 249 : idx_next = make_ssa_name (sizetype);
3770 249 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3771 : bitint_big_endian
3772 0 : ? size_int (-1) : size_one_node);
3773 249 : insert_before (g);
3774 249 : m_gsi = gsi_for_stmt (final_stmt);
3775 249 : tree nidx = make_ssa_name (sizetype);
3776 249 : phi = create_phi_node (nidx, gsi_bb (m_gsi));
3777 249 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3778 249 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3779 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3780 249 : add_phi_arg (phi, idx, edge_false, UNKNOWN_LOCATION);
3781 249 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3782 249 : idx = nidx;
3783 : }
3784 774 : g = gimple_build_assign (make_ssa_name (sizetype),
3785 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3786 : idx, n2);
3787 387 : insert_before (g);
3788 387 : idx = gimple_assign_lhs (g);
3789 387 : tree sext = ext;
3790 387 : if (!TYPE_UNSIGNED (type))
3791 187 : sext = add_cast (signed_type_for (m_limb_type), ext);
3792 387 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (sext)),
3793 : RSHIFT_EXPR, sext, n1);
3794 387 : insert_before (g);
3795 387 : t1 = gimple_assign_lhs (g);
3796 387 : if (!TYPE_UNSIGNED (type))
3797 : {
3798 187 : t1 = add_cast (m_limb_type, t1);
3799 187 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (sext)),
3800 : RSHIFT_EXPR, sext,
3801 187 : build_int_cst (TREE_TYPE (n),
3802 187 : limb_prec - 1));
3803 187 : insert_before (g);
3804 187 : ext = add_cast (m_limb_type, gimple_assign_lhs (g));
3805 : }
3806 : else
3807 200 : ext = build_zero_cst (m_limb_type);
3808 387 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3809 387 : g = gimple_build_assign (l, t1);
3810 387 : insert_before (g);
3811 387 : g = gimple_build_assign (make_ssa_name (sizetype), PLUS_EXPR, idx,
3812 : bitint_big_endian
3813 0 : ? size_int (-1) : size_one_node);
3814 387 : insert_before (g);
3815 387 : tree p2 = p;
3816 387 : if (bitint_big_endian)
3817 : {
3818 0 : tree new_idx = gimple_assign_lhs (g);
3819 0 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
3820 : NULL_TREE, NULL_TREE);
3821 0 : idx = new_idx;
3822 : }
3823 : else
3824 : {
3825 387 : if (bitint_extended == bitint_ext_full
3826 0 : && abi_limb_prec > limb_prec)
3827 0 : p2 = build_int_cst (sizetype,
3828 0 : CEIL (prec, abi_limb_prec)
3829 0 : * abi_limb_prec / limb_prec - 1);
3830 387 : idx = gimple_assign_lhs (g);
3831 387 : g = gimple_build_cond (LE_EXPR, idx, p2, NULL_TREE, NULL_TREE);
3832 : }
3833 387 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3834 387 : idx = create_loop (idx, &idx_next);
3835 387 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3836 387 : g = gimple_build_assign (l, ext);
3837 387 : insert_before (g);
3838 387 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3839 : bitint_big_endian
3840 0 : ? size_int (-1) : size_one_node);
3841 387 : insert_before (g);
3842 387 : if (bitint_big_endian)
3843 0 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
3844 : NULL_TREE, NULL_TREE);
3845 : else
3846 387 : g = gimple_build_cond (LE_EXPR, idx_next, p2, NULL_TREE, NULL_TREE);
3847 387 : insert_before (g);
3848 : }
3849 : else
3850 : {
3851 : /* Lower
3852 : dst = src << n;
3853 : as
3854 : unsigned n1 = n % limb_prec;
3855 : size_t n2 = n / limb_prec;
3856 : size_t n3 = n1 != 0;
3857 : unsigned n4 = (limb_prec - n1) % limb_prec;
3858 : size_t idx;
3859 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
3860 : for (idx = p; (ssize_t) idx >= (ssize_t) (n2 + n3); --idx)
3861 : dst[idx] = (src[idx - n2] << n1) | (src[idx - n2 - n3] >> n4);
3862 : if (n1)
3863 : {
3864 : dst[idx] = src[idx - n2] << n1;
3865 : --idx;
3866 : }
3867 : for (; (ssize_t) idx >= 0; --idx)
3868 : dst[idx] = 0; */
3869 197 : tree n2pn3;
3870 197 : if (TREE_CODE (n2) == INTEGER_CST && TREE_CODE (n3) == INTEGER_CST)
3871 71 : n2pn3 = int_const_binop (PLUS_EXPR, n2, n3);
3872 : else
3873 : {
3874 126 : n2pn3 = make_ssa_name (sizetype);
3875 126 : g = gimple_build_assign (n2pn3, PLUS_EXPR, n2, n3);
3876 126 : insert_before (g);
3877 : }
3878 197 : if (bitint_big_endian)
3879 : {
3880 0 : if (TREE_CODE (n2pn3) == INTEGER_CST)
3881 0 : n2pn3 = int_const_binop (MINUS_EXPR, p, n2pn3);
3882 : else
3883 : {
3884 0 : g = gimple_build_assign (make_ssa_name (sizetype),
3885 : MINUS_EXPR, p, n2pn3);
3886 0 : insert_before (g);
3887 0 : n2pn3 = gimple_assign_lhs (g);
3888 : }
3889 : }
3890 : /* For LSHIFT_EXPR, we can use handle_operand with non-INTEGER_CST
3891 : idx even to access the most significant partial limb. */
3892 197 : m_var_msb = true;
3893 197 : if (integer_zerop (n3))
3894 : /* For n3 == 0 p >= n2 + n3 is always true for all valid shift
3895 : counts. Emit if (true) condition that can be optimized later. */
3896 45 : g = gimple_build_cond (NE_EXPR, boolean_true_node, boolean_false_node,
3897 : NULL_TREE, NULL_TREE);
3898 152 : else if (bitint_big_endian)
3899 0 : g = gimple_build_cond (NE_EXPR, n2pn3, size_int (-1), NULL_TREE,
3900 : NULL_TREE);
3901 : else
3902 152 : g = gimple_build_cond (LE_EXPR, n2pn3, p, NULL_TREE, NULL_TREE);
3903 197 : edge edge_true, edge_false;
3904 197 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3905 197 : tree idx_next;
3906 197 : tree idx = create_loop (bitint_big_endian ? size_zero_node : p,
3907 : &idx_next);
3908 197 : tree idxmn2 = make_ssa_name (sizetype);
3909 197 : tree idxmn2mn3 = make_ssa_name (sizetype);
3910 394 : g = gimple_build_assign (idxmn2,
3911 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3912 : idx, n2);
3913 197 : insert_before (g);
3914 394 : g = gimple_build_assign (idxmn2mn3,
3915 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3916 : idxmn2, n3);
3917 197 : insert_before (g);
3918 197 : m_data_cnt = 0;
3919 197 : tree t1 = handle_operand (rhs1, idxmn2);
3920 197 : m_first = false;
3921 197 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3922 : LSHIFT_EXPR, t1, n1);
3923 197 : insert_before (g);
3924 197 : t1 = gimple_assign_lhs (g);
3925 197 : if (!integer_zerop (n3))
3926 : {
3927 152 : m_data_cnt = 0;
3928 152 : tree t2 = handle_operand (rhs1, idxmn2mn3);
3929 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3930 : RSHIFT_EXPR, t2, n4);
3931 152 : insert_before (g);
3932 152 : t2 = gimple_assign_lhs (g);
3933 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3934 : BIT_IOR_EXPR, t1, t2);
3935 152 : insert_before (g);
3936 152 : t1 = gimple_assign_lhs (g);
3937 : }
3938 197 : tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3939 197 : g = gimple_build_assign (l, t1);
3940 197 : insert_before (g);
3941 394 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3942 : bitint_big_endian
3943 197 : ? size_one_node : size_int (-1));
3944 197 : insert_before (g);
3945 197 : tree sn2pn3 = add_cast (ssizetype, n2pn3);
3946 394 : g = gimple_build_cond (bitint_big_endian ? LE_EXPR : GE_EXPR,
3947 : add_cast (ssizetype, idx_next), sn2pn3,
3948 : NULL_TREE, NULL_TREE);
3949 197 : insert_before (g);
3950 197 : idx = make_ssa_name (sizetype);
3951 197 : m_gsi = gsi_for_stmt (final_stmt);
3952 197 : gphi *phi = create_phi_node (idx, gsi_bb (m_gsi));
3953 197 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3954 197 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3955 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3956 197 : add_phi_arg (phi, bitint_big_endian ? size_zero_node : p,
3957 : edge_false, UNKNOWN_LOCATION);
3958 197 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3959 197 : m_data_cnt = 0;
3960 197 : if (!integer_zerop (n3))
3961 : {
3962 152 : g = gimple_build_cond (NE_EXPR, n3, size_zero_node,
3963 : NULL_TREE, NULL_TREE);
3964 152 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3965 152 : idxmn2 = make_ssa_name (sizetype);
3966 304 : g = gimple_build_assign (idxmn2,
3967 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3968 : idx, n2);
3969 152 : insert_before (g);
3970 152 : m_data_cnt = 0;
3971 152 : t1 = handle_operand (rhs1, idxmn2);
3972 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3973 : LSHIFT_EXPR, t1, n1);
3974 152 : insert_before (g);
3975 152 : t1 = gimple_assign_lhs (g);
3976 152 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3977 152 : g = gimple_build_assign (l, t1);
3978 152 : insert_before (g);
3979 152 : idx_next = make_ssa_name (sizetype);
3980 304 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3981 : bitint_big_endian
3982 152 : ? size_one_node : size_int (-1));
3983 152 : insert_before (g);
3984 152 : m_gsi = gsi_for_stmt (final_stmt);
3985 152 : tree nidx = make_ssa_name (sizetype);
3986 152 : phi = create_phi_node (nidx, gsi_bb (m_gsi));
3987 152 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3988 152 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3989 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3990 152 : add_phi_arg (phi, idx, edge_false, UNKNOWN_LOCATION);
3991 152 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3992 152 : idx = nidx;
3993 : }
3994 197 : if (bitint_big_endian)
3995 0 : g = gimple_build_cond (LE_EXPR, idx, p, NULL_TREE, NULL_TREE);
3996 : else
3997 197 : g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx),
3998 : ssize_int (0), NULL_TREE, NULL_TREE);
3999 197 : if_then (g, profile_probability::likely (), edge_true, edge_false);
4000 197 : idx = create_loop (idx, &idx_next);
4001 197 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
4002 197 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
4003 197 : insert_before (g);
4004 394 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
4005 : bitint_big_endian
4006 197 : ? size_one_node : size_int (-1));
4007 197 : insert_before (g);
4008 197 : if (bitint_big_endian)
4009 0 : g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
4010 : else
4011 197 : g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx_next),
4012 : ssize_int (0), NULL_TREE, NULL_TREE);
4013 197 : insert_before (g);
4014 197 : if (bitint_extended && prec % limb_prec != 0)
4015 : {
4016 : /* The most significant limb has been updated either in the
4017 : loop or in the if after it. To simplify the code, just
4018 : read it back from memory and extend. */
4019 0 : m_gsi = gsi_after_labels (edge_false->dest);
4020 0 : idx = bitint_big_endian ? size_zero_node : p;
4021 0 : tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
4022 0 : tree type = limb_access_type (TREE_TYPE (lhs), idx);
4023 0 : tree v = make_ssa_name (m_limb_type);
4024 0 : g = gimple_build_assign (v, l);
4025 0 : insert_before (g);
4026 0 : v = add_cast (type, v);
4027 0 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
4028 0 : v = add_cast (m_limb_type, v);
4029 0 : g = gimple_build_assign (l, v);
4030 0 : insert_before (g);
4031 0 : if (bitint_extended == bitint_ext_full
4032 0 : && abi_limb_prec > limb_prec
4033 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4034 0 : > CEIL (prec, limb_prec) * limb_prec))
4035 : {
4036 0 : tree p2 = build_int_cst (sizetype,
4037 : CEIL (prec, abi_limb_prec)
4038 0 : * abi_limb_prec / limb_prec - 1);
4039 0 : if (TYPE_UNSIGNED (TREE_TYPE (lhs)))
4040 0 : v = build_zero_cst (m_limb_type);
4041 : else
4042 : {
4043 0 : v = add_cast (signed_type_for (m_limb_type), v);
4044 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (v)),
4045 : RSHIFT_EXPR, v,
4046 : build_int_cst (unsigned_type_node,
4047 0 : limb_prec - 1));
4048 0 : insert_before (g);
4049 0 : v = add_cast (m_limb_type, gimple_assign_lhs (g));
4050 : }
4051 0 : l = limb_access (TREE_TYPE (lhs), obj, p2, true);
4052 0 : g = gimple_build_assign (l, v);
4053 0 : insert_before (g);
4054 : }
4055 : }
4056 197 : else if (bitint_extended == bitint_ext_full
4057 0 : && abi_limb_prec > limb_prec
4058 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4059 0 : > CEIL (prec, limb_prec) * limb_prec))
4060 : {
4061 0 : m_gsi = gsi_after_labels (edge_false->dest);
4062 0 : tree p2 = build_int_cst (sizetype,
4063 : CEIL (prec, abi_limb_prec)
4064 0 : * abi_limb_prec / limb_prec - 1);
4065 0 : tree v;
4066 0 : if (TYPE_UNSIGNED (TREE_TYPE (lhs)))
4067 0 : v = build_zero_cst (m_limb_type);
4068 : else
4069 : {
4070 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p, true);
4071 0 : v = make_ssa_name (m_limb_type);
4072 0 : g = gimple_build_assign (v, l);
4073 0 : insert_before (g);
4074 0 : v = add_cast (signed_type_for (m_limb_type), v);
4075 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (v)),
4076 : RSHIFT_EXPR, v,
4077 : build_int_cst (unsigned_type_node,
4078 0 : limb_prec - 1));
4079 0 : insert_before (g);
4080 0 : v = add_cast (m_limb_type, gimple_assign_lhs (g));
4081 : }
4082 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
4083 0 : g = gimple_build_assign (l, v);
4084 0 : insert_before (g);
4085 : }
4086 : }
4087 584 : }
4088 :
4089 : /* Lower large/huge _BitInt multiplication or division. */
4090 :
4091 : void
4092 362 : bitint_large_huge::lower_muldiv_stmt (tree obj, gimple *stmt)
4093 : {
4094 362 : tree rhs1 = gimple_assign_rhs1 (stmt);
4095 362 : tree rhs2 = gimple_assign_rhs2 (stmt);
4096 362 : tree lhs = gimple_assign_lhs (stmt);
4097 362 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
4098 362 : tree type = TREE_TYPE (rhs1);
4099 362 : gcc_assert (BITINT_TYPE_P (type)
4100 : && bitint_precision_kind (type) >= bitint_prec_large);
4101 362 : int prec = TYPE_PRECISION (type), prec1, prec2;
4102 362 : bool ext_ms_limb = false;
4103 362 : bool do_ext = false;
4104 362 : rhs1 = handle_operand_addr (rhs1, stmt, NULL, &prec1);
4105 362 : rhs2 = handle_operand_addr (rhs2, stmt, NULL, &prec2);
4106 362 : if (obj == NULL_TREE)
4107 : {
4108 157 : int part = var_to_partition (m_map, lhs);
4109 157 : gcc_assert (m_vars[part] != NULL_TREE);
4110 157 : obj = m_vars[part];
4111 157 : lhs = build_fold_addr_expr (obj);
4112 : }
4113 : else
4114 : {
4115 205 : lhs = build_fold_addr_expr (obj);
4116 205 : lhs = force_gimple_operand_gsi (&m_gsi, lhs, true,
4117 : NULL_TREE, true, GSI_SAME_STMT);
4118 : }
4119 362 : if (bitint_extended && TYPE_OVERFLOW_WRAPS (type))
4120 : {
4121 0 : if (rhs_code == MULT_EXPR)
4122 : do_ext = true;
4123 : /* For signed division with -fwrapv, minimum negative / -1 needs
4124 : is minimum negative and the padding bits above it should be all
4125 : set. */
4126 0 : else if (!TYPE_UNSIGNED (type)
4127 0 : && (rhs_code == TRUNC_DIV_EXPR || rhs_code == EXACT_DIV_EXPR))
4128 362 : do_ext = true;
4129 : }
4130 362 : if (bitint_extended == bitint_ext_full
4131 0 : && abi_limb_prec > limb_prec
4132 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4133 0 : > CEIL (prec, limb_prec) * limb_prec))
4134 : {
4135 : /* unsigned multiplication needs to wrap around, so we can't
4136 : increase prec. Similarly for -fwrapv. */
4137 0 : if (do_ext)
4138 : ext_ms_limb = true;
4139 : else
4140 0 : prec = CEIL (prec, abi_limb_prec) * abi_limb_prec;
4141 : }
4142 362 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
4143 362 : gimple *g;
4144 362 : switch (rhs_code)
4145 : {
4146 207 : case MULT_EXPR:
4147 207 : g = gimple_build_call_internal (IFN_MULBITINT, 6,
4148 207 : lhs, build_int_cst (sitype, prec),
4149 207 : rhs1, build_int_cst (sitype, prec1),
4150 207 : rhs2, build_int_cst (sitype, prec2));
4151 207 : insert_before (g);
4152 207 : break;
4153 103 : case TRUNC_DIV_EXPR:
4154 103 : case EXACT_DIV_EXPR:
4155 103 : g = gimple_build_call_internal (IFN_DIVMODBITINT, 8,
4156 103 : lhs, build_int_cst (sitype, prec),
4157 : null_pointer_node,
4158 : build_int_cst (sitype, 0),
4159 103 : rhs1, build_int_cst (sitype, prec1),
4160 103 : rhs2, build_int_cst (sitype, prec2));
4161 103 : if (!stmt_ends_bb_p (stmt))
4162 102 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4163 103 : insert_before (g);
4164 103 : break;
4165 52 : case TRUNC_MOD_EXPR:
4166 52 : g = gimple_build_call_internal (IFN_DIVMODBITINT, 8, null_pointer_node,
4167 : build_int_cst (sitype, 0),
4168 52 : lhs, build_int_cst (sitype, prec),
4169 52 : rhs1, build_int_cst (sitype, prec1),
4170 52 : rhs2, build_int_cst (sitype, prec2));
4171 52 : if (!stmt_ends_bb_p (stmt))
4172 49 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4173 52 : insert_before (g);
4174 52 : break;
4175 0 : default:
4176 0 : gcc_unreachable ();
4177 : }
4178 362 : if (stmt_ends_bb_p (stmt))
4179 : {
4180 4 : maybe_duplicate_eh_stmt (g, stmt);
4181 4 : edge e1;
4182 4 : edge_iterator ei;
4183 4 : basic_block bb = gimple_bb (stmt);
4184 :
4185 4 : FOR_EACH_EDGE (e1, ei, bb->succs)
4186 4 : if (e1->flags & EDGE_EH)
4187 : break;
4188 4 : if (e1)
4189 : {
4190 4 : edge e2 = split_block (gsi_bb (m_gsi), g);
4191 4 : m_gsi = gsi_after_labels (e2->dest);
4192 4 : add_eh_edge (e2->src, e1);
4193 : }
4194 : }
4195 362 : if (do_ext
4196 362 : && ((prec % limb_prec) != 0 || (ext_ms_limb && !TYPE_UNSIGNED (type))))
4197 : {
4198 : /* Unsigned multiplication wraps, but libgcc function will return the
4199 : bits beyond prec within the top limb as another limb of the full
4200 : multiplication. So, clear the padding bits here. */
4201 0 : tree idx = size_int (bitint_big_endian ? 0 : prec / limb_prec);
4202 0 : tree l = limb_access (type, obj, idx, true);
4203 0 : tree ctype = limb_access_type (type, idx);
4204 0 : tree v = make_ssa_name (m_limb_type);
4205 0 : g = gimple_build_assign (v, l);
4206 0 : insert_before (g);
4207 0 : tree v2 = v;
4208 0 : if ((prec % limb_prec) != 0)
4209 : {
4210 0 : v = add_cast (ctype, v);
4211 0 : l = limb_access (type, obj, idx, true);
4212 0 : v = add_cast (m_limb_type, v);
4213 0 : v2 = v;
4214 0 : g = gimple_build_assign (l, v);
4215 0 : insert_before (g);
4216 : }
4217 0 : if (ext_ms_limb && !TYPE_UNSIGNED (type))
4218 : {
4219 0 : v2 = add_cast (signed_type_for (m_limb_type), v2);
4220 0 : tree lpm1 = build_int_cst (unsigned_type_node, limb_prec - 1);
4221 0 : v = make_ssa_name (TREE_TYPE (v2));
4222 0 : g = gimple_build_assign (v, RSHIFT_EXPR, v2, lpm1);
4223 0 : insert_before (g);
4224 0 : unsigned int i
4225 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4226 0 : v = add_cast (m_limb_type, v);
4227 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1),
4228 : true), v);
4229 0 : insert_before (g);
4230 0 : ext_ms_limb = false;
4231 : }
4232 : }
4233 362 : if (ext_ms_limb)
4234 : {
4235 0 : unsigned int i = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4236 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1), true),
4237 : build_zero_cst (m_limb_type));
4238 0 : insert_before (g);
4239 : }
4240 362 : }
4241 :
4242 : /* Lower large/huge _BitInt conversion to/from floating point. */
4243 :
4244 : void
4245 319 : bitint_large_huge::lower_float_conv_stmt (tree obj, gimple *stmt)
4246 : {
4247 319 : tree rhs1 = gimple_assign_rhs1 (stmt);
4248 319 : tree lhs = gimple_assign_lhs (stmt);
4249 319 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
4250 319 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
4251 319 : gimple *g;
4252 319 : if (rhs_code == FIX_TRUNC_EXPR)
4253 : {
4254 179 : tree type = TREE_TYPE (lhs);
4255 179 : int prec = TYPE_PRECISION (type);
4256 179 : bool extend_ms_limb = false;
4257 179 : if (bitint_extended == bitint_ext_full
4258 0 : && abi_limb_prec > limb_prec
4259 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4260 0 : > CEIL (prec, limb_prec) * limb_prec))
4261 179 : extend_ms_limb = true;
4262 179 : if (!TYPE_UNSIGNED (type))
4263 93 : prec = -prec;
4264 179 : if (obj == NULL_TREE)
4265 : {
4266 135 : int part = var_to_partition (m_map, lhs);
4267 135 : gcc_assert (m_vars[part] != NULL_TREE);
4268 135 : obj = m_vars[part];
4269 135 : lhs = build_fold_addr_expr (obj);
4270 : }
4271 : else
4272 : {
4273 44 : lhs = build_fold_addr_expr (obj);
4274 44 : lhs = force_gimple_operand_gsi (&m_gsi, lhs, true,
4275 : NULL_TREE, true, GSI_SAME_STMT);
4276 : }
4277 179 : scalar_mode from_mode
4278 179 : = as_a <scalar_mode> (TYPE_MODE (TREE_TYPE (rhs1)));
4279 : #ifdef HAVE_SFmode
4280 : /* IEEE single is a full superset of both IEEE half and
4281 : bfloat formats, convert to float first and then to _BitInt
4282 : to avoid the need of another 2 library routines. */
4283 179 : if ((REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format
4284 179 : || REAL_MODE_FORMAT (from_mode) == &ieee_half_format)
4285 191 : && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
4286 : {
4287 12 : tree type = lang_hooks.types.type_for_mode (SFmode, 0);
4288 12 : if (type)
4289 12 : rhs1 = add_cast (type, rhs1);
4290 : }
4291 : #endif
4292 179 : g = gimple_build_call_internal (IFN_FLOATTOBITINT, 3,
4293 179 : lhs, build_int_cst (sitype, prec),
4294 : rhs1);
4295 179 : insert_before (g);
4296 179 : if (extend_ms_limb)
4297 : {
4298 0 : unsigned int i
4299 0 : = (CEIL (prec < 0 ? -prec : prec, abi_limb_prec)
4300 0 : * abi_limb_prec / limb_prec);
4301 0 : tree val;
4302 0 : if (prec < 0)
4303 : {
4304 0 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4305 : limb_access (type, obj,
4306 0 : size_int (i - 2),
4307 : true));
4308 0 : insert_before (g);
4309 0 : val = add_cast (signed_type_for (m_limb_type),
4310 : gimple_assign_lhs (g));
4311 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (val)),
4312 : RSHIFT_EXPR, val,
4313 : build_int_cst (unsigned_type_node,
4314 0 : limb_prec - 1));
4315 0 : insert_before (g);
4316 0 : val = add_cast (m_limb_type, gimple_assign_lhs (g));
4317 : }
4318 : else
4319 0 : val = build_zero_cst (m_limb_type);
4320 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1),
4321 : true), val);
4322 0 : insert_before (g);
4323 : }
4324 : }
4325 : else
4326 : {
4327 140 : int prec;
4328 140 : rhs1 = handle_operand_addr (rhs1, stmt, NULL, &prec);
4329 140 : g = gimple_build_call_internal (IFN_BITINTTOFLOAT, 2,
4330 140 : rhs1, build_int_cst (sitype, prec));
4331 140 : gimple_call_set_lhs (g, lhs);
4332 140 : if (!stmt_ends_bb_p (stmt))
4333 139 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4334 140 : gsi_replace (&m_gsi, g, true);
4335 : }
4336 319 : }
4337 :
4338 : /* Helper method for lower_addsub_overflow and lower_mul_overflow.
4339 : If check_zero is true, caller wants to check if all bits in [start, end)
4340 : are zero, otherwise if bits in [start, end) are either all zero or
4341 : all ones. L is the limb with index LIMB, START and END are measured
4342 : in bits. */
4343 :
4344 : tree
4345 6453 : bitint_large_huge::arith_overflow_extract_bits (unsigned int start,
4346 : unsigned int end, tree l,
4347 : unsigned int limb,
4348 : bool check_zero)
4349 : {
4350 6453 : unsigned startlimb = start / limb_prec;
4351 6453 : unsigned endlimb = (end - 1) / limb_prec;
4352 6453 : gimple *g;
4353 :
4354 6453 : if ((start % limb_prec) == 0 && (end % limb_prec) == 0)
4355 : return l;
4356 6184 : if (startlimb == endlimb && limb == startlimb)
4357 : {
4358 2039 : if (check_zero)
4359 : {
4360 1486 : wide_int w = wi::shifted_mask (start % limb_prec,
4361 1486 : end - start, false, limb_prec);
4362 2972 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4363 : BIT_AND_EXPR, l,
4364 1486 : wide_int_to_tree (m_limb_type, w));
4365 1486 : insert_before (g);
4366 1486 : return gimple_assign_lhs (g);
4367 1486 : }
4368 553 : unsigned int shift = start % limb_prec;
4369 553 : if ((end % limb_prec) != 0)
4370 : {
4371 336 : unsigned int lshift = (-end) % limb_prec;
4372 336 : shift += lshift;
4373 336 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4374 : LSHIFT_EXPR, l,
4375 : build_int_cst (unsigned_type_node,
4376 336 : lshift));
4377 336 : insert_before (g);
4378 336 : l = gimple_assign_lhs (g);
4379 : }
4380 553 : l = add_cast (signed_type_for (m_limb_type), l);
4381 553 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4382 : RSHIFT_EXPR, l,
4383 553 : build_int_cst (unsigned_type_node, shift));
4384 553 : insert_before (g);
4385 553 : return add_cast (m_limb_type, gimple_assign_lhs (g));
4386 : }
4387 4145 : else if (limb == startlimb)
4388 : {
4389 1990 : if ((start % limb_prec) == 0)
4390 : return l;
4391 1876 : if (!check_zero)
4392 950 : l = add_cast (signed_type_for (m_limb_type), l);
4393 1876 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4394 : RSHIFT_EXPR, l,
4395 : build_int_cst (unsigned_type_node,
4396 1876 : start % limb_prec));
4397 1876 : insert_before (g);
4398 1876 : l = gimple_assign_lhs (g);
4399 1876 : if (!check_zero)
4400 950 : l = add_cast (m_limb_type, l);
4401 1876 : return l;
4402 : }
4403 2155 : else if (limb == endlimb)
4404 : {
4405 1704 : if ((end % limb_prec) == 0)
4406 : return l;
4407 1687 : if (check_zero)
4408 : {
4409 880 : wide_int w = wi::mask (end % limb_prec, false, limb_prec);
4410 1760 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4411 : BIT_AND_EXPR, l,
4412 880 : wide_int_to_tree (m_limb_type, w));
4413 880 : insert_before (g);
4414 880 : return gimple_assign_lhs (g);
4415 880 : }
4416 807 : unsigned int shift = (-end) % limb_prec;
4417 807 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4418 : LSHIFT_EXPR, l,
4419 807 : build_int_cst (unsigned_type_node, shift));
4420 807 : insert_before (g);
4421 807 : l = add_cast (signed_type_for (m_limb_type), gimple_assign_lhs (g));
4422 807 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4423 : RSHIFT_EXPR, l,
4424 807 : build_int_cst (unsigned_type_node, shift));
4425 807 : insert_before (g);
4426 807 : return add_cast (m_limb_type, gimple_assign_lhs (g));
4427 : }
4428 : return l;
4429 : }
4430 :
4431 : /* Helper method for lower_addsub_overflow and lower_mul_overflow. Store
4432 : result including overflow flag into the right locations. */
4433 :
4434 : void
4435 4212 : bitint_large_huge::finish_arith_overflow (tree var, tree obj, tree type,
4436 : tree ovf, tree lhs, tree orig_obj,
4437 : gimple *stmt, unsigned nelts,
4438 : tree_code code)
4439 : {
4440 4212 : gimple *g;
4441 :
4442 4212 : if (obj == NULL_TREE
4443 4212 : && (!BITINT_TYPE_P (type)
4444 231 : || bitint_precision_kind (type) < bitint_prec_large))
4445 : {
4446 : /* Add support for 3 or more limbs filled in from normal integral
4447 : type if this assert fails. If no target chooses limb mode smaller
4448 : than half of largest supported normal integral type, this will not
4449 : be needed. */
4450 250 : gcc_assert (TYPE_PRECISION (type) <= 2 * limb_prec);
4451 250 : tree lhs_type = type;
4452 110 : if (BITINT_TYPE_P (type)
4453 250 : && bitint_precision_kind (type) == bitint_prec_middle)
4454 46 : lhs_type = build_nonstandard_integer_type (TYPE_PRECISION (type),
4455 46 : TYPE_UNSIGNED (type));
4456 250 : tree r1 = limb_access (NULL_TREE, var,
4457 : bitint_big_endian
4458 0 : ? size_int (nelts - 1) : size_zero_node, true);
4459 250 : g = gimple_build_assign (make_ssa_name (m_limb_type), r1);
4460 250 : insert_before (g);
4461 250 : r1 = gimple_assign_lhs (g);
4462 250 : if (!useless_type_conversion_p (lhs_type, TREE_TYPE (r1)))
4463 250 : r1 = add_cast (lhs_type, r1);
4464 250 : if (TYPE_PRECISION (lhs_type) > limb_prec)
4465 : {
4466 90 : tree r2 = limb_access (NULL_TREE, var,
4467 : bitint_big_endian
4468 0 : ? size_int (nelts - 2) : size_one_node, true);
4469 90 : g = gimple_build_assign (make_ssa_name (m_limb_type), r2);
4470 90 : insert_before (g);
4471 90 : r2 = gimple_assign_lhs (g);
4472 90 : r2 = add_cast (lhs_type, r2);
4473 90 : g = gimple_build_assign (make_ssa_name (lhs_type), LSHIFT_EXPR, r2,
4474 : build_int_cst (unsigned_type_node,
4475 90 : limb_prec));
4476 90 : insert_before (g);
4477 90 : g = gimple_build_assign (make_ssa_name (lhs_type), BIT_IOR_EXPR, r1,
4478 : gimple_assign_lhs (g));
4479 90 : insert_before (g);
4480 90 : r1 = gimple_assign_lhs (g);
4481 : }
4482 250 : if (lhs_type != type)
4483 46 : r1 = add_cast (type, r1);
4484 250 : ovf = add_cast (lhs_type, ovf);
4485 250 : if (lhs_type != type)
4486 46 : ovf = add_cast (type, ovf);
4487 250 : g = gimple_build_assign (lhs, COMPLEX_EXPR, r1, ovf);
4488 250 : m_gsi = gsi_for_stmt (stmt);
4489 250 : gsi_replace (&m_gsi, g, true);
4490 : }
4491 : else
4492 : {
4493 3962 : unsigned HOST_WIDE_INT obj_nelts = 0;
4494 3962 : tree atype = NULL_TREE;
4495 3962 : if (obj)
4496 : {
4497 3871 : obj_nelts = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))) / limb_prec;
4498 3871 : if (orig_obj == NULL_TREE)
4499 2277 : obj_nelts >>= 1;
4500 3871 : atype = build_array_type_nelts (m_limb_type, obj_nelts);
4501 : }
4502 3962 : if (bitint_extended && (var || obj))
4503 : {
4504 0 : unsigned prec = TYPE_PRECISION (type);
4505 0 : unsigned prec_limbs = CEIL (prec, limb_prec);
4506 0 : bool ext_ms_limb
4507 : = (bitint_extended == bitint_ext_full
4508 0 : && abi_limb_prec > limb_prec
4509 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4510 0 : > CEIL (prec, limb_prec) * limb_prec));
4511 : /* For .{ADD,SUB}_OVERFLOW the partial limb if any is
4512 : already extended in lower_addsub_overflow. */
4513 0 : if ((code == MULT_EXPR && (prec % limb_prec) != 0)
4514 0 : || (ext_ms_limb && !TYPE_UNSIGNED (type)))
4515 : {
4516 0 : tree plm1idx = size_int (bitint_big_endian
4517 : ? nelts - obj_nelts : prec_limbs - 1);
4518 0 : tree plm1type
4519 0 : = limb_access_type (type, bitint_big_endian
4520 : ? size_zero_node : plm1idx);
4521 0 : tree l = limb_access (bitint_big_endian ? NULL_TREE : type,
4522 : var ? var : obj, plm1idx, true);
4523 0 : tree rhs = make_ssa_name (TREE_TYPE (l));
4524 0 : g = gimple_build_assign (rhs, l);
4525 0 : insert_before (g);
4526 0 : if (code == MULT_EXPR && (prec % limb_prec) != 0)
4527 : {
4528 0 : if (!useless_type_conversion_p (plm1type, TREE_TYPE (rhs)))
4529 0 : rhs = add_cast (plm1type, rhs);
4530 0 : if (!useless_type_conversion_p (TREE_TYPE (l),
4531 0 : TREE_TYPE (rhs)))
4532 0 : rhs = add_cast (TREE_TYPE (l), rhs);
4533 0 : l = limb_access (bitint_big_endian ? NULL_TREE : type,
4534 : var ? var : obj, plm1idx, true);
4535 0 : g = gimple_build_assign (l, rhs);
4536 0 : insert_before (g);
4537 : }
4538 0 : if (ext_ms_limb && !TYPE_UNSIGNED (type))
4539 : {
4540 0 : rhs = add_cast (signed_type_for (m_limb_type), rhs);
4541 0 : tree lpm1 = build_int_cst (unsigned_type_node,
4542 0 : limb_prec - 1);
4543 0 : tree v = make_ssa_name (TREE_TYPE (rhs));
4544 0 : g = gimple_build_assign (v, RSHIFT_EXPR, rhs, lpm1);
4545 0 : insert_before (g);
4546 0 : unsigned int i
4547 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4548 0 : v = add_cast (m_limb_type, v);
4549 0 : g = gimple_build_assign (limb_access (type, var ? var : obj,
4550 0 : size_int (i - 1),
4551 : true), v);
4552 0 : insert_before (g);
4553 0 : ext_ms_limb = false;
4554 : }
4555 : }
4556 0 : if (ext_ms_limb)
4557 : {
4558 0 : unsigned int i
4559 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4560 0 : g = gimple_build_assign (limb_access (type, var ? var : obj,
4561 0 : size_int (i - 1), true),
4562 : build_zero_cst (m_limb_type));
4563 0 : insert_before (g);
4564 : }
4565 : }
4566 3962 : if (var && obj)
4567 : {
4568 519 : tree v1, v2;
4569 519 : tree off;
4570 519 : if (orig_obj == NULL_TREE)
4571 : {
4572 32 : off = build_zero_cst (build_pointer_type (TREE_TYPE (obj)));
4573 32 : v1 = build2 (MEM_REF, atype,
4574 : build_fold_addr_expr (unshare_expr (obj)), off);
4575 : }
4576 487 : else if (!useless_type_conversion_p (atype, TREE_TYPE (obj)))
4577 9 : v1 = build1 (VIEW_CONVERT_EXPR, atype, unshare_expr (obj));
4578 : else
4579 478 : v1 = unshare_expr (obj);
4580 519 : off = build_int_cst (build_pointer_type (TREE_TYPE (var)),
4581 : bitint_big_endian
4582 519 : ? (nelts - obj_nelts) * m_limb_size : 0);
4583 519 : v2 = build2 (MEM_REF, atype, build_fold_addr_expr (var), off);
4584 519 : g = gimple_build_assign (v1, v2);
4585 519 : insert_before (g);
4586 : }
4587 3443 : else if (obj && bitint_big_endian && nelts != obj_nelts)
4588 : {
4589 0 : gcc_assert (nelts > obj_nelts);
4590 0 : tree fn = builtin_decl_implicit (BUILT_IN_MEMMOVE);
4591 0 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (obj)),
4592 0 : (nelts - obj_nelts) * m_limb_size);
4593 0 : tree src = build2 (MEM_REF, atype,
4594 : build_fold_addr_expr (unshare_expr (obj)), off);
4595 0 : g = gimple_build_call (fn, 3,
4596 : build_fold_addr_expr (unshare_expr (obj)),
4597 : src, build_int_cst (size_type_node,
4598 0 : obj_nelts * m_limb_size));
4599 0 : insert_before (g);
4600 : }
4601 3962 : if (orig_obj == NULL_TREE && obj)
4602 : {
4603 2277 : ovf = add_cast (m_limb_type, ovf);
4604 2277 : tree l = limb_access (NULL_TREE, obj,
4605 2277 : size_int (bitint_big_endian
4606 : ? obj_nelts * 2 - 1 : obj_nelts),
4607 : true);
4608 2277 : g = gimple_build_assign (l, ovf);
4609 2277 : insert_before (g);
4610 2277 : if (obj_nelts > 1)
4611 : {
4612 2277 : atype = build_array_type_nelts (m_limb_type, obj_nelts - 1);
4613 2277 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (obj)),
4614 2277 : (obj_nelts + !bitint_big_endian)
4615 2277 : * m_limb_size);
4616 2277 : tree v1 = build2 (MEM_REF, atype,
4617 : build_fold_addr_expr (unshare_expr (obj)),
4618 : off);
4619 2277 : g = gimple_build_assign (v1, build_zero_cst (atype));
4620 2277 : insert_before (g);
4621 : }
4622 : }
4623 1685 : else if (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE)
4624 : {
4625 1640 : imm_use_iterator ui;
4626 1640 : use_operand_p use_p;
4627 1640 : FOR_EACH_IMM_USE_FAST (use_p, ui, lhs)
4628 : {
4629 1640 : g = USE_STMT (use_p);
4630 1640 : if (!is_gimple_assign (g)
4631 1640 : || gimple_assign_rhs_code (g) != IMAGPART_EXPR)
4632 0 : continue;
4633 1640 : tree lhs2 = gimple_assign_lhs (g);
4634 1640 : gimple *use_stmt;
4635 1640 : single_imm_use (lhs2, &use_p, &use_stmt);
4636 1640 : lhs2 = gimple_assign_lhs (use_stmt);
4637 1640 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
4638 1640 : if (useless_type_conversion_p (TREE_TYPE (lhs2), TREE_TYPE (ovf)))
4639 1619 : g = gimple_build_assign (lhs2, ovf);
4640 : else
4641 21 : g = gimple_build_assign (lhs2, NOP_EXPR, ovf);
4642 1640 : gsi_replace (&gsi, g, true);
4643 1640 : if (gsi_stmt (m_gsi) == use_stmt)
4644 91 : m_gsi = gsi_for_stmt (g);
4645 1640 : break;
4646 1640 : }
4647 : }
4648 45 : else if (ovf != boolean_false_node)
4649 : {
4650 45 : g = gimple_build_cond (NE_EXPR, ovf, boolean_false_node,
4651 : NULL_TREE, NULL_TREE);
4652 45 : edge edge_true, edge_false;
4653 45 : if_then (g, profile_probability::very_unlikely (),
4654 : edge_true, edge_false);
4655 45 : tree zero = build_zero_cst (TREE_TYPE (lhs));
4656 45 : tree fn = ubsan_build_overflow_builtin (code, m_loc,
4657 45 : TREE_TYPE (lhs),
4658 : zero, zero, NULL);
4659 45 : force_gimple_operand_gsi (&m_gsi, fn, true, NULL_TREE,
4660 : true, GSI_SAME_STMT);
4661 45 : m_gsi = gsi_after_labels (edge_true->dest);
4662 : }
4663 : }
4664 4212 : if (var)
4665 : {
4666 781 : tree clobber = build_clobber (TREE_TYPE (var), CLOBBER_STORAGE_END);
4667 781 : g = gimple_build_assign (var, clobber);
4668 781 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
4669 : }
4670 4212 : }
4671 :
4672 : /* Helper function for lower_addsub_overflow and lower_mul_overflow.
4673 : Given precisions of result TYPE (PREC), argument 0 precision PREC0,
4674 : argument 1 precision PREC1 and minimum precision for the result
4675 : PREC2, compute *START, *END, *CHECK_ZERO and return OVF. */
4676 :
4677 : static tree
4678 4212 : arith_overflow (tree_code code, tree type, int prec, int prec0, int prec1,
4679 : int prec2, unsigned *start, unsigned *end, bool *check_zero)
4680 : {
4681 4212 : *start = 0;
4682 4212 : *end = 0;
4683 4212 : *check_zero = true;
4684 : /* Ignore this special rule for subtraction, even if both
4685 : prec0 >= 0 and prec1 >= 0, their subtraction can be negative
4686 : in infinite precision. */
4687 4212 : if (code != MINUS_EXPR && prec0 >= 0 && prec1 >= 0)
4688 : {
4689 : /* Result in [0, prec2) is unsigned, if prec > prec2,
4690 : all bits above it will be zero. */
4691 681 : if ((prec - !TYPE_UNSIGNED (type)) >= prec2)
4692 0 : return boolean_false_node;
4693 : else
4694 : {
4695 : /* ovf if any of bits in [start, end) is non-zero. */
4696 681 : *start = prec - !TYPE_UNSIGNED (type);
4697 681 : *end = prec2;
4698 : }
4699 : }
4700 3531 : else if (TYPE_UNSIGNED (type))
4701 : {
4702 : /* If result in [0, prec2) is signed and if prec > prec2,
4703 : all bits above it will be sign bit copies. */
4704 1950 : if (prec >= prec2)
4705 : {
4706 : /* ovf if bit prec - 1 is non-zero. */
4707 184 : *start = prec - 1;
4708 184 : *end = prec;
4709 : }
4710 : else
4711 : {
4712 : /* ovf if any of bits in [start, end) is non-zero. */
4713 1766 : *start = prec;
4714 1766 : *end = prec2;
4715 : }
4716 : }
4717 1581 : else if (prec >= prec2)
4718 2 : return boolean_false_node;
4719 : else
4720 : {
4721 : /* ovf if [start, end) bits aren't all zeros or all ones. */
4722 1579 : *start = prec - 1;
4723 1579 : *end = prec2;
4724 1579 : *check_zero = false;
4725 : }
4726 : return NULL_TREE;
4727 : }
4728 :
4729 : /* Lower a .{ADD,SUB}_OVERFLOW call with at least one large/huge _BitInt
4730 : argument or return type _Complex large/huge _BitInt. */
4731 :
4732 : void
4733 2733 : bitint_large_huge::lower_addsub_overflow (tree obj, gimple *stmt)
4734 : {
4735 2733 : tree arg0 = gimple_call_arg (stmt, 0);
4736 2733 : tree arg1 = gimple_call_arg (stmt, 1);
4737 2733 : tree lhs = gimple_call_lhs (stmt);
4738 2733 : gimple *g;
4739 :
4740 2733 : if (!lhs)
4741 : {
4742 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
4743 0 : gsi_remove (&gsi, true);
4744 0 : return;
4745 : }
4746 2733 : gimple *final_stmt = gsi_stmt (m_gsi);
4747 2733 : tree type = TREE_TYPE (lhs);
4748 2733 : if (TREE_CODE (type) == COMPLEX_TYPE)
4749 2703 : type = TREE_TYPE (type);
4750 2733 : int prec = TYPE_PRECISION (type);
4751 2733 : int prec0 = range_to_prec (arg0, stmt);
4752 2733 : int prec1 = range_to_prec (arg1, stmt);
4753 : /* If PREC0 >= 0 && PREC1 >= 0 and CODE is not MINUS_EXPR, PREC2 is
4754 : the minimum unsigned precision of any possible operation's
4755 : result, otherwise it is minimum signed precision.
4756 : Some examples:
4757 : If PREC0 or PREC1 is 8, it means that argument is [0, 0xff],
4758 : if PREC0 or PREC1 is 10, it means that argument is [0, 0x3ff],
4759 : if PREC0 or PREC1 is -8, it means that argument is [-0x80, 0x7f],
4760 : if PREC0 or PREC1 is -10, it means that argument is [-0x200, 0x1ff].
4761 : PREC0 CODE PREC1 RESULT PREC2 SIGNED vs. UNSIGNED
4762 : 8 + 8 [0, 0x1fe] 9 UNSIGNED
4763 : 8 + 10 [0, 0x4fe] 11 UNSIGNED
4764 : -8 + -8 [-0x100, 0xfe] 9 SIGNED
4765 : -8 + -10 [-0x280, 0x27e] 11 SIGNED
4766 : 8 + -8 [-0x80, 0x17e] 10 SIGNED
4767 : 8 + -10 [-0x200, 0x2fe] 11 SIGNED
4768 : 10 + -8 [-0x80, 0x47e] 12 SIGNED
4769 : 8 - 8 [-0xff, 0xff] 9 SIGNED
4770 : 8 - 10 [-0x3ff, 0xff] 11 SIGNED
4771 : 10 - 8 [-0xff, 0x3ff] 11 SIGNED
4772 : -8 - -8 [-0xff, 0xff] 9 SIGNED
4773 : -8 - -10 [-0x27f, 0x27f] 11 SIGNED
4774 : -10 - -8 [-0x27f, 0x27f] 11 SIGNED
4775 : 8 - -8 [-0x7f, 0x17f] 10 SIGNED
4776 : 8 - -10 [-0x1ff, 0x2ff] 11 SIGNED
4777 : 10 - -8 [-0x7f, 0x47f] 12 SIGNED
4778 : -8 - 8 [-0x17f, 0x7f] 10 SIGNED
4779 : -8 - 10 [-0x47f, 0x7f] 12 SIGNED
4780 : -10 - 8 [-0x2ff, 0x1ff] 11 SIGNED */
4781 2733 : int prec2 = MAX (prec0 < 0 ? -prec0 : prec0,
4782 : prec1 < 0 ? -prec1 : prec1);
4783 : /* If operands are either both signed or both unsigned,
4784 : we need just one additional bit. */
4785 3777 : prec2 = (((prec0 < 0) == (prec1 < 0)
4786 : /* If one operand is signed and one unsigned and
4787 : the signed one has larger precision, we need
4788 : just one extra bit, otherwise two. */
4789 707 : || (prec0 < 0 ? (prec2 == -prec0 && prec2 != prec1)
4790 337 : : (prec2 == -prec1 && prec2 != prec0)))
4791 2733 : ? prec2 + 1 : prec2 + 2);
4792 2733 : int prec3 = MAX (prec0 < 0 ? -prec0 : prec0,
4793 : prec1 < 0 ? -prec1 : prec1);
4794 2733 : prec3 = MAX (prec3, prec);
4795 2733 : tree var = NULL_TREE;
4796 2733 : tree orig_obj = obj;
4797 2733 : if (obj == NULL_TREE
4798 1753 : && BITINT_TYPE_P (type)
4799 1652 : && bitint_precision_kind (type) >= bitint_prec_large
4800 1536 : && m_names
4801 4245 : && bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
4802 : {
4803 1457 : int part = var_to_partition (m_map, lhs);
4804 1457 : gcc_assert (m_vars[part] != NULL_TREE);
4805 1457 : obj = m_vars[part];
4806 1457 : if (TREE_TYPE (lhs) == type)
4807 14 : orig_obj = obj;
4808 : }
4809 101 : if (!BITINT_TYPE_P (type)
4810 2733 : || bitint_precision_kind (type) < bitint_prec_large)
4811 : {
4812 217 : unsigned HOST_WIDE_INT nelts = CEIL (prec, limb_prec);
4813 217 : tree atype = build_array_type_nelts (m_limb_type, nelts);
4814 217 : var = create_tmp_var (atype);
4815 : }
4816 :
4817 2733 : enum tree_code code;
4818 2733 : switch (gimple_call_internal_fn (stmt))
4819 : {
4820 : case IFN_ADD_OVERFLOW:
4821 : case IFN_UBSAN_CHECK_ADD:
4822 : code = PLUS_EXPR;
4823 : break;
4824 1399 : case IFN_SUB_OVERFLOW:
4825 1399 : case IFN_UBSAN_CHECK_SUB:
4826 1399 : code = MINUS_EXPR;
4827 1399 : break;
4828 0 : default:
4829 0 : gcc_unreachable ();
4830 : }
4831 2733 : unsigned start, end;
4832 2733 : bool check_zero;
4833 2733 : tree ovf = arith_overflow (code, type, prec, prec0, prec1, prec2,
4834 : &start, &end, &check_zero);
4835 :
4836 2733 : unsigned startlimb, endlimb;
4837 2733 : if (ovf)
4838 : {
4839 : startlimb = ~0U;
4840 : endlimb = ~0U;
4841 : }
4842 : else
4843 : {
4844 2733 : startlimb = start / limb_prec;
4845 2733 : endlimb = (end - 1) / limb_prec;
4846 : }
4847 :
4848 2733 : int prec4 = ovf != NULL_TREE ? prec : prec3;
4849 2733 : bitint_prec_kind kind = bitint_precision_kind (prec4);
4850 2733 : unsigned cnt, rem = 0, fin = 0, nelts;
4851 2733 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
4852 5466 : bool last_ovf = (ovf == NULL_TREE
4853 2733 : && CEIL (prec2, limb_prec) > CEIL (prec3, limb_prec));
4854 2733 : if (kind != bitint_prec_huge)
4855 1539 : nelts = cnt = CEIL (prec4, limb_prec) + last_ovf;
4856 : else
4857 : {
4858 1194 : rem = prec4 % (2 * limb_prec);
4859 1194 : fin = (prec4 - rem) / limb_prec;
4860 1194 : cnt = 2 + CEIL (rem, limb_prec) + last_ovf;
4861 1194 : nelts = fin + cnt - 2;
4862 1194 : idx = idx_first = create_loop (bitint_big_endian
4863 1194 : ? size_int (nelts - 1) : size_zero_node,
4864 : &idx_next);
4865 : }
4866 :
4867 2733 : if (kind == bitint_prec_huge)
4868 1194 : m_upwards_2limb = fin;
4869 2733 : m_upwards = true;
4870 :
4871 2733 : tree type0 = TREE_TYPE (arg0);
4872 2733 : tree type1 = TREE_TYPE (arg1);
4873 2733 : int prec5 = prec3;
4874 2733 : if (bitint_precision_kind (prec5) < bitint_prec_large)
4875 10 : prec5 = MAX (TYPE_PRECISION (type0), TYPE_PRECISION (type1));
4876 2733 : if (TYPE_PRECISION (type0) < prec5)
4877 : {
4878 146 : type0 = build_bitint_type (prec5, TYPE_UNSIGNED (type0));
4879 146 : if (TREE_CODE (arg0) == INTEGER_CST)
4880 27 : arg0 = fold_convert (type0, arg0);
4881 : }
4882 2733 : if (TYPE_PRECISION (type1) < prec5)
4883 : {
4884 156 : type1 = build_bitint_type (prec5, TYPE_UNSIGNED (type1));
4885 156 : if (TREE_CODE (arg1) == INTEGER_CST)
4886 76 : arg1 = fold_convert (type1, arg1);
4887 : }
4888 2733 : unsigned int data_cnt = 0;
4889 2733 : tree last_rhs1 = NULL_TREE, last_rhs2 = NULL_TREE;
4890 2733 : tree cmp = build_zero_cst (m_limb_type);
4891 2733 : unsigned prec_limbs = CEIL ((unsigned) prec, limb_prec);
4892 2733 : tree ovf_out = NULL_TREE, cmp_out = NULL_TREE;
4893 11904 : for (unsigned i = 0; i < cnt; i++)
4894 : {
4895 9171 : m_data_cnt = 0;
4896 9171 : tree rhs1, rhs2;
4897 9171 : if (kind != bitint_prec_huge)
4898 5303 : idx = size_int (bitint_big_endian ? nelts - 1 - i : i);
4899 3868 : else if (i >= 2)
4900 1480 : idx = size_int (bitint_big_endian ? nelts + 1 - fin - i : fin + i - 2);
4901 9171 : if (!last_ovf || i < cnt - 1)
4902 : {
4903 8225 : tree idx0 = idx, idx1 = idx;
4904 8225 : if (bitint_big_endian
4905 8225 : && CEIL ((unsigned) TYPE_PRECISION (type0), limb_prec) != nelts)
4906 : {
4907 0 : HOST_WIDE_INT diff
4908 0 : = ((HOST_WIDE_INT) CEIL (TYPE_PRECISION (type0), limb_prec)
4909 0 : - (HOST_WIDE_INT) nelts);
4910 0 : if (tree_fits_uhwi_p (idx))
4911 0 : idx0 = size_int (tree_to_uhwi (idx) + diff);
4912 : else
4913 : {
4914 0 : idx0 = make_ssa_name (sizetype);
4915 0 : g = gimple_build_assign (idx0, PLUS_EXPR, idx,
4916 0 : size_int (diff));
4917 0 : insert_before (g);
4918 : }
4919 : }
4920 8225 : if (type0 != TREE_TYPE (arg0))
4921 334 : rhs1 = handle_cast (type0, arg0, idx0);
4922 : else
4923 7891 : rhs1 = handle_operand (arg0, idx0);
4924 8225 : if (bitint_big_endian
4925 8225 : && CEIL ((unsigned) TYPE_PRECISION (type1), limb_prec) != nelts)
4926 : {
4927 0 : HOST_WIDE_INT diff
4928 0 : = ((HOST_WIDE_INT) CEIL (TYPE_PRECISION (type1), limb_prec)
4929 0 : - (HOST_WIDE_INT) nelts);
4930 0 : if (tree_fits_uhwi_p (idx))
4931 0 : idx1 = size_int (tree_to_uhwi (idx) + diff);
4932 : else
4933 : {
4934 0 : idx1 = make_ssa_name (sizetype);
4935 0 : g = gimple_build_assign (idx1, PLUS_EXPR, idx,
4936 0 : size_int (diff));
4937 0 : insert_before (g);
4938 : }
4939 : }
4940 8225 : if (type1 != TREE_TYPE (arg1))
4941 250 : rhs2 = handle_cast (type1, arg1, idx1);
4942 : else
4943 7975 : rhs2 = handle_operand (arg1, idx1);
4944 8225 : if (i == 0)
4945 2733 : data_cnt = m_data_cnt;
4946 8225 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
4947 1983 : rhs1 = add_cast (m_limb_type, rhs1);
4948 8225 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs2)))
4949 1983 : rhs2 = add_cast (m_limb_type, rhs2);
4950 : last_rhs1 = rhs1;
4951 : last_rhs2 = rhs2;
4952 : }
4953 : else
4954 : {
4955 946 : m_data_cnt = data_cnt;
4956 946 : if (TYPE_UNSIGNED (type0) || prec0 >= 0)
4957 421 : rhs1 = build_zero_cst (m_limb_type);
4958 : else
4959 : {
4960 525 : rhs1 = add_cast (signed_type_for (m_limb_type), last_rhs1);
4961 525 : if (TREE_CODE (rhs1) == INTEGER_CST)
4962 52 : rhs1 = build_int_cst (m_limb_type,
4963 74 : tree_int_cst_sgn (rhs1) < 0 ? -1 : 0);
4964 : else
4965 : {
4966 946 : tree lpm1 = build_int_cst (unsigned_type_node,
4967 473 : limb_prec - 1);
4968 473 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
4969 : RSHIFT_EXPR, rhs1, lpm1);
4970 473 : insert_before (g);
4971 473 : rhs1 = add_cast (m_limb_type, gimple_assign_lhs (g));
4972 : }
4973 : }
4974 946 : if (TYPE_UNSIGNED (type1) || prec1 >= 0)
4975 543 : rhs2 = build_zero_cst (m_limb_type);
4976 : else
4977 : {
4978 403 : rhs2 = add_cast (signed_type_for (m_limb_type), last_rhs2);
4979 403 : if (TREE_CODE (rhs2) == INTEGER_CST)
4980 114 : rhs2 = build_int_cst (m_limb_type,
4981 153 : tree_int_cst_sgn (rhs2) < 0 ? -1 : 0);
4982 : else
4983 : {
4984 578 : tree lpm1 = build_int_cst (unsigned_type_node,
4985 289 : limb_prec - 1);
4986 289 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs2)),
4987 : RSHIFT_EXPR, rhs2, lpm1);
4988 289 : insert_before (g);
4989 289 : rhs2 = add_cast (m_limb_type, gimple_assign_lhs (g));
4990 : }
4991 : }
4992 : }
4993 9171 : tree rhs = handle_plus_minus (code, rhs1, rhs2, idx);
4994 9171 : if (ovf != boolean_false_node)
4995 : {
4996 9171 : if (tree_fits_uhwi_p (idx))
4997 : {
4998 6783 : unsigned limb = tree_to_uhwi (idx);
4999 6783 : if (bitint_big_endian)
5000 0 : limb = nelts - 1 - limb;
5001 6783 : if (limb >= startlimb && limb <= endlimb)
5002 : {
5003 3370 : tree l = arith_overflow_extract_bits (start, end, rhs,
5004 : limb, check_zero);
5005 3370 : tree this_ovf = make_ssa_name (boolean_type_node);
5006 3370 : if (ovf == NULL_TREE && !check_zero)
5007 : {
5008 907 : cmp = l;
5009 907 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5010 : PLUS_EXPR, l,
5011 : build_int_cst (m_limb_type, 1));
5012 907 : insert_before (g);
5013 907 : g = gimple_build_assign (this_ovf, GT_EXPR,
5014 : gimple_assign_lhs (g),
5015 : build_int_cst (m_limb_type, 1));
5016 : }
5017 : else
5018 2463 : g = gimple_build_assign (this_ovf, NE_EXPR, l, cmp);
5019 3370 : insert_before (g);
5020 3370 : if (ovf == NULL_TREE)
5021 : ovf = this_ovf;
5022 : else
5023 : {
5024 1064 : tree b = make_ssa_name (boolean_type_node);
5025 1064 : g = gimple_build_assign (b, BIT_IOR_EXPR, ovf, this_ovf);
5026 1064 : insert_before (g);
5027 1064 : ovf = b;
5028 : }
5029 : }
5030 : }
5031 2388 : else if (startlimb < fin)
5032 : {
5033 854 : if (m_first && startlimb + 2 < fin)
5034 : {
5035 324 : tree data_out;
5036 324 : ovf = prepare_data_in_out (boolean_false_node, idx, &data_out);
5037 324 : ovf_out = m_data.pop ();
5038 324 : m_data.pop ();
5039 324 : if (!check_zero)
5040 : {
5041 169 : cmp = prepare_data_in_out (cmp, idx, &data_out);
5042 169 : cmp_out = m_data.pop ();
5043 169 : m_data.pop ();
5044 : }
5045 : }
5046 854 : if (i != 0 || startlimb != fin - 1)
5047 : {
5048 839 : tree_code cmp_code;
5049 839 : bool single_comparison
5050 839 : = (startlimb + 2 >= fin || (startlimb & 1) != (i & 1));
5051 : if (!single_comparison)
5052 : cmp_code = GE_EXPR;
5053 515 : else if ((startlimb & 1) == (i & 1))
5054 : cmp_code = EQ_EXPR;
5055 : else
5056 412 : cmp_code = GT_EXPR;
5057 839 : if (bitint_big_endian)
5058 0 : g = gimple_build_cond (swap_tree_comparison (cmp_code),
5059 0 : idx, size_int (nelts - 1
5060 : - startlimb),
5061 : NULL_TREE, NULL_TREE);
5062 : else
5063 839 : g = gimple_build_cond (cmp_code, idx, size_int (startlimb),
5064 : NULL_TREE, NULL_TREE);
5065 839 : edge edge_true_true, edge_true_false, edge_false;
5066 839 : gimple *g2 = NULL;
5067 839 : if (!single_comparison)
5068 324 : g2 = gimple_build_cond (NE_EXPR, idx,
5069 324 : size_int (bitint_big_endian
5070 : ? nelts - 1 - startlimb
5071 : : startlimb),
5072 : NULL_TREE, NULL_TREE);
5073 839 : if_then_if_then_else (g, g2, profile_probability::likely (),
5074 : profile_probability::likely (),
5075 : edge_true_true, edge_true_false,
5076 : edge_false);
5077 839 : unsigned tidx = startlimb + (cmp_code == GT_EXPR);
5078 839 : tree l = arith_overflow_extract_bits (start, end, rhs, tidx,
5079 : check_zero);
5080 839 : tree this_ovf = make_ssa_name (boolean_type_node);
5081 839 : if (cmp_code != GT_EXPR && !check_zero)
5082 : {
5083 173 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5084 : PLUS_EXPR, l,
5085 : build_int_cst (m_limb_type, 1));
5086 173 : insert_before (g);
5087 173 : g = gimple_build_assign (this_ovf, GT_EXPR,
5088 : gimple_assign_lhs (g),
5089 : build_int_cst (m_limb_type, 1));
5090 : }
5091 : else
5092 666 : g = gimple_build_assign (this_ovf, NE_EXPR, l, cmp);
5093 839 : insert_before (g);
5094 839 : if (cmp_code == GT_EXPR)
5095 : {
5096 412 : tree t = make_ssa_name (boolean_type_node);
5097 412 : g = gimple_build_assign (t, BIT_IOR_EXPR, ovf, this_ovf);
5098 412 : insert_before (g);
5099 412 : this_ovf = t;
5100 : }
5101 839 : tree this_ovf2 = NULL_TREE;
5102 839 : if (!single_comparison)
5103 : {
5104 324 : m_gsi = gsi_after_labels (edge_true_true->src);
5105 324 : tree t = make_ssa_name (boolean_type_node);
5106 324 : g = gimple_build_assign (t, NE_EXPR, rhs, cmp);
5107 324 : insert_before (g);
5108 324 : this_ovf2 = make_ssa_name (boolean_type_node);
5109 324 : g = gimple_build_assign (this_ovf2, BIT_IOR_EXPR,
5110 : ovf, t);
5111 324 : insert_before (g);
5112 : }
5113 839 : m_gsi = gsi_after_labels (edge_true_false->dest);
5114 839 : tree t;
5115 839 : if (i == 1 && ovf_out)
5116 : t = ovf_out;
5117 : else
5118 515 : t = make_ssa_name (boolean_type_node);
5119 839 : gphi *phi = create_phi_node (t, edge_true_false->dest);
5120 839 : add_phi_arg (phi, this_ovf, edge_true_false,
5121 : UNKNOWN_LOCATION);
5122 839 : add_phi_arg (phi, ovf ? ovf
5123 : : boolean_false_node, edge_false,
5124 : UNKNOWN_LOCATION);
5125 839 : if (edge_true_true)
5126 324 : add_phi_arg (phi, this_ovf2, edge_true_true,
5127 : UNKNOWN_LOCATION);
5128 839 : ovf = t;
5129 839 : if (!check_zero && cmp_code != GT_EXPR)
5130 : {
5131 173 : t = cmp_out ? cmp_out : make_ssa_name (m_limb_type);
5132 173 : phi = create_phi_node (t, edge_true_false->dest);
5133 173 : add_phi_arg (phi, l, edge_true_false, UNKNOWN_LOCATION);
5134 173 : add_phi_arg (phi, cmp, edge_false, UNKNOWN_LOCATION);
5135 173 : if (edge_true_true)
5136 169 : add_phi_arg (phi, cmp, edge_true_true,
5137 : UNKNOWN_LOCATION);
5138 : cmp = t;
5139 : }
5140 : }
5141 : }
5142 : }
5143 :
5144 9171 : if (var || obj)
5145 : {
5146 8943 : if (tree_fits_uhwi_p (idx)
5147 6673 : && (bitint_big_endian
5148 6673 : ? nelts - 1 - tree_to_uhwi (idx)
5149 6673 : : tree_to_uhwi (idx)) >= prec_limbs)
5150 : ;
5151 7555 : else if (!tree_fits_uhwi_p (idx)
5152 2270 : && (unsigned) prec < (fin - (i == 0)) * limb_prec)
5153 : {
5154 1458 : bool single_comparison
5155 729 : = (((unsigned) prec % limb_prec) == 0
5156 571 : || prec_limbs + 1 >= fin
5157 1231 : || (prec_limbs & 1) == (i & 1));
5158 729 : if (bitint_big_endian)
5159 0 : g = gimple_build_cond (GE_EXPR, idx,
5160 0 : size_int (nelts - prec_limbs),
5161 : NULL_TREE, NULL_TREE);
5162 : else
5163 729 : g = gimple_build_cond (LE_EXPR, idx, size_int (prec_limbs - 1),
5164 : NULL_TREE, NULL_TREE);
5165 729 : gimple *g2 = NULL;
5166 729 : if (!single_comparison)
5167 251 : g2 = gimple_build_cond (EQ_EXPR, idx,
5168 251 : size_int (bitint_big_endian
5169 : ? nelts - prec_limbs
5170 : : prec_limbs - 1),
5171 : NULL_TREE, NULL_TREE);
5172 729 : edge edge_true_true, edge_true_false, edge_false;
5173 729 : if_then_if_then_else (g, g2, profile_probability::likely (),
5174 : profile_probability::unlikely (),
5175 : edge_true_true, edge_true_false,
5176 : edge_false);
5177 729 : tree idxl = idx;
5178 729 : if (bitint_big_endian && prec_limbs != nelts)
5179 : {
5180 0 : HOST_WIDE_INT diff = ((HOST_WIDE_INT) prec_limbs
5181 0 : - (HOST_WIDE_INT) nelts);
5182 0 : if (tree_fits_uhwi_p (idx))
5183 0 : idxl = size_int (tree_to_uhwi (idx) + diff);
5184 : else
5185 : {
5186 0 : idxl = make_ssa_name (sizetype);
5187 0 : g = gimple_build_assign (idxl, PLUS_EXPR, idx,
5188 0 : size_int (diff));
5189 0 : insert_before (g);
5190 : }
5191 : }
5192 1082 : tree l = limb_access (type, var ? var : obj, idxl, true);
5193 729 : g = gimple_build_assign (l, rhs);
5194 729 : insert_before (g);
5195 729 : if (!single_comparison)
5196 : {
5197 251 : m_gsi = gsi_after_labels (edge_true_true->src);
5198 251 : tree plm1idx = size_int (bitint_big_endian
5199 : ? 0 : prec_limbs - 1);
5200 251 : tree plm1type = limb_access_type (type, plm1idx);
5201 251 : l = limb_access (type, var ? var : obj, plm1idx, true);
5202 251 : if (!useless_type_conversion_p (plm1type, TREE_TYPE (rhs)))
5203 251 : rhs = add_cast (plm1type, rhs);
5204 251 : if (!useless_type_conversion_p (TREE_TYPE (l),
5205 251 : TREE_TYPE (rhs)))
5206 251 : rhs = add_cast (TREE_TYPE (l), rhs);
5207 251 : g = gimple_build_assign (l, rhs);
5208 251 : insert_before (g);
5209 : }
5210 729 : m_gsi = gsi_after_labels (edge_true_false->dest);
5211 729 : }
5212 : else
5213 : {
5214 6826 : tree idxl = idx;
5215 6826 : if (bitint_big_endian && prec_limbs != nelts)
5216 : {
5217 0 : HOST_WIDE_INT diff = ((HOST_WIDE_INT) prec_limbs
5218 0 : - (HOST_WIDE_INT) nelts);
5219 0 : if (tree_fits_uhwi_p (idx))
5220 0 : idxl = size_int (tree_to_uhwi (idx) + diff);
5221 : else
5222 : {
5223 0 : idxl = make_ssa_name (sizetype);
5224 0 : g = gimple_build_assign (idxl, PLUS_EXPR, idx,
5225 0 : size_int (diff));
5226 0 : insert_before (g);
5227 : }
5228 : }
5229 13623 : tree l = limb_access (type, var ? var : obj, idxl, true);
5230 6826 : if (bitint_extended && tree_fits_uhwi_p (idxl))
5231 : {
5232 0 : tree atype = limb_access_type (type, idxl);
5233 0 : if (!useless_type_conversion_p (atype, TREE_TYPE (rhs)))
5234 0 : rhs = add_cast (atype, rhs);
5235 : }
5236 6826 : if (!useless_type_conversion_p (TREE_TYPE (l), TREE_TYPE (rhs)))
5237 0 : rhs = add_cast (TREE_TYPE (l), rhs);
5238 6826 : g = gimple_build_assign (l, rhs);
5239 6826 : insert_before (g);
5240 : }
5241 : }
5242 9171 : m_first = false;
5243 9171 : if (kind == bitint_prec_huge && i <= 1)
5244 : {
5245 2388 : if (i == 0)
5246 : {
5247 1194 : idx = make_ssa_name (sizetype);
5248 1194 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
5249 : bitint_big_endian
5250 0 : ? size_int (-1) : size_one_node);
5251 1194 : insert_before (g);
5252 : }
5253 : else
5254 : {
5255 1194 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
5256 2388 : size_int (bitint_big_endian ? -2 : 2));
5257 1194 : insert_before (g);
5258 1194 : if (bitint_big_endian)
5259 0 : g = gimple_build_cond (NE_EXPR, idx_first,
5260 0 : size_int (nelts + 1 - fin),
5261 : NULL_TREE, NULL_TREE);
5262 : else
5263 1194 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (fin),
5264 : NULL_TREE, NULL_TREE);
5265 1194 : insert_before (g);
5266 1194 : m_gsi = gsi_for_stmt (final_stmt);
5267 1194 : m_bb = NULL;
5268 : }
5269 : }
5270 : }
5271 :
5272 2733 : finish_arith_overflow (var, obj, type, ovf, lhs, orig_obj, stmt,
5273 : prec_limbs, code);
5274 : }
5275 :
5276 : /* Lower a .MUL_OVERFLOW call with at least one large/huge _BitInt
5277 : argument or return type _Complex large/huge _BitInt. */
5278 :
5279 : void
5280 1479 : bitint_large_huge::lower_mul_overflow (tree obj, gimple *stmt)
5281 : {
5282 1479 : tree arg0 = gimple_call_arg (stmt, 0);
5283 1479 : tree arg1 = gimple_call_arg (stmt, 1);
5284 1479 : tree lhs = gimple_call_lhs (stmt);
5285 1479 : if (!lhs)
5286 : {
5287 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5288 0 : gsi_remove (&gsi, true);
5289 0 : return;
5290 : }
5291 1479 : gimple *final_stmt = gsi_stmt (m_gsi);
5292 1479 : tree type = TREE_TYPE (lhs);
5293 1479 : if (TREE_CODE (type) == COMPLEX_TYPE)
5294 1464 : type = TREE_TYPE (type);
5295 1479 : int prec = TYPE_PRECISION (type), prec0, prec1;
5296 1479 : arg0 = handle_operand_addr (arg0, stmt, NULL, &prec0);
5297 1479 : arg1 = handle_operand_addr (arg1, stmt, NULL, &prec1);
5298 1479 : int prec2 = ((prec0 < 0 ? -prec0 : prec0)
5299 1479 : + (prec1 < 0 ? -prec1 : prec1));
5300 1479 : if (prec0 == 1 || prec1 == 1)
5301 35 : --prec2;
5302 1479 : tree var = NULL_TREE;
5303 1479 : tree orig_obj = obj;
5304 1479 : bool force_var = false;
5305 1479 : if (obj == NULL_TREE
5306 886 : && BITINT_TYPE_P (type)
5307 877 : && bitint_precision_kind (type) >= bitint_prec_large
5308 853 : && m_names
5309 2320 : && bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
5310 : {
5311 841 : int part = var_to_partition (m_map, lhs);
5312 841 : gcc_assert (m_vars[part] != NULL_TREE);
5313 841 : obj = m_vars[part];
5314 841 : if (TREE_TYPE (lhs) == type)
5315 7 : orig_obj = obj;
5316 : }
5317 638 : else if (obj != NULL_TREE && DECL_P (obj))
5318 : {
5319 1755 : for (int i = 0; i < 2; ++i)
5320 : {
5321 1170 : tree arg = i ? arg1 : arg0;
5322 1170 : if (TREE_CODE (arg) == ADDR_EXPR)
5323 1170 : arg = TREE_OPERAND (arg, 0);
5324 1170 : if (get_base_address (arg) == obj)
5325 : {
5326 : force_var = true;
5327 : break;
5328 : }
5329 : }
5330 : }
5331 1479 : if (obj == NULL_TREE
5332 1479 : || force_var
5333 1434 : || !BITINT_TYPE_P (type)
5334 1434 : || bitint_precision_kind (type) < bitint_prec_large
5335 3747 : || prec2 > (CEIL (prec, limb_prec) * limb_prec * (orig_obj ? 1 : 2)))
5336 : {
5337 564 : unsigned HOST_WIDE_INT nelts = CEIL (MAX (prec, prec2), limb_prec);
5338 564 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5339 564 : var = create_tmp_var (atype);
5340 : }
5341 1479 : tree addr = build_fold_addr_expr (var ? var : obj);
5342 1479 : addr = force_gimple_operand_gsi (&m_gsi, addr, true,
5343 : NULL_TREE, true, GSI_SAME_STMT);
5344 1479 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
5345 1479 : gimple *g
5346 2958 : = gimple_build_call_internal (IFN_MULBITINT, 6,
5347 : addr, build_int_cst (sitype,
5348 1543 : MAX (prec2, prec)),
5349 1479 : arg0, build_int_cst (sitype, prec0),
5350 1479 : arg1, build_int_cst (sitype, prec1));
5351 1479 : insert_before (g);
5352 :
5353 1479 : unsigned start, end;
5354 1479 : bool check_zero;
5355 1479 : tree ovf = arith_overflow (MULT_EXPR, type, prec, prec0, prec1, prec2,
5356 : &start, &end, &check_zero);
5357 1479 : if (ovf == NULL_TREE)
5358 : {
5359 1477 : unsigned startlimb = start / limb_prec;
5360 1477 : unsigned endlimb = (end - 1) / limb_prec;
5361 1477 : unsigned nelts = CEIL (MAX (prec, prec2), limb_prec);
5362 1477 : unsigned cnt;
5363 1477 : bool use_loop = false;
5364 1477 : if (startlimb == endlimb)
5365 : cnt = 1;
5366 1190 : else if (startlimb + 1 == endlimb)
5367 : cnt = 2;
5368 1013 : else if ((end % limb_prec) == 0)
5369 : {
5370 : cnt = 2;
5371 : use_loop = true;
5372 : }
5373 : else
5374 : {
5375 767 : cnt = 3;
5376 767 : use_loop = startlimb + 2 < endlimb;
5377 : }
5378 767 : if (cnt == 1)
5379 : {
5380 507 : tree l = limb_access (NULL_TREE, var ? var : obj,
5381 287 : size_int (bitint_big_endian
5382 : ? nelts - 1 - startlimb
5383 : : startlimb), true);
5384 287 : g = gimple_build_assign (make_ssa_name (m_limb_type), l);
5385 287 : insert_before (g);
5386 287 : l = arith_overflow_extract_bits (start, end, gimple_assign_lhs (g),
5387 : startlimb, check_zero);
5388 287 : ovf = make_ssa_name (boolean_type_node);
5389 287 : if (check_zero)
5390 247 : g = gimple_build_assign (ovf, NE_EXPR, l,
5391 : build_zero_cst (m_limb_type));
5392 : else
5393 : {
5394 40 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5395 : PLUS_EXPR, l,
5396 : build_int_cst (m_limb_type, 1));
5397 40 : insert_before (g);
5398 40 : g = gimple_build_assign (ovf, GT_EXPR, gimple_assign_lhs (g),
5399 : build_int_cst (m_limb_type, 1));
5400 : }
5401 287 : insert_before (g);
5402 : }
5403 : else
5404 : {
5405 1190 : basic_block edge_bb = NULL;
5406 1190 : gimple_stmt_iterator gsi = m_gsi;
5407 1190 : gsi_prev (&gsi);
5408 1190 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5409 1190 : edge_bb = e->src;
5410 1190 : m_gsi = gsi_end_bb (edge_bb);
5411 :
5412 1190 : tree cmp = build_zero_cst (m_limb_type);
5413 4337 : for (unsigned i = 0; i < cnt; i++)
5414 : {
5415 3147 : tree idx, idx_next = NULL_TREE;
5416 3147 : if (i == 0)
5417 1190 : idx = size_int (bitint_big_endian
5418 : ? nelts - 1 - startlimb : startlimb);
5419 1957 : else if (i == 2)
5420 767 : idx = size_int (bitint_big_endian
5421 : ? nelts - 1 - endlimb : endlimb);
5422 1190 : else if (use_loop)
5423 665 : idx = create_loop (size_int (bitint_big_endian
5424 : ? nelts - startlimb - 2
5425 : : startlimb + 1), &idx_next);
5426 : else
5427 525 : idx = size_int (bitint_big_endian
5428 : ? nelts - startlimb - 2 : startlimb + 1);
5429 4988 : tree l = limb_access (NULL_TREE, var ? var : obj, idx, true);
5430 3147 : g = gimple_build_assign (make_ssa_name (m_limb_type), l);
5431 3147 : insert_before (g);
5432 3147 : l = gimple_assign_lhs (g);
5433 3147 : if (i == 0 || i == 2)
5434 2724 : l = arith_overflow_extract_bits (start, end, l,
5435 : i == 0 ? startlimb : endlimb,
5436 : check_zero);
5437 1957 : if (i == 0 && !check_zero)
5438 : {
5439 459 : cmp = l;
5440 459 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5441 : PLUS_EXPR, l,
5442 : build_int_cst (m_limb_type, 1));
5443 459 : insert_before (g);
5444 459 : g = gimple_build_cond (GT_EXPR, gimple_assign_lhs (g),
5445 : build_int_cst (m_limb_type, 1),
5446 : NULL_TREE, NULL_TREE);
5447 : }
5448 : else
5449 2688 : g = gimple_build_cond (NE_EXPR, l, cmp, NULL_TREE, NULL_TREE);
5450 3147 : insert_before (g);
5451 3147 : edge e1 = split_block (gsi_bb (m_gsi), g);
5452 3147 : e1->flags = EDGE_FALSE_VALUE;
5453 3147 : edge e2 = make_edge (e1->src, gimple_bb (final_stmt),
5454 : EDGE_TRUE_VALUE);
5455 3147 : e1->probability = profile_probability::likely ();
5456 3147 : e2->probability = e1->probability.invert ();
5457 3147 : if (i == 0)
5458 1190 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5459 3147 : m_gsi = gsi_after_labels (e1->dest);
5460 3147 : if (i == 1 && use_loop)
5461 : {
5462 665 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
5463 : bitint_big_endian
5464 0 : ? size_int (-1) : size_one_node);
5465 665 : insert_before (g);
5466 665 : if (bitint_big_endian)
5467 0 : g = gimple_build_cond (NE_EXPR, idx,
5468 0 : size_int (nelts - endlimb
5469 : - (cnt == 2)),
5470 : NULL_TREE, NULL_TREE);
5471 : else
5472 665 : g = gimple_build_cond (NE_EXPR, idx_next,
5473 665 : size_int (endlimb + (cnt == 2)),
5474 : NULL_TREE, NULL_TREE);
5475 665 : insert_before (g);
5476 665 : edge true_edge, false_edge;
5477 665 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
5478 : &true_edge,
5479 : &false_edge);
5480 665 : m_gsi = gsi_after_labels (false_edge->dest);
5481 665 : m_bb = NULL;
5482 : }
5483 : }
5484 :
5485 1190 : ovf = make_ssa_name (boolean_type_node);
5486 1190 : basic_block bb = gimple_bb (final_stmt);
5487 1190 : gphi *phi = create_phi_node (ovf, bb);
5488 1190 : edge e1 = find_edge (gsi_bb (m_gsi), bb);
5489 1190 : edge_iterator ei;
5490 5527 : FOR_EACH_EDGE (e, ei, bb->preds)
5491 : {
5492 4337 : tree val = e == e1 ? boolean_false_node : boolean_true_node;
5493 4337 : add_phi_arg (phi, val, e, UNKNOWN_LOCATION);
5494 : }
5495 1190 : m_gsi = gsi_for_stmt (final_stmt);
5496 : }
5497 : }
5498 :
5499 1479 : finish_arith_overflow (var, obj, type, ovf, lhs, orig_obj, stmt,
5500 1479 : CEIL (MAX (prec, prec2), limb_prec), MULT_EXPR);
5501 : }
5502 :
5503 : /* Lower REALPART_EXPR or IMAGPART_EXPR stmt extracting part of result from
5504 : .{ADD,SUB,MUL}_OVERFLOW call. */
5505 :
5506 : void
5507 6002 : bitint_large_huge::lower_cplxpart_stmt (tree obj, gimple *stmt)
5508 : {
5509 6002 : tree rhs1 = gimple_assign_rhs1 (stmt);
5510 6002 : rhs1 = TREE_OPERAND (rhs1, 0);
5511 6002 : if (obj == NULL_TREE)
5512 : {
5513 5929 : int part = var_to_partition (m_map, gimple_assign_lhs (stmt));
5514 5929 : gcc_assert (m_vars[part] != NULL_TREE);
5515 : obj = m_vars[part];
5516 : }
5517 6002 : if (TREE_CODE (rhs1) == SSA_NAME
5518 6002 : && (m_names == NULL
5519 6001 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
5520 : {
5521 1549 : lower_call (obj, SSA_NAME_DEF_STMT (rhs1));
5522 1549 : return;
5523 : }
5524 4453 : int part = var_to_partition (m_map, rhs1);
5525 4453 : gcc_assert (m_vars[part] != NULL_TREE);
5526 4453 : tree var = m_vars[part];
5527 4453 : unsigned HOST_WIDE_INT nelts
5528 4453 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))) / limb_prec;
5529 4453 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5530 4453 : if (!useless_type_conversion_p (atype, TREE_TYPE (obj)))
5531 102 : obj = build1 (VIEW_CONVERT_EXPR, atype, obj);
5532 4453 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (var)),
5533 4453 : gimple_assign_rhs_code (stmt) == REALPART_EXPR
5534 4453 : ? 0 : nelts * m_limb_size);
5535 4453 : tree v2 = build2 (MEM_REF, atype, build_fold_addr_expr (var), off);
5536 4453 : gimple *g = gimple_build_assign (obj, v2);
5537 4453 : insert_before (g);
5538 : }
5539 :
5540 : /* Lower COMPLEX_EXPR stmt. */
5541 :
5542 : void
5543 18 : bitint_large_huge::lower_complexexpr_stmt (gimple *stmt)
5544 : {
5545 18 : tree lhs = gimple_assign_lhs (stmt);
5546 18 : tree rhs1 = gimple_assign_rhs1 (stmt);
5547 18 : tree rhs2 = gimple_assign_rhs2 (stmt);
5548 18 : int part = var_to_partition (m_map, lhs);
5549 18 : gcc_assert (m_vars[part] != NULL_TREE);
5550 18 : lhs = m_vars[part];
5551 18 : unsigned HOST_WIDE_INT nelts
5552 18 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (rhs1))) / limb_prec;
5553 18 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5554 18 : tree zero = build_zero_cst (build_pointer_type (TREE_TYPE (lhs)));
5555 18 : tree v1 = build2 (MEM_REF, atype, build_fold_addr_expr (lhs), zero);
5556 18 : tree v2;
5557 18 : if (TREE_CODE (rhs1) == SSA_NAME)
5558 : {
5559 18 : part = var_to_partition (m_map, rhs1);
5560 18 : gcc_assert (m_vars[part] != NULL_TREE);
5561 : v2 = m_vars[part];
5562 : }
5563 0 : else if (integer_zerop (rhs1))
5564 0 : v2 = build_zero_cst (atype);
5565 : else
5566 0 : v2 = tree_output_constant_def (rhs1);
5567 18 : if (!useless_type_conversion_p (atype, TREE_TYPE (v2)))
5568 18 : v2 = build1 (VIEW_CONVERT_EXPR, atype, v2);
5569 18 : gimple *g = gimple_build_assign (v1, v2);
5570 18 : insert_before (g);
5571 18 : tree off = fold_convert (build_pointer_type (TREE_TYPE (lhs)),
5572 : TYPE_SIZE_UNIT (atype));
5573 18 : v1 = build2 (MEM_REF, atype, build_fold_addr_expr (lhs), off);
5574 18 : if (TREE_CODE (rhs2) == SSA_NAME)
5575 : {
5576 0 : part = var_to_partition (m_map, rhs2);
5577 0 : gcc_assert (m_vars[part] != NULL_TREE);
5578 : v2 = m_vars[part];
5579 : }
5580 18 : else if (integer_zerop (rhs2))
5581 18 : v2 = build_zero_cst (atype);
5582 : else
5583 0 : v2 = tree_output_constant_def (rhs2);
5584 18 : if (!useless_type_conversion_p (atype, TREE_TYPE (v2)))
5585 0 : v2 = build1 (VIEW_CONVERT_EXPR, atype, v2);
5586 18 : g = gimple_build_assign (v1, v2);
5587 18 : insert_before (g);
5588 18 : }
5589 :
5590 : /* Lower a .{CLZ,CTZ,CLRSB,FFS,PARITY,POPCOUNT} call with one large/huge _BitInt
5591 : argument. */
5592 :
5593 : void
5594 101 : bitint_large_huge::lower_bit_query (gimple *stmt)
5595 : {
5596 101 : tree arg0 = gimple_call_arg (stmt, 0);
5597 101 : tree arg1 = (gimple_call_num_args (stmt) == 2
5598 101 : ? gimple_call_arg (stmt, 1) : NULL_TREE);
5599 101 : tree lhs = gimple_call_lhs (stmt);
5600 101 : gimple *g;
5601 :
5602 101 : if (!lhs)
5603 : {
5604 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5605 0 : gsi_remove (&gsi, true);
5606 0 : return;
5607 : }
5608 101 : tree type = TREE_TYPE (arg0);
5609 101 : gcc_assert (BITINT_TYPE_P (type));
5610 101 : bitint_prec_kind kind = bitint_precision_kind (type);
5611 101 : gcc_assert (kind >= bitint_prec_large);
5612 101 : enum internal_fn ifn = gimple_call_internal_fn (stmt);
5613 101 : enum built_in_function fcode = END_BUILTINS;
5614 101 : gcc_assert (TYPE_PRECISION (unsigned_type_node) == limb_prec
5615 : || TYPE_PRECISION (long_unsigned_type_node) == limb_prec
5616 : || TYPE_PRECISION (long_long_unsigned_type_node) == limb_prec);
5617 101 : switch (ifn)
5618 : {
5619 26 : case IFN_CLZ:
5620 26 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5621 : fcode = BUILT_IN_CLZ;
5622 26 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5623 : fcode = BUILT_IN_CLZL;
5624 : else
5625 0 : fcode = BUILT_IN_CLZLL;
5626 : break;
5627 11 : case IFN_FFS:
5628 : /* .FFS (X) is .CTZ (X, -1) + 1, though under the hood
5629 : we don't add the addend at the end. */
5630 11 : arg1 = integer_zero_node;
5631 : /* FALLTHRU */
5632 39 : case IFN_CTZ:
5633 39 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5634 : fcode = BUILT_IN_CTZ;
5635 39 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5636 : fcode = BUILT_IN_CTZL;
5637 : else
5638 0 : fcode = BUILT_IN_CTZLL;
5639 39 : m_upwards = true;
5640 39 : break;
5641 9 : case IFN_CLRSB:
5642 9 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5643 : fcode = BUILT_IN_CLRSB;
5644 9 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5645 : fcode = BUILT_IN_CLRSBL;
5646 : else
5647 0 : fcode = BUILT_IN_CLRSBLL;
5648 : break;
5649 16 : case IFN_PARITY:
5650 16 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5651 : fcode = BUILT_IN_PARITY;
5652 16 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5653 : fcode = BUILT_IN_PARITYL;
5654 : else
5655 0 : fcode = BUILT_IN_PARITYLL;
5656 16 : m_upwards = true;
5657 16 : break;
5658 11 : case IFN_POPCOUNT:
5659 11 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5660 : fcode = BUILT_IN_POPCOUNT;
5661 11 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5662 : fcode = BUILT_IN_POPCOUNTL;
5663 : else
5664 0 : fcode = BUILT_IN_POPCOUNTLL;
5665 11 : m_upwards = true;
5666 11 : break;
5667 0 : default:
5668 0 : gcc_unreachable ();
5669 : }
5670 101 : tree fndecl = builtin_decl_explicit (fcode), res = NULL_TREE;
5671 101 : unsigned cnt = 0, rem = 0, end = 0, prec = TYPE_PRECISION (type);
5672 101 : unsigned nelts = CEIL (prec, limb_prec);
5673 101 : struct bq_details { edge e; tree val, addend; } *bqp = NULL;
5674 101 : basic_block edge_bb = NULL;
5675 101 : if (m_upwards)
5676 : {
5677 66 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
5678 66 : if (kind == bitint_prec_large)
5679 : cnt = nelts;
5680 : else
5681 : {
5682 39 : rem = (prec % (2 * limb_prec));
5683 39 : end = (prec - rem) / limb_prec;
5684 39 : cnt = 2 + CEIL (rem, limb_prec);
5685 39 : idx = idx_first = create_loop (bitint_big_endian
5686 39 : ? size_int (nelts - 1)
5687 : : size_zero_node, &idx_next);
5688 : }
5689 :
5690 66 : if (ifn == IFN_CTZ || ifn == IFN_FFS)
5691 : {
5692 39 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5693 39 : gsi_prev (&gsi);
5694 39 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5695 39 : edge_bb = e->src;
5696 39 : if (kind == bitint_prec_large)
5697 32 : m_gsi = gsi_end_bb (edge_bb);
5698 39 : bqp = XALLOCAVEC (struct bq_details, cnt);
5699 : }
5700 : else
5701 27 : m_after_stmt = stmt;
5702 66 : if (kind != bitint_prec_large)
5703 39 : m_upwards_2limb = end;
5704 :
5705 244 : for (unsigned i = 0; i < cnt; i++)
5706 : {
5707 178 : m_data_cnt = 0;
5708 178 : if (kind == bitint_prec_large)
5709 83 : idx = size_int (bitint_big_endian ? nelts - 1 - i : i);
5710 95 : else if (i >= 2)
5711 17 : idx = size_int (bitint_big_endian
5712 : ? nelts - 1 - end - (i > 2) : end + (i > 2));
5713 :
5714 178 : tree rhs1 = handle_operand (arg0, idx);
5715 178 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
5716 : {
5717 32 : if (!TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5718 5 : rhs1 = add_cast (unsigned_type_for (TREE_TYPE (rhs1)), rhs1);
5719 32 : rhs1 = add_cast (m_limb_type, rhs1);
5720 : }
5721 :
5722 178 : tree in, out, tem;
5723 178 : if (ifn == IFN_PARITY)
5724 43 : in = prepare_data_in_out (build_zero_cst (m_limb_type), idx, &out);
5725 135 : else if (ifn == IFN_FFS)
5726 29 : in = prepare_data_in_out (integer_one_node, idx, &out);
5727 : else
5728 106 : in = prepare_data_in_out (integer_zero_node, idx, &out);
5729 :
5730 178 : switch (ifn)
5731 : {
5732 104 : case IFN_CTZ:
5733 104 : case IFN_FFS:
5734 104 : g = gimple_build_cond (NE_EXPR, rhs1,
5735 : build_zero_cst (m_limb_type),
5736 : NULL_TREE, NULL_TREE);
5737 104 : insert_before (g);
5738 104 : edge e1, e2;
5739 104 : e1 = split_block (gsi_bb (m_gsi), g);
5740 104 : e1->flags = EDGE_FALSE_VALUE;
5741 104 : e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
5742 104 : e1->probability = profile_probability::unlikely ();
5743 104 : e2->probability = e1->probability.invert ();
5744 104 : if (i == 0)
5745 39 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5746 104 : m_gsi = gsi_after_labels (e1->dest);
5747 104 : bqp[i].e = e2;
5748 104 : bqp[i].val = rhs1;
5749 104 : if (tree_fits_uhwi_p (idx))
5750 58 : bqp[i].addend
5751 58 : = build_int_cst (integer_type_node,
5752 : (bitint_big_endian
5753 58 : ? nelts - 1 - tree_to_uhwi (idx)
5754 58 : : tree_to_uhwi (idx)) * limb_prec
5755 58 : + (ifn == IFN_FFS));
5756 : else
5757 : {
5758 46 : bqp[i].addend = in;
5759 46 : if (i == 1)
5760 23 : res = out;
5761 : else
5762 23 : res = make_ssa_name (integer_type_node);
5763 46 : g = gimple_build_assign (res, PLUS_EXPR, in,
5764 : build_int_cst (integer_type_node,
5765 46 : limb_prec));
5766 46 : insert_before (g);
5767 46 : m_data[m_data_cnt] = res;
5768 : }
5769 : break;
5770 43 : case IFN_PARITY:
5771 43 : if (!integer_zerop (in))
5772 : {
5773 37 : if (kind == bitint_prec_huge && i == 1)
5774 10 : res = out;
5775 : else
5776 27 : res = make_ssa_name (m_limb_type);
5777 37 : g = gimple_build_assign (res, BIT_XOR_EXPR, in, rhs1);
5778 37 : insert_before (g);
5779 : }
5780 : else
5781 : res = rhs1;
5782 43 : m_data[m_data_cnt] = res;
5783 43 : break;
5784 31 : case IFN_POPCOUNT:
5785 31 : g = gimple_build_call (fndecl, 1, rhs1);
5786 31 : tem = make_ssa_name (integer_type_node);
5787 31 : gimple_call_set_lhs (g, tem);
5788 31 : insert_before (g);
5789 31 : if (!integer_zerop (in))
5790 : {
5791 26 : if (kind == bitint_prec_huge && i == 1)
5792 6 : res = out;
5793 : else
5794 20 : res = make_ssa_name (integer_type_node);
5795 26 : g = gimple_build_assign (res, PLUS_EXPR, in, tem);
5796 26 : insert_before (g);
5797 : }
5798 : else
5799 : res = tem;
5800 31 : m_data[m_data_cnt] = res;
5801 31 : break;
5802 0 : default:
5803 0 : gcc_unreachable ();
5804 : }
5805 :
5806 178 : m_first = false;
5807 178 : if (kind == bitint_prec_huge && i <= 1)
5808 : {
5809 78 : if (i == 0)
5810 : {
5811 39 : idx = make_ssa_name (sizetype);
5812 39 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
5813 : bitint_big_endian
5814 0 : ? size_int (-1) : size_one_node);
5815 39 : insert_before (g);
5816 : }
5817 : else
5818 : {
5819 39 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
5820 78 : size_int (bitint_big_endian
5821 : ? -2 : 2));
5822 39 : insert_before (g);
5823 39 : if (bitint_big_endian)
5824 0 : g = gimple_build_cond (NE_EXPR, idx_first,
5825 0 : size_int (cnt - 1),
5826 : NULL_TREE, NULL_TREE);
5827 : else
5828 39 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (end),
5829 : NULL_TREE, NULL_TREE);
5830 39 : insert_before (g);
5831 39 : if (ifn == IFN_CTZ || ifn == IFN_FFS)
5832 23 : m_gsi = gsi_after_labels (edge_bb);
5833 : else
5834 16 : m_gsi = gsi_for_stmt (stmt);
5835 39 : m_bb = NULL;
5836 : }
5837 : }
5838 : }
5839 : }
5840 : else
5841 : {
5842 35 : tree idx = NULL_TREE, idx_next = NULL_TREE, first = NULL_TREE;
5843 35 : int sub_one = 0;
5844 35 : if (kind == bitint_prec_large)
5845 : cnt = nelts;
5846 : else
5847 : {
5848 18 : rem = prec % limb_prec;
5849 18 : if (rem == 0 && (!TYPE_UNSIGNED (type) || ifn == IFN_CLRSB))
5850 : rem = limb_prec;
5851 18 : end = (prec - rem) / limb_prec;
5852 18 : cnt = 1 + (rem != 0);
5853 18 : if (ifn == IFN_CLRSB)
5854 5 : sub_one = 1;
5855 : }
5856 :
5857 35 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5858 35 : gsi_prev (&gsi);
5859 35 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5860 35 : edge_bb = e->src;
5861 35 : m_gsi = gsi_end_bb (edge_bb);
5862 :
5863 35 : if (ifn == IFN_CLZ)
5864 26 : bqp = XALLOCAVEC (struct bq_details, cnt);
5865 : else
5866 : {
5867 9 : gsi = gsi_for_stmt (stmt);
5868 9 : gsi_prev (&gsi);
5869 9 : e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5870 9 : edge_bb = e->src;
5871 9 : bqp = XALLOCAVEC (struct bq_details, 2 * cnt);
5872 : }
5873 :
5874 116 : for (unsigned i = 0; i < cnt; i++)
5875 : {
5876 81 : m_data_cnt = 0;
5877 81 : if (kind == bitint_prec_large)
5878 51 : idx = size_int (bitint_big_endian ? i : cnt - i - 1);
5879 30 : else if (i == cnt - 1)
5880 18 : idx = create_loop (size_int (bitint_big_endian ? i : end - 1),
5881 : &idx_next);
5882 : else
5883 12 : idx = bitint_big_endian ? size_zero_node : size_int (end);
5884 :
5885 81 : tree rhs1 = handle_operand (arg0, idx);
5886 81 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
5887 : {
5888 19 : if (ifn == IFN_CLZ && !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5889 0 : rhs1 = add_cast (unsigned_type_for (TREE_TYPE (rhs1)), rhs1);
5890 19 : else if (ifn == IFN_CLRSB && TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5891 0 : rhs1 = add_cast (signed_type_for (TREE_TYPE (rhs1)), rhs1);
5892 19 : rhs1 = add_cast (m_limb_type, rhs1);
5893 : }
5894 :
5895 81 : if (ifn == IFN_CLZ)
5896 : {
5897 59 : g = gimple_build_cond (NE_EXPR, rhs1,
5898 : build_zero_cst (m_limb_type),
5899 : NULL_TREE, NULL_TREE);
5900 59 : insert_before (g);
5901 59 : edge e1 = split_block (gsi_bb (m_gsi), g);
5902 59 : e1->flags = EDGE_FALSE_VALUE;
5903 59 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
5904 59 : e1->probability = profile_probability::unlikely ();
5905 59 : e2->probability = e1->probability.invert ();
5906 59 : if (i == 0)
5907 26 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5908 59 : m_gsi = gsi_after_labels (e1->dest);
5909 59 : bqp[i].e = e2;
5910 59 : bqp[i].val = rhs1;
5911 : }
5912 : else
5913 : {
5914 22 : if (i == 0)
5915 : {
5916 9 : first = rhs1;
5917 9 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5918 : PLUS_EXPR, rhs1,
5919 : build_int_cst (m_limb_type, 1));
5920 9 : insert_before (g);
5921 9 : g = gimple_build_cond (GT_EXPR, gimple_assign_lhs (g),
5922 : build_int_cst (m_limb_type, 1),
5923 : NULL_TREE, NULL_TREE);
5924 9 : insert_before (g);
5925 : }
5926 : else
5927 : {
5928 13 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5929 : BIT_XOR_EXPR, rhs1, first);
5930 13 : insert_before (g);
5931 13 : tree stype = signed_type_for (m_limb_type);
5932 13 : g = gimple_build_cond (LT_EXPR,
5933 : add_cast (stype,
5934 : gimple_assign_lhs (g)),
5935 : build_zero_cst (stype),
5936 : NULL_TREE, NULL_TREE);
5937 13 : insert_before (g);
5938 13 : edge e1 = split_block (gsi_bb (m_gsi), g);
5939 13 : e1->flags = EDGE_FALSE_VALUE;
5940 13 : edge e2 = make_edge (e1->src, gimple_bb (stmt),
5941 : EDGE_TRUE_VALUE);
5942 13 : e1->probability = profile_probability::unlikely ();
5943 13 : e2->probability = e1->probability.invert ();
5944 13 : if (i == 1)
5945 9 : set_immediate_dominator (CDI_DOMINATORS, e2->dest,
5946 : e2->src);
5947 13 : m_gsi = gsi_after_labels (e1->dest);
5948 13 : bqp[2 * i].e = e2;
5949 13 : g = gimple_build_cond (NE_EXPR, rhs1, first,
5950 : NULL_TREE, NULL_TREE);
5951 13 : insert_before (g);
5952 : }
5953 22 : edge e1 = split_block (gsi_bb (m_gsi), g);
5954 22 : e1->flags = EDGE_FALSE_VALUE;
5955 22 : edge e2 = make_edge (e1->src, edge_bb, EDGE_TRUE_VALUE);
5956 22 : e1->probability = profile_probability::unlikely ();
5957 22 : e2->probability = e1->probability.invert ();
5958 22 : if (i == 0)
5959 9 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5960 22 : m_gsi = gsi_after_labels (e1->dest);
5961 22 : bqp[2 * i + 1].e = e2;
5962 22 : bqp[i].val = rhs1;
5963 : }
5964 81 : if (tree_fits_uhwi_p (idx))
5965 126 : bqp[i].addend
5966 63 : = build_int_cst (integer_type_node,
5967 63 : (int) prec
5968 63 : - (((int) (bitint_big_endian
5969 0 : ? nelts - 1 - tree_to_uhwi (idx)
5970 63 : : tree_to_uhwi (idx)) + 1)
5971 63 : * limb_prec) - sub_one);
5972 : else
5973 : {
5974 18 : tree in, out;
5975 18 : in = build_int_cst (integer_type_node, rem - sub_one);
5976 18 : m_first = true;
5977 18 : in = prepare_data_in_out (in, idx, &out);
5978 18 : out = m_data[m_data_cnt + 1];
5979 18 : bqp[i].addend = in;
5980 18 : g = gimple_build_assign (out, PLUS_EXPR, in,
5981 : build_int_cst (integer_type_node,
5982 18 : limb_prec));
5983 18 : insert_before (g);
5984 18 : m_data[m_data_cnt] = out;
5985 : }
5986 :
5987 81 : m_first = false;
5988 81 : if (kind == bitint_prec_huge && i == cnt - 1)
5989 : {
5990 36 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
5991 : bitint_big_endian
5992 18 : ? size_one_node : size_int (-1));
5993 18 : insert_before (g);
5994 18 : g = gimple_build_cond (NE_EXPR, idx,
5995 : bitint_big_endian
5996 0 : ? size_int (nelts - 1) : size_zero_node,
5997 : NULL_TREE, NULL_TREE);
5998 18 : insert_before (g);
5999 18 : edge true_edge, false_edge;
6000 18 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
6001 : &true_edge, &false_edge);
6002 18 : m_gsi = gsi_after_labels (false_edge->dest);
6003 18 : m_bb = NULL;
6004 : }
6005 : }
6006 : }
6007 101 : switch (ifn)
6008 : {
6009 65 : case IFN_CLZ:
6010 65 : case IFN_CTZ:
6011 65 : case IFN_FFS:
6012 65 : gphi *phi1, *phi2, *phi3;
6013 65 : basic_block bb;
6014 65 : bb = gsi_bb (m_gsi);
6015 65 : remove_edge (find_edge (bb, gimple_bb (stmt)));
6016 65 : phi1 = create_phi_node (make_ssa_name (m_limb_type),
6017 : gimple_bb (stmt));
6018 65 : phi2 = create_phi_node (make_ssa_name (integer_type_node),
6019 : gimple_bb (stmt));
6020 228 : for (unsigned i = 0; i < cnt; i++)
6021 : {
6022 163 : add_phi_arg (phi1, bqp[i].val, bqp[i].e, UNKNOWN_LOCATION);
6023 163 : add_phi_arg (phi2, bqp[i].addend, bqp[i].e, UNKNOWN_LOCATION);
6024 : }
6025 65 : if (arg1 == NULL_TREE)
6026 : {
6027 35 : g = gimple_build_builtin_unreachable (m_loc);
6028 35 : insert_before (g);
6029 : }
6030 65 : m_gsi = gsi_for_stmt (stmt);
6031 65 : g = gimple_build_call (fndecl, 1, gimple_phi_result (phi1));
6032 65 : gimple_call_set_lhs (g, make_ssa_name (integer_type_node));
6033 65 : insert_before (g);
6034 65 : if (arg1 == NULL_TREE)
6035 35 : g = gimple_build_assign (lhs, PLUS_EXPR,
6036 : gimple_phi_result (phi2),
6037 : gimple_call_lhs (g));
6038 : else
6039 : {
6040 30 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6041 : PLUS_EXPR, gimple_phi_result (phi2),
6042 : gimple_call_lhs (g));
6043 30 : insert_before (g);
6044 30 : edge e1 = split_block (gimple_bb (stmt), g);
6045 30 : edge e2 = make_edge (bb, e1->dest, EDGE_FALLTHRU);
6046 30 : e2->probability = profile_probability::always ();
6047 30 : set_immediate_dominator (CDI_DOMINATORS, e1->dest,
6048 : get_immediate_dominator (CDI_DOMINATORS,
6049 : e1->src));
6050 30 : phi3 = create_phi_node (make_ssa_name (integer_type_node), e1->dest);
6051 30 : add_phi_arg (phi3, gimple_assign_lhs (g), e1, UNKNOWN_LOCATION);
6052 30 : add_phi_arg (phi3, arg1, e2, UNKNOWN_LOCATION);
6053 30 : m_gsi = gsi_for_stmt (stmt);
6054 30 : g = gimple_build_assign (lhs, gimple_phi_result (phi3));
6055 : }
6056 65 : gsi_replace (&m_gsi, g, true);
6057 65 : break;
6058 9 : case IFN_CLRSB:
6059 9 : bb = gsi_bb (m_gsi);
6060 9 : remove_edge (find_edge (bb, edge_bb));
6061 9 : edge e;
6062 9 : e = make_edge (bb, gimple_bb (stmt), EDGE_FALLTHRU);
6063 9 : e->probability = profile_probability::always ();
6064 9 : set_immediate_dominator (CDI_DOMINATORS, gimple_bb (stmt),
6065 : get_immediate_dominator (CDI_DOMINATORS,
6066 : edge_bb));
6067 9 : phi1 = create_phi_node (make_ssa_name (m_limb_type),
6068 : edge_bb);
6069 9 : phi2 = create_phi_node (make_ssa_name (integer_type_node),
6070 : edge_bb);
6071 9 : phi3 = create_phi_node (make_ssa_name (integer_type_node),
6072 : gimple_bb (stmt));
6073 31 : for (unsigned i = 0; i < cnt; i++)
6074 : {
6075 22 : add_phi_arg (phi1, bqp[i].val, bqp[2 * i + 1].e, UNKNOWN_LOCATION);
6076 22 : add_phi_arg (phi2, bqp[i].addend, bqp[2 * i + 1].e,
6077 : UNKNOWN_LOCATION);
6078 22 : tree a = bqp[i].addend;
6079 22 : if (i && kind == bitint_prec_large)
6080 8 : a = int_const_binop (PLUS_EXPR, a, integer_minus_one_node);
6081 22 : if (i)
6082 13 : add_phi_arg (phi3, a, bqp[2 * i].e, UNKNOWN_LOCATION);
6083 : }
6084 9 : add_phi_arg (phi3, build_int_cst (integer_type_node, prec - 1), e,
6085 : UNKNOWN_LOCATION);
6086 9 : m_gsi = gsi_after_labels (edge_bb);
6087 9 : g = gimple_build_call (fndecl, 1,
6088 : add_cast (signed_type_for (m_limb_type),
6089 : gimple_phi_result (phi1)));
6090 9 : gimple_call_set_lhs (g, make_ssa_name (integer_type_node));
6091 9 : insert_before (g);
6092 9 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6093 : PLUS_EXPR, gimple_call_lhs (g),
6094 : gimple_phi_result (phi2));
6095 9 : insert_before (g);
6096 9 : if (kind != bitint_prec_large)
6097 : {
6098 5 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6099 : PLUS_EXPR, gimple_assign_lhs (g),
6100 : integer_one_node);
6101 5 : insert_before (g);
6102 : }
6103 9 : add_phi_arg (phi3, gimple_assign_lhs (g),
6104 : find_edge (edge_bb, gimple_bb (stmt)), UNKNOWN_LOCATION);
6105 9 : m_gsi = gsi_for_stmt (stmt);
6106 9 : g = gimple_build_assign (lhs, gimple_phi_result (phi3));
6107 9 : gsi_replace (&m_gsi, g, true);
6108 9 : break;
6109 16 : case IFN_PARITY:
6110 16 : g = gimple_build_call (fndecl, 1, res);
6111 16 : gimple_call_set_lhs (g, lhs);
6112 16 : gsi_replace (&m_gsi, g, true);
6113 16 : break;
6114 11 : case IFN_POPCOUNT:
6115 11 : g = gimple_build_assign (lhs, res);
6116 11 : gsi_replace (&m_gsi, g, true);
6117 11 : break;
6118 0 : default:
6119 0 : gcc_unreachable ();
6120 : }
6121 : }
6122 :
6123 : /* Lower a .{BSWAP,BITREVERSE} call with one large/huge _BitInt argument. */
6124 :
6125 : void
6126 15 : bitint_large_huge::lower_bswap_bitreverse (tree obj, gimple *stmt)
6127 : {
6128 15 : tree arg = gimple_call_arg (stmt, 0);
6129 15 : tree lhs = gimple_call_lhs (stmt);
6130 15 : gimple *g;
6131 :
6132 15 : if (!lhs)
6133 : {
6134 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6135 0 : gsi_remove (&gsi, true);
6136 0 : return;
6137 : }
6138 15 : tree type = TREE_TYPE (arg);
6139 15 : gcc_assert (BITINT_TYPE_P (type));
6140 15 : bitint_prec_kind kind = bitint_precision_kind (type);
6141 15 : gcc_assert (kind >= bitint_prec_large);
6142 15 : if (!obj)
6143 : {
6144 2 : int part = var_to_partition (m_map, lhs);
6145 2 : gcc_assert (m_vars[part] != NULL_TREE);
6146 : obj = m_vars[part];
6147 : }
6148 15 : enum internal_fn ifn = gimple_call_internal_fn (stmt);
6149 15 : enum built_in_function bcode = END_BUILTINS;
6150 15 : switch (limb_prec)
6151 : {
6152 0 : case 16:
6153 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP16 : BUILT_IN_BITREVERSE16;
6154 : break;
6155 0 : case 32:
6156 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP32 : BUILT_IN_BITREVERSE32;
6157 : break;
6158 15 : case 64:
6159 15 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP64 : BUILT_IN_BITREVERSE64;
6160 : break;
6161 0 : case 128:
6162 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP128 : BUILT_IN_BITREVERSE128;
6163 : break;
6164 0 : default:
6165 0 : gcc_unreachable ();
6166 : }
6167 15 : tree fndecl = builtin_decl_explicit (bcode);
6168 15 : unsigned prec = TYPE_PRECISION (type);
6169 30 : tree p = build_int_cst (sizetype,
6170 15 : prec / limb_prec - (prec % limb_prec == 0));
6171 : /* For IFN .BSWAP or .BITREVERSE and
6172 : FN corresponding __builtin_bswapN or __builtin_bitreverseN where N
6173 : is limb_prec, lower
6174 : dst = IFN (src);
6175 : as
6176 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
6177 : if constexpr ((prec % limb_prec) == 0)
6178 : {
6179 : for (idx = 0; idx <= p; ++idx)
6180 : dst[p - idx] = FN (src[idx]);
6181 : }
6182 : else
6183 : {
6184 : unsigned n1 = prec % limb_prec;
6185 : unsigned n2 = limb_prec - n1;
6186 : dst[p] = FN (src[0]) >> n2;
6187 : for (idx = p - 1; (ssize_t) idx >= 0; --idx)
6188 : dst[idx] = FN ((src[p - idx] << n2) | (src[p - idx - 1] >> n1));
6189 : } */
6190 15 : if (prec % limb_prec == 0)
6191 : {
6192 3 : tree idx_next;
6193 3 : tree idx = create_loop (size_zero_node, &idx_next);
6194 3 : tree pmidx = make_ssa_name (sizetype);
6195 3 : g = gimple_build_assign (pmidx, MINUS_EXPR, p, idx);
6196 3 : insert_before (g);
6197 3 : m_data_cnt = 0;
6198 3 : tree t = handle_operand (arg, idx);
6199 3 : m_first = false;
6200 3 : g = gimple_build_call (fndecl, 1, t);
6201 3 : t = make_ssa_name (m_limb_type);
6202 3 : gimple_call_set_lhs (g, t);
6203 3 : insert_before (g);
6204 3 : tree l = limb_access (TREE_TYPE (lhs), obj, pmidx, true);
6205 3 : g = gimple_build_assign (l, t);
6206 3 : insert_before (g);
6207 3 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_one_node);
6208 3 : insert_before (g);
6209 3 : g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
6210 3 : insert_before (g);
6211 : }
6212 : else
6213 : {
6214 12 : tree n1 = build_int_cst (unsigned_type_node, prec % limb_prec);
6215 24 : tree n2 = build_int_cst (unsigned_type_node,
6216 12 : limb_prec - (prec % limb_prec));
6217 12 : m_data_cnt = 0;
6218 12 : tree t = handle_operand (arg, bitint_big_endian ? p : size_zero_node);
6219 12 : m_first = false;
6220 : /* Nothing is needed to bswap 8 bits or bitreverse 1 bit, so just
6221 : mask off higher bits in that case. */
6222 19 : if ((prec % limb_prec) != (ifn == IFN_BSWAP ? 8 : 1))
6223 : {
6224 8 : g = gimple_build_call (fndecl, 1, t);
6225 8 : t = make_ssa_name (m_limb_type);
6226 8 : gimple_call_set_lhs (g, t);
6227 8 : insert_before (g);
6228 8 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
6229 : t, n2);
6230 : }
6231 : else
6232 4 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_AND_EXPR,
6233 : t, build_int_cst (m_limb_type,
6234 : ifn == IFN_BSWAP
6235 6 : ? 0xff : 1));
6236 12 : insert_before (g);
6237 12 : t = gimple_assign_lhs (g);
6238 12 : tree l = limb_access (TREE_TYPE (lhs), obj,
6239 : bitint_big_endian ? size_zero_node : p, true);
6240 12 : g = gimple_build_assign (l, t);
6241 12 : insert_before (g);
6242 12 : tree pm1 = build_int_cst (sizetype, prec / limb_prec - 1);
6243 12 : tree idx_next;
6244 12 : tree idx = create_loop (pm1, &idx_next);
6245 12 : tree pmidx = make_ssa_name (sizetype);
6246 12 : g = gimple_build_assign (pmidx, MINUS_EXPR, p, idx);
6247 12 : insert_before (g);
6248 12 : m_data_cnt = 0;
6249 24 : t = handle_operand (arg, bitint_big_endian ? idx : pmidx);
6250 12 : m_data_cnt = 0;
6251 12 : g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
6252 : t, n2);
6253 12 : insert_before (g);
6254 12 : t = gimple_assign_lhs (g);
6255 12 : tree t2 = make_ssa_name (sizetype);
6256 12 : if (bitint_big_endian)
6257 0 : g = gimple_build_assign (t2, PLUS_EXPR, idx, size_one_node);
6258 : else
6259 12 : g = gimple_build_assign (t2, PLUS_EXPR, pmidx, size_int (-1));
6260 12 : insert_before (g);
6261 12 : t2 = handle_operand (arg, t2);
6262 12 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
6263 : t2, n1);
6264 12 : insert_before (g);
6265 12 : t2 = gimple_assign_lhs (g);
6266 12 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
6267 : t, t2);
6268 12 : insert_before (g);
6269 12 : t = gimple_assign_lhs (g);
6270 12 : g = gimple_build_call (fndecl, 1, t);
6271 12 : t = make_ssa_name (m_limb_type);
6272 12 : gimple_call_set_lhs (g, t);
6273 12 : insert_before (g);
6274 24 : l = limb_access (TREE_TYPE (lhs), obj,
6275 : bitint_big_endian ? pmidx : idx, true);
6276 12 : g = gimple_build_assign (l, t);
6277 12 : insert_before (g);
6278 12 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_int (-1));
6279 12 : insert_before (g);
6280 12 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node, NULL_TREE,
6281 : NULL_TREE);
6282 12 : insert_before (g);
6283 : }
6284 15 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6285 15 : if (bitint_extended == bitint_ext_full
6286 0 : && abi_limb_prec > limb_prec
6287 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
6288 0 : > CEIL (prec, limb_prec) * limb_prec))
6289 : {
6290 0 : m_gsi = gsi;
6291 0 : tree p2 = build_int_cst (sizetype,
6292 : CEIL (prec, abi_limb_prec)
6293 0 : * abi_limb_prec / limb_prec - 1);
6294 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
6295 0 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
6296 0 : insert_before (g);
6297 : }
6298 : }
6299 :
6300 : /* Lower a call statement with one or more large/huge _BitInt
6301 : arguments or large/huge _BitInt return value. */
6302 :
6303 : void
6304 8856 : bitint_large_huge::lower_call (tree obj, gimple *stmt)
6305 : {
6306 8856 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6307 8856 : unsigned int nargs = gimple_call_num_args (stmt);
6308 8856 : if (gimple_call_internal_p (stmt))
6309 4328 : switch (gimple_call_internal_fn (stmt))
6310 : {
6311 2733 : case IFN_ADD_OVERFLOW:
6312 2733 : case IFN_SUB_OVERFLOW:
6313 2733 : case IFN_UBSAN_CHECK_ADD:
6314 2733 : case IFN_UBSAN_CHECK_SUB:
6315 2733 : lower_addsub_overflow (obj, stmt);
6316 7061 : return;
6317 1479 : case IFN_MUL_OVERFLOW:
6318 1479 : case IFN_UBSAN_CHECK_MUL:
6319 1479 : lower_mul_overflow (obj, stmt);
6320 1479 : return;
6321 101 : case IFN_CLZ:
6322 101 : case IFN_CTZ:
6323 101 : case IFN_CLRSB:
6324 101 : case IFN_FFS:
6325 101 : case IFN_PARITY:
6326 101 : case IFN_POPCOUNT:
6327 101 : lower_bit_query (stmt);
6328 101 : return;
6329 15 : case IFN_BSWAP:
6330 15 : case IFN_BITREVERSE:
6331 15 : lower_bswap_bitreverse (obj, stmt);
6332 15 : return;
6333 : default:
6334 : break;
6335 : }
6336 4528 : bool returns_twice = (gimple_call_flags (stmt) & ECF_RETURNS_TWICE) != 0;
6337 10200 : for (unsigned int i = 0; i < nargs; ++i)
6338 : {
6339 5672 : tree arg = gimple_call_arg (stmt, i);
6340 9001 : if (TREE_CODE (arg) != SSA_NAME
6341 2440 : || !BITINT_TYPE_P (TREE_TYPE (arg))
6342 8050 : || bitint_precision_kind (TREE_TYPE (arg)) <= bitint_prec_middle)
6343 3329 : continue;
6344 2343 : if (SSA_NAME_IS_DEFAULT_DEF (arg)
6345 2343 : && (!SSA_NAME_VAR (arg) || VAR_P (SSA_NAME_VAR (arg))))
6346 : {
6347 1 : tree var = create_tmp_reg (TREE_TYPE (arg));
6348 1 : arg = get_or_create_ssa_default_def (cfun, var);
6349 : }
6350 : else
6351 : {
6352 2342 : int p = var_to_partition (m_map, arg);
6353 2342 : tree v = m_vars[p];
6354 2342 : gcc_assert (v != NULL_TREE);
6355 2342 : if (!types_compatible_p (TREE_TYPE (arg), TREE_TYPE (v)))
6356 2322 : v = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (arg), v);
6357 2342 : arg = make_ssa_name (TREE_TYPE (arg));
6358 2342 : gimple *g = gimple_build_assign (arg, v);
6359 2342 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
6360 2342 : if (returns_twice && bb_has_abnormal_pred (gimple_bb (stmt)))
6361 : {
6362 11 : m_returns_twice_calls.safe_push (stmt);
6363 11 : returns_twice = false;
6364 : }
6365 : }
6366 2343 : gimple_call_set_arg (stmt, i, arg);
6367 2343 : if (m_preserved == NULL)
6368 412 : m_preserved = BITMAP_ALLOC (NULL);
6369 2343 : bitmap_set_bit (m_preserved, SSA_NAME_VERSION (arg));
6370 : }
6371 4528 : tree lhs = gimple_call_lhs (stmt);
6372 4528 : if (lhs
6373 4397 : && TREE_CODE (lhs) == SSA_NAME
6374 4397 : && BITINT_TYPE_P (TREE_TYPE (lhs))
6375 8847 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large)
6376 : {
6377 4293 : int p = var_to_partition (m_map, lhs);
6378 4293 : tree v = m_vars[p];
6379 4293 : gcc_assert (v != NULL_TREE);
6380 4293 : if (!types_compatible_p (TREE_TYPE (lhs), TREE_TYPE (v)))
6381 4293 : v = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), v);
6382 4293 : gimple_call_set_lhs (stmt, v);
6383 4293 : SSA_NAME_DEF_STMT (lhs) = gimple_build_nop ();
6384 : }
6385 4528 : update_stmt (stmt);
6386 : }
6387 :
6388 : /* Lower __asm STMT which involves large/huge _BitInt values. */
6389 :
6390 : void
6391 3 : bitint_large_huge::lower_asm (gimple *stmt)
6392 : {
6393 3 : gasm *g = as_a <gasm *> (stmt);
6394 3 : unsigned noutputs = gimple_asm_noutputs (g);
6395 3 : unsigned ninputs = gimple_asm_ninputs (g);
6396 :
6397 5 : for (unsigned i = 0; i < noutputs; ++i)
6398 : {
6399 2 : tree t = gimple_asm_output_op (g, i);
6400 2 : tree s = TREE_VALUE (t);
6401 2 : if (TREE_CODE (s) == SSA_NAME
6402 1 : && BITINT_TYPE_P (TREE_TYPE (s))
6403 3 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
6404 : {
6405 1 : int part = var_to_partition (m_map, s);
6406 1 : gcc_assert (m_vars[part] != NULL_TREE);
6407 1 : TREE_VALUE (t) = m_vars[part];
6408 : }
6409 : }
6410 8 : for (unsigned i = 0; i < ninputs; ++i)
6411 : {
6412 5 : tree t = gimple_asm_input_op (g, i);
6413 5 : tree s = TREE_VALUE (t);
6414 5 : if (TREE_CODE (s) == SSA_NAME
6415 4 : && BITINT_TYPE_P (TREE_TYPE (s))
6416 9 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
6417 : {
6418 4 : if (SSA_NAME_IS_DEFAULT_DEF (s)
6419 4 : && (!SSA_NAME_VAR (s) || VAR_P (SSA_NAME_VAR (s))))
6420 : {
6421 1 : TREE_VALUE (t) = create_tmp_var (TREE_TYPE (s), "bitint");
6422 1 : mark_addressable (TREE_VALUE (t));
6423 : }
6424 : else
6425 : {
6426 3 : int part = var_to_partition (m_map, s);
6427 3 : gcc_assert (m_vars[part] != NULL_TREE);
6428 3 : TREE_VALUE (t) = m_vars[part];
6429 : }
6430 : }
6431 : }
6432 3 : update_stmt (stmt);
6433 3 : }
6434 :
6435 : /* Lower statement STMT which involves large/huge _BitInt values
6436 : into code accessing individual limbs. */
6437 :
6438 : void
6439 42875 : bitint_large_huge::lower_stmt (gimple *stmt)
6440 : {
6441 42875 : m_first = true;
6442 42875 : m_lhs = NULL_TREE;
6443 42875 : m_data.truncate (0);
6444 42875 : m_data_cnt = 0;
6445 42875 : m_gsi = gsi_for_stmt (stmt);
6446 42875 : m_after_stmt = NULL;
6447 42875 : m_bb = NULL;
6448 42875 : m_init_gsi = m_gsi;
6449 42875 : gsi_prev (&m_init_gsi);
6450 42875 : m_preheader_bb = NULL;
6451 42875 : m_upwards_2limb = 0;
6452 42875 : m_upwards = false;
6453 42875 : m_var_msb = false;
6454 42875 : m_cast_conditional = false;
6455 42875 : m_bitfld_load = 0;
6456 42875 : m_loc = gimple_location (stmt);
6457 42875 : if (is_gimple_call (stmt))
6458 : {
6459 7179 : lower_call (NULL_TREE, stmt);
6460 7179 : return;
6461 : }
6462 35696 : if (gimple_code (stmt) == GIMPLE_ASM)
6463 : {
6464 3 : lower_asm (stmt);
6465 3 : return;
6466 : }
6467 35693 : tree lhs = NULL_TREE, cmp_op1 = NULL_TREE, cmp_op2 = NULL_TREE;
6468 35693 : tree_code cmp_code = comparison_op (stmt, &cmp_op1, &cmp_op2);
6469 35693 : bool eq_p = (cmp_code == EQ_EXPR || cmp_code == NE_EXPR);
6470 35693 : bool mergeable_cast_p = false;
6471 35693 : bool final_cast_p = false;
6472 35693 : if (gimple_assign_cast_p (stmt))
6473 : {
6474 5448 : lhs = gimple_assign_lhs (stmt);
6475 5448 : tree rhs1 = gimple_assign_rhs1 (stmt);
6476 5448 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
6477 46 : rhs1 = TREE_OPERAND (rhs1, 0);
6478 9668 : if (BITINT_TYPE_P (TREE_TYPE (lhs))
6479 1228 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
6480 6640 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1)))
6481 : mergeable_cast_p = true;
6482 4546 : else if (BITINT_TYPE_P (TREE_TYPE (rhs1))
6483 4256 : && bitint_precision_kind (TREE_TYPE (rhs1)) >= bitint_prec_large
6484 8657 : && (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6485 36 : || POINTER_TYPE_P (TREE_TYPE (lhs))
6486 35 : || gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR))
6487 : {
6488 4256 : final_cast_p = true;
6489 4256 : if (((TREE_CODE (TREE_TYPE (lhs)) == INTEGER_TYPE
6490 570 : && TYPE_PRECISION (TREE_TYPE (lhs)) > MAX_FIXED_MODE_SIZE)
6491 4256 : || (!INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6492 36 : && !POINTER_TYPE_P (TREE_TYPE (lhs))))
6493 4291 : && gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)
6494 : {
6495 : /* Handle VIEW_CONVERT_EXPRs to not generally supported
6496 : huge INTEGER_TYPEs like uint256_t or uint512_t. These
6497 : are usually emitted from memcpy folding and backends
6498 : support moves with them but that is usually it.
6499 : Similarly handle VCEs to vector/complex types etc. */
6500 35 : gcc_assert (TREE_CODE (rhs1) == SSA_NAME);
6501 35 : if (SSA_NAME_IS_DEFAULT_DEF (rhs1)
6502 35 : && (!SSA_NAME_VAR (rhs1) || VAR_P (SSA_NAME_VAR (rhs1))))
6503 : {
6504 0 : tree var = create_tmp_reg (TREE_TYPE (lhs));
6505 0 : rhs1 = get_or_create_ssa_default_def (cfun, var);
6506 0 : gimple_assign_set_rhs1 (stmt, rhs1);
6507 0 : gimple_assign_set_rhs_code (stmt, SSA_NAME);
6508 : }
6509 35 : else if (m_names == NULL
6510 35 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))
6511 : {
6512 0 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6513 0 : gcc_assert (gimple_assign_load_p (g));
6514 0 : tree mem = gimple_assign_rhs1 (g);
6515 0 : tree ltype = TREE_TYPE (lhs);
6516 0 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (mem));
6517 0 : if (as != TYPE_ADDR_SPACE (ltype))
6518 0 : ltype
6519 0 : = build_qualified_type (ltype,
6520 0 : TYPE_QUALS (ltype)
6521 0 : | ENCODE_QUAL_ADDR_SPACE (as));
6522 0 : rhs1 = build1 (VIEW_CONVERT_EXPR, ltype, unshare_expr (mem));
6523 0 : gimple_assign_set_rhs1 (stmt, rhs1);
6524 : }
6525 : else
6526 : {
6527 35 : int part = var_to_partition (m_map, rhs1);
6528 35 : gcc_assert (m_vars[part] != NULL_TREE);
6529 35 : rhs1 = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs),
6530 : m_vars[part]);
6531 35 : gimple_assign_set_rhs1 (stmt, rhs1);
6532 : }
6533 35 : update_stmt (stmt);
6534 35 : return;
6535 : }
6536 4221 : if (TREE_CODE (rhs1) == SSA_NAME
6537 4221 : && (m_names == NULL
6538 4184 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
6539 : {
6540 1723 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6541 1723 : if (is_gimple_assign (g)
6542 1723 : && gimple_assign_rhs_code (g) == IMAGPART_EXPR)
6543 : {
6544 1640 : tree rhs2 = TREE_OPERAND (gimple_assign_rhs1 (g), 0);
6545 1640 : if (TREE_CODE (rhs2) == SSA_NAME
6546 1640 : && (m_names == NULL
6547 1603 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs2))))
6548 : {
6549 1640 : g = SSA_NAME_DEF_STMT (rhs2);
6550 1640 : int ovf = optimizable_arith_overflow (g);
6551 1640 : if (ovf == 2)
6552 : /* If .{ADD,SUB,MUL}_OVERFLOW has both REALPART_EXPR
6553 : and IMAGPART_EXPR uses, where the latter is cast to
6554 : non-_BitInt, it will be optimized when handling
6555 : the REALPART_EXPR. */
6556 : return;
6557 91 : if (ovf == 1)
6558 : {
6559 91 : lower_call (NULL_TREE, g);
6560 91 : return;
6561 : }
6562 : }
6563 : }
6564 : }
6565 : }
6566 145 : else if (BITINT_TYPE_P (TREE_TYPE (lhs))
6567 145 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
6568 145 : && !INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
6569 145 : && !POINTER_TYPE_P (TREE_TYPE (rhs1))
6570 290 : && gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)
6571 : {
6572 10 : int part = var_to_partition (m_map, lhs);
6573 10 : gcc_assert (m_vars[part] != NULL_TREE);
6574 10 : lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (rhs1), m_vars[part]);
6575 10 : insert_before (gimple_build_assign (lhs, rhs1));
6576 10 : return;
6577 : }
6578 : }
6579 34008 : if (gimple_store_p (stmt))
6580 : {
6581 9006 : tree rhs1 = gimple_assign_rhs1 (stmt);
6582 9006 : if (TREE_CODE (rhs1) == SSA_NAME
6583 9006 : && (m_names == NULL
6584 7946 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
6585 : {
6586 1639 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6587 1639 : m_loc = gimple_location (g);
6588 1639 : lhs = gimple_assign_lhs (stmt);
6589 1639 : if (is_gimple_assign (g) && !mergeable_op (g))
6590 643 : switch (gimple_assign_rhs_code (g))
6591 : {
6592 119 : case LSHIFT_EXPR:
6593 119 : case RSHIFT_EXPR:
6594 119 : lower_shift_stmt (lhs, g);
6595 478 : handled:
6596 478 : m_gsi = gsi_for_stmt (stmt);
6597 478 : unlink_stmt_vdef (stmt);
6598 956 : release_ssa_name (gimple_vdef (stmt));
6599 478 : gsi_remove (&m_gsi, true);
6600 478 : return;
6601 205 : case MULT_EXPR:
6602 205 : case TRUNC_DIV_EXPR:
6603 205 : case EXACT_DIV_EXPR:
6604 205 : case TRUNC_MOD_EXPR:
6605 205 : lower_muldiv_stmt (lhs, g);
6606 205 : goto handled;
6607 44 : case FIX_TRUNC_EXPR:
6608 44 : lower_float_conv_stmt (lhs, g);
6609 44 : goto handled;
6610 73 : case REALPART_EXPR:
6611 73 : case IMAGPART_EXPR:
6612 73 : lower_cplxpart_stmt (lhs, g);
6613 73 : goto handled;
6614 8 : case VIEW_CONVERT_EXPR:
6615 8 : {
6616 8 : tree rhs1 = gimple_assign_rhs1 (g);
6617 8 : rhs1 = TREE_OPERAND (rhs1, 0);
6618 8 : if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
6619 7 : && !POINTER_TYPE_P (TREE_TYPE (rhs1)))
6620 : {
6621 7 : tree ltype = TREE_TYPE (rhs1);
6622 7 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (lhs));
6623 7 : ltype
6624 14 : = build_qualified_type (ltype,
6625 7 : TYPE_QUALS (TREE_TYPE (lhs))
6626 7 : | ENCODE_QUAL_ADDR_SPACE (as));
6627 7 : lhs = build1 (VIEW_CONVERT_EXPR, ltype, lhs);
6628 7 : gimple_assign_set_lhs (stmt, lhs);
6629 7 : gimple_assign_set_rhs1 (stmt, rhs1);
6630 7 : gimple_assign_set_rhs_code (stmt, TREE_CODE (rhs1));
6631 7 : update_stmt (stmt);
6632 7 : return;
6633 : }
6634 : }
6635 : break;
6636 : default:
6637 : break;
6638 : }
6639 996 : else if (optimizable_arith_overflow (g) == 3)
6640 : {
6641 24 : lower_call (lhs, g);
6642 24 : goto handled;
6643 : }
6644 972 : else if (is_gimple_call (g) && gimple_call_internal_p (g))
6645 13 : switch (gimple_call_internal_fn (g))
6646 : {
6647 13 : case IFN_BSWAP:
6648 13 : case IFN_BITREVERSE:
6649 13 : lower_call (lhs, g);
6650 13 : goto handled;
6651 : default:
6652 : break;
6653 : }
6654 :
6655 1154 : m_loc = gimple_location (stmt);
6656 : }
6657 : }
6658 33523 : if (mergeable_op (stmt)
6659 22090 : || gimple_store_p (stmt)
6660 22090 : || gimple_assign_load_p (stmt)
6661 : || eq_p
6662 17280 : || mergeable_cast_p
6663 43672 : || (is_gimple_assign (stmt)
6664 9761 : && gimple_assign_rhs_code (stmt) == PAREN_EXPR))
6665 : {
6666 23376 : lhs = lower_mergeable_stmt (stmt, cmp_code, cmp_op1, cmp_op2);
6667 23376 : if (!eq_p)
6668 : return;
6669 : }
6670 10147 : else if (cmp_code != ERROR_MARK)
6671 722 : lhs = lower_comparison_stmt (stmt, cmp_code, cmp_op1, cmp_op2);
6672 16697 : if (cmp_code != ERROR_MARK)
6673 : {
6674 7272 : if (gimple_code (stmt) == GIMPLE_COND)
6675 : {
6676 6498 : gcond *cstmt = as_a <gcond *> (stmt);
6677 6498 : gimple_cond_set_lhs (cstmt, lhs);
6678 6498 : gimple_cond_set_rhs (cstmt, boolean_false_node);
6679 6498 : gimple_cond_set_code (cstmt, cmp_code);
6680 6498 : update_stmt (stmt);
6681 6498 : return;
6682 : }
6683 774 : if (gimple_assign_rhs_code (stmt) == COND_EXPR)
6684 : {
6685 0 : tree cond = build2 (cmp_code, boolean_type_node, lhs,
6686 : boolean_false_node);
6687 0 : gimple_assign_set_rhs1 (stmt, cond);
6688 0 : lhs = gimple_assign_lhs (stmt);
6689 0 : gcc_assert (!BITINT_TYPE_P (TREE_TYPE (lhs))
6690 : || (bitint_precision_kind (TREE_TYPE (lhs))
6691 : <= bitint_prec_middle));
6692 0 : update_stmt (stmt);
6693 0 : return;
6694 : }
6695 774 : gimple_assign_set_rhs1 (stmt, lhs);
6696 774 : gimple_assign_set_rhs2 (stmt, boolean_false_node);
6697 774 : gimple_assign_set_rhs_code (stmt, cmp_code);
6698 774 : update_stmt (stmt);
6699 774 : return;
6700 : }
6701 9425 : if (final_cast_p)
6702 : {
6703 2581 : tree lhs_type = TREE_TYPE (lhs);
6704 : /* Add support for 3 or more limbs filled in from normal integral
6705 : type if this assert fails. If no target chooses limb mode smaller
6706 : than half of largest supported normal integral type, this will not
6707 : be needed. */
6708 2581 : gcc_assert (TYPE_PRECISION (lhs_type) <= 2 * limb_prec);
6709 2581 : gimple *g;
6710 2545 : if ((BITINT_TYPE_P (lhs_type)
6711 36 : && bitint_precision_kind (lhs_type) == bitint_prec_middle)
6712 5147 : || POINTER_TYPE_P (lhs_type))
6713 16 : lhs_type = build_nonstandard_integer_type (TYPE_PRECISION (lhs_type),
6714 16 : TYPE_UNSIGNED (lhs_type));
6715 2581 : m_data_cnt = 0;
6716 2581 : tree rhs1 = gimple_assign_rhs1 (stmt);
6717 2581 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (rhs1));
6718 2581 : unsigned int cnt = CEIL (prec, limb_prec);
6719 2581 : tree r1 = handle_operand (rhs1, size_int (bitint_big_endian
6720 : ? cnt - 1 : 0));
6721 2581 : if (!useless_type_conversion_p (lhs_type, TREE_TYPE (r1)))
6722 2472 : r1 = add_cast (lhs_type, r1);
6723 2581 : if (TYPE_PRECISION (lhs_type) > limb_prec)
6724 : {
6725 70 : m_data_cnt = 0;
6726 70 : m_first = false;
6727 70 : tree r2 = handle_operand (rhs1, size_int (bitint_big_endian
6728 : ? cnt - 2 : 1));
6729 70 : r2 = add_cast (lhs_type, r2);
6730 70 : g = gimple_build_assign (make_ssa_name (lhs_type), LSHIFT_EXPR, r2,
6731 : build_int_cst (unsigned_type_node,
6732 70 : limb_prec));
6733 70 : insert_before (g);
6734 70 : g = gimple_build_assign (make_ssa_name (lhs_type), BIT_IOR_EXPR, r1,
6735 : gimple_assign_lhs (g));
6736 70 : insert_before (g);
6737 70 : r1 = gimple_assign_lhs (g);
6738 : }
6739 2581 : if (lhs_type != TREE_TYPE (lhs))
6740 16 : g = gimple_build_assign (lhs, NOP_EXPR, r1);
6741 : else
6742 2565 : g = gimple_build_assign (lhs, r1);
6743 2581 : gsi_replace (&m_gsi, g, true);
6744 2581 : return;
6745 : }
6746 6844 : if (is_gimple_assign (stmt))
6747 6844 : switch (gimple_assign_rhs_code (stmt))
6748 : {
6749 465 : case LSHIFT_EXPR:
6750 465 : case RSHIFT_EXPR:
6751 465 : lower_shift_stmt (NULL_TREE, stmt);
6752 465 : return;
6753 157 : case MULT_EXPR:
6754 157 : case TRUNC_DIV_EXPR:
6755 157 : case EXACT_DIV_EXPR:
6756 157 : case TRUNC_MOD_EXPR:
6757 157 : lower_muldiv_stmt (NULL_TREE, stmt);
6758 157 : return;
6759 275 : case FIX_TRUNC_EXPR:
6760 275 : case FLOAT_EXPR:
6761 275 : lower_float_conv_stmt (NULL_TREE, stmt);
6762 275 : return;
6763 5929 : case REALPART_EXPR:
6764 5929 : case IMAGPART_EXPR:
6765 5929 : lower_cplxpart_stmt (NULL_TREE, stmt);
6766 5929 : return;
6767 18 : case COMPLEX_EXPR:
6768 18 : lower_complexexpr_stmt (stmt);
6769 18 : return;
6770 : default:
6771 : break;
6772 : }
6773 0 : gcc_unreachable ();
6774 : }
6775 :
6776 : /* Helper for walk_non_aliased_vuses. Determine if we arrived at
6777 : the desired memory state. */
6778 :
6779 : void *
6780 2186 : vuse_eq (ao_ref *, tree vuse1, void *data)
6781 : {
6782 2186 : tree vuse2 = (tree) data;
6783 2186 : if (vuse1 == vuse2)
6784 820 : return data;
6785 :
6786 : return NULL;
6787 : }
6788 :
6789 : /* Return true if STMT uses a library function and needs to take
6790 : address of its inputs. We need to avoid bit-fields in those
6791 : cases. Similarly, we need to avoid overlap between destination
6792 : and source limb arrays. */
6793 :
6794 : bool
6795 15083 : stmt_needs_operand_addr (gimple *stmt)
6796 : {
6797 15083 : if (is_gimple_assign (stmt))
6798 10285 : switch (gimple_assign_rhs_code (stmt))
6799 : {
6800 585 : case MULT_EXPR:
6801 585 : case TRUNC_DIV_EXPR:
6802 585 : case EXACT_DIV_EXPR:
6803 585 : case TRUNC_MOD_EXPR:
6804 585 : case FLOAT_EXPR:
6805 585 : return true;
6806 : default:
6807 : break;
6808 : }
6809 4798 : else if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
6810 46 : switch (gimple_call_internal_fn (stmt))
6811 : {
6812 17 : case IFN_MUL_OVERFLOW:
6813 17 : case IFN_UBSAN_CHECK_MUL:
6814 17 : return true;
6815 : /* These two actually don't take address, but reshuffle
6816 : all bytes or bits, so need similar treatment. */
6817 4 : case IFN_BSWAP:
6818 4 : case IFN_BITREVERSE:
6819 4 : return true;
6820 : default:
6821 : break;
6822 : }
6823 : return false;
6824 : }
6825 :
6826 : /* Dominator walker used to discover which large/huge _BitInt
6827 : loads could be sunk into all their uses. */
6828 :
6829 624 : class bitint_dom_walker : public dom_walker
6830 : {
6831 : public:
6832 312 : bitint_dom_walker (bitmap names, bitmap loads)
6833 624 : : dom_walker (CDI_DOMINATORS), m_names (names), m_loads (loads) {}
6834 :
6835 : edge before_dom_children (basic_block) final override;
6836 :
6837 : private:
6838 : bitmap m_names, m_loads;
6839 : };
6840 :
6841 : edge
6842 4508 : bitint_dom_walker::before_dom_children (basic_block bb)
6843 : {
6844 4508 : gphi *phi = get_virtual_phi (bb);
6845 4508 : tree vop;
6846 4508 : if (phi)
6847 794 : vop = gimple_phi_result (phi);
6848 3714 : else if (bb == ENTRY_BLOCK_PTR_FOR_FN (cfun))
6849 : vop = NULL_TREE;
6850 : else
6851 3402 : vop = (tree) get_immediate_dominator (CDI_DOMINATORS, bb)->aux;
6852 :
6853 4508 : auto_vec<tree, 16> worklist;
6854 9016 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
6855 19980 : !gsi_end_p (gsi); gsi_next (&gsi))
6856 : {
6857 15472 : gimple *stmt = gsi_stmt (gsi);
6858 15472 : if (is_gimple_debug (stmt))
6859 2764 : continue;
6860 :
6861 15517 : if (!vop && gimple_vuse (stmt))
6862 : vop = gimple_vuse (stmt);
6863 :
6864 15083 : tree cvop = vop;
6865 28308 : if (gimple_vdef (stmt))
6866 15083 : vop = gimple_vdef (stmt);
6867 :
6868 15083 : tree lhs = gimple_get_lhs (stmt);
6869 17458 : if (lhs
6870 10958 : && TREE_CODE (lhs) == SSA_NAME
6871 8737 : && BITINT_TYPE_P (TREE_TYPE (lhs))
6872 5860 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
6873 20795 : && !bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
6874 : /* If lhs of stmt is large/huge _BitInt SSA_NAME not in m_names,
6875 : it means it will be handled in a loop or straight line code
6876 : at the location of its (ultimate) immediate use, so for
6877 : vop checking purposes check these only at the ultimate
6878 : immediate use. */
6879 2375 : continue;
6880 :
6881 12708 : ssa_op_iter oi;
6882 12708 : use_operand_p use_p;
6883 21548 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, oi, SSA_OP_USE)
6884 : {
6885 8840 : tree s = USE_FROM_PTR (use_p);
6886 14260 : if (BITINT_TYPE_P (TREE_TYPE (s))
6887 8840 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
6888 3265 : worklist.safe_push (s);
6889 : }
6890 :
6891 12708 : bool needs_operand_addr = stmt_needs_operand_addr (stmt);
6892 31514 : while (worklist.length () > 0)
6893 : {
6894 6098 : tree s = worklist.pop ();
6895 :
6896 6098 : if (!bitmap_bit_p (m_names, SSA_NAME_VERSION (s)))
6897 : {
6898 2375 : gimple *g = SSA_NAME_DEF_STMT (s);
6899 2375 : needs_operand_addr |= stmt_needs_operand_addr (g);
6900 5447 : FOR_EACH_SSA_USE_OPERAND (use_p, g, oi, SSA_OP_USE)
6901 : {
6902 3072 : tree s2 = USE_FROM_PTR (use_p);
6903 3262 : if (BITINT_TYPE_P (TREE_TYPE (s2))
6904 3072 : && (bitint_precision_kind (TREE_TYPE (s2))
6905 : >= bitint_prec_large))
6906 2833 : worklist.safe_push (s2);
6907 : }
6908 3095 : continue;
6909 2375 : }
6910 3723 : if (!SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
6911 3723 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (s)))
6912 : {
6913 226 : tree rhs = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s));
6914 396 : if (TREE_CODE (rhs) == SSA_NAME
6915 226 : && bitmap_bit_p (m_loads, SSA_NAME_VERSION (rhs)))
6916 : s = rhs;
6917 : else
6918 170 : continue;
6919 : }
6920 3497 : else if (!bitmap_bit_p (m_loads, SSA_NAME_VERSION (s)))
6921 548 : continue;
6922 :
6923 3005 : gimple *g = SSA_NAME_DEF_STMT (s);
6924 3005 : tree rhs1 = gimple_assign_rhs1 (g);
6925 3005 : if (needs_operand_addr
6926 218 : && TREE_CODE (rhs1) == COMPONENT_REF
6927 3022 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (rhs1, 1)))
6928 : {
6929 4 : tree fld = TREE_OPERAND (rhs1, 1);
6930 : /* For little-endian, we can allow as inputs bit-fields
6931 : which start at a limb boundary. */
6932 6 : if (!bitint_big_endian
6933 4 : && DECL_OFFSET_ALIGN (fld) >= TYPE_ALIGN (TREE_TYPE (rhs1))
6934 4 : && tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld))
6935 4 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
6936 4 : % limb_prec) == 0)
6937 : ;
6938 : else
6939 : {
6940 2 : bitmap_clear_bit (m_loads, SSA_NAME_VERSION (s));
6941 2 : continue;
6942 : }
6943 : }
6944 :
6945 3003 : ao_ref ref;
6946 3003 : ao_ref_init (&ref, rhs1);
6947 3003 : tree lvop = gimple_vuse (g);
6948 3003 : unsigned limit = 64;
6949 3003 : tree vuse = cvop;
6950 3003 : if (vop != cvop
6951 1334 : && is_gimple_assign (stmt)
6952 1332 : && gimple_store_p (stmt)
6953 4335 : && (needs_operand_addr
6954 1139 : || !operand_equal_p (lhs, gimple_assign_rhs1 (g), 0)))
6955 : vuse = vop;
6956 3003 : if (vuse != lvop
6957 3003 : && walk_non_aliased_vuses (&ref, vuse, false, vuse_eq,
6958 : NULL, NULL, NULL, limit, lvop) == NULL)
6959 528 : bitmap_clear_bit (m_loads, SSA_NAME_VERSION (s));
6960 : }
6961 : }
6962 :
6963 4508 : bb->aux = (void *) vop;
6964 4508 : return NULL;
6965 4508 : }
6966 :
6967 : }
6968 :
6969 : /* Replacement for normal processing of STMT in tree-ssa-coalesce.cc
6970 : build_ssa_conflict_graph.
6971 : The differences are:
6972 : 1) don't process assignments with large/huge _BitInt lhs not in NAMES
6973 : 2) for large/huge _BitInt multiplication/division/modulo process def
6974 : only after processing uses rather than before to make uses conflict
6975 : with the definition
6976 : 3) for large/huge _BitInt uses not in NAMES mark the uses of their
6977 : SSA_NAME_DEF_STMT (recursively), because those uses will be sunk into
6978 : the final statement. */
6979 :
6980 : void
6981 84195 : build_bitint_stmt_ssa_conflicts (gimple *stmt, live_track *live,
6982 : ssa_conflicts *graph, bitmap names,
6983 : void (*def) (live_track *, tree,
6984 : ssa_conflicts *),
6985 : void (*use) (live_track *, tree),
6986 : void (*clear) (live_track *, tree))
6987 : {
6988 84195 : bool muldiv_p = false;
6989 84195 : tree lhs = NULL_TREE;
6990 84195 : if (is_gimple_assign (stmt))
6991 : {
6992 46215 : lhs = gimple_assign_lhs (stmt);
6993 46215 : if (TREE_CODE (lhs) == SSA_NAME)
6994 : {
6995 33396 : tree type = TREE_TYPE (lhs);
6996 33396 : if (TREE_CODE (type) == COMPLEX_TYPE)
6997 63 : type = TREE_TYPE (type);
6998 12837 : if (BITINT_TYPE_P (type)
6999 33401 : && bitint_precision_kind (type) >= bitint_prec_large)
7000 : {
7001 19946 : if (!bitmap_bit_p (names, SSA_NAME_VERSION (lhs)))
7002 4894 : return;
7003 :
7004 : /* A copy between 2 partitions does not introduce an interference
7005 : by itself. If they did, you would never be able to coalesce
7006 : two things which are copied. If the two variables really do
7007 : conflict, they will conflict elsewhere in the program.
7008 :
7009 : This is handled by simply removing the SRC of the copy from
7010 : the live list, and processing the stmt normally.
7011 :
7012 : Don't do this if lhs is not in names though, in such cases
7013 : it is actually used at some point later in the basic
7014 : block. */
7015 15052 : if (gimple_assign_copy_p (stmt))
7016 : {
7017 1686 : tree rhs1 = gimple_assign_rhs1 (stmt);
7018 1686 : if (TREE_CODE (rhs1) == SSA_NAME)
7019 45 : clear (live, rhs1);
7020 : }
7021 :
7022 15052 : switch (gimple_assign_rhs_code (stmt))
7023 : {
7024 157 : case MULT_EXPR:
7025 157 : case TRUNC_DIV_EXPR:
7026 157 : case EXACT_DIV_EXPR:
7027 157 : case TRUNC_MOD_EXPR:
7028 157 : muldiv_p = true;
7029 : default:
7030 : break;
7031 : }
7032 : }
7033 : }
7034 : }
7035 37980 : else if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
7036 4301 : switch (gimple_call_internal_fn (stmt))
7037 : {
7038 2699 : case IFN_ADD_OVERFLOW:
7039 2699 : case IFN_SUB_OVERFLOW:
7040 2699 : case IFN_UBSAN_CHECK_ADD:
7041 2699 : case IFN_UBSAN_CHECK_SUB:
7042 2699 : if (bitint_big_endian)
7043 : {
7044 0 : lhs = gimple_call_lhs (stmt);
7045 0 : if (lhs)
7046 79301 : muldiv_p = true;
7047 : }
7048 : break;
7049 1478 : case IFN_MUL_OVERFLOW:
7050 1478 : case IFN_UBSAN_CHECK_MUL:
7051 1478 : case IFN_BSWAP:
7052 1478 : case IFN_BITREVERSE:
7053 1478 : lhs = gimple_call_lhs (stmt);
7054 1478 : if (lhs)
7055 79301 : muldiv_p = true;
7056 : break;
7057 : default:
7058 : break;
7059 : }
7060 :
7061 158602 : auto_vec<tree, 16> worklist;
7062 79301 : ssa_op_iter iter;
7063 79301 : tree var;
7064 : /* On little-endian, mergeable ops process limbs from 0 up so except
7065 : for multiplication/division/modulo there is no risk in using the
7066 : same underlying variable for lhs and some operand, even when casts
7067 : are involved, the lhs limb is stored only after processing the source
7068 : limbs with the same index.
7069 : For multiplication/division/modulo, the libgcc library function requires
7070 : no aliasing between result and sources.
7071 : On big-endian, even mergeable ops limb processing can be problematic
7072 : though, because it can apply various index corrections e.g. when there
7073 : is a cast from operand with different number of limbs. So, make the
7074 : lhs conflict with all the operands which are (for now virtually) used on
7075 : the current stmt if there is any mismatch in the number of limbs between
7076 : operands and the lhs. */
7077 79301 : if (bitint_big_endian && lhs && !muldiv_p)
7078 : {
7079 0 : tree ltype = TREE_TYPE (lhs);
7080 0 : if (TREE_CODE (ltype) == COMPLEX_TYPE)
7081 : muldiv_p = true;
7082 0 : else if (TREE_CODE (lhs) == SSA_NAME
7083 0 : && BITINT_TYPE_P (ltype)
7084 0 : && bitint_precision_kind (ltype) >= bitint_prec_large)
7085 : {
7086 0 : unsigned lnelts = CEIL (TYPE_PRECISION (ltype), limb_prec);
7087 0 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_USE)
7088 : {
7089 0 : tree type = TREE_TYPE (var);
7090 0 : if (TREE_CODE (type) == COMPLEX_TYPE)
7091 0 : type = TREE_TYPE (type);
7092 0 : if (BITINT_TYPE_P (type)
7093 0 : && bitint_precision_kind (type) >= bitint_prec_large)
7094 : {
7095 0 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7096 : {
7097 0 : unsigned nelts = CEIL (TYPE_PRECISION (type), limb_prec);
7098 0 : if (TREE_CODE (TREE_TYPE (var)) == COMPLEX_TYPE
7099 0 : || lnelts != nelts)
7100 : {
7101 0 : muldiv_p = true;
7102 : break;
7103 : }
7104 : }
7105 : else
7106 0 : worklist.safe_push (var);
7107 : }
7108 : }
7109 :
7110 0 : while (!muldiv_p && worklist.length () > 0)
7111 : {
7112 0 : tree s = worklist.pop ();
7113 0 : FOR_EACH_SSA_TREE_OPERAND (var, SSA_NAME_DEF_STMT (s), iter,
7114 : SSA_OP_USE)
7115 : {
7116 0 : tree type = TREE_TYPE (var);
7117 0 : if (TREE_CODE (type) == COMPLEX_TYPE)
7118 0 : type = TREE_TYPE (type);
7119 0 : if (BITINT_TYPE_P (type)
7120 0 : && bitint_precision_kind (type) >= bitint_prec_large)
7121 : {
7122 0 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7123 : {
7124 0 : unsigned nelts = CEIL (TYPE_PRECISION (type),
7125 : limb_prec);
7126 0 : if (TREE_CODE (TREE_TYPE (var)) == COMPLEX_TYPE
7127 0 : || lnelts != nelts)
7128 : {
7129 : muldiv_p = true;
7130 : break;
7131 : }
7132 : }
7133 : else
7134 0 : worklist.safe_push (var);
7135 : }
7136 : }
7137 : }
7138 0 : worklist.truncate (0);
7139 : }
7140 : }
7141 :
7142 79301 : if (!muldiv_p)
7143 : {
7144 : /* For stmts with more than one SSA_NAME definition pretend all the
7145 : SSA_NAME outputs but the first one are live at this point, so
7146 : that conflicts are added in between all those even when they are
7147 : actually not really live after the asm, because expansion might
7148 : copy those into pseudos after the asm and if multiple outputs
7149 : share the same partition, it might overwrite those that should
7150 : be live. E.g.
7151 : asm volatile (".." : "=r" (a) : "=r" (b) : "0" (a), "1" (a));
7152 : return a;
7153 : See PR70593. */
7154 77666 : bool first = true;
7155 113661 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
7156 35995 : if (first)
7157 : first = false;
7158 : else
7159 0 : use (live, var);
7160 :
7161 113661 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
7162 35995 : def (live, var, graph);
7163 : }
7164 :
7165 135181 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_USE)
7166 : {
7167 55880 : tree type = TREE_TYPE (var);
7168 55880 : if (TREE_CODE (type) == COMPLEX_TYPE)
7169 6313 : type = TREE_TYPE (type);
7170 18845 : if (BITINT_TYPE_P (type)
7171 55897 : && bitint_precision_kind (type) >= bitint_prec_large)
7172 : {
7173 36053 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7174 30955 : use (live, var);
7175 : else
7176 5098 : worklist.safe_push (var);
7177 : }
7178 : }
7179 :
7180 87396 : while (worklist.length () > 0)
7181 : {
7182 8095 : tree s = worklist.pop ();
7183 16928 : FOR_EACH_SSA_TREE_OPERAND (var, SSA_NAME_DEF_STMT (s), iter, SSA_OP_USE)
7184 : {
7185 8833 : tree type = TREE_TYPE (var);
7186 8833 : if (TREE_CODE (type) == COMPLEX_TYPE)
7187 1675 : type = TREE_TYPE (type);
7188 536 : if (BITINT_TYPE_P (type)
7189 8838 : && bitint_precision_kind (type) >= bitint_prec_large)
7190 : {
7191 8167 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7192 5170 : use (live, var);
7193 : else
7194 2997 : worklist.safe_push (var);
7195 : }
7196 : }
7197 : }
7198 :
7199 79301 : if (muldiv_p)
7200 1635 : def (live, lhs, graph);
7201 : }
7202 :
7203 : /* If STMT is .{ADD,SUB,MUL}_OVERFLOW with INTEGER_CST arguments,
7204 : return the largest bitint_prec_kind of them, otherwise return
7205 : bitint_prec_small. */
7206 :
7207 : static bitint_prec_kind
7208 208412 : arith_overflow_arg_kind (gimple *stmt)
7209 : {
7210 208412 : bitint_prec_kind ret = bitint_prec_small;
7211 208412 : if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
7212 88453 : switch (gimple_call_internal_fn (stmt))
7213 : {
7214 : case IFN_ADD_OVERFLOW:
7215 : case IFN_SUB_OVERFLOW:
7216 : case IFN_MUL_OVERFLOW:
7217 224451 : for (int i = 0; i < 2; ++i)
7218 : {
7219 149634 : tree a = gimple_call_arg (stmt, i);
7220 149634 : if (TREE_CODE (a) == INTEGER_CST
7221 149634 : && BITINT_TYPE_P (TREE_TYPE (a)))
7222 : {
7223 5930 : bitint_prec_kind kind = bitint_precision_kind (TREE_TYPE (a));
7224 149634 : ret = MAX (ret, kind);
7225 : }
7226 : }
7227 : break;
7228 : default:
7229 : break;
7230 : }
7231 208412 : return ret;
7232 : }
7233 :
7234 : /* Entry point for _BitInt(N) operation lowering during optimization. */
7235 :
7236 : static unsigned int
7237 1494831 : gimple_lower_bitint (void)
7238 : {
7239 1494831 : small_max_prec = mid_min_prec = large_min_prec = huge_min_prec = 0;
7240 1494831 : limb_prec = abi_limb_prec = 0;
7241 1494831 : bitint_big_endian = false;
7242 :
7243 1494831 : unsigned int i;
7244 64216538 : for (i = 0; i < num_ssa_names; ++i)
7245 : {
7246 62729051 : tree s = ssa_name (i);
7247 62729051 : if (s == NULL)
7248 13361038 : continue;
7249 49368013 : tree type = TREE_TYPE (s);
7250 49368013 : if (TREE_CODE (type) == COMPLEX_TYPE)
7251 : {
7252 197736 : if (arith_overflow_arg_kind (SSA_NAME_DEF_STMT (s))
7253 : != bitint_prec_small)
7254 : break;
7255 197623 : type = TREE_TYPE (type);
7256 : }
7257 49351096 : if (BITINT_TYPE_P (type)
7258 49367907 : && bitint_precision_kind (type) != bitint_prec_small)
7259 : break;
7260 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
7261 : into memory. Such functions could have no large/huge SSA_NAMEs. */
7262 49360736 : if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
7263 : {
7264 21569239 : gimple *g = SSA_NAME_DEF_STMT (s);
7265 21569239 : if (is_gimple_assign (g) && gimple_store_p (g))
7266 : {
7267 10955753 : tree t = gimple_assign_rhs1 (g);
7268 21909055 : if (BITINT_TYPE_P (TREE_TYPE (t))
7269 10955753 : && (bitint_precision_kind (TREE_TYPE (t))
7270 : >= bitint_prec_large))
7271 : break;
7272 : }
7273 : }
7274 : /* Similarly, e.g. with -frounding-math casts from _BitInt INTEGER_CSTs
7275 : to floating point types need to be rewritten. */
7276 27791497 : else if (SCALAR_FLOAT_TYPE_P (type))
7277 : {
7278 2314213 : gimple *g = SSA_NAME_DEF_STMT (s);
7279 2314213 : if (is_gimple_assign (g) && gimple_assign_rhs_code (g) == FLOAT_EXPR)
7280 : {
7281 128198 : tree t = gimple_assign_rhs1 (g);
7282 128198 : if (TREE_CODE (t) == INTEGER_CST
7283 110 : && BITINT_TYPE_P (TREE_TYPE (t))
7284 128199 : && (bitint_precision_kind (TREE_TYPE (t))
7285 : != bitint_prec_small))
7286 : break;
7287 : }
7288 : }
7289 : }
7290 2989662 : if (i == num_ssa_names)
7291 : return 0;
7292 :
7293 7344 : basic_block bb;
7294 7344 : auto_vec<gimple *, 4> switch_statements;
7295 46147 : FOR_EACH_BB_FN (bb, cfun)
7296 : {
7297 115643 : if (gswitch *swtch = safe_dyn_cast <gswitch *> (*gsi_last_bb (bb)))
7298 : {
7299 23 : tree idx = gimple_switch_index (swtch);
7300 31 : if (!BITINT_TYPE_P (TREE_TYPE (idx))
7301 23 : || bitint_precision_kind (TREE_TYPE (idx)) < bitint_prec_large)
7302 12 : continue;
7303 :
7304 11 : if (optimize)
7305 6 : group_case_labels_stmt (swtch);
7306 11 : if (gimple_switch_num_labels (swtch) == 1)
7307 : {
7308 0 : single_succ_edge (bb)->flags |= EDGE_FALLTHRU;
7309 0 : gimple_stmt_iterator gsi = gsi_for_stmt (swtch);
7310 0 : gsi_remove (&gsi, true);
7311 : }
7312 : else
7313 11 : switch_statements.safe_push (swtch);
7314 : }
7315 : }
7316 :
7317 7344 : if (!switch_statements.is_empty ())
7318 : {
7319 11 : bool expanded = false;
7320 11 : gimple *stmt;
7321 11 : unsigned int j;
7322 11 : i = 0;
7323 22 : FOR_EACH_VEC_ELT (switch_statements, j, stmt)
7324 : {
7325 11 : gswitch *swtch = as_a<gswitch *> (stmt);
7326 11 : tree_switch_conversion::switch_decision_tree dt (swtch);
7327 11 : expanded |= dt.analyze_switch_statement ();
7328 11 : }
7329 :
7330 11 : if (expanded)
7331 : {
7332 11 : free_dominance_info (CDI_DOMINATORS);
7333 11 : free_dominance_info (CDI_POST_DOMINATORS);
7334 11 : mark_virtual_operands_for_renaming (cfun);
7335 11 : cleanup_tree_cfg (TODO_update_ssa);
7336 : }
7337 : }
7338 :
7339 7344 : if (flag_sanitize & (SANITIZE_ADDRESS | SANITIZE_HWADDRESS))
7340 : {
7341 14 : hash_map<tree, tree> shadow_vars_mapping;
7342 14 : bool need_commit_edge_insert = false;
7343 14 : bool any_asan_poison = false;
7344 129 : for (unsigned j = 0; j < num_ssa_names; ++j)
7345 : {
7346 115 : tree s = ssa_name (j);
7347 115 : if (s == NULL)
7348 31 : continue;
7349 84 : tree type = TREE_TYPE (s);
7350 37 : if (BITINT_TYPE_P (type)
7351 47 : && gimple_call_internal_p (SSA_NAME_DEF_STMT (s),
7352 : IFN_ASAN_POISON)
7353 112 : && bitint_precision_kind (type) >= bitint_prec_large)
7354 : {
7355 28 : any_asan_poison = true;
7356 28 : gimple_stmt_iterator gsi = gsi_for_stmt (SSA_NAME_DEF_STMT (s));
7357 28 : asan_expand_poison_ifn (&gsi, &need_commit_edge_insert,
7358 : shadow_vars_mapping);
7359 : }
7360 : }
7361 14 : if (any_asan_poison)
7362 : {
7363 14 : if (need_commit_edge_insert)
7364 0 : gsi_commit_edge_inserts ();
7365 14 : i = 0;
7366 14 : free_dominance_info (CDI_DOMINATORS);
7367 14 : free_dominance_info (CDI_POST_DOMINATORS);
7368 14 : mark_virtual_operands_for_renaming (cfun);
7369 14 : cleanup_tree_cfg (TODO_update_ssa);
7370 : }
7371 14 : }
7372 :
7373 7344 : struct bitint_large_huge large_huge;
7374 7344 : bool has_large_huge_parm_result = false;
7375 7344 : bool has_large_huge = false;
7376 7344 : unsigned int ret = 0, first_large_huge = ~0U;
7377 7344 : bool edge_insertions = false;
7378 126551 : for (; i < num_ssa_names; ++i)
7379 : {
7380 119207 : tree s = ssa_name (i);
7381 119207 : if (s == NULL)
7382 2789 : continue;
7383 116418 : tree type = TREE_TYPE (s);
7384 116418 : if (TREE_CODE (type) == COMPLEX_TYPE)
7385 : {
7386 5353 : if (arith_overflow_arg_kind (SSA_NAME_DEF_STMT (s))
7387 : >= bitint_prec_large)
7388 1958 : has_large_huge = true;
7389 5353 : type = TREE_TYPE (type);
7390 : }
7391 70274 : if (BITINT_TYPE_P (type)
7392 116444 : && bitint_precision_kind (type) >= bitint_prec_large)
7393 : {
7394 36138 : if (first_large_huge == ~0U)
7395 5801 : first_large_huge = i;
7396 36138 : gimple *stmt = SSA_NAME_DEF_STMT (s), *g;
7397 36138 : gimple_stmt_iterator gsi;
7398 36138 : tree_code rhs_code;
7399 : /* Unoptimize certain constructs to simpler alternatives to
7400 : avoid having to lower all of them. */
7401 36138 : if (is_gimple_assign (stmt) && gimple_bb (stmt))
7402 22259 : switch (rhs_code = gimple_assign_rhs_code (stmt))
7403 : {
7404 : default:
7405 : break;
7406 362 : case MULT_EXPR:
7407 362 : case TRUNC_DIV_EXPR:
7408 362 : case EXACT_DIV_EXPR:
7409 362 : case TRUNC_MOD_EXPR:
7410 362 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s))
7411 : {
7412 2 : location_t loc = gimple_location (stmt);
7413 2 : gsi = gsi_for_stmt (stmt);
7414 2 : tree rhs1 = gimple_assign_rhs1 (stmt);
7415 2 : tree rhs2 = gimple_assign_rhs2 (stmt);
7416 : /* For multiplication and division with (ab)
7417 : lhs and one or both operands force the operands
7418 : into new SSA_NAMEs to avoid coalescing failures. */
7419 2 : if (TREE_CODE (rhs1) == SSA_NAME
7420 2 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1))
7421 : {
7422 2 : first_large_huge = 0;
7423 2 : tree t = make_ssa_name (TREE_TYPE (rhs1));
7424 2 : g = gimple_build_assign (t, SSA_NAME, rhs1);
7425 2 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7426 2 : gimple_set_location (g, loc);
7427 2 : gimple_assign_set_rhs1 (stmt, t);
7428 2 : if (rhs1 == rhs2)
7429 : {
7430 0 : gimple_assign_set_rhs2 (stmt, t);
7431 0 : rhs2 = t;
7432 : }
7433 2 : update_stmt (stmt);
7434 : }
7435 2 : if (TREE_CODE (rhs2) == SSA_NAME
7436 2 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs2))
7437 : {
7438 0 : first_large_huge = 0;
7439 0 : tree t = make_ssa_name (TREE_TYPE (rhs2));
7440 0 : g = gimple_build_assign (t, SSA_NAME, rhs2);
7441 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7442 0 : gimple_set_location (g, loc);
7443 0 : gimple_assign_set_rhs2 (stmt, t);
7444 0 : update_stmt (stmt);
7445 : }
7446 : }
7447 : break;
7448 3 : case LROTATE_EXPR:
7449 3 : case RROTATE_EXPR:
7450 3 : {
7451 3 : first_large_huge = 0;
7452 3 : location_t loc = gimple_location (stmt);
7453 3 : gsi = gsi_for_stmt (stmt);
7454 3 : tree rhs1 = gimple_assign_rhs1 (stmt);
7455 3 : tree type = TREE_TYPE (rhs1);
7456 3 : tree n = gimple_assign_rhs2 (stmt), m;
7457 3 : tree p = build_int_cst (TREE_TYPE (n),
7458 3 : TYPE_PRECISION (type));
7459 3 : if (TREE_CODE (n) == INTEGER_CST)
7460 : {
7461 0 : if (integer_zerop (n))
7462 : m = n;
7463 : else
7464 0 : m = fold_build2 (MINUS_EXPR, TREE_TYPE (n), p, n);
7465 : }
7466 : else
7467 : {
7468 3 : tree tem = make_ssa_name (TREE_TYPE (n));
7469 3 : g = gimple_build_assign (tem, MINUS_EXPR, p, n);
7470 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7471 3 : gimple_set_location (g, loc);
7472 3 : m = make_ssa_name (TREE_TYPE (n));
7473 3 : g = gimple_build_assign (m, TRUNC_MOD_EXPR, tem, p);
7474 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7475 3 : gimple_set_location (g, loc);
7476 : }
7477 3 : if (!TYPE_UNSIGNED (type))
7478 : {
7479 0 : tree utype = build_bitint_type (TYPE_PRECISION (type),
7480 : 1);
7481 0 : if (TREE_CODE (rhs1) == INTEGER_CST)
7482 0 : rhs1 = fold_convert (utype, rhs1);
7483 : else
7484 : {
7485 0 : tree t = make_ssa_name (type);
7486 0 : g = gimple_build_assign (t, NOP_EXPR, rhs1);
7487 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7488 0 : gimple_set_location (g, loc);
7489 : }
7490 : }
7491 4 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7492 : rhs_code == LROTATE_EXPR
7493 : ? LSHIFT_EXPR : RSHIFT_EXPR,
7494 : rhs1, n);
7495 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7496 3 : gimple_set_location (g, loc);
7497 3 : tree op1 = gimple_assign_lhs (g);
7498 4 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7499 : rhs_code == LROTATE_EXPR
7500 : ? RSHIFT_EXPR : LSHIFT_EXPR,
7501 : rhs1, m);
7502 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7503 3 : gimple_set_location (g, loc);
7504 3 : tree op2 = gimple_assign_lhs (g);
7505 3 : tree lhs = gimple_assign_lhs (stmt);
7506 3 : if (!TYPE_UNSIGNED (type))
7507 : {
7508 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (op1)),
7509 : BIT_IOR_EXPR, op1, op2);
7510 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7511 0 : gimple_set_location (g, loc);
7512 0 : g = gimple_build_assign (lhs, NOP_EXPR,
7513 : gimple_assign_lhs (g));
7514 : }
7515 : else
7516 3 : g = gimple_build_assign (lhs, BIT_IOR_EXPR, op1, op2);
7517 3 : gsi_replace (&gsi, g, true);
7518 3 : gimple_set_location (g, loc);
7519 : }
7520 3 : break;
7521 17 : case ABS_EXPR:
7522 17 : case ABSU_EXPR:
7523 17 : case MIN_EXPR:
7524 17 : case MAX_EXPR:
7525 17 : case COND_EXPR:
7526 17 : first_large_huge = 0;
7527 17 : gsi = gsi_for_stmt (stmt);
7528 17 : tree lhs = gimple_assign_lhs (stmt);
7529 17 : tree rhs1 = gimple_assign_rhs1 (stmt), rhs2 = NULL_TREE;
7530 17 : location_t loc = gimple_location (stmt);
7531 17 : if (rhs_code == ABS_EXPR)
7532 4 : g = gimple_build_cond (LT_EXPR, rhs1,
7533 4 : build_zero_cst (TREE_TYPE (rhs1)),
7534 : NULL_TREE, NULL_TREE);
7535 13 : else if (rhs_code == ABSU_EXPR)
7536 : {
7537 4 : rhs2 = make_ssa_name (TREE_TYPE (lhs));
7538 4 : g = gimple_build_assign (rhs2, NOP_EXPR, rhs1);
7539 4 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7540 4 : gimple_set_location (g, loc);
7541 4 : g = gimple_build_cond (LT_EXPR, rhs1,
7542 4 : build_zero_cst (TREE_TYPE (rhs1)),
7543 : NULL_TREE, NULL_TREE);
7544 4 : rhs1 = rhs2;
7545 : }
7546 9 : else if (rhs_code == MIN_EXPR || rhs_code == MAX_EXPR)
7547 : {
7548 9 : rhs2 = gimple_assign_rhs2 (stmt);
7549 9 : if (TREE_CODE (rhs1) == INTEGER_CST)
7550 0 : std::swap (rhs1, rhs2);
7551 9 : g = gimple_build_cond (LT_EXPR, rhs1, rhs2,
7552 : NULL_TREE, NULL_TREE);
7553 9 : if (rhs_code == MAX_EXPR)
7554 5 : std::swap (rhs1, rhs2);
7555 : }
7556 : else
7557 : {
7558 0 : g = gimple_build_cond (NE_EXPR, rhs1,
7559 0 : build_zero_cst (TREE_TYPE (rhs1)),
7560 : NULL_TREE, NULL_TREE);
7561 0 : rhs1 = gimple_assign_rhs2 (stmt);
7562 0 : rhs2 = gimple_assign_rhs3 (stmt);
7563 : }
7564 17 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7565 17 : gimple_set_location (g, loc);
7566 17 : edge e1 = split_block (gsi_bb (gsi), g);
7567 17 : edge e2 = split_block (e1->dest, (gimple *) NULL);
7568 17 : edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
7569 17 : e3->probability = profile_probability::even ();
7570 17 : e1->flags = EDGE_TRUE_VALUE;
7571 17 : e1->probability = e3->probability.invert ();
7572 17 : if (dom_info_available_p (CDI_DOMINATORS))
7573 9 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
7574 17 : if (rhs_code == ABS_EXPR || rhs_code == ABSU_EXPR)
7575 : {
7576 8 : gsi = gsi_after_labels (e1->dest);
7577 8 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7578 : NEGATE_EXPR, rhs1);
7579 8 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7580 8 : gimple_set_location (g, loc);
7581 8 : rhs2 = gimple_assign_lhs (g);
7582 8 : std::swap (rhs1, rhs2);
7583 : }
7584 17 : gsi = gsi_for_stmt (stmt);
7585 17 : gsi_remove (&gsi, true);
7586 17 : gphi *phi = create_phi_node (lhs, e2->dest);
7587 17 : add_phi_arg (phi, rhs1, e2, UNKNOWN_LOCATION);
7588 17 : add_phi_arg (phi, rhs2, e3, UNKNOWN_LOCATION);
7589 17 : break;
7590 : }
7591 13879 : else if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
7592 76 : && is_gimple_call (stmt)
7593 13899 : && gimple_call_internal_p (stmt))
7594 20 : switch (gimple_call_internal_fn (stmt))
7595 : {
7596 12 : case IFN_UBSAN_CHECK_ADD:
7597 12 : case IFN_UBSAN_CHECK_SUB:
7598 12 : if (!bitint_big_endian)
7599 : break;
7600 : /* FALLTHRU */
7601 : case IFN_UBSAN_CHECK_MUL:
7602 : case IFN_BSWAP:
7603 : case IFN_BITREVERSE:
7604 22 : for (unsigned i = 0; i < gimple_call_num_args (stmt); ++i)
7605 : {
7606 14 : location_t loc = gimple_location (stmt);
7607 14 : gsi = gsi_for_stmt (stmt);
7608 : /* Similar case to multiplication/division with (ab)
7609 : above for internal functions which set muldiv_p. */
7610 14 : tree arg = gimple_call_arg (stmt, i);
7611 14 : if (TREE_CODE (arg) == SSA_NAME
7612 14 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg))
7613 : {
7614 8 : first_large_huge = 0;
7615 8 : tree t = make_ssa_name (TREE_TYPE (arg));
7616 8 : g = gimple_build_assign (t, SSA_NAME, arg);
7617 8 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7618 8 : gimple_set_location (g, loc);
7619 8 : gimple_call_set_arg (stmt, i, t);
7620 8 : if (i == 0
7621 8 : && gimple_call_num_args (stmt) >= 2
7622 14 : && gimple_call_arg (stmt, 1) == arg)
7623 0 : gimple_call_set_arg (stmt, 1, t);
7624 8 : update_stmt (stmt);
7625 : }
7626 : }
7627 : break;
7628 : default:
7629 : break;
7630 : }
7631 : }
7632 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
7633 : into memory. Such functions could have no large/huge SSA_NAMEs. */
7634 80280 : else if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
7635 : {
7636 51190 : gimple *g = SSA_NAME_DEF_STMT (s);
7637 51190 : if (is_gimple_assign (g) && gimple_store_p (g))
7638 : {
7639 16381 : tree t = gimple_assign_rhs1 (g);
7640 18564 : if (BITINT_TYPE_P (TREE_TYPE (t))
7641 16381 : && (bitint_precision_kind (TREE_TYPE (t))
7642 : >= bitint_prec_large))
7643 : has_large_huge = true;
7644 : }
7645 : }
7646 : /* Similarly, e.g. with -frounding-math casts from _BitInt INTEGER_CSTs
7647 : to floating point types need to be rewritten. */
7648 29090 : else if (SCALAR_FLOAT_TYPE_P (type))
7649 : {
7650 686 : gimple *g = SSA_NAME_DEF_STMT (s);
7651 686 : if (is_gimple_assign (g) && gimple_assign_rhs_code (g) == FLOAT_EXPR)
7652 : {
7653 179 : tree t = gimple_assign_rhs1 (g);
7654 179 : if (TREE_CODE (t) == INTEGER_CST
7655 1 : && BITINT_TYPE_P (TREE_TYPE (t))
7656 180 : && (bitint_precision_kind (TREE_TYPE (t))
7657 : >= bitint_prec_large))
7658 : has_large_huge = true;
7659 : }
7660 : }
7661 : }
7662 103127 : for (i = first_large_huge; i < num_ssa_names; ++i)
7663 : {
7664 95783 : tree s = ssa_name (i);
7665 95783 : if (s == NULL)
7666 2461 : continue;
7667 93322 : tree type = TREE_TYPE (s);
7668 93322 : if (TREE_CODE (type) == COMPLEX_TYPE)
7669 4124 : type = TREE_TYPE (type);
7670 56546 : if (BITINT_TYPE_P (type)
7671 93346 : && bitint_precision_kind (type) >= bitint_prec_large)
7672 : {
7673 36138 : use_operand_p use_p;
7674 36138 : gimple *use_stmt;
7675 36138 : has_large_huge = true;
7676 37802 : if (optimize
7677 53160 : && optimizable_arith_overflow (SSA_NAME_DEF_STMT (s)))
7678 6572 : continue;
7679 : /* Ignore large/huge _BitInt SSA_NAMEs which have single use in
7680 : the same bb and could be handled in the same loop with the
7681 : immediate use. */
7682 34474 : if (optimize
7683 15358 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
7684 15291 : && single_imm_use (s, &use_p, &use_stmt)
7685 49153 : && gimple_bb (SSA_NAME_DEF_STMT (s)) == gimple_bb (use_stmt))
7686 : {
7687 10324 : if (mergeable_op (SSA_NAME_DEF_STMT (s)))
7688 : {
7689 2219 : if (mergeable_op (use_stmt))
7690 1910 : continue;
7691 309 : tree_code cmp_code = comparison_op (use_stmt, NULL, NULL);
7692 309 : if (cmp_code == EQ_EXPR || cmp_code == NE_EXPR)
7693 26 : continue;
7694 283 : if (gimple_assign_cast_p (use_stmt))
7695 : {
7696 128 : tree lhs = gimple_assign_lhs (use_stmt);
7697 256 : if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
7698 : /* Don't merge with VIEW_CONVERT_EXPRs to
7699 : huge INTEGER_TYPEs used sometimes in memcpy
7700 : expansion. */
7701 244 : && (TREE_CODE (TREE_TYPE (lhs)) != INTEGER_TYPE
7702 6 : || (TYPE_PRECISION (TREE_TYPE (lhs))
7703 12 : <= MAX_FIXED_MODE_SIZE)))
7704 116 : continue;
7705 : }
7706 155 : else if (gimple_store_p (use_stmt)
7707 0 : && is_gimple_assign (use_stmt)
7708 0 : && !gimple_has_volatile_ops (use_stmt)
7709 155 : && !stmt_ends_bb_p (use_stmt))
7710 0 : continue;
7711 : }
7712 8272 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (s)))
7713 : {
7714 862 : tree rhs1 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s));
7715 862 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
7716 : {
7717 20 : rhs1 = TREE_OPERAND (rhs1, 0);
7718 20 : if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
7719 16 : && !POINTER_TYPE_P (TREE_TYPE (rhs1))
7720 16 : && gimple_store_p (use_stmt))
7721 7 : continue;
7722 : }
7723 1710 : if (INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
7724 764 : && ((is_gimple_assign (use_stmt)
7725 717 : && (gimple_assign_rhs_code (use_stmt)
7726 : != COMPLEX_EXPR))
7727 47 : || gimple_code (use_stmt) == GIMPLE_COND)
7728 754 : && (!gimple_store_p (use_stmt)
7729 130 : || (is_gimple_assign (use_stmt)
7730 130 : && !gimple_has_volatile_ops (use_stmt)
7731 130 : && !stmt_ends_bb_p (use_stmt)))
7732 1609 : && (TREE_CODE (rhs1) != SSA_NAME
7733 754 : || !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)))
7734 : {
7735 754 : if (is_gimple_assign (use_stmt))
7736 717 : switch (gimple_assign_rhs_code (use_stmt))
7737 : {
7738 54 : case TRUNC_DIV_EXPR:
7739 54 : case EXACT_DIV_EXPR:
7740 54 : case TRUNC_MOD_EXPR:
7741 54 : case FLOAT_EXPR:
7742 : /* For division, modulo and casts to floating
7743 : point, avoid representing unsigned operands
7744 : using negative prec if they were sign-extended
7745 : from narrower precision. */
7746 54 : if (TYPE_UNSIGNED (TREE_TYPE (s))
7747 28 : && !TYPE_UNSIGNED (TREE_TYPE (rhs1))
7748 63 : && (TYPE_PRECISION (TREE_TYPE (s))
7749 9 : > TYPE_PRECISION (TREE_TYPE (rhs1))))
7750 8 : goto force_name;
7751 : /* FALLTHRU */
7752 113 : case MULT_EXPR:
7753 128 : if (!BITINT_TYPE_P (TREE_TYPE (rhs1))
7754 113 : || (bitint_precision_kind (TREE_TYPE (rhs1))
7755 : < bitint_prec_large))
7756 48 : continue;
7757 : /* Uses which use handle_operand_addr can't
7758 : deal with nested casts. */
7759 65 : if (TREE_CODE (rhs1) == SSA_NAME
7760 65 : && gimple_assign_cast_p
7761 65 : (SSA_NAME_DEF_STMT (rhs1))
7762 43 : && has_single_use (rhs1)
7763 108 : && (gimple_bb (SSA_NAME_DEF_STMT (rhs1))
7764 43 : == gimple_bb (SSA_NAME_DEF_STMT (s))))
7765 43 : goto force_name;
7766 : break;
7767 0 : case VIEW_CONVERT_EXPR:
7768 0 : {
7769 0 : tree lhs = gimple_assign_lhs (use_stmt);
7770 : /* Don't merge with VIEW_CONVERT_EXPRs to
7771 : non-integral types. */
7772 0 : if (!INTEGRAL_TYPE_P (TREE_TYPE (lhs)))
7773 0 : goto force_name;
7774 : /* Don't merge with VIEW_CONVERT_EXPRs to
7775 : huge INTEGER_TYPEs used sometimes in memcpy
7776 : expansion. */
7777 0 : if (TREE_CODE (TREE_TYPE (lhs)) == INTEGER_TYPE
7778 0 : && (TYPE_PRECISION (TREE_TYPE (lhs))
7779 0 : > MAX_FIXED_MODE_SIZE))
7780 0 : goto force_name;
7781 : }
7782 : break;
7783 : default:
7784 : break;
7785 : }
7786 832 : if (!BITINT_TYPE_P (TREE_TYPE (rhs1))
7787 655 : || (bitint_precision_kind (TREE_TYPE (rhs1))
7788 : < bitint_prec_large))
7789 239 : continue;
7790 416 : if ((TYPE_PRECISION (TREE_TYPE (rhs1))
7791 416 : >= TYPE_PRECISION (TREE_TYPE (s)))
7792 416 : && mergeable_op (use_stmt))
7793 60 : continue;
7794 : /* Prevent merging a widening non-mergeable cast
7795 : on result of some narrower mergeable op
7796 : together with later mergeable operations. E.g.
7797 : result of _BitInt(223) addition shouldn't be
7798 : sign-extended to _BitInt(513) and have another
7799 : _BitInt(513) added to it, as handle_plus_minus
7800 : with its PHI node handling inside of handle_cast
7801 : will not work correctly. An exception is if
7802 : use_stmt is a store, this is handled directly
7803 : in lower_mergeable_stmt. */
7804 705 : if (TREE_CODE (rhs1) != SSA_NAME
7805 356 : || !has_single_use (rhs1)
7806 270 : || (gimple_bb (SSA_NAME_DEF_STMT (rhs1))
7807 270 : != gimple_bb (SSA_NAME_DEF_STMT (s)))
7808 214 : || !mergeable_op (SSA_NAME_DEF_STMT (rhs1))
7809 441 : || gimple_store_p (use_stmt))
7810 349 : continue;
7811 7 : if ((TYPE_PRECISION (TREE_TYPE (rhs1))
7812 7 : < TYPE_PRECISION (TREE_TYPE (s)))
7813 9 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (rhs1)))
7814 : {
7815 : /* Another exception is if the widening cast is
7816 : from mergeable same precision cast from something
7817 : not mergeable. */
7818 0 : tree rhs2
7819 0 : = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (rhs1));
7820 0 : if (BITINT_TYPE_P (TREE_TYPE (rhs2))
7821 0 : && (TYPE_PRECISION (TREE_TYPE (rhs1))
7822 0 : == TYPE_PRECISION (TREE_TYPE (rhs2))))
7823 : {
7824 0 : if (TREE_CODE (rhs2) != SSA_NAME
7825 0 : || !has_single_use (rhs2)
7826 0 : || (gimple_bb (SSA_NAME_DEF_STMT (rhs2))
7827 0 : != gimple_bb (SSA_NAME_DEF_STMT (s)))
7828 0 : || !mergeable_op (SSA_NAME_DEF_STMT (rhs2)))
7829 0 : continue;
7830 : }
7831 : }
7832 : }
7833 : }
7834 7518 : if (is_gimple_assign (SSA_NAME_DEF_STMT (s)))
7835 5411 : switch (gimple_assign_rhs_code (SSA_NAME_DEF_STMT (s)))
7836 : {
7837 1794 : case REALPART_EXPR:
7838 1794 : case IMAGPART_EXPR:
7839 1794 : {
7840 1794 : gimple *ds = SSA_NAME_DEF_STMT (s);
7841 1794 : tree rhs1 = gimple_assign_rhs1 (ds);
7842 1794 : rhs1 = TREE_OPERAND (rhs1, 0);
7843 1794 : if (TREE_CODE (rhs1) == SSA_NAME)
7844 : {
7845 1794 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
7846 1794 : if (optimizable_arith_overflow (g))
7847 : {
7848 1650 : if (gimple_assign_rhs_code (ds) == IMAGPART_EXPR)
7849 1640 : continue;
7850 10 : if (gimple_store_p (use_stmt))
7851 : {
7852 : /* Punt if the cast use of IMAGPART_EXPR stmt
7853 : appears before the store use_stmt, because
7854 : optimizable arith overflow can't be
7855 : lowered at the store location in that case.
7856 : See PR121828. */
7857 10 : gimple_stmt_iterator gsi
7858 10 : = gsi_for_stmt (use_stmt);
7859 10 : unsigned int cnt = 0;
7860 12 : do
7861 : {
7862 12 : gsi_prev_nondebug (&gsi);
7863 12 : if (gsi_end_p (gsi))
7864 : break;
7865 12 : gimple *g2 = gsi_stmt (gsi);
7866 12 : if (g2 == ds)
7867 : break;
7868 3 : if (++cnt == 64)
7869 : break;
7870 3 : if (!gimple_assign_cast_p (g2))
7871 2 : continue;
7872 1 : tree rhs2 = gimple_assign_rhs1 (g2);
7873 1 : if (TREE_CODE (rhs2) != SSA_NAME)
7874 0 : continue;
7875 1 : gimple *g3 = SSA_NAME_DEF_STMT (rhs2);
7876 1 : if (!is_gimple_assign (g3))
7877 0 : continue;
7878 1 : if (gimple_assign_rhs_code (g3)
7879 : != IMAGPART_EXPR)
7880 0 : continue;
7881 1 : rhs2 = gimple_assign_rhs1 (g3);
7882 1 : rhs2 = TREE_OPERAND (rhs2, 0);
7883 1 : if (rhs2 != rhs1)
7884 0 : continue;
7885 : cnt = 64;
7886 : break;
7887 : }
7888 : while (1);
7889 10 : if (cnt == 64)
7890 : break;
7891 : }
7892 : }
7893 : }
7894 : }
7895 : /* FALLTHRU */
7896 837 : case LSHIFT_EXPR:
7897 837 : case RSHIFT_EXPR:
7898 837 : case MULT_EXPR:
7899 837 : case TRUNC_DIV_EXPR:
7900 837 : case EXACT_DIV_EXPR:
7901 837 : case TRUNC_MOD_EXPR:
7902 837 : case FIX_TRUNC_EXPR:
7903 837 : if (gimple_store_p (use_stmt)
7904 444 : && is_gimple_assign (use_stmt)
7905 444 : && !gimple_has_volatile_ops (use_stmt)
7906 1281 : && !stmt_ends_bb_p (use_stmt))
7907 : {
7908 444 : tree lhs = gimple_assign_lhs (use_stmt);
7909 : /* As multiply/division passes address of the lhs
7910 : to library function and that assumes it can extend
7911 : it to whole number of limbs, avoid merging those
7912 : with bit-field stores. Don't allow it for
7913 : shifts etc. either, so that the bit-field store
7914 : handling doesn't have to be done everywhere. */
7915 444 : if (TREE_CODE (lhs) == COMPONENT_REF
7916 444 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
7917 : break;
7918 441 : continue;
7919 441 : }
7920 : break;
7921 : default:
7922 : break;
7923 : }
7924 2107 : else if (is_gimple_call (SSA_NAME_DEF_STMT (s))
7925 2107 : && gimple_call_internal_p (SSA_NAME_DEF_STMT (s)))
7926 15 : switch (gimple_call_internal_fn (SSA_NAME_DEF_STMT (s)))
7927 : {
7928 13 : case IFN_BSWAP:
7929 13 : case IFN_BITREVERSE:
7930 13 : if (gimple_store_p (use_stmt)
7931 13 : && is_gimple_assign (use_stmt)
7932 13 : && !gimple_has_volatile_ops (use_stmt)
7933 26 : && !stmt_ends_bb_p (use_stmt))
7934 : {
7935 13 : tree lhs = gimple_assign_lhs (use_stmt);
7936 13 : if (TREE_CODE (lhs) == COMPONENT_REF
7937 13 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
7938 : break;
7939 13 : continue;
7940 13 : }
7941 : break;
7942 : default:
7943 : break;
7944 : }
7945 : }
7946 :
7947 : /* Also ignore uninitialized uses. */
7948 29574 : if (SSA_NAME_IS_DEFAULT_DEF (s)
7949 29574 : && (!SSA_NAME_VAR (s) || VAR_P (SSA_NAME_VAR (s))))
7950 59 : continue;
7951 :
7952 29566 : force_name:
7953 29566 : if (!large_huge.m_names)
7954 5678 : large_huge.m_names = BITMAP_ALLOC (NULL);
7955 29566 : bitmap_set_bit (large_huge.m_names, SSA_NAME_VERSION (s));
7956 29566 : if (has_single_use (s))
7957 : {
7958 : tree s2 = s;
7959 : /* The coalescing hook special cases SSA_NAME copies.
7960 : Make sure not to mark in m_single_use_names single
7961 : use SSA_NAMEs copied from non-single use SSA_NAMEs. */
7962 25864 : while (gimple_assign_copy_p (SSA_NAME_DEF_STMT (s2)))
7963 : {
7964 938 : s2 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s2));
7965 938 : if (TREE_CODE (s2) != SSA_NAME)
7966 : break;
7967 41 : if (!has_single_use (s2))
7968 : {
7969 : s2 = NULL_TREE;
7970 : break;
7971 : }
7972 : }
7973 25851 : if (s2)
7974 : {
7975 25823 : if (!large_huge.m_single_use_names)
7976 5564 : large_huge.m_single_use_names = BITMAP_ALLOC (NULL);
7977 25823 : bitmap_set_bit (large_huge.m_single_use_names,
7978 25823 : SSA_NAME_VERSION (s));
7979 : }
7980 : }
7981 29566 : if (SSA_NAME_VAR (s)
7982 7351 : && ((TREE_CODE (SSA_NAME_VAR (s)) == PARM_DECL
7983 5412 : && SSA_NAME_IS_DEFAULT_DEF (s))
7984 1976 : || TREE_CODE (SSA_NAME_VAR (s)) == RESULT_DECL))
7985 : has_large_huge_parm_result = true;
7986 29566 : if (optimize
7987 10450 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
7988 10383 : && gimple_assign_load_p (SSA_NAME_DEF_STMT (s))
7989 6062 : && !gimple_has_volatile_ops (SSA_NAME_DEF_STMT (s))
7990 32548 : && !stmt_ends_bb_p (SSA_NAME_DEF_STMT (s)))
7991 : {
7992 2982 : use_operand_p use_p;
7993 2982 : imm_use_iterator iter;
7994 2982 : bool optimizable_load = true;
7995 9011 : FOR_EACH_IMM_USE_FAST (use_p, iter, s)
7996 : {
7997 3145 : gimple *use_stmt = USE_STMT (use_p);
7998 3145 : if (is_gimple_debug (use_stmt))
7999 0 : continue;
8000 3145 : if (is_gimple_call (use_stmt)
8001 3145 : && gimple_call_internal_p (use_stmt))
8002 66 : switch (gimple_call_internal_fn (use_stmt))
8003 : {
8004 10 : case IFN_CLZ:
8005 10 : case IFN_CTZ:
8006 10 : case IFN_CLRSB:
8007 10 : case IFN_FFS:
8008 10 : case IFN_PARITY:
8009 10 : case IFN_POPCOUNT:
8010 10 : case IFN_BSWAP:
8011 10 : case IFN_BITREVERSE:
8012 10 : continue;
8013 : default:
8014 : break;
8015 : }
8016 3135 : if (gimple_code (use_stmt) == GIMPLE_PHI
8017 3123 : || is_gimple_call (use_stmt)
8018 3038 : || gimple_code (use_stmt) == GIMPLE_ASM
8019 6172 : || (is_gimple_assign (use_stmt)
8020 1903 : && (gimple_assign_rhs_code (use_stmt)
8021 : == COMPLEX_EXPR)))
8022 : {
8023 : optimizable_load = false;
8024 : break;
8025 : }
8026 2982 : }
8027 :
8028 2982 : ssa_op_iter oi;
8029 4027 : FOR_EACH_SSA_USE_OPERAND (use_p, SSA_NAME_DEF_STMT (s),
8030 : oi, SSA_OP_USE)
8031 : {
8032 1045 : tree s2 = USE_FROM_PTR (use_p);
8033 1045 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s2))
8034 : {
8035 : optimizable_load = false;
8036 : break;
8037 : }
8038 : }
8039 :
8040 2982 : if (optimizable_load && !stmt_ends_bb_p (SSA_NAME_DEF_STMT (s)))
8041 : {
8042 2884 : if (!large_huge.m_loads)
8043 312 : large_huge.m_loads = BITMAP_ALLOC (NULL);
8044 2884 : bitmap_set_bit (large_huge.m_loads, SSA_NAME_VERSION (s));
8045 : }
8046 : }
8047 : }
8048 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
8049 : into memory. Such functions could have no large/huge SSA_NAMEs. */
8050 57184 : else if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
8051 : {
8052 40400 : gimple *g = SSA_NAME_DEF_STMT (s);
8053 40400 : if (is_gimple_assign (g) && gimple_store_p (g))
8054 : {
8055 13739 : tree t = gimple_assign_rhs1 (g);
8056 15618 : if (BITINT_TYPE_P (TREE_TYPE (t))
8057 13739 : && bitint_precision_kind (TREE_TYPE (t)) >= bitint_prec_large)
8058 : has_large_huge = true;
8059 : }
8060 : }
8061 : }
8062 :
8063 7344 : if (large_huge.m_names || has_large_huge)
8064 : {
8065 5876 : ret = TODO_update_ssa_only_virtuals | TODO_cleanup_cfg;
8066 5876 : calculate_dominance_info (CDI_DOMINATORS);
8067 5876 : if (optimize)
8068 3093 : enable_ranger (cfun);
8069 5876 : if (large_huge.m_loads)
8070 : {
8071 312 : basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
8072 312 : entry->aux = NULL;
8073 624 : bitint_dom_walker (large_huge.m_names,
8074 312 : large_huge.m_loads).walk (entry);
8075 312 : bitmap_and_compl_into (large_huge.m_names, large_huge.m_loads);
8076 312 : clear_aux_for_blocks ();
8077 312 : BITMAP_FREE (large_huge.m_loads);
8078 : }
8079 5876 : large_huge.m_limb_type = build_nonstandard_integer_type (limb_prec, 1);
8080 5876 : large_huge.m_limb_size
8081 5876 : = tree_to_uhwi (TYPE_SIZE_UNIT (large_huge.m_limb_type));
8082 : }
8083 7344 : if (large_huge.m_names)
8084 : {
8085 5678 : large_huge.m_map
8086 11356 : = init_var_map (num_ssa_names, NULL, large_huge.m_names);
8087 5678 : coalesce_ssa_name (large_huge.m_map);
8088 5678 : partition_view_normal (large_huge.m_map);
8089 5678 : if (dump_file && (dump_flags & TDF_DETAILS))
8090 : {
8091 0 : fprintf (dump_file, "After Coalescing:\n");
8092 0 : dump_var_map (dump_file, large_huge.m_map);
8093 : }
8094 5678 : large_huge.m_vars
8095 5678 : = XCNEWVEC (tree, num_var_partitions (large_huge.m_map));
8096 5678 : bitmap_iterator bi;
8097 5678 : if (has_large_huge_parm_result)
8098 19718 : EXECUTE_IF_SET_IN_BITMAP (large_huge.m_names, 0, i, bi)
8099 : {
8100 15514 : tree s = ssa_name (i);
8101 15514 : if (SSA_NAME_VAR (s)
8102 6208 : && ((TREE_CODE (SSA_NAME_VAR (s)) == PARM_DECL
8103 5412 : && SSA_NAME_IS_DEFAULT_DEF (s))
8104 833 : || TREE_CODE (SSA_NAME_VAR (s)) == RESULT_DECL))
8105 : {
8106 5375 : int p = var_to_partition (large_huge.m_map, s);
8107 5375 : if (large_huge.m_vars[p] == NULL_TREE)
8108 : {
8109 5375 : large_huge.m_vars[p] = SSA_NAME_VAR (s);
8110 5375 : mark_addressable (SSA_NAME_VAR (s));
8111 : }
8112 : }
8113 : }
8114 5678 : tree atype = NULL_TREE;
8115 5678 : if (dump_file && (dump_flags & TDF_DETAILS))
8116 0 : fprintf (dump_file, "Mapping SSA_NAMEs to decls:\n");
8117 32890 : EXECUTE_IF_SET_IN_BITMAP (large_huge.m_names, 0, i, bi)
8118 : {
8119 27212 : tree s = ssa_name (i);
8120 27212 : int p = var_to_partition (large_huge.m_map, s);
8121 27212 : if (large_huge.m_vars[p] == NULL_TREE)
8122 : {
8123 18912 : if (atype == NULL_TREE
8124 33042 : || !tree_int_cst_equal (TYPE_SIZE (atype),
8125 14130 : TYPE_SIZE (TREE_TYPE (s))))
8126 : {
8127 7759 : unsigned HOST_WIDE_INT nelts
8128 7759 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (s))) / limb_prec;
8129 7759 : atype = build_array_type_nelts (large_huge.m_limb_type,
8130 7759 : nelts);
8131 : }
8132 18912 : large_huge.m_vars[p] = create_tmp_var (atype, "bitint");
8133 18912 : mark_addressable (large_huge.m_vars[p]);
8134 : }
8135 27212 : if (dump_file && (dump_flags & TDF_DETAILS))
8136 : {
8137 0 : print_generic_expr (dump_file, s, TDF_SLIM);
8138 0 : fprintf (dump_file, " -> ");
8139 0 : print_generic_expr (dump_file, large_huge.m_vars[p], TDF_SLIM);
8140 0 : fprintf (dump_file, "\n");
8141 : }
8142 : }
8143 : }
8144 :
8145 46434 : FOR_EACH_BB_REVERSE_FN (bb, cfun)
8146 : {
8147 39090 : gimple_stmt_iterator prev;
8148 197720 : for (gimple_stmt_iterator gsi = gsi_last_bb (bb); !gsi_end_p (gsi);
8149 119540 : gsi = prev)
8150 : {
8151 119540 : prev = gsi;
8152 119540 : gsi_prev (&prev);
8153 119540 : ssa_op_iter iter;
8154 119540 : gimple *stmt = gsi_stmt (gsi);
8155 119540 : if (is_gimple_debug (stmt))
8156 81634 : continue;
8157 114571 : bitint_prec_kind kind = bitint_prec_small;
8158 114571 : tree t;
8159 344747 : FOR_EACH_SSA_TREE_OPERAND (t, stmt, iter, SSA_OP_ALL_OPERANDS)
8160 230176 : if (BITINT_TYPE_P (TREE_TYPE (t)))
8161 : {
8162 79186 : bitint_prec_kind this_kind
8163 79186 : = bitint_precision_kind (TREE_TYPE (t));
8164 230384 : kind = MAX (kind, this_kind);
8165 : }
8166 114571 : if (is_gimple_assign (stmt) && gimple_store_p (stmt))
8167 : {
8168 16403 : t = gimple_assign_rhs1 (stmt);
8169 16403 : if (BITINT_TYPE_P (TREE_TYPE (t)))
8170 : {
8171 14202 : bitint_prec_kind this_kind
8172 14202 : = bitint_precision_kind (TREE_TYPE (t));
8173 14202 : kind = MAX (kind, this_kind);
8174 : }
8175 : }
8176 114571 : if (is_gimple_assign (stmt)
8177 114571 : && gimple_assign_rhs_code (stmt) == FLOAT_EXPR)
8178 : {
8179 181 : t = gimple_assign_rhs1 (stmt);
8180 210 : if (BITINT_TYPE_P (TREE_TYPE (t))
8181 183 : && TREE_CODE (t) == INTEGER_CST)
8182 : {
8183 1 : bitint_prec_kind this_kind
8184 1 : = bitint_precision_kind (TREE_TYPE (t));
8185 1 : kind = MAX (kind, this_kind);
8186 : }
8187 : }
8188 114571 : if (is_gimple_call (stmt))
8189 : {
8190 25859 : t = gimple_call_lhs (stmt);
8191 25859 : if (t && TREE_CODE (TREE_TYPE (t)) == COMPLEX_TYPE)
8192 : {
8193 5323 : bitint_prec_kind this_kind = arith_overflow_arg_kind (stmt);
8194 5323 : kind = MAX (kind, this_kind);
8195 5323 : if (BITINT_TYPE_P (TREE_TYPE (TREE_TYPE (t))))
8196 : {
8197 5198 : this_kind
8198 5198 : = bitint_precision_kind (TREE_TYPE (TREE_TYPE (t)));
8199 5198 : kind = MAX (kind, this_kind);
8200 : }
8201 : }
8202 : }
8203 114249 : if (kind == bitint_prec_small)
8204 45056 : continue;
8205 69515 : switch (gimple_code (stmt))
8206 : {
8207 11136 : case GIMPLE_CALL:
8208 : /* For now. We'll need to handle some internal functions and
8209 : perhaps some builtins. */
8210 11136 : if (kind == bitint_prec_middle)
8211 2280 : continue;
8212 : break;
8213 4 : case GIMPLE_ASM:
8214 4 : if (kind == bitint_prec_middle)
8215 1 : continue;
8216 : break;
8217 1124 : case GIMPLE_RETURN:
8218 1124 : continue;
8219 48628 : case GIMPLE_ASSIGN:
8220 48628 : if (gimple_clobber_p (stmt))
8221 3511 : continue;
8222 45117 : if (kind >= bitint_prec_large)
8223 : break;
8224 8732 : if (gimple_assign_single_p (stmt))
8225 : /* No need to lower copies, loads or stores. */
8226 5788 : continue;
8227 2944 : if (gimple_assign_cast_p (stmt))
8228 : {
8229 2379 : tree lhs = gimple_assign_lhs (stmt);
8230 2379 : tree rhs = gimple_assign_rhs1 (stmt);
8231 4758 : if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
8232 2379 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs))
8233 4752 : && (TYPE_PRECISION (TREE_TYPE (lhs))
8234 2373 : == TYPE_PRECISION (TREE_TYPE (rhs))))
8235 : /* No need to lower casts to same precision. */
8236 28 : continue;
8237 : }
8238 : break;
8239 : default:
8240 : break;
8241 1124 : }
8242 :
8243 11539 : if (kind == bitint_prec_middle)
8244 : {
8245 5041 : tree type = NULL_TREE;
8246 : /* Middle _BitInt(N) is rewritten to casts to INTEGER_TYPEs
8247 : with the same precision and back. */
8248 5041 : unsigned int nops = gimple_num_ops (stmt);
8249 16928 : for (unsigned int i = is_gimple_assign (stmt) ? 1 : 0;
8250 16928 : i < nops; ++i)
8251 11887 : if (tree op = gimple_op (stmt, i))
8252 : {
8253 7645 : tree nop = maybe_cast_middle_bitint (&gsi, op, type);
8254 7645 : if (nop != op)
8255 6765 : gimple_set_op (stmt, i, nop);
8256 880 : else if (COMPARISON_CLASS_P (op))
8257 : {
8258 0 : TREE_OPERAND (op, 0)
8259 0 : = maybe_cast_middle_bitint (&gsi,
8260 0 : TREE_OPERAND (op, 0),
8261 : type);
8262 0 : TREE_OPERAND (op, 1)
8263 0 : = maybe_cast_middle_bitint (&gsi,
8264 0 : TREE_OPERAND (op, 1),
8265 : type);
8266 : }
8267 880 : else if (TREE_CODE (op) == CASE_LABEL_EXPR)
8268 : {
8269 24 : CASE_LOW (op)
8270 24 : = maybe_cast_middle_bitint (&gsi, CASE_LOW (op),
8271 : type);
8272 48 : CASE_HIGH (op)
8273 48 : = maybe_cast_middle_bitint (&gsi, CASE_HIGH (op),
8274 : type);
8275 : }
8276 : }
8277 5041 : if (tree lhs = gimple_get_lhs (stmt))
8278 4464 : if (BITINT_TYPE_P (TREE_TYPE (lhs))
8279 2916 : && (bitint_precision_kind (TREE_TYPE (lhs))
8280 : == bitint_prec_middle))
8281 : {
8282 1367 : int prec = TYPE_PRECISION (TREE_TYPE (lhs));
8283 1367 : int uns = TYPE_UNSIGNED (TREE_TYPE (lhs));
8284 1367 : type = build_nonstandard_integer_type (prec, uns);
8285 1367 : tree lhs2 = make_ssa_name (type);
8286 1367 : gimple_set_lhs (stmt, lhs2);
8287 1367 : gimple *g = gimple_build_assign (lhs, NOP_EXPR, lhs2);
8288 1367 : if (stmt_ends_bb_p (stmt))
8289 : {
8290 4 : edge e = find_fallthru_edge (gsi_bb (gsi)->succs);
8291 4 : gsi_insert_on_edge (e, g);
8292 4 : edge_insertions = true;
8293 : }
8294 : else
8295 1363 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
8296 : }
8297 5041 : update_stmt (stmt);
8298 5041 : continue;
8299 5041 : }
8300 :
8301 51742 : if (tree lhs = gimple_get_lhs (stmt))
8302 45110 : if (TREE_CODE (lhs) == SSA_NAME)
8303 : {
8304 36104 : tree type = TREE_TYPE (lhs);
8305 36104 : if (TREE_CODE (type) == COMPLEX_TYPE)
8306 4197 : type = TREE_TYPE (type);
8307 5435 : if (BITINT_TYPE_P (type)
8308 30681 : && bitint_precision_kind (type) >= bitint_prec_large
8309 66583 : && (large_huge.m_names == NULL
8310 30274 : || !bitmap_bit_p (large_huge.m_names,
8311 30274 : SSA_NAME_VERSION (lhs))))
8312 8867 : continue;
8313 : }
8314 :
8315 42875 : large_huge.lower_stmt (stmt);
8316 : }
8317 :
8318 39090 : tree atype = NULL_TREE;
8319 47728 : for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
8320 8638 : gsi_next (&gsi))
8321 : {
8322 8638 : gphi *phi = gsi.phi ();
8323 8638 : tree lhs = gimple_phi_result (phi);
8324 17044 : if (!BITINT_TYPE_P (TREE_TYPE (lhs))
8325 8638 : || bitint_precision_kind (TREE_TYPE (lhs)) < bitint_prec_large)
8326 8447 : continue;
8327 191 : int p1 = var_to_partition (large_huge.m_map, lhs);
8328 191 : gcc_assert (large_huge.m_vars[p1] != NULL_TREE);
8329 : tree v1 = large_huge.m_vars[p1];
8330 688 : for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
8331 : {
8332 497 : tree arg = gimple_phi_arg_def (phi, i);
8333 497 : edge e = gimple_phi_arg_edge (phi, i);
8334 497 : gimple *g;
8335 497 : switch (TREE_CODE (arg))
8336 : {
8337 97 : case INTEGER_CST:
8338 97 : if (integer_zerop (arg) && VAR_P (v1))
8339 : {
8340 45 : tree zero = build_zero_cst (TREE_TYPE (v1));
8341 45 : g = gimple_build_assign (v1, zero);
8342 45 : gsi_insert_on_edge (e, g);
8343 45 : edge_insertions = true;
8344 159 : break;
8345 : }
8346 52 : int ext;
8347 52 : unsigned int min_prec, prec, rem;
8348 52 : tree c;
8349 52 : prec = TYPE_PRECISION (TREE_TYPE (arg));
8350 52 : rem = prec % (2 * limb_prec);
8351 52 : min_prec = bitint_min_cst_precision (arg, ext);
8352 52 : if (min_prec > prec - rem - 2 * limb_prec
8353 12 : && min_prec > (unsigned) limb_prec)
8354 : /* Constant which has enough significant bits that it
8355 : isn't worth trying to save .rodata space by extending
8356 : from smaller number. */
8357 : min_prec = prec;
8358 : else
8359 : {
8360 43 : min_prec = CEIL (min_prec, limb_prec) * limb_prec;
8361 43 : if (min_prec > (unsigned) limb_prec
8362 3 : && abi_limb_prec > limb_prec)
8363 : {
8364 : /* For targets with ABI limb precision higher than
8365 : limb precision round to ABI limb precision,
8366 : otherwise c can contain padding bits. */
8367 0 : min_prec
8368 0 : = CEIL (min_prec, abi_limb_prec) * abi_limb_prec;
8369 0 : if (min_prec > prec - rem - 2 * limb_prec)
8370 9 : min_prec = prec;
8371 : }
8372 : }
8373 52 : if (min_prec == 0)
8374 : c = NULL_TREE;
8375 49 : else if (min_prec == prec)
8376 9 : c = tree_output_constant_def (arg);
8377 40 : else if (min_prec == (unsigned) limb_prec)
8378 37 : c = fold_convert (large_huge.m_limb_type, arg);
8379 : else
8380 : {
8381 3 : tree ctype = build_bitint_type (min_prec, 1);
8382 3 : c = tree_output_constant_def (fold_convert (ctype, arg));
8383 : }
8384 49 : if (c)
8385 : {
8386 49 : if (VAR_P (v1) && min_prec == prec)
8387 : {
8388 8 : tree v2 = build1 (VIEW_CONVERT_EXPR,
8389 8 : TREE_TYPE (v1), c);
8390 8 : g = gimple_build_assign (v1, v2);
8391 8 : gsi_insert_on_edge (e, g);
8392 8 : edge_insertions = true;
8393 8 : break;
8394 : }
8395 41 : if (TREE_CODE (TREE_TYPE (c)) == INTEGER_TYPE)
8396 : {
8397 37 : if (bitint_big_endian)
8398 : {
8399 0 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8400 0 : tree sz1 = TYPE_SIZE_UNIT (TREE_TYPE (v1));
8401 0 : tree sz2 = TYPE_SIZE_UNIT (TREE_TYPE (c));
8402 0 : tree off = build_int_cst (ptype,
8403 0 : tree_to_uhwi (sz1)
8404 0 : - tree_to_uhwi (sz2));
8405 0 : tree vd = build2 (MEM_REF, TREE_TYPE (c),
8406 : build_fold_addr_expr (v1),
8407 : off);
8408 0 : g = gimple_build_assign (vd, c);
8409 : }
8410 : else
8411 37 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8412 37 : TREE_TYPE (c),
8413 : v1), c);
8414 : }
8415 : else
8416 : {
8417 4 : unsigned HOST_WIDE_INT nelts
8418 4 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (c)))
8419 4 : / limb_prec;
8420 4 : tree vtype
8421 8 : = build_array_type_nelts (large_huge.m_limb_type,
8422 4 : nelts);
8423 4 : tree vd;
8424 4 : if (bitint_big_endian)
8425 : {
8426 0 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8427 0 : tree sz1 = TYPE_SIZE_UNIT (TREE_TYPE (v1));
8428 0 : tree sz2 = TYPE_SIZE_UNIT (vtype);
8429 0 : tree off = build_int_cst (ptype,
8430 0 : tree_to_uhwi (sz1)
8431 0 : - tree_to_uhwi (sz2));
8432 0 : vd = build2 (MEM_REF, vtype,
8433 : build_fold_addr_expr (v1), off);
8434 : }
8435 : else
8436 4 : vd = build1 (VIEW_CONVERT_EXPR, vtype, v1);
8437 4 : g = gimple_build_assign (vd,
8438 : build1 (VIEW_CONVERT_EXPR,
8439 : vtype, c));
8440 : }
8441 41 : gsi_insert_on_edge (e, g);
8442 41 : if (min_prec == prec)
8443 : {
8444 : edge_insertions = true;
8445 : break;
8446 : }
8447 : }
8448 43 : if (ext == 0)
8449 : {
8450 37 : unsigned HOST_WIDE_INT nelts
8451 37 : = (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (v1)))
8452 37 : - min_prec) / limb_prec;
8453 37 : tree vtype
8454 74 : = build_array_type_nelts (large_huge.m_limb_type,
8455 37 : nelts);
8456 37 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8457 37 : tree off;
8458 37 : if (c && !bitint_big_endian)
8459 36 : off = fold_convert (ptype,
8460 : TYPE_SIZE_UNIT (TREE_TYPE (c)));
8461 : else
8462 1 : off = build_zero_cst (ptype);
8463 37 : tree vd = build2 (MEM_REF, vtype,
8464 : build_fold_addr_expr (v1), off);
8465 37 : g = gimple_build_assign (vd, build_zero_cst (vtype));
8466 : }
8467 : else
8468 : {
8469 6 : tree vd = v1;
8470 6 : if (c && !bitint_big_endian)
8471 : {
8472 4 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8473 4 : tree off
8474 4 : = fold_convert (ptype,
8475 : TYPE_SIZE_UNIT (TREE_TYPE (c)));
8476 4 : vd = build2 (MEM_REF, large_huge.m_limb_type,
8477 : build_fold_addr_expr (v1), off);
8478 : }
8479 6 : vd = build_fold_addr_expr (vd);
8480 6 : unsigned HOST_WIDE_INT nbytes
8481 6 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (v1)));
8482 6 : if (c)
8483 4 : nbytes
8484 4 : -= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (c)));
8485 6 : tree fn = builtin_decl_implicit (BUILT_IN_MEMSET);
8486 6 : g = gimple_build_call (fn, 3, vd,
8487 : integer_minus_one_node,
8488 : build_int_cst (sizetype,
8489 6 : nbytes));
8490 : }
8491 43 : gsi_insert_on_edge (e, g);
8492 43 : edge_insertions = true;
8493 43 : break;
8494 0 : default:
8495 0 : gcc_unreachable ();
8496 400 : case SSA_NAME:
8497 400 : if (gimple_code (SSA_NAME_DEF_STMT (arg)) == GIMPLE_NOP)
8498 : {
8499 9 : if (large_huge.m_names == NULL
8500 18 : || !bitmap_bit_p (large_huge.m_names,
8501 9 : SSA_NAME_VERSION (arg)))
8502 383 : continue;
8503 : }
8504 400 : int p2 = var_to_partition (large_huge.m_map, arg);
8505 400 : if (p1 == p2)
8506 383 : continue;
8507 17 : gcc_assert (large_huge.m_vars[p2] != NULL_TREE);
8508 17 : tree v2 = large_huge.m_vars[p2];
8509 17 : if (VAR_P (v1) && VAR_P (v2))
8510 17 : g = gimple_build_assign (v1, v2);
8511 0 : else if (VAR_P (v1))
8512 0 : g = gimple_build_assign (v1, build1 (VIEW_CONVERT_EXPR,
8513 0 : TREE_TYPE (v1), v2));
8514 0 : else if (VAR_P (v2))
8515 0 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8516 0 : TREE_TYPE (v2), v1), v2);
8517 : else
8518 : {
8519 0 : if (atype == NULL_TREE
8520 0 : || !tree_int_cst_equal (TYPE_SIZE (atype),
8521 0 : TYPE_SIZE (TREE_TYPE (lhs))))
8522 : {
8523 0 : unsigned HOST_WIDE_INT nelts
8524 0 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (lhs)))
8525 0 : / limb_prec;
8526 0 : atype
8527 0 : = build_array_type_nelts (large_huge.m_limb_type,
8528 0 : nelts);
8529 : }
8530 0 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8531 : atype, v1),
8532 : build1 (VIEW_CONVERT_EXPR,
8533 : atype, v2));
8534 : }
8535 17 : gsi_insert_on_edge (e, g);
8536 17 : edge_insertions = true;
8537 17 : break;
8538 : }
8539 : }
8540 : }
8541 : }
8542 :
8543 7344 : if (large_huge.m_names || has_large_huge)
8544 : {
8545 5876 : gimple *nop = NULL;
8546 386821 : for (i = 0; i < num_ssa_names; ++i)
8547 : {
8548 380945 : tree s = ssa_name (i);
8549 380945 : if (s == NULL_TREE)
8550 15523 : continue;
8551 365422 : tree type = TREE_TYPE (s);
8552 365422 : if (TREE_CODE (type) == COMPLEX_TYPE)
8553 17605 : type = TREE_TYPE (type);
8554 325543 : if (BITINT_TYPE_P (type)
8555 365449 : && bitint_precision_kind (type) >= bitint_prec_large)
8556 : {
8557 40824 : if (large_huge.m_preserved
8558 45322 : && bitmap_bit_p (large_huge.m_preserved,
8559 6841 : SSA_NAME_VERSION (s)))
8560 2343 : continue;
8561 36138 : gimple *g = SSA_NAME_DEF_STMT (s);
8562 36138 : if (gimple_code (g) == GIMPLE_NOP)
8563 : {
8564 9727 : if (SSA_NAME_VAR (s))
8565 5444 : set_ssa_default_def (cfun, SSA_NAME_VAR (s), NULL_TREE);
8566 9727 : release_ssa_name (s);
8567 9727 : continue;
8568 : }
8569 26411 : if (gimple_bb (g) == NULL)
8570 : {
8571 0 : release_ssa_name (s);
8572 0 : continue;
8573 : }
8574 26411 : if (gimple_code (g) != GIMPLE_ASM)
8575 : {
8576 26410 : gimple_stmt_iterator gsi = gsi_for_stmt (g);
8577 26410 : bool save_vta = flag_var_tracking_assignments;
8578 26410 : flag_var_tracking_assignments = false;
8579 26410 : gsi_remove (&gsi, true);
8580 26410 : flag_var_tracking_assignments = save_vta;
8581 : }
8582 26411 : if (nop == NULL)
8583 4908 : nop = gimple_build_nop ();
8584 26411 : SSA_NAME_DEF_STMT (s) = nop;
8585 26411 : release_ssa_name (s);
8586 : }
8587 : }
8588 5876 : if (optimize)
8589 3093 : disable_ranger (cfun);
8590 : }
8591 :
8592 7344 : if (edge_insertions)
8593 55 : gsi_commit_edge_inserts ();
8594 :
8595 : /* Fix up arguments of ECF_RETURNS_TWICE calls. Those were temporarily
8596 : inserted before the call, but that is invalid IL, so move them to the
8597 : right place and add corresponding PHIs. */
8598 7344 : if (!large_huge.m_returns_twice_calls.is_empty ())
8599 : {
8600 9 : auto_vec<gimple *, 16> arg_stmts;
8601 29 : while (!large_huge.m_returns_twice_calls.is_empty ())
8602 : {
8603 11 : gimple *stmt = large_huge.m_returns_twice_calls.pop ();
8604 11 : gimple_stmt_iterator gsi = gsi_after_labels (gimple_bb (stmt));
8605 36 : while (gsi_stmt (gsi) != stmt)
8606 : {
8607 25 : if (is_gimple_debug (gsi_stmt (gsi)))
8608 2 : gsi_next (&gsi);
8609 : else
8610 : {
8611 23 : arg_stmts.safe_push (gsi_stmt (gsi));
8612 23 : gsi_remove (&gsi, false);
8613 : }
8614 : }
8615 11 : gimple *g;
8616 11 : basic_block bb = NULL;
8617 11 : edge e = NULL, ead = NULL;
8618 34 : FOR_EACH_VEC_ELT (arg_stmts, i, g)
8619 : {
8620 23 : gsi_safe_insert_before (&gsi, g);
8621 23 : if (i == 0)
8622 : {
8623 11 : bb = gimple_bb (stmt);
8624 11 : gcc_checking_assert (EDGE_COUNT (bb->preds) == 2);
8625 11 : e = EDGE_PRED (bb, 0);
8626 11 : ead = EDGE_PRED (bb, 1);
8627 11 : if ((ead->flags & EDGE_ABNORMAL) == 0)
8628 0 : std::swap (e, ead);
8629 11 : gcc_checking_assert ((e->flags & EDGE_ABNORMAL) == 0
8630 : && (ead->flags & EDGE_ABNORMAL));
8631 : }
8632 23 : tree lhs = gimple_assign_lhs (g);
8633 23 : tree arg = lhs;
8634 23 : gphi *phi = create_phi_node (copy_ssa_name (arg), bb);
8635 23 : add_phi_arg (phi, arg, e, UNKNOWN_LOCATION);
8636 23 : tree var = create_tmp_reg (TREE_TYPE (arg));
8637 23 : suppress_warning (var, OPT_Wuninitialized);
8638 23 : arg = get_or_create_ssa_default_def (cfun, var);
8639 23 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg) = 1;
8640 23 : add_phi_arg (phi, arg, ead, UNKNOWN_LOCATION);
8641 23 : arg = gimple_phi_result (phi);
8642 23 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg) = 1;
8643 23 : imm_use_iterator iter;
8644 23 : gimple *use_stmt;
8645 92 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
8646 : {
8647 46 : if (use_stmt == phi)
8648 23 : continue;
8649 23 : gcc_checking_assert (use_stmt == stmt);
8650 23 : use_operand_p use_p;
8651 69 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
8652 23 : SET_USE (use_p, arg);
8653 23 : }
8654 : }
8655 11 : update_stmt (stmt);
8656 11 : arg_stmts.truncate (0);
8657 : }
8658 9 : }
8659 :
8660 7344 : return ret;
8661 7344 : }
8662 :
8663 : namespace {
8664 :
8665 : const pass_data pass_data_lower_bitint =
8666 : {
8667 : GIMPLE_PASS, /* type */
8668 : "bitintlower", /* name */
8669 : OPTGROUP_NONE, /* optinfo_flags */
8670 : TV_NONE, /* tv_id */
8671 : PROP_ssa, /* properties_required */
8672 : PROP_gimple_lbitint, /* properties_provided */
8673 : 0, /* properties_destroyed */
8674 : 0, /* todo_flags_start */
8675 : 0, /* todo_flags_finish */
8676 : };
8677 :
8678 : class pass_lower_bitint : public gimple_opt_pass
8679 : {
8680 : public:
8681 587656 : pass_lower_bitint (gcc::context *ctxt)
8682 1175312 : : gimple_opt_pass (pass_data_lower_bitint, ctxt)
8683 : {}
8684 :
8685 : /* opt_pass methods: */
8686 293828 : opt_pass * clone () final override { return new pass_lower_bitint (m_ctxt); }
8687 1051283 : unsigned int execute (function *) final override
8688 : {
8689 1051283 : return gimple_lower_bitint ();
8690 : }
8691 :
8692 : }; // class pass_lower_bitint
8693 :
8694 : } // anon namespace
8695 :
8696 : gimple_opt_pass *
8697 293828 : make_pass_lower_bitint (gcc::context *ctxt)
8698 : {
8699 293828 : return new pass_lower_bitint (ctxt);
8700 : }
8701 :
8702 :
8703 : namespace {
8704 :
8705 : const pass_data pass_data_lower_bitint_O0 =
8706 : {
8707 : GIMPLE_PASS, /* type */
8708 : "bitintlower0", /* name */
8709 : OPTGROUP_NONE, /* optinfo_flags */
8710 : TV_NONE, /* tv_id */
8711 : PROP_cfg, /* properties_required */
8712 : PROP_gimple_lbitint, /* properties_provided */
8713 : 0, /* properties_destroyed */
8714 : 0, /* todo_flags_start */
8715 : 0, /* todo_flags_finish */
8716 : };
8717 :
8718 : class pass_lower_bitint_O0 : public gimple_opt_pass
8719 : {
8720 : public:
8721 293828 : pass_lower_bitint_O0 (gcc::context *ctxt)
8722 587656 : : gimple_opt_pass (pass_data_lower_bitint_O0, ctxt)
8723 : {}
8724 :
8725 : /* opt_pass methods: */
8726 1494726 : bool gate (function *fun) final override
8727 : {
8728 : /* With errors, normal optimization passes are not run. If we don't
8729 : lower bitint operations at all, rtl expansion will abort. */
8730 1494726 : return !(fun->curr_properties & PROP_gimple_lbitint);
8731 : }
8732 :
8733 443548 : unsigned int execute (function *) final override
8734 : {
8735 443548 : return gimple_lower_bitint ();
8736 : }
8737 :
8738 : }; // class pass_lower_bitint_O0
8739 :
8740 : } // anon namespace
8741 :
8742 : gimple_opt_pass *
8743 293828 : make_pass_lower_bitint_O0 (gcc::context *ctxt)
8744 : {
8745 293828 : return new pass_lower_bitint_O0 (ctxt);
8746 : }
|
