Line data Source code
1 : /* Lower _BitInt(N) operations to scalar operations.
2 : Copyright (C) 2023-2026 Free Software Foundation, Inc.
3 : Contributed by Jakub Jelinek <jakub@redhat.com>.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it
8 : under the terms of the GNU General Public License as published by the
9 : Free Software Foundation; either version 3, or (at your option) any
10 : later version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT
13 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "cfghooks.h"
29 : #include "tree-pass.h"
30 : #include "ssa.h"
31 : #include "fold-const.h"
32 : #include "gimplify.h"
33 : #include "gimple-iterator.h"
34 : #include "tree-cfg.h"
35 : #include "tree-dfa.h"
36 : #include "cfgloop.h"
37 : #include "cfganal.h"
38 : #include "target.h"
39 : #include "tree-ssa-live.h"
40 : #include "tree-ssa-coalesce.h"
41 : #include "domwalk.h"
42 : #include "memmodel.h"
43 : #include "optabs.h"
44 : #include "varasm.h"
45 : #include "gimple-range.h"
46 : #include "value-range.h"
47 : #include "langhooks.h"
48 : #include "gimplify-me.h"
49 : #include "diagnostic-core.h"
50 : #include "tree-eh.h"
51 : #include "tree-pretty-print.h"
52 : #include "alloc-pool.h"
53 : #include "tree-into-ssa.h"
54 : #include "tree-cfgcleanup.h"
55 : #include "tree-switch-conversion.h"
56 : #include "ubsan.h"
57 : #include "stor-layout.h"
58 : #include "gimple-lower-bitint.h"
59 : #include "attribs.h"
60 : #include "asan.h"
61 :
62 : /* Split BITINT_TYPE precisions in 4 categories. Small _BitInt, where
63 : target hook says it is a single limb, middle _BitInt which per ABI
64 : does not, but there is some INTEGER_TYPE in which arithmetics can be
65 : performed (operations on such _BitInt are lowered to casts to that
66 : arithmetic type and cast back; e.g. on x86_64 limb is DImode, but
67 : target supports TImode, so _BitInt(65) to _BitInt(128) are middle
68 : ones), large _BitInt which should by straight line code and
69 : finally huge _BitInt which should be handled by loops over the limbs. */
70 :
71 : enum bitint_prec_kind {
72 : bitint_prec_small,
73 : bitint_prec_middle,
74 : bitint_prec_large,
75 : bitint_prec_huge
76 : };
77 :
78 : /* Caches to speed up bitint_precision_kind. */
79 :
80 : static int small_max_prec, mid_min_prec, large_min_prec, huge_min_prec;
81 : static int limb_prec, abi_limb_prec;
82 : static bool bitint_big_endian;
83 : static enum bitint_ext bitint_extended;
84 :
85 : /* Categorize _BitInt(PREC) as small, middle, large or huge. */
86 :
87 : static bitint_prec_kind
88 488750 : bitint_precision_kind (int prec)
89 : {
90 488750 : if (prec <= small_max_prec)
91 : return bitint_prec_small;
92 470468 : if (huge_min_prec && prec >= huge_min_prec)
93 : return bitint_prec_huge;
94 265201 : if (large_min_prec && prec >= large_min_prec)
95 : return bitint_prec_large;
96 52494 : if (mid_min_prec && prec >= mid_min_prec)
97 : return bitint_prec_middle;
98 :
99 10042 : struct bitint_info info;
100 10042 : bool ok = targetm.c.bitint_type_info (prec, &info);
101 10042 : gcc_assert (ok);
102 10042 : scalar_int_mode limb_mode = as_a <scalar_int_mode> (info.limb_mode);
103 10042 : if (prec <= GET_MODE_PRECISION (limb_mode))
104 : {
105 2596 : small_max_prec = prec;
106 2596 : return bitint_prec_small;
107 : }
108 7446 : bitint_big_endian = info.big_endian;
109 7446 : bitint_extended = info.extended;
110 7446 : if (info.limb_mode == info.abi_limb_mode && bitint_extended == bitint_ext_full)
111 0 : bitint_extended = bitint_ext_partial;
112 7446 : if (!large_min_prec
113 14802 : && GET_MODE_PRECISION (limb_mode) <= MAX_FIXED_MODE_SIZE)
114 14712 : large_min_prec = MAX_FIXED_MODE_SIZE + 1;
115 7446 : if (!limb_prec)
116 7356 : limb_prec = GET_MODE_PRECISION (limb_mode);
117 7446 : if (!abi_limb_prec)
118 7356 : abi_limb_prec
119 7356 : = GET_MODE_PRECISION (as_a <scalar_int_mode> (info.abi_limb_mode));
120 : /* For bitint_ext_full with different limb_mode from abi_limb_mode we
121 : currently only support only abi_limb_mode twice the precision of
122 : limb_mode, and don't support big endian in that case either. */
123 7446 : gcc_assert (bitint_extended != bitint_ext_full
124 : || (abi_limb_prec == 2 * limb_prec
125 : && !bitint_big_endian));
126 7446 : if (!huge_min_prec)
127 : {
128 14712 : if (4 * limb_prec >= MAX_FIXED_MODE_SIZE)
129 7356 : huge_min_prec = 4 * limb_prec;
130 : else
131 0 : huge_min_prec = MAX_FIXED_MODE_SIZE + 1;
132 : }
133 14892 : if (prec <= MAX_FIXED_MODE_SIZE)
134 : {
135 1609 : if (!mid_min_prec || prec < mid_min_prec)
136 1609 : mid_min_prec = prec;
137 : return bitint_prec_middle;
138 : }
139 5837 : if (huge_min_prec && prec >= huge_min_prec)
140 2360 : return bitint_prec_huge;
141 : return bitint_prec_large;
142 : }
143 :
144 : /* Same for a TYPE. */
145 :
146 : static bitint_prec_kind
147 483284 : bitint_precision_kind (tree type)
148 : {
149 483284 : return bitint_precision_kind (TYPE_PRECISION (type));
150 : }
151 :
152 : /* Return minimum precision needed to describe INTEGER_CST
153 : CST. All bits above that precision up to precision of
154 : TREE_TYPE (CST) are cleared if EXT is set to 0, or set
155 : if EXT is set to -1. */
156 :
157 : static unsigned
158 5397 : bitint_min_cst_precision (tree cst, int &ext)
159 : {
160 5397 : ext = tree_int_cst_sgn (cst) < 0 ? -1 : 0;
161 5397 : wide_int w = wi::to_wide (cst);
162 5397 : unsigned min_prec = wi::min_precision (w, TYPE_SIGN (TREE_TYPE (cst)));
163 : /* For signed values, we don't need to count the sign bit,
164 : we'll use constant 0 or -1 for the upper bits. */
165 5397 : if (!TYPE_UNSIGNED (TREE_TYPE (cst)))
166 3281 : --min_prec;
167 : else
168 : {
169 : /* For unsigned values, also try signed min_precision
170 : in case the constant has lots of most significant bits set. */
171 2116 : unsigned min_prec2 = wi::min_precision (w, SIGNED) - 1;
172 2116 : if (min_prec2 < min_prec)
173 : {
174 1001 : ext = -1;
175 1001 : return min_prec2;
176 : }
177 : }
178 : return min_prec;
179 5397 : }
180 :
181 : namespace {
182 :
183 : /* If OP is middle _BitInt, cast it to corresponding INTEGER_TYPE
184 : cached in TYPE and return it. */
185 :
186 : tree
187 7768 : maybe_cast_middle_bitint (gimple_stmt_iterator *gsi, tree op, tree &type)
188 : {
189 7768 : if (op == NULL_TREE
190 7740 : || !BITINT_TYPE_P (TREE_TYPE (op))
191 14631 : || bitint_precision_kind (TREE_TYPE (op)) != bitint_prec_middle)
192 : return op;
193 :
194 6860 : int prec = TYPE_PRECISION (TREE_TYPE (op));
195 6860 : int uns = TYPE_UNSIGNED (TREE_TYPE (op));
196 6860 : if (type == NULL_TREE
197 2538 : || TYPE_PRECISION (type) != prec
198 9398 : || TYPE_UNSIGNED (type) != uns)
199 4322 : type = build_nonstandard_integer_type (prec, uns);
200 :
201 6860 : if (TREE_CODE (op) != SSA_NAME)
202 : {
203 2351 : tree nop = fold_convert (type, op);
204 2351 : if (is_gimple_val (nop))
205 : return nop;
206 : }
207 :
208 4509 : tree nop = make_ssa_name (type);
209 4509 : gimple *g = gimple_build_assign (nop, NOP_EXPR, op);
210 4509 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
211 4509 : return nop;
212 : }
213 :
214 : /* Return true if STMT can be handled in a loop from least to most
215 : significant limb together with its dependencies. */
216 :
217 : bool
218 48074 : mergeable_op (gimple *stmt)
219 : {
220 48074 : if (!is_gimple_assign (stmt))
221 : return false;
222 39351 : switch (gimple_assign_rhs_code (stmt))
223 : {
224 : case PLUS_EXPR:
225 : case MINUS_EXPR:
226 : case NEGATE_EXPR:
227 : case BIT_AND_EXPR:
228 : case BIT_IOR_EXPR:
229 : case BIT_XOR_EXPR:
230 : case BIT_NOT_EXPR:
231 : case SSA_NAME:
232 : case INTEGER_CST:
233 : case BIT_FIELD_REF:
234 : return true;
235 410 : case LSHIFT_EXPR:
236 410 : {
237 410 : tree cnt = gimple_assign_rhs2 (stmt);
238 410 : if (tree_fits_uhwi_p (cnt)
239 198 : && tree_to_uhwi (cnt) < (unsigned HOST_WIDE_INT) limb_prec)
240 : return true;
241 : }
242 : break;
243 6332 : CASE_CONVERT:
244 6332 : case VIEW_CONVERT_EXPR:
245 6332 : {
246 6332 : tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt));
247 6332 : tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (stmt));
248 6332 : if (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
249 6290 : && BITINT_TYPE_P (lhs_type)
250 3715 : && BITINT_TYPE_P (rhs_type)
251 3360 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
252 3324 : && bitint_precision_kind (rhs_type) >= bitint_prec_large
253 9495 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
254 3163 : == CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
255 : {
256 2195 : if (TYPE_PRECISION (rhs_type) >= TYPE_PRECISION (lhs_type))
257 : return true;
258 183 : if ((unsigned) TYPE_PRECISION (lhs_type) % (2 * limb_prec) != 0)
259 : return true;
260 17 : if (bitint_precision_kind (lhs_type) == bitint_prec_large)
261 : return true;
262 : }
263 : break;
264 : }
265 : default:
266 : break;
267 : }
268 : return false;
269 : }
270 :
271 : /* Return non-zero if stmt is .{ADD,SUB,MUL}_OVERFLOW call with
272 : _Complex large/huge _BitInt lhs which has at most two immediate uses,
273 : at most one use in REALPART_EXPR stmt in the same bb and exactly one
274 : IMAGPART_EXPR use in the same bb with a single use which casts it to
275 : non-BITINT_TYPE integral type. If there is a REALPART_EXPR use,
276 : return 2. Such cases (most common uses of those builtins) can be
277 : optimized by marking their lhs and lhs of IMAGPART_EXPR and maybe lhs
278 : of REALPART_EXPR as not needed to be backed up by a stack variable.
279 : For .UBSAN_CHECK_{ADD,SUB,MUL} return 3. */
280 :
281 : int
282 21470 : optimizable_arith_overflow (gimple *stmt)
283 : {
284 21470 : bool is_ubsan = false;
285 21470 : if (!is_gimple_call (stmt) || !gimple_call_internal_p (stmt))
286 : return false;
287 5236 : switch (gimple_call_internal_fn (stmt))
288 : {
289 : case IFN_ADD_OVERFLOW:
290 : case IFN_SUB_OVERFLOW:
291 : case IFN_MUL_OVERFLOW:
292 : break;
293 63 : case IFN_UBSAN_CHECK_ADD:
294 63 : case IFN_UBSAN_CHECK_SUB:
295 63 : case IFN_UBSAN_CHECK_MUL:
296 63 : is_ubsan = true;
297 63 : break;
298 : default:
299 : return 0;
300 : }
301 5210 : tree lhs = gimple_call_lhs (stmt);
302 5210 : if (!lhs)
303 : return 0;
304 5210 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
305 : return 0;
306 5195 : tree type = is_ubsan ? TREE_TYPE (lhs) : TREE_TYPE (TREE_TYPE (lhs));
307 0 : if (!BITINT_TYPE_P (type)
308 5195 : || bitint_precision_kind (type) < bitint_prec_large)
309 : return 0;
310 :
311 5195 : if (is_ubsan)
312 : {
313 48 : use_operand_p use_p;
314 48 : gimple *use_stmt;
315 48 : if (!single_imm_use (lhs, &use_p, &use_stmt)
316 48 : || gimple_bb (use_stmt) != gimple_bb (stmt)
317 48 : || !gimple_store_p (use_stmt)
318 48 : || !is_gimple_assign (use_stmt)
319 48 : || gimple_has_volatile_ops (use_stmt)
320 96 : || stmt_ends_bb_p (use_stmt))
321 0 : return 0;
322 : return 3;
323 : }
324 :
325 5147 : imm_use_iterator ui;
326 5147 : use_operand_p use_p;
327 5147 : int seen = 0;
328 5147 : gimple *realpart = NULL, *cast = NULL;
329 14740 : FOR_EACH_IMM_USE_FAST (use_p, ui, lhs)
330 : {
331 9807 : gimple *g = USE_STMT (use_p);
332 9807 : if (is_gimple_debug (g))
333 0 : continue;
334 9807 : if (!is_gimple_assign (g) || gimple_bb (g) != gimple_bb (stmt))
335 : return 0;
336 9807 : if (gimple_assign_rhs_code (g) == REALPART_EXPR)
337 : {
338 4660 : if ((seen & 1) != 0)
339 : return 0;
340 4660 : seen |= 1;
341 4660 : realpart = g;
342 : }
343 5147 : else if (gimple_assign_rhs_code (g) == IMAGPART_EXPR)
344 : {
345 5147 : if ((seen & 2) != 0)
346 214 : return 0;
347 5147 : seen |= 2;
348 :
349 5147 : use_operand_p use2_p;
350 5147 : gimple *use_stmt;
351 5147 : tree lhs2 = gimple_assign_lhs (g);
352 5147 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs2))
353 : return 0;
354 5147 : if (!single_imm_use (lhs2, &use2_p, &use_stmt)
355 5147 : || gimple_bb (use_stmt) != gimple_bb (stmt)
356 10294 : || !gimple_assign_cast_p (use_stmt))
357 : return 0;
358 :
359 4937 : lhs2 = gimple_assign_lhs (use_stmt);
360 9874 : if (!INTEGRAL_TYPE_P (TREE_TYPE (lhs2))
361 9874 : || BITINT_TYPE_P (TREE_TYPE (lhs2)))
362 : return 0;
363 4933 : cast = use_stmt;
364 : }
365 : else
366 : return 0;
367 214 : }
368 4933 : if ((seen & 2) == 0)
369 : return 0;
370 4933 : if (seen == 3)
371 : {
372 : /* Punt if the cast stmt appears before realpart stmt, because
373 : if both appear, the lowering wants to emit all the code
374 : at the location of realpart stmt. */
375 4660 : gimple_stmt_iterator gsi = gsi_for_stmt (realpart);
376 4660 : unsigned int cnt = 0;
377 4663 : do
378 : {
379 4663 : gsi_prev_nondebug (&gsi);
380 4663 : if (gsi_end_p (gsi) || gsi_stmt (gsi) == cast)
381 : return 0;
382 4660 : if (gsi_stmt (gsi) == stmt)
383 : return 2;
384 : /* If realpart is too far from stmt, punt as well.
385 : Usually it will appear right after it. */
386 3 : if (++cnt == 32)
387 : return 0;
388 : }
389 : while (1);
390 : }
391 : return 1;
392 : }
393 :
394 : /* If STMT is some kind of comparison (GIMPLE_COND, comparison assignment)
395 : comparing large/huge _BitInt types, return the comparison code and if
396 : non-NULL fill in the comparison operands to *POP1 and *POP2. */
397 :
398 : tree_code
399 36021 : comparison_op (gimple *stmt, tree *pop1, tree *pop2)
400 : {
401 36021 : tree op1 = NULL_TREE, op2 = NULL_TREE;
402 36021 : tree_code code = ERROR_MARK;
403 36021 : if (gimple_code (stmt) == GIMPLE_COND)
404 : {
405 6552 : code = gimple_cond_code (stmt);
406 6552 : op1 = gimple_cond_lhs (stmt);
407 6552 : op2 = gimple_cond_rhs (stmt);
408 : }
409 29469 : else if (is_gimple_assign (stmt))
410 : {
411 29442 : code = gimple_assign_rhs_code (stmt);
412 29442 : op1 = gimple_assign_rhs1 (stmt);
413 29442 : if (TREE_CODE_CLASS (code) == tcc_comparison
414 29442 : || TREE_CODE_CLASS (code) == tcc_binary)
415 2214 : op2 = gimple_assign_rhs2 (stmt);
416 : }
417 36021 : if (TREE_CODE_CLASS (code) != tcc_comparison)
418 : return ERROR_MARK;
419 7334 : tree type = TREE_TYPE (op1);
420 10 : if (!BITINT_TYPE_P (type)
421 7344 : || bitint_precision_kind (type) < bitint_prec_large)
422 : return ERROR_MARK;
423 7334 : if (pop1)
424 : {
425 7272 : *pop1 = op1;
426 7272 : *pop2 = op2;
427 : }
428 : return code;
429 : }
430 :
431 : /* Class used during large/huge _BitInt lowering containing all the
432 : state for the methods. */
433 :
434 : struct bitint_large_huge
435 : {
436 7354 : bitint_large_huge ()
437 7354 : : m_names (NULL), m_loads (NULL), m_preserved (NULL),
438 7354 : m_single_use_names (NULL), m_map (NULL), m_vars (NULL),
439 7354 : m_limb_type (NULL_TREE), m_data (vNULL),
440 7354 : m_returns_twice_calls (vNULL) {}
441 :
442 : ~bitint_large_huge ();
443 :
444 : void insert_before (gimple *);
445 : tree limb_access_type (tree, tree);
446 : tree limb_access (tree, tree, tree, bool, bool = false);
447 : tree build_bit_field_ref (tree, tree, unsigned HOST_WIDE_INT,
448 : unsigned HOST_WIDE_INT);
449 : void if_then (gimple *, profile_probability, edge &, edge &);
450 : void if_then_else (gimple *, profile_probability, edge &, edge &);
451 : void if_then_if_then_else (gimple *g, gimple *,
452 : profile_probability, profile_probability,
453 : edge &, edge &, edge &);
454 : tree handle_operand (tree, tree);
455 : tree prepare_data_in_out (tree, tree, tree *, tree = NULL_TREE);
456 : tree add_cast (tree, tree);
457 : tree handle_plus_minus (tree_code, tree, tree, tree);
458 : tree handle_lshift (tree, tree, tree);
459 : tree handle_cast (tree, tree, tree);
460 : tree handle_bit_field_ref (tree, tree);
461 : tree handle_load (gimple *, tree);
462 : tree handle_stmt (gimple *, tree);
463 : tree handle_operand_addr (tree, gimple *, int *, int *);
464 : tree create_loop (tree, tree *);
465 : tree lower_mergeable_stmt (gimple *, tree_code &, tree, tree);
466 : tree lower_comparison_stmt (gimple *, tree_code &, tree, tree);
467 : void lower_shift_stmt (tree, gimple *);
468 : void lower_muldiv_stmt (tree, gimple *);
469 : void lower_float_conv_stmt (tree, gimple *);
470 : tree arith_overflow_extract_bits (unsigned int, unsigned int, tree,
471 : unsigned int, bool);
472 : void finish_arith_overflow (tree, tree, tree, tree, tree, tree, gimple *,
473 : unsigned, tree_code);
474 : void lower_addsub_overflow (tree, gimple *);
475 : void lower_mul_overflow (tree, gimple *);
476 : void lower_cplxpart_stmt (tree, gimple *);
477 : void lower_complexexpr_stmt (gimple *);
478 : void lower_bit_query (gimple *);
479 : void lower_bswap_bitreverse (tree, gimple *);
480 : void lower_call (tree, gimple *);
481 : void lower_asm (gimple *);
482 : void lower_stmt (gimple *);
483 :
484 : /* Bitmap of large/huge _BitInt SSA_NAMEs except those can be
485 : merged with their uses. */
486 : bitmap m_names;
487 : /* Subset of those for lhs of load statements. These will be
488 : cleared in m_names if the loads will be mergeable with all
489 : their uses. */
490 : bitmap m_loads;
491 : /* Bitmap of large/huge _BitInt SSA_NAMEs that should survive
492 : to later passes (arguments or return values of calls). */
493 : bitmap m_preserved;
494 : /* Subset of m_names which have a single use. As the lowering
495 : can replace various original statements with their lowered
496 : form even before it is done iterating over all basic blocks,
497 : testing has_single_use for the purpose of emitting clobbers
498 : doesn't work properly. */
499 : bitmap m_single_use_names;
500 : /* Used for coalescing/partitioning of large/huge _BitInt SSA_NAMEs
501 : set in m_names. */
502 : var_map m_map;
503 : /* Mapping of the partitions to corresponding decls. */
504 : tree *m_vars;
505 : /* Unsigned integer type with limb precision. */
506 : tree m_limb_type;
507 : /* Its TYPE_SIZE_UNIT. */
508 : unsigned HOST_WIDE_INT m_limb_size;
509 : /* Location of a gimple stmt which is being currently lowered. */
510 : location_t m_loc;
511 : /* Current stmt iterator where code is being lowered currently. */
512 : gimple_stmt_iterator m_gsi;
513 : /* Statement after which any clobbers should be added if non-NULL. */
514 : gimple *m_after_stmt;
515 : /* Set when creating loops to the loop header bb and its preheader. */
516 : basic_block m_bb, m_preheader_bb;
517 : /* Stmt iterator after which initialization statements should be emitted. */
518 : gimple_stmt_iterator m_init_gsi;
519 : /* Decl into which a mergeable statement stores result. */
520 : tree m_lhs;
521 : /* handle_operand/handle_stmt can be invoked in various ways.
522 :
523 : lower_mergeable_stmt for large _BitInt calls those with constant
524 : idx only, expanding to straight line code, for huge _BitInt
525 : emits a loop from least significant limb upwards, where each loop
526 : iteration handles 2 limbs, plus there can be up to one full limb
527 : and one partial limb processed after the loop, where handle_operand
528 : and/or handle_stmt are called with constant idx. m_upwards_2limb
529 : is set for this case, false otherwise. m_upwards is true if it
530 : is either large or huge _BitInt handled by lower_mergeable_stmt,
531 : i.e. indexes always increase.
532 :
533 : Another way is used by lower_comparison_stmt, which walks limbs
534 : from most significant to least significant, partial limb if any
535 : processed first with constant idx and then loop processing a single
536 : limb per iteration with non-constant idx.
537 :
538 : Another way is used in lower_shift_stmt, where for LSHIFT_EXPR
539 : destination limbs are processed from most significant to least
540 : significant or for RSHIFT_EXPR the other way around, in loops or
541 : straight line code, but idx usually is non-constant (so from
542 : handle_operand/handle_stmt POV random access). The LSHIFT_EXPR
543 : handling there can access even partial limbs using non-constant
544 : idx (then m_var_msb should be true, for all the other cases
545 : including lower_mergeable_stmt/lower_comparison_stmt that is
546 : not the case and so m_var_msb should be false.
547 :
548 : m_first should be set the first time handle_operand/handle_stmt
549 : is called and clear when it is called for some other limb with
550 : the same argument. If the lowering of an operand (e.g. INTEGER_CST)
551 : or statement (e.g. +/-/<< with < limb_prec constant) needs some
552 : state between the different calls, when m_first is true it should
553 : push some trees to m_data vector and also make sure m_data_cnt is
554 : incremented by how many trees were pushed, and when m_first is
555 : false, it can use the m_data[m_data_cnt] etc. data or update them,
556 : just needs to bump m_data_cnt by the same amount as when it was
557 : called with m_first set. The toplevel calls to
558 : handle_operand/handle_stmt should set m_data_cnt to 0 and truncate
559 : m_data vector when setting m_first to true.
560 :
561 : m_cast_conditional and m_bitfld_load are used when handling a
562 : bit-field load inside of a widening cast. handle_cast sometimes
563 : needs to do runtime comparisons and handle_operand only conditionally
564 : or even in two separate conditional blocks for one idx (once with
565 : constant index after comparing the runtime one for equality with the
566 : constant). In these cases, m_cast_conditional is set to true and
567 : the bit-field load then communicates its m_data_cnt to handle_cast
568 : using m_bitfld_load. */
569 : bool m_first;
570 : bool m_var_msb;
571 : unsigned m_upwards_2limb;
572 : bool m_upwards;
573 : bool m_cast_conditional;
574 : unsigned m_bitfld_load;
575 : vec<tree> m_data;
576 : unsigned int m_data_cnt;
577 : vec<gimple *> m_returns_twice_calls;
578 : };
579 :
580 7354 : bitint_large_huge::~bitint_large_huge ()
581 : {
582 7354 : BITMAP_FREE (m_names);
583 7354 : BITMAP_FREE (m_loads);
584 7354 : BITMAP_FREE (m_preserved);
585 7354 : BITMAP_FREE (m_single_use_names);
586 7354 : if (m_map)
587 5688 : delete_var_map (m_map);
588 7354 : XDELETEVEC (m_vars);
589 7354 : m_data.release ();
590 7354 : m_returns_twice_calls.release ();
591 7354 : }
592 :
593 : /* Insert gimple statement G before current location
594 : and set its gimple_location. */
595 :
596 : void
597 364023 : bitint_large_huge::insert_before (gimple *g)
598 : {
599 364023 : gimple_set_location (g, m_loc);
600 364023 : gsi_insert_before (&m_gsi, g, GSI_SAME_STMT);
601 364023 : }
602 :
603 : /* Return type for accessing limb IDX of BITINT_TYPE TYPE.
604 : This is normally m_limb_type, except for a partial most
605 : significant limb if any. */
606 :
607 : tree
608 130816 : bitint_large_huge::limb_access_type (tree type, tree idx)
609 : {
610 130816 : if (type == NULL_TREE)
611 5913 : return m_limb_type;
612 124903 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
613 124903 : unsigned int prec = TYPE_PRECISION (type);
614 124903 : gcc_assert (i * limb_prec < prec
615 : || (bitint_extended == bitint_ext_full
616 : && abi_limb_prec > limb_prec
617 : && i * limb_prec
618 : < CEIL (prec, abi_limb_prec) * abi_limb_prec));
619 249806 : if (bitint_big_endian
620 124903 : ? (i != 0 || (prec % limb_prec) == 0)
621 124903 : : (i + 1) * limb_prec <= prec)
622 81509 : return m_limb_type;
623 : else
624 86788 : return build_nonstandard_integer_type (prec % limb_prec,
625 43394 : TYPE_UNSIGNED (type));
626 : }
627 :
628 : /* Return a tree how to access limb IDX of VAR corresponding to BITINT_TYPE
629 : TYPE. If WRITE_P is true, it will be a store, otherwise a read. */
630 :
631 : tree
632 156134 : bitint_large_huge::limb_access (tree type, tree var, tree idx, bool write_p,
633 : bool abi_load_p)
634 : {
635 156134 : tree atype = (tree_fits_uhwi_p (idx)
636 156134 : ? limb_access_type (type, idx) : m_limb_type);
637 :
638 156134 : tree ltype = (bitint_extended && abi_load_p) ? atype : m_limb_type;
639 :
640 156134 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (var));
641 156134 : tree ret;
642 156134 : if (DECL_P (var) && tree_fits_uhwi_p (idx))
643 : {
644 95990 : if (as != TYPE_ADDR_SPACE (ltype))
645 0 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
646 0 : | ENCODE_QUAL_ADDR_SPACE (as));
647 95990 : tree ptype = build_pointer_type (strip_array_types (TREE_TYPE (var)));
648 95990 : unsigned HOST_WIDE_INT off = tree_to_uhwi (idx) * m_limb_size;
649 95990 : if (bitint_big_endian)
650 0 : off += m_limb_size - tree_to_uhwi (TYPE_SIZE_UNIT (ltype));
651 95990 : ret = build2 (MEM_REF, ltype,
652 : build_fold_addr_expr (var),
653 95990 : build_int_cst (ptype, off));
654 95990 : TREE_THIS_VOLATILE (ret) = TREE_THIS_VOLATILE (var);
655 95990 : TREE_SIDE_EFFECTS (ret) = TREE_SIDE_EFFECTS (var);
656 95990 : }
657 60144 : else if (TREE_CODE (var) == MEM_REF && tree_fits_uhwi_p (idx))
658 : {
659 4302 : if (as != TYPE_ADDR_SPACE (ltype))
660 0 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
661 0 : | ENCODE_QUAL_ADDR_SPACE (as));
662 4302 : unsigned HOST_WIDE_INT off = tree_to_uhwi (idx) * m_limb_size;
663 4302 : if (bitint_big_endian)
664 0 : off += m_limb_size - tree_to_uhwi (TYPE_SIZE_UNIT (ltype));
665 4302 : ret
666 8604 : = build2 (MEM_REF, ltype, unshare_expr (TREE_OPERAND (var, 0)),
667 8604 : size_binop (PLUS_EXPR, TREE_OPERAND (var, 1),
668 : build_int_cst (TREE_TYPE (TREE_OPERAND (var, 1)),
669 : off)));
670 4302 : TREE_THIS_VOLATILE (ret) = TREE_THIS_VOLATILE (var);
671 4302 : TREE_SIDE_EFFECTS (ret) = TREE_SIDE_EFFECTS (var);
672 4302 : TREE_THIS_NOTRAP (ret) = TREE_THIS_NOTRAP (var);
673 4302 : }
674 : else
675 : {
676 55842 : ltype = m_limb_type;
677 55842 : if (as != TYPE_ADDR_SPACE (ltype))
678 17 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
679 17 : | ENCODE_QUAL_ADDR_SPACE (as));
680 55842 : var = unshare_expr (var);
681 55842 : if (TREE_CODE (TREE_TYPE (var)) != ARRAY_TYPE
682 83852 : || !useless_type_conversion_p (m_limb_type,
683 28010 : TREE_TYPE (TREE_TYPE (var))))
684 : {
685 28774 : unsigned HOST_WIDE_INT nelts
686 28774 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var))), limb_prec);
687 28774 : tree atype = build_array_type_nelts (ltype, nelts);
688 28774 : var = build1 (VIEW_CONVERT_EXPR, atype, var);
689 : }
690 55842 : ret = build4 (ARRAY_REF, ltype, var, idx, NULL_TREE, NULL_TREE);
691 : }
692 156134 : if (!write_p && !useless_type_conversion_p (atype, ltype))
693 : {
694 19032 : gimple *g = gimple_build_assign (make_ssa_name (m_limb_type), ret);
695 19032 : insert_before (g);
696 19032 : ret = gimple_assign_lhs (g);
697 19032 : ret = build1 (NOP_EXPR, atype, ret);
698 : }
699 156134 : return ret;
700 : }
701 :
702 : /* Build a BIT_FIELD_REF to access BITSIZE bits with FTYPE type at
703 : offset BITPOS inside of OBJ. */
704 :
705 : tree
706 273 : bitint_large_huge::build_bit_field_ref (tree ftype, tree obj,
707 : unsigned HOST_WIDE_INT bitsize,
708 : unsigned HOST_WIDE_INT bitpos)
709 : {
710 546 : if (INTEGRAL_TYPE_P (TREE_TYPE (obj))
711 282 : && !type_has_mode_precision_p (TREE_TYPE (obj)))
712 : {
713 9 : unsigned HOST_WIDE_INT nelts
714 9 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))), limb_prec);
715 9 : tree ltype = m_limb_type;
716 9 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (obj));
717 9 : if (as != TYPE_ADDR_SPACE (ltype))
718 0 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
719 0 : | ENCODE_QUAL_ADDR_SPACE (as));
720 9 : tree atype = build_array_type_nelts (ltype, nelts);
721 9 : obj = build1 (VIEW_CONVERT_EXPR, atype, obj);
722 : }
723 273 : return build3 (BIT_FIELD_REF, ftype, obj, bitsize_int (bitsize),
724 273 : bitsize_int (bitpos));
725 : }
726 :
727 : /* Emit a half diamond,
728 : if (COND)
729 : |\
730 : | \
731 : | \
732 : | new_bb1
733 : | /
734 : | /
735 : |/
736 : or if (COND) new_bb1;
737 : PROB is the probability that the condition is true.
738 : Updates m_gsi to start of new_bb1.
739 : Sets EDGE_TRUE to edge from new_bb1 to successor and
740 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND) bb. */
741 :
742 : void
743 4318 : bitint_large_huge::if_then (gimple *cond, profile_probability prob,
744 : edge &edge_true, edge &edge_false)
745 : {
746 4318 : insert_before (cond);
747 4318 : edge e1 = split_block (gsi_bb (m_gsi), cond);
748 4318 : edge e2 = split_block (e1->dest, (gimple *) NULL);
749 4318 : edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
750 4318 : e1->flags = EDGE_TRUE_VALUE;
751 4318 : e1->probability = prob;
752 4318 : e3->probability = prob.invert ();
753 4318 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
754 4318 : edge_true = e2;
755 4318 : edge_false = e3;
756 4318 : m_gsi = gsi_after_labels (e1->dest);
757 4318 : }
758 :
759 : /* Emit a full diamond,
760 : if (COND)
761 : /\
762 : / \
763 : / \
764 : new_bb1 new_bb2
765 : \ /
766 : \ /
767 : \/
768 : or if (COND) new_bb2; else new_bb1;
769 : PROB is the probability that the condition is true.
770 : Updates m_gsi to start of new_bb2.
771 : Sets EDGE_TRUE to edge from new_bb1 to successor and
772 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND) bb. */
773 :
774 : void
775 112 : bitint_large_huge::if_then_else (gimple *cond, profile_probability prob,
776 : edge &edge_true, edge &edge_false)
777 : {
778 112 : insert_before (cond);
779 112 : edge e1 = split_block (gsi_bb (m_gsi), cond);
780 112 : edge e2 = split_block (e1->dest, (gimple *) NULL);
781 112 : basic_block bb = create_empty_bb (e1->dest);
782 112 : add_bb_to_loop (bb, e1->dest->loop_father);
783 112 : edge e3 = make_edge (e1->src, bb, EDGE_TRUE_VALUE);
784 112 : e1->flags = EDGE_FALSE_VALUE;
785 112 : e3->probability = prob;
786 112 : e1->probability = prob.invert ();
787 112 : bb->count = e1->src->count.apply_probability (prob);
788 112 : set_immediate_dominator (CDI_DOMINATORS, bb, e1->src);
789 112 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
790 112 : edge_true = make_single_succ_edge (bb, e2->dest, EDGE_FALLTHRU);
791 112 : edge_false = e2;
792 112 : m_gsi = gsi_after_labels (bb);
793 112 : }
794 :
795 : /* Emit a half diamond with full diamond in it
796 : if (COND1)
797 : |\
798 : | \
799 : | \
800 : | if (COND2)
801 : | / \
802 : | / \
803 : |new_bb1 new_bb2
804 : | | /
805 : \ | /
806 : \ | /
807 : \ | /
808 : \|/
809 : or if (COND1) { if (COND2) new_bb2; else new_bb1; }
810 : PROB1 is the probability that the condition 1 is true.
811 : PROB2 is the probability that the condition 2 is true.
812 : Updates m_gsi to start of new_bb1.
813 : Sets EDGE_TRUE_TRUE to edge from new_bb2 to successor,
814 : EDGE_TRUE_FALSE to edge from new_bb1 to successor and
815 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND1) bb.
816 : If COND2 is NULL, this is equivalent to
817 : if_then (COND1, PROB1, EDGE_TRUE_FALSE, EDGE_FALSE);
818 : EDGE_TRUE_TRUE = NULL; */
819 :
820 : void
821 2098 : bitint_large_huge::if_then_if_then_else (gimple *cond1, gimple *cond2,
822 : profile_probability prob1,
823 : profile_probability prob2,
824 : edge &edge_true_true,
825 : edge &edge_true_false,
826 : edge &edge_false)
827 : {
828 2098 : edge e2, e3, e4 = NULL;
829 2098 : if_then (cond1, prob1, e2, e3);
830 2098 : if (cond2 == NULL)
831 : {
832 1257 : edge_true_true = NULL;
833 1257 : edge_true_false = e2;
834 1257 : edge_false = e3;
835 1257 : return;
836 : }
837 841 : insert_before (cond2);
838 841 : e2 = split_block (gsi_bb (m_gsi), cond2);
839 841 : basic_block bb = create_empty_bb (e2->dest);
840 841 : add_bb_to_loop (bb, e2->dest->loop_father);
841 841 : e4 = make_edge (e2->src, bb, EDGE_TRUE_VALUE);
842 841 : set_immediate_dominator (CDI_DOMINATORS, bb, e2->src);
843 841 : e4->probability = prob2;
844 841 : e2->flags = EDGE_FALSE_VALUE;
845 841 : e2->probability = prob2.invert ();
846 841 : bb->count = e2->src->count.apply_probability (prob2);
847 841 : e4 = make_single_succ_edge (bb, e3->dest, EDGE_FALLTHRU);
848 841 : e2 = find_edge (e2->dest, e3->dest);
849 841 : edge_true_true = e4;
850 841 : edge_true_false = e2;
851 841 : edge_false = e3;
852 841 : m_gsi = gsi_after_labels (e2->src);
853 : }
854 :
855 : /* Emit code to access limb IDX from OP. */
856 :
857 : tree
858 111479 : bitint_large_huge::handle_operand (tree op, tree idx)
859 : {
860 111479 : switch (TREE_CODE (op))
861 : {
862 77045 : case SSA_NAME:
863 77045 : if (m_names == NULL
864 77045 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (op)))
865 : {
866 14507 : if (SSA_NAME_IS_DEFAULT_DEF (op))
867 : {
868 47 : if (m_first)
869 : {
870 16 : tree v = create_tmp_reg (m_limb_type);
871 16 : if (SSA_NAME_VAR (op) && VAR_P (SSA_NAME_VAR (op)))
872 : {
873 16 : DECL_NAME (v) = DECL_NAME (SSA_NAME_VAR (op));
874 16 : DECL_SOURCE_LOCATION (v)
875 16 : = DECL_SOURCE_LOCATION (SSA_NAME_VAR (op));
876 : }
877 16 : v = get_or_create_ssa_default_def (cfun, v);
878 16 : m_data.safe_push (v);
879 : }
880 47 : tree ret = m_data[m_data_cnt];
881 47 : m_data_cnt++;
882 47 : if (tree_fits_uhwi_p (idx))
883 : {
884 45 : tree type = limb_access_type (TREE_TYPE (op), idx);
885 45 : ret = add_cast (type, ret);
886 : }
887 : return ret;
888 : }
889 14460 : location_t loc_save = m_loc;
890 14460 : m_loc = gimple_location (SSA_NAME_DEF_STMT (op));
891 14460 : tree ret = handle_stmt (SSA_NAME_DEF_STMT (op), idx);
892 14460 : m_loc = loc_save;
893 14460 : return ret;
894 : }
895 62538 : int p;
896 62538 : gimple *g;
897 62538 : tree t;
898 62538 : p = var_to_partition (m_map, op);
899 62538 : gcc_assert (m_vars[p] != NULL_TREE);
900 62538 : t = limb_access (TREE_TYPE (op), m_vars[p], idx, false);
901 62538 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t);
902 62538 : insert_before (g);
903 62538 : t = gimple_assign_lhs (g);
904 62538 : if (m_first
905 22641 : && m_single_use_names
906 21769 : && m_vars[p] != m_lhs
907 21669 : && m_after_stmt
908 71300 : && bitmap_bit_p (m_single_use_names, SSA_NAME_VERSION (op)))
909 : {
910 8446 : tree clobber = build_clobber (TREE_TYPE (m_vars[p]),
911 : CLOBBER_STORAGE_END);
912 8446 : g = gimple_build_assign (m_vars[p], clobber);
913 8446 : gimple_stmt_iterator gsi = gsi_for_stmt (m_after_stmt);
914 8446 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
915 : }
916 : return t;
917 34434 : case INTEGER_CST:
918 34434 : if (tree_fits_uhwi_p (idx))
919 : {
920 23839 : tree c, type = limb_access_type (TREE_TYPE (op), idx);
921 23839 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
922 23839 : if (m_first)
923 : {
924 6214 : m_data.safe_push (NULL_TREE);
925 6214 : m_data.safe_push (NULL_TREE);
926 : }
927 23839 : if (bitint_big_endian)
928 0 : i = CEIL (TYPE_PRECISION (TREE_TYPE (op)), limb_prec) - 1 - i;
929 23839 : if (limb_prec != HOST_BITS_PER_WIDE_INT)
930 : {
931 0 : wide_int w = wi::rshift (wi::to_wide (op), i * limb_prec,
932 0 : TYPE_SIGN (TREE_TYPE (op)));
933 0 : c = wide_int_to_tree (type,
934 0 : wide_int::from (w, TYPE_PRECISION (type),
935 : UNSIGNED));
936 0 : }
937 23839 : else if (i >= TREE_INT_CST_EXT_NUNITS (op))
938 7511 : c = build_int_cst (type,
939 13334 : tree_int_cst_sgn (op) < 0 ? -1 : 0);
940 : else
941 16328 : c = build_int_cst (type, TREE_INT_CST_ELT (op, i));
942 23839 : m_data_cnt += 2;
943 23839 : return c;
944 : }
945 10595 : if (m_first
946 10595 : || (m_data[m_data_cnt] == NULL_TREE
947 159 : && m_data[m_data_cnt + 1] == NULL_TREE))
948 : {
949 5343 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
950 5343 : unsigned int rem = prec % ((m_upwards_2limb ? 2 : 1) * limb_prec);
951 5343 : int ext;
952 5343 : unsigned min_prec = bitint_min_cst_precision (op, ext);
953 5343 : if (m_first)
954 : {
955 5184 : m_data.safe_push (NULL_TREE);
956 5184 : m_data.safe_push (NULL_TREE);
957 : }
958 5343 : if (integer_zerop (op))
959 : {
960 837 : tree c = build_zero_cst (m_limb_type);
961 837 : m_data[m_data_cnt] = c;
962 837 : m_data[m_data_cnt + 1] = c;
963 : }
964 4506 : else if (integer_all_onesp (op))
965 : {
966 678 : tree c = build_all_ones_cst (m_limb_type);
967 678 : m_data[m_data_cnt] = c;
968 678 : m_data[m_data_cnt + 1] = c;
969 : }
970 3828 : else if (m_upwards_2limb && min_prec <= (unsigned) limb_prec)
971 : {
972 : /* Single limb constant. Use a phi with that limb from
973 : the preheader edge and 0 or -1 constant from the other edge
974 : and for the second limb in the loop. */
975 894 : tree out;
976 894 : gcc_assert (m_first);
977 894 : m_data.pop ();
978 894 : m_data.pop ();
979 894 : prepare_data_in_out (fold_convert (m_limb_type, op), idx, &out,
980 894 : build_int_cst (m_limb_type, ext));
981 894 : }
982 2934 : else if (min_prec > prec - rem - 2 * limb_prec)
983 : {
984 : /* Constant which has enough significant bits that it isn't
985 : worth trying to save .rodata space by extending from smaller
986 : number. */
987 2415 : tree type;
988 2415 : if (m_var_msb)
989 25 : type = TREE_TYPE (op);
990 : else
991 : /* If we have a guarantee the most significant partial limb
992 : (if any) will be only accessed through handle_operand
993 : with INTEGER_CST idx, we don't need to include the partial
994 : limb in .rodata. */
995 2390 : type = build_bitint_type (prec - rem, 1);
996 2415 : tree c = tree_output_constant_def (fold_convert (type, op));
997 2415 : m_data[m_data_cnt] = c;
998 2415 : m_data[m_data_cnt + 1] = NULL_TREE;
999 : }
1000 519 : else if (m_upwards_2limb)
1001 : {
1002 : /* Constant with smaller number of bits. Trade conditional
1003 : code for .rodata space by extending from smaller number. */
1004 444 : min_prec = CEIL (min_prec, 2 * limb_prec) * (2 * limb_prec);
1005 444 : tree type = build_bitint_type (min_prec, 1);
1006 444 : tree c = tree_output_constant_def (fold_convert (type, op));
1007 444 : tree ridx = idx;
1008 444 : if (bitint_big_endian)
1009 : {
1010 0 : ridx = make_ssa_name (sizetype);
1011 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1012 0 : size_int (min_prec / limb_prec
1013 : - ((HOST_WIDE_INT)
1014 : CEIL (prec,
1015 : limb_prec))));
1016 0 : insert_before (g);
1017 : }
1018 444 : tree ridx2 = make_ssa_name (sizetype);
1019 444 : g = gimple_build_assign (ridx2, PLUS_EXPR, ridx,
1020 : bitint_big_endian
1021 0 : ? size_int (-1) : size_one_node);
1022 444 : insert_before (g);
1023 444 : if (bitint_big_endian)
1024 0 : g = gimple_build_cond (GE_EXPR, idx,
1025 0 : size_int (CEIL (prec, limb_prec)
1026 : - min_prec / limb_prec),
1027 : NULL_TREE, NULL_TREE);
1028 : else
1029 444 : g = gimple_build_cond (LT_EXPR, idx,
1030 444 : size_int (min_prec / limb_prec),
1031 : NULL_TREE, NULL_TREE);
1032 444 : edge edge_true, edge_false;
1033 888 : if_then (g, (min_prec >= (prec - rem) / 2
1034 312 : ? profile_probability::likely ()
1035 132 : : profile_probability::unlikely ()),
1036 : edge_true, edge_false);
1037 444 : tree c1 = limb_access (TREE_TYPE (op), c, ridx, false);
1038 444 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c1)), c1);
1039 444 : insert_before (g);
1040 444 : c1 = gimple_assign_lhs (g);
1041 444 : tree c2 = limb_access (TREE_TYPE (op), c, ridx2, false);
1042 444 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c2)), c2);
1043 444 : insert_before (g);
1044 444 : c2 = gimple_assign_lhs (g);
1045 444 : tree c3 = build_int_cst (m_limb_type, ext);
1046 444 : m_gsi = gsi_after_labels (edge_true->dest);
1047 444 : m_data[m_data_cnt] = make_ssa_name (m_limb_type);
1048 444 : m_data[m_data_cnt + 1] = make_ssa_name (m_limb_type);
1049 444 : gphi *phi = create_phi_node (m_data[m_data_cnt],
1050 : edge_true->dest);
1051 444 : add_phi_arg (phi, c1, edge_true, UNKNOWN_LOCATION);
1052 444 : add_phi_arg (phi, c3, edge_false, UNKNOWN_LOCATION);
1053 444 : phi = create_phi_node (m_data[m_data_cnt + 1], edge_true->dest);
1054 444 : add_phi_arg (phi, c2, edge_true, UNKNOWN_LOCATION);
1055 444 : add_phi_arg (phi, c3, edge_false, UNKNOWN_LOCATION);
1056 : }
1057 : else
1058 : {
1059 : /* Constant with smaller number of bits. Trade conditional
1060 : code for .rodata space by extending from smaller number.
1061 : Version for loops with random access to the limbs or
1062 : downwards loops. */
1063 75 : min_prec = CEIL (min_prec, limb_prec) * limb_prec;
1064 75 : tree c;
1065 75 : if (min_prec <= (unsigned) limb_prec)
1066 21 : c = fold_convert (m_limb_type, op);
1067 : else
1068 : {
1069 54 : tree type = build_bitint_type (min_prec, 1);
1070 54 : c = tree_output_constant_def (fold_convert (type, op));
1071 : }
1072 75 : m_data[m_data_cnt] = c;
1073 75 : m_data[m_data_cnt + 1] = integer_type_node;
1074 : }
1075 5343 : t = m_data[m_data_cnt];
1076 : }
1077 : else
1078 5252 : t = m_data[m_data_cnt + 1];
1079 10595 : if (m_data[m_data_cnt + 1] == NULL_TREE)
1080 : {
1081 4774 : tree ridx = idx;
1082 4774 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
1083 4774 : tree c = m_data[m_data_cnt];
1084 4774 : unsigned int min_prec = TYPE_PRECISION (TREE_TYPE (c));
1085 4774 : if (bitint_big_endian
1086 0 : && CEIL (min_prec, limb_prec) != CEIL (prec, limb_prec))
1087 : {
1088 0 : ridx = make_ssa_name (sizetype);
1089 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1090 0 : size_int (CEIL (min_prec, limb_prec)
1091 : - ((HOST_WIDE_INT)
1092 : CEIL (prec, limb_prec))));
1093 0 : insert_before (g);
1094 : }
1095 4774 : t = limb_access (TREE_TYPE (op), c, ridx, false);
1096 4774 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t);
1097 4774 : insert_before (g);
1098 4774 : t = gimple_assign_lhs (g);
1099 : }
1100 5821 : else if (m_data[m_data_cnt + 1] == integer_type_node)
1101 : {
1102 115 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
1103 115 : unsigned rem = prec % ((m_upwards_2limb ? 2 : 1) * limb_prec);
1104 115 : int ext = wi::neg_p (wi::to_wide (op)) ? -1 : 0;
1105 115 : tree c = m_data[m_data_cnt];
1106 115 : unsigned min_prec = TYPE_PRECISION (TREE_TYPE (c));
1107 115 : if (bitint_big_endian)
1108 0 : g = gimple_build_cond (GE_EXPR, idx,
1109 0 : size_int (CEIL (prec, limb_prec)
1110 : - min_prec / limb_prec),
1111 : NULL_TREE, NULL_TREE);
1112 : else
1113 115 : g = gimple_build_cond (LT_EXPR, idx,
1114 115 : size_int (min_prec / limb_prec),
1115 : NULL_TREE, NULL_TREE);
1116 115 : edge edge_true, edge_false;
1117 230 : if_then (g, (min_prec >= (prec - rem) / 2
1118 29 : ? profile_probability::likely ()
1119 86 : : profile_probability::unlikely ()),
1120 : edge_true, edge_false);
1121 115 : if (min_prec > (unsigned) limb_prec)
1122 : {
1123 70 : tree ridx = idx;
1124 70 : if (bitint_big_endian)
1125 : {
1126 0 : ridx = make_ssa_name (sizetype);
1127 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1128 0 : size_int (min_prec / limb_prec
1129 : - ((HOST_WIDE_INT)
1130 : CEIL (prec,
1131 : limb_prec))));
1132 0 : insert_before (g);
1133 : }
1134 70 : c = limb_access (TREE_TYPE (op), c, ridx, false);
1135 70 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c)), c);
1136 70 : insert_before (g);
1137 70 : c = gimple_assign_lhs (g);
1138 : }
1139 115 : tree c2 = build_int_cst (m_limb_type, ext);
1140 115 : m_gsi = gsi_after_labels (edge_true->dest);
1141 115 : t = make_ssa_name (m_limb_type);
1142 115 : gphi *phi = create_phi_node (t, edge_true->dest);
1143 115 : add_phi_arg (phi, c, edge_true, UNKNOWN_LOCATION);
1144 115 : add_phi_arg (phi, c2, edge_false, UNKNOWN_LOCATION);
1145 : }
1146 10595 : m_data_cnt += 2;
1147 10595 : return t;
1148 0 : default:
1149 0 : gcc_unreachable ();
1150 : }
1151 : }
1152 :
1153 : /* Helper method, add a PHI node with VAL from preheader edge if
1154 : inside of a loop and m_first. Keep state in a pair of m_data
1155 : elements. If VAL_OUT is non-NULL, use that as PHI argument from
1156 : the latch edge, otherwise create a new SSA_NAME for it and let
1157 : caller initialize it. */
1158 :
1159 : tree
1160 15851 : bitint_large_huge::prepare_data_in_out (tree val, tree idx, tree *data_out,
1161 : tree val_out)
1162 : {
1163 15851 : if (!m_first)
1164 : {
1165 9425 : *data_out = tree_fits_uhwi_p (idx) ? NULL_TREE : m_data[m_data_cnt + 1];
1166 9425 : return m_data[m_data_cnt];
1167 : }
1168 :
1169 6426 : *data_out = NULL_TREE;
1170 6426 : if (tree_fits_uhwi_p (idx))
1171 : {
1172 2004 : m_data.safe_push (val);
1173 2004 : m_data.safe_push (NULL_TREE);
1174 2004 : return val;
1175 : }
1176 :
1177 4422 : tree in = make_ssa_name (TREE_TYPE (val));
1178 4422 : gphi *phi = create_phi_node (in, m_bb);
1179 4422 : edge e1 = find_edge (m_preheader_bb, m_bb);
1180 4422 : edge e2 = EDGE_PRED (m_bb, 0);
1181 4422 : if (e1 == e2)
1182 4422 : e2 = EDGE_PRED (m_bb, 1);
1183 4422 : add_phi_arg (phi, val, e1, UNKNOWN_LOCATION);
1184 4422 : tree out = val_out ? val_out : make_ssa_name (TREE_TYPE (val));
1185 4422 : add_phi_arg (phi, out, e2, UNKNOWN_LOCATION);
1186 4422 : m_data.safe_push (in);
1187 4422 : m_data.safe_push (out);
1188 4422 : return in;
1189 : }
1190 :
1191 : /* Return VAL cast to TYPE. If VAL is INTEGER_CST, just
1192 : convert it without emitting any code, otherwise emit
1193 : the conversion statement before the current location. */
1194 :
1195 : tree
1196 38731 : bitint_large_huge::add_cast (tree type, tree val)
1197 : {
1198 38731 : if (TREE_CODE (val) == INTEGER_CST)
1199 4577 : return fold_convert (type, val);
1200 :
1201 34154 : tree lhs = make_ssa_name (type);
1202 34154 : gimple *g = gimple_build_assign (lhs, NOP_EXPR, val);
1203 34154 : insert_before (g);
1204 34154 : return lhs;
1205 : }
1206 :
1207 : /* Helper of handle_stmt method, handle PLUS_EXPR or MINUS_EXPR. */
1208 :
1209 : tree
1210 13353 : bitint_large_huge::handle_plus_minus (tree_code code, tree rhs1, tree rhs2,
1211 : tree idx)
1212 : {
1213 13353 : tree lhs, data_out, ctype;
1214 13353 : tree rhs1_type = TREE_TYPE (rhs1);
1215 13353 : gimple *g;
1216 13353 : tree data_in = prepare_data_in_out (build_zero_cst (m_limb_type), idx,
1217 : &data_out);
1218 :
1219 19139 : if (optab_handler (code == PLUS_EXPR ? uaddc5_optab : usubc5_optab,
1220 13353 : TYPE_MODE (m_limb_type)) != CODE_FOR_nothing)
1221 : {
1222 13353 : ctype = build_complex_type (m_limb_type);
1223 13353 : if (!types_compatible_p (rhs1_type, m_limb_type))
1224 : {
1225 1071 : if (!TYPE_UNSIGNED (rhs1_type))
1226 : {
1227 360 : tree type = unsigned_type_for (rhs1_type);
1228 360 : rhs1 = add_cast (type, rhs1);
1229 360 : rhs2 = add_cast (type, rhs2);
1230 : }
1231 1071 : rhs1 = add_cast (m_limb_type, rhs1);
1232 1071 : rhs2 = add_cast (m_limb_type, rhs2);
1233 : }
1234 13353 : lhs = make_ssa_name (ctype);
1235 19139 : g = gimple_build_call_internal (code == PLUS_EXPR
1236 : ? IFN_UADDC : IFN_USUBC,
1237 : 3, rhs1, rhs2, data_in);
1238 13353 : gimple_call_set_lhs (g, lhs);
1239 13353 : insert_before (g);
1240 13353 : if (data_out == NULL_TREE)
1241 11134 : data_out = make_ssa_name (m_limb_type);
1242 13353 : g = gimple_build_assign (data_out, IMAGPART_EXPR,
1243 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1244 13353 : insert_before (g);
1245 : }
1246 0 : else if (types_compatible_p (rhs1_type, m_limb_type))
1247 : {
1248 0 : ctype = build_complex_type (m_limb_type);
1249 0 : lhs = make_ssa_name (ctype);
1250 0 : g = gimple_build_call_internal (code == PLUS_EXPR
1251 : ? IFN_ADD_OVERFLOW : IFN_SUB_OVERFLOW,
1252 : 2, rhs1, rhs2);
1253 0 : gimple_call_set_lhs (g, lhs);
1254 0 : insert_before (g);
1255 0 : if (data_out == NULL_TREE)
1256 0 : data_out = make_ssa_name (m_limb_type);
1257 0 : if (!integer_zerop (data_in))
1258 : {
1259 0 : rhs1 = make_ssa_name (m_limb_type);
1260 0 : g = gimple_build_assign (rhs1, REALPART_EXPR,
1261 : build1 (REALPART_EXPR, m_limb_type, lhs));
1262 0 : insert_before (g);
1263 0 : rhs2 = make_ssa_name (m_limb_type);
1264 0 : g = gimple_build_assign (rhs2, IMAGPART_EXPR,
1265 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1266 0 : insert_before (g);
1267 0 : lhs = make_ssa_name (ctype);
1268 0 : g = gimple_build_call_internal (code == PLUS_EXPR
1269 : ? IFN_ADD_OVERFLOW
1270 : : IFN_SUB_OVERFLOW,
1271 : 2, rhs1, data_in);
1272 0 : gimple_call_set_lhs (g, lhs);
1273 0 : insert_before (g);
1274 0 : data_in = make_ssa_name (m_limb_type);
1275 0 : g = gimple_build_assign (data_in, IMAGPART_EXPR,
1276 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1277 0 : insert_before (g);
1278 0 : g = gimple_build_assign (data_out, PLUS_EXPR, rhs2, data_in);
1279 0 : insert_before (g);
1280 : }
1281 : else
1282 : {
1283 0 : g = gimple_build_assign (data_out, IMAGPART_EXPR,
1284 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1285 0 : insert_before (g);
1286 : }
1287 : }
1288 : else
1289 : {
1290 : /* Always perform the two additions or subtractions in
1291 : unsigned type, avoid introducing a temporary UB. While
1292 : the end result of the 2 additions or 2 subtractions in
1293 : a valid program should not overflow, temporarily it can.
1294 : See PR126503. */
1295 0 : tree utype = unsigned_type_for (rhs1_type);
1296 0 : tree in = add_cast (utype, data_in);
1297 0 : lhs = make_ssa_name (utype);
1298 0 : if (utype != rhs1_type)
1299 : {
1300 0 : rhs1 = add_cast (utype, rhs1);
1301 0 : rhs2 = add_cast (utype, rhs2);
1302 : }
1303 0 : g = gimple_build_assign (lhs, code, rhs1, rhs2);
1304 0 : insert_before (g);
1305 0 : rhs1 = make_ssa_name (utype);
1306 0 : g = gimple_build_assign (rhs1, code, lhs, in);
1307 0 : insert_before (g);
1308 0 : m_data[m_data_cnt] = NULL_TREE;
1309 0 : m_data_cnt += 2;
1310 0 : return utype != rhs1_type ? add_cast (rhs1_type, rhs1) : rhs1;
1311 : }
1312 13353 : rhs1 = make_ssa_name (m_limb_type);
1313 13353 : g = gimple_build_assign (rhs1, REALPART_EXPR,
1314 : build1 (REALPART_EXPR, m_limb_type, lhs));
1315 13353 : insert_before (g);
1316 13353 : if (!types_compatible_p (rhs1_type, m_limb_type))
1317 1071 : rhs1 = add_cast (rhs1_type, rhs1);
1318 13353 : m_data[m_data_cnt] = data_out;
1319 13353 : m_data_cnt += 2;
1320 13353 : return rhs1;
1321 : }
1322 :
1323 : /* Helper function for handle_stmt method, handle LSHIFT_EXPR by
1324 : count in [0, limb_prec - 1] range. */
1325 :
1326 : tree
1327 152 : bitint_large_huge::handle_lshift (tree rhs1, tree rhs2, tree idx)
1328 : {
1329 152 : unsigned HOST_WIDE_INT cnt = tree_to_uhwi (rhs2);
1330 152 : gcc_checking_assert (cnt < (unsigned) limb_prec);
1331 152 : if (cnt == 0)
1332 : return rhs1;
1333 :
1334 152 : tree lhs, data_out, rhs1_type = TREE_TYPE (rhs1);
1335 152 : gimple *g;
1336 152 : tree data_in = prepare_data_in_out (build_zero_cst (m_limb_type), idx,
1337 : &data_out);
1338 :
1339 152 : if (!integer_zerop (data_in))
1340 : {
1341 136 : lhs = make_ssa_name (m_limb_type);
1342 136 : g = gimple_build_assign (lhs, RSHIFT_EXPR, data_in,
1343 : build_int_cst (unsigned_type_node,
1344 136 : limb_prec - cnt));
1345 136 : insert_before (g);
1346 136 : if (!types_compatible_p (rhs1_type, m_limb_type))
1347 35 : lhs = add_cast (rhs1_type, lhs);
1348 : data_in = lhs;
1349 : }
1350 152 : if (types_compatible_p (rhs1_type, m_limb_type))
1351 : {
1352 117 : if (data_out == NULL_TREE)
1353 82 : data_out = make_ssa_name (m_limb_type);
1354 117 : g = gimple_build_assign (data_out, rhs1);
1355 117 : insert_before (g);
1356 : }
1357 152 : if (cnt < (unsigned) TYPE_PRECISION (rhs1_type))
1358 : {
1359 137 : lhs = make_ssa_name (rhs1_type);
1360 137 : g = gimple_build_assign (lhs, LSHIFT_EXPR, rhs1, rhs2);
1361 137 : insert_before (g);
1362 137 : if (!integer_zerop (data_in))
1363 : {
1364 121 : rhs1 = lhs;
1365 121 : lhs = make_ssa_name (rhs1_type);
1366 121 : g = gimple_build_assign (lhs, BIT_IOR_EXPR, rhs1, data_in);
1367 121 : insert_before (g);
1368 : }
1369 : }
1370 : else
1371 : lhs = data_in;
1372 152 : m_data[m_data_cnt] = data_out;
1373 152 : m_data_cnt += 2;
1374 152 : return lhs;
1375 : }
1376 :
1377 : /* Helper function for handle_stmt method, handle an integral
1378 : to integral conversion. */
1379 :
1380 : tree
1381 7558 : bitint_large_huge::handle_cast (tree lhs_type, tree rhs1, tree idx)
1382 : {
1383 7558 : tree rhs_type = TREE_TYPE (rhs1);
1384 7558 : gimple *g;
1385 7558 : if ((TREE_CODE (rhs1) == SSA_NAME || TREE_CODE (rhs1) == INTEGER_CST)
1386 7558 : && BITINT_TYPE_P (lhs_type)
1387 7558 : && BITINT_TYPE_P (rhs_type)
1388 6565 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
1389 14123 : && bitint_precision_kind (rhs_type) >= bitint_prec_large)
1390 : {
1391 5926 : if (TYPE_PRECISION (rhs_type) >= TYPE_PRECISION (lhs_type)
1392 : /* If lhs has bigger precision than rhs, we can use
1393 : the simple case only if there is a guarantee that
1394 : the most significant limb is handled in straight
1395 : line code. If m_var_msb (on left shifts) or
1396 : if m_upwards_2limb * limb_prec is equal to
1397 : lhs precision or if not m_upwards_2limb and lhs_type
1398 : has precision which is multiple of limb_prec that is
1399 : not the case. */
1400 5926 : || (!m_var_msb
1401 1524 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1402 1524 : == CEIL (TYPE_PRECISION (rhs_type), limb_prec))
1403 370 : && ((!m_upwards_2limb
1404 182 : && (TYPE_PRECISION (lhs_type) % limb_prec != 0))
1405 267 : || (m_upwards_2limb
1406 376 : && (m_upwards_2limb * limb_prec
1407 188 : < TYPE_PRECISION (lhs_type))))))
1408 : {
1409 4667 : tree ridx = idx;
1410 4667 : if (bitint_big_endian
1411 4667 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1412 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1413 : {
1414 0 : HOST_WIDE_INT diff = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1415 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1416 0 : if (tree_fits_uhwi_p (idx))
1417 0 : ridx = size_int (tree_to_uhwi (idx) + diff);
1418 : else
1419 : {
1420 0 : tree t = make_ssa_name (sizetype);
1421 0 : g = gimple_build_assign (t, PLUS_EXPR, idx, size_int (diff));
1422 0 : insert_before (g);
1423 0 : ridx = t;
1424 : }
1425 : }
1426 4667 : rhs1 = handle_operand (rhs1, ridx);
1427 4667 : if (tree_fits_uhwi_p (idx))
1428 : {
1429 2386 : tree type = limb_access_type (lhs_type, idx);
1430 2386 : if (!types_compatible_p (type, TREE_TYPE (rhs1)))
1431 1249 : rhs1 = add_cast (type, rhs1);
1432 : }
1433 : return rhs1;
1434 : }
1435 1259 : tree t;
1436 : /* Indexes lower than this don't need any special processing. */
1437 1259 : unsigned low = ((unsigned) TYPE_PRECISION (rhs_type)
1438 1259 : - !TYPE_UNSIGNED (rhs_type)) / limb_prec;
1439 : /* Indexes >= than this always contain an extension. */
1440 1259 : unsigned high = CEIL ((unsigned) TYPE_PRECISION (rhs_type), limb_prec);
1441 1259 : unsigned lcnt = CEIL ((unsigned) TYPE_PRECISION (lhs_type), limb_prec);
1442 1259 : unsigned lowe = bitint_big_endian ? lcnt - 1 - low : low;
1443 1259 : bool save_first = m_first;
1444 1259 : if (m_first)
1445 : {
1446 414 : m_data.safe_push (NULL_TREE);
1447 414 : m_data.safe_push (NULL_TREE);
1448 414 : m_data.safe_push (NULL_TREE);
1449 414 : if (TYPE_UNSIGNED (rhs_type))
1450 : /* No need to keep state between iterations. */
1451 : ;
1452 193 : else if (m_upwards && !m_upwards_2limb)
1453 : /* We need to keep state between iterations, but
1454 : not within any loop, everything is straight line
1455 : code with only increasing indexes. */
1456 : ;
1457 153 : else if (!m_upwards_2limb)
1458 : {
1459 3 : unsigned save_data_cnt = m_data_cnt;
1460 3 : gimple_stmt_iterator save_gsi = m_gsi;
1461 3 : m_gsi = m_init_gsi;
1462 3 : if (gsi_end_p (m_gsi))
1463 0 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
1464 : else
1465 3 : gsi_next (&m_gsi);
1466 3 : m_data_cnt = save_data_cnt + 3;
1467 3 : t = handle_operand (rhs1, size_int (bitint_big_endian
1468 : ? high - 1 - low : low));
1469 3 : m_first = false;
1470 3 : m_data[save_data_cnt + 2]
1471 3 : = build_int_cst (NULL_TREE, m_data_cnt);
1472 3 : m_data_cnt = save_data_cnt;
1473 3 : t = add_cast (signed_type_for (m_limb_type), t);
1474 3 : tree lpm1 = build_int_cst (unsigned_type_node, limb_prec - 1);
1475 3 : tree n = make_ssa_name (TREE_TYPE (t));
1476 3 : g = gimple_build_assign (n, RSHIFT_EXPR, t, lpm1);
1477 3 : insert_before (g);
1478 3 : m_data[save_data_cnt + 1] = add_cast (m_limb_type, n);
1479 3 : m_init_gsi = m_gsi;
1480 3 : if (gsi_end_p (m_init_gsi))
1481 0 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
1482 : else
1483 3 : gsi_prev (&m_init_gsi);
1484 3 : m_gsi = save_gsi;
1485 : }
1486 150 : else if (m_upwards_2limb * limb_prec < TYPE_PRECISION (rhs_type))
1487 : /* We need to keep state between iterations, but
1488 : fortunately not within the loop, only afterwards. */
1489 : ;
1490 : else
1491 : {
1492 146 : tree out;
1493 146 : m_data.truncate (m_data_cnt);
1494 146 : prepare_data_in_out (build_zero_cst (m_limb_type), idx, &out);
1495 146 : m_data.safe_push (NULL_TREE);
1496 : }
1497 : }
1498 :
1499 1259 : unsigned save_data_cnt = m_data_cnt;
1500 1259 : m_data_cnt += 3;
1501 1259 : if (!tree_fits_uhwi_p (idx))
1502 : {
1503 688 : if (m_upwards_2limb
1504 668 : && low >= m_upwards_2limb - m_first)
1505 : {
1506 158 : if (bitint_big_endian
1507 158 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1508 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1509 : {
1510 0 : HOST_WIDE_INT diff
1511 0 : = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1512 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1513 0 : tree t = make_ssa_name (sizetype);
1514 0 : g = gimple_build_assign (t, PLUS_EXPR, idx, size_int (diff));
1515 0 : insert_before (g);
1516 0 : idx = t;
1517 : }
1518 158 : rhs1 = handle_operand (rhs1, idx);
1519 158 : if (m_first)
1520 131 : m_data[save_data_cnt + 2]
1521 262 : = build_int_cst (NULL_TREE, m_data_cnt);
1522 158 : m_first = save_first;
1523 158 : return rhs1;
1524 : }
1525 1326 : bool single_comparison
1526 530 : = low == high || (m_upwards_2limb && (low & 1) == m_first);
1527 266 : tree idxc = idx;
1528 266 : if (!single_comparison
1529 266 : && m_upwards_2limb
1530 246 : && !m_first
1531 112 : && low + 1 == m_upwards_2limb)
1532 : /* In this case we know that idx <= low always,
1533 : so effectively we just needs a single comparison,
1534 : idx < low or idx == low, but we'd need to emit different
1535 : code for the 2 branches than single_comparison normally
1536 : emits. So, instead of special-casing that, emit a
1537 : low <= low comparison which cfg cleanup will clean up
1538 : at the end of the pass. */
1539 89 : idxc = size_int (lowe);
1540 530 : if (bitint_big_endian)
1541 0 : g = gimple_build_cond (single_comparison ? GT_EXPR : GE_EXPR,
1542 0 : idxc, size_int (lowe),
1543 : NULL_TREE, NULL_TREE);
1544 : else
1545 796 : g = gimple_build_cond (single_comparison ? LT_EXPR : LE_EXPR,
1546 530 : idxc, size_int (low), NULL_TREE, NULL_TREE);
1547 530 : edge edge_true_true, edge_true_false, edge_false;
1548 796 : if_then_if_then_else (g, (single_comparison ? NULL
1549 266 : : gimple_build_cond (EQ_EXPR, idx,
1550 266 : size_int (lowe),
1551 : NULL_TREE,
1552 : NULL_TREE)),
1553 : profile_probability::likely (),
1554 : profile_probability::unlikely (),
1555 : edge_true_true, edge_true_false, edge_false);
1556 530 : bool save_cast_conditional = m_cast_conditional;
1557 530 : m_cast_conditional = true;
1558 530 : m_bitfld_load = 0;
1559 530 : tree t1 = idx, t2 = NULL_TREE;
1560 530 : if (bitint_big_endian
1561 530 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1562 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1563 : {
1564 0 : HOST_WIDE_INT diff = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1565 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1566 0 : t1 = make_ssa_name (sizetype);
1567 0 : g = gimple_build_assign (t1, PLUS_EXPR, idx, size_int (diff));
1568 0 : insert_before (g);
1569 : }
1570 530 : t1 = handle_operand (rhs1, t1);
1571 530 : if (m_first)
1572 208 : m_data[save_data_cnt + 2]
1573 416 : = build_int_cst (NULL_TREE, m_data_cnt);
1574 530 : tree ext = NULL_TREE;
1575 530 : tree bitfld = NULL_TREE;
1576 530 : if (!single_comparison)
1577 : {
1578 266 : m_gsi = gsi_after_labels (edge_true_true->src);
1579 266 : m_first = false;
1580 266 : m_data_cnt = save_data_cnt + 3;
1581 266 : if (m_bitfld_load)
1582 : {
1583 4 : bitfld = m_data[m_bitfld_load];
1584 4 : m_data[m_bitfld_load] = m_data[m_bitfld_load + 2];
1585 4 : m_bitfld_load = 0;
1586 : }
1587 266 : t2 = handle_operand (rhs1, size_int (bitint_big_endian
1588 : ? high - 1 - low : low));
1589 266 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (t2)))
1590 221 : t2 = add_cast (m_limb_type, t2);
1591 266 : if (!TYPE_UNSIGNED (rhs_type) && m_upwards_2limb)
1592 : {
1593 146 : ext = add_cast (signed_type_for (m_limb_type), t2);
1594 292 : tree lpm1 = build_int_cst (unsigned_type_node,
1595 146 : limb_prec - 1);
1596 146 : tree n = make_ssa_name (TREE_TYPE (ext));
1597 146 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
1598 146 : insert_before (g);
1599 146 : ext = add_cast (m_limb_type, n);
1600 : }
1601 : }
1602 530 : tree t3;
1603 530 : if (TYPE_UNSIGNED (rhs_type))
1604 278 : t3 = build_zero_cst (m_limb_type);
1605 252 : else if (m_upwards_2limb && (save_first || ext != NULL_TREE))
1606 168 : t3 = m_data[save_data_cnt];
1607 : else
1608 84 : t3 = m_data[save_data_cnt + 1];
1609 530 : m_gsi = gsi_after_labels (edge_true_false->dest);
1610 530 : t = make_ssa_name (m_limb_type);
1611 530 : gphi *phi = create_phi_node (t, edge_true_false->dest);
1612 530 : add_phi_arg (phi, t1, edge_true_false, UNKNOWN_LOCATION);
1613 530 : add_phi_arg (phi, t3, edge_false, UNKNOWN_LOCATION);
1614 530 : if (edge_true_true)
1615 266 : add_phi_arg (phi, t2, edge_true_true, UNKNOWN_LOCATION);
1616 530 : if (ext)
1617 : {
1618 146 : tree t4 = make_ssa_name (m_limb_type);
1619 146 : phi = create_phi_node (t4, edge_true_false->dest);
1620 146 : add_phi_arg (phi, build_zero_cst (m_limb_type), edge_true_false,
1621 : UNKNOWN_LOCATION);
1622 146 : add_phi_arg (phi, m_data[save_data_cnt], edge_false,
1623 : UNKNOWN_LOCATION);
1624 146 : add_phi_arg (phi, ext, edge_true_true, UNKNOWN_LOCATION);
1625 146 : if (!save_cast_conditional)
1626 : {
1627 136 : g = gimple_build_assign (m_data[save_data_cnt + 1], t4);
1628 136 : insert_before (g);
1629 : }
1630 : else
1631 10 : for (basic_block bb = gsi_bb (m_gsi);;)
1632 : {
1633 10 : edge e1 = single_succ_edge (bb);
1634 10 : edge e2 = find_edge (e1->dest, m_bb), e3;
1635 10 : tree t5 = (e2 ? m_data[save_data_cnt + 1]
1636 10 : : make_ssa_name (m_limb_type));
1637 10 : phi = create_phi_node (t5, e1->dest);
1638 10 : edge_iterator ei;
1639 30 : FOR_EACH_EDGE (e3, ei, e1->dest->preds)
1640 30 : add_phi_arg (phi, (e3 == e1 ? t4
1641 10 : : build_zero_cst (m_limb_type)),
1642 : e3, UNKNOWN_LOCATION);
1643 10 : if (e2)
1644 : break;
1645 0 : t4 = t5;
1646 0 : bb = e1->dest;
1647 0 : }
1648 : }
1649 530 : if (m_bitfld_load)
1650 : {
1651 8 : tree t4;
1652 8 : if (!save_first && !save_cast_conditional)
1653 2 : t4 = m_data[m_bitfld_load + 1];
1654 : else
1655 6 : t4 = make_ssa_name (m_limb_type);
1656 8 : phi = create_phi_node (t4, edge_true_false->dest);
1657 12 : add_phi_arg (phi,
1658 4 : edge_true_true ? bitfld : m_data[m_bitfld_load],
1659 : edge_true_false, UNKNOWN_LOCATION);
1660 8 : add_phi_arg (phi, m_data[m_bitfld_load + 2],
1661 : edge_false, UNKNOWN_LOCATION);
1662 8 : if (edge_true_true)
1663 4 : add_phi_arg (phi, m_data[m_bitfld_load], edge_true_true,
1664 : UNKNOWN_LOCATION);
1665 8 : if (save_cast_conditional)
1666 4 : for (basic_block bb = gsi_bb (m_gsi);;)
1667 : {
1668 4 : edge e1 = single_succ_edge (bb);
1669 4 : edge e2 = find_edge (e1->dest, m_bb), e3;
1670 4 : tree t5 = ((e2 && !save_first) ? m_data[m_bitfld_load + 1]
1671 4 : : make_ssa_name (m_limb_type));
1672 4 : phi = create_phi_node (t5, e1->dest);
1673 4 : edge_iterator ei;
1674 14 : FOR_EACH_EDGE (e3, ei, e1->dest->preds)
1675 16 : add_phi_arg (phi, (e3 == e1 ? t4
1676 6 : : build_zero_cst (m_limb_type)),
1677 : e3, UNKNOWN_LOCATION);
1678 4 : t4 = t5;
1679 4 : if (e2)
1680 : break;
1681 0 : bb = e1->dest;
1682 0 : }
1683 8 : m_data[m_bitfld_load] = t4;
1684 8 : m_data[m_bitfld_load + 2] = t4;
1685 8 : m_bitfld_load = 0;
1686 : }
1687 530 : m_cast_conditional = save_cast_conditional;
1688 530 : m_first = save_first;
1689 530 : return t;
1690 : }
1691 : else
1692 : {
1693 571 : unsigned tidx = tree_to_uhwi (idx);
1694 571 : if (bitint_big_endian)
1695 0 : tidx = lcnt - 1 - tidx;
1696 571 : if (tidx < low)
1697 : {
1698 152 : t = handle_operand (rhs1, (bitint_big_endian
1699 0 : ? size_int (high - 1 - tidx) : idx));
1700 152 : if (m_first)
1701 71 : m_data[save_data_cnt + 2]
1702 142 : = build_int_cst (NULL_TREE, m_data_cnt);
1703 : }
1704 419 : else if (tidx < high)
1705 : {
1706 68 : t = handle_operand (rhs1, size_int (bitint_big_endian
1707 : ? high - 1 - low : low));
1708 68 : if (m_first)
1709 1 : m_data[save_data_cnt + 2]
1710 2 : = build_int_cst (NULL_TREE, m_data_cnt);
1711 68 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (t)))
1712 60 : t = add_cast (m_limb_type, t);
1713 68 : tree ext = NULL_TREE;
1714 68 : if (!TYPE_UNSIGNED (rhs_type) && m_upwards)
1715 : {
1716 44 : ext = add_cast (signed_type_for (m_limb_type), t);
1717 88 : tree lpm1 = build_int_cst (unsigned_type_node,
1718 44 : limb_prec - 1);
1719 44 : tree n = make_ssa_name (TREE_TYPE (ext));
1720 44 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
1721 44 : insert_before (g);
1722 44 : ext = add_cast (m_limb_type, n);
1723 44 : m_data[save_data_cnt + 1] = ext;
1724 : }
1725 : }
1726 : else
1727 : {
1728 351 : if (TYPE_UNSIGNED (rhs_type) && m_first)
1729 : {
1730 0 : handle_operand (rhs1, (bitint_big_endian
1731 0 : ? size_int (high - 1)
1732 : : size_zero_node));
1733 0 : m_data[save_data_cnt + 2]
1734 0 : = build_int_cst (NULL_TREE, m_data_cnt);
1735 : }
1736 : else
1737 351 : m_data_cnt = tree_to_uhwi (m_data[save_data_cnt + 2]);
1738 351 : if (TYPE_UNSIGNED (rhs_type))
1739 184 : t = build_zero_cst (m_limb_type);
1740 167 : else if (m_bb
1741 16 : && m_data[save_data_cnt]
1742 180 : && ((tidx & 1) == 0 || tidx != low + 1))
1743 : t = m_data[save_data_cnt];
1744 : else
1745 161 : t = m_data[save_data_cnt + 1];
1746 : }
1747 571 : tree type = limb_access_type (lhs_type, idx);
1748 571 : if (!useless_type_conversion_p (type, m_limb_type))
1749 292 : t = add_cast (type, t);
1750 571 : m_first = save_first;
1751 571 : return t;
1752 : }
1753 : }
1754 0 : else if (BITINT_TYPE_P (lhs_type)
1755 1632 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
1756 3264 : && INTEGRAL_TYPE_P (rhs_type))
1757 : {
1758 : /* Add support for 3 or more limbs filled in from normal integral
1759 : type if this assert fails. If no target chooses limb mode smaller
1760 : than half of largest supported normal integral type, this will not
1761 : be needed. */
1762 1632 : gcc_assert (TYPE_PRECISION (rhs_type) <= 2 * limb_prec);
1763 1632 : tree r1 = NULL_TREE, r2 = NULL_TREE, rext = NULL_TREE;
1764 1632 : if (m_first)
1765 : {
1766 594 : gimple_stmt_iterator save_gsi = m_gsi;
1767 594 : m_gsi = m_init_gsi;
1768 594 : if (gsi_end_p (m_gsi))
1769 58 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
1770 : else
1771 536 : gsi_next (&m_gsi);
1772 375 : if (BITINT_TYPE_P (rhs_type)
1773 594 : && bitint_precision_kind (rhs_type) == bitint_prec_middle)
1774 : {
1775 63 : tree type = NULL_TREE;
1776 63 : rhs1 = maybe_cast_middle_bitint (&m_gsi, rhs1, type);
1777 63 : rhs_type = TREE_TYPE (rhs1);
1778 : }
1779 594 : r1 = rhs1;
1780 594 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
1781 526 : r1 = add_cast (m_limb_type, rhs1);
1782 594 : if (TYPE_PRECISION (rhs_type) > limb_prec)
1783 : {
1784 109 : g = gimple_build_assign (make_ssa_name (rhs_type),
1785 : RSHIFT_EXPR, rhs1,
1786 : build_int_cst (unsigned_type_node,
1787 109 : limb_prec));
1788 109 : insert_before (g);
1789 109 : r2 = add_cast (m_limb_type, gimple_assign_lhs (g));
1790 : }
1791 594 : if (TYPE_UNSIGNED (rhs_type))
1792 287 : rext = build_zero_cst (m_limb_type);
1793 : else
1794 : {
1795 307 : rext = add_cast (signed_type_for (m_limb_type), r2 ? r2 : r1);
1796 307 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rext)),
1797 : RSHIFT_EXPR, rext,
1798 : build_int_cst (unsigned_type_node,
1799 307 : limb_prec - 1));
1800 307 : insert_before (g);
1801 307 : rext = add_cast (m_limb_type, gimple_assign_lhs (g));
1802 : }
1803 594 : m_init_gsi = m_gsi;
1804 594 : if (gsi_end_p (m_init_gsi))
1805 598 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
1806 : else
1807 295 : gsi_prev (&m_init_gsi);
1808 594 : m_gsi = save_gsi;
1809 : }
1810 1632 : tree t;
1811 1632 : if (m_upwards_2limb)
1812 : {
1813 776 : if (m_first)
1814 : {
1815 297 : tree out1, out2;
1816 297 : prepare_data_in_out (r1, idx, &out1, rext);
1817 297 : if (TYPE_PRECISION (rhs_type) > limb_prec)
1818 : {
1819 70 : prepare_data_in_out (r2, idx, &out2, rext);
1820 70 : m_data.pop ();
1821 70 : t = m_data.pop ();
1822 70 : m_data[m_data_cnt + 1] = t;
1823 : }
1824 : else
1825 227 : m_data[m_data_cnt + 1] = rext;
1826 297 : m_data.safe_push (rext);
1827 297 : t = m_data[m_data_cnt];
1828 : }
1829 479 : else if (!tree_fits_uhwi_p (idx))
1830 297 : t = m_data[m_data_cnt + 1];
1831 : else
1832 : {
1833 182 : tree type = limb_access_type (lhs_type, idx);
1834 182 : t = m_data[m_data_cnt + 2];
1835 182 : if (!useless_type_conversion_p (type, m_limb_type))
1836 153 : t = add_cast (type, t);
1837 : }
1838 776 : m_data_cnt += 3;
1839 776 : return t;
1840 : }
1841 856 : else if (m_first)
1842 : {
1843 297 : m_data.safe_push (r1);
1844 297 : m_data.safe_push (r2);
1845 297 : m_data.safe_push (rext);
1846 : }
1847 856 : unsigned lcnt = CEIL ((unsigned) TYPE_PRECISION (lhs_type), limb_prec);
1848 856 : if (tree_fits_uhwi_p (idx))
1849 : {
1850 812 : tree type = limb_access_type (lhs_type, idx);
1851 812 : if (bitint_big_endian
1852 812 : ? tree_to_uhwi (idx) == lcnt - 1 : integer_zerop (idx))
1853 269 : t = m_data[m_data_cnt];
1854 543 : else if (TYPE_PRECISION (rhs_type) > limb_prec
1855 543 : && (bitint_big_endian
1856 72 : ? tree_to_uhwi (idx) == lcnt - 2
1857 72 : : integer_onep (idx)))
1858 33 : t = m_data[m_data_cnt + 1];
1859 : else
1860 510 : t = m_data[m_data_cnt + 2];
1861 812 : if (!useless_type_conversion_p (type, m_limb_type))
1862 250 : t = add_cast (type, t);
1863 812 : m_data_cnt += 3;
1864 812 : return t;
1865 : }
1866 44 : g = gimple_build_cond (NE_EXPR, idx,
1867 : bitint_big_endian
1868 0 : ? size_int (lcnt - 1) : size_zero_node,
1869 : NULL_TREE, NULL_TREE);
1870 44 : edge e2, e3, e4 = NULL;
1871 44 : if_then (g, profile_probability::likely (), e2, e3);
1872 44 : if (m_data[m_data_cnt + 1])
1873 : {
1874 14 : g = gimple_build_cond (EQ_EXPR, idx,
1875 : bitint_big_endian
1876 0 : ? size_int (lcnt - 2) : size_one_node,
1877 : NULL_TREE, NULL_TREE);
1878 14 : insert_before (g);
1879 14 : edge e5 = split_block (gsi_bb (m_gsi), g);
1880 14 : e4 = make_edge (e5->src, e2->dest, EDGE_TRUE_VALUE);
1881 14 : e2 = find_edge (e5->dest, e2->dest);
1882 14 : e4->probability = profile_probability::unlikely ();
1883 14 : e5->flags = EDGE_FALSE_VALUE;
1884 14 : e5->probability = e4->probability.invert ();
1885 : }
1886 44 : m_gsi = gsi_after_labels (e2->dest);
1887 44 : t = make_ssa_name (m_limb_type);
1888 44 : gphi *phi = create_phi_node (t, e2->dest);
1889 44 : add_phi_arg (phi, m_data[m_data_cnt + 2], e2, UNKNOWN_LOCATION);
1890 44 : add_phi_arg (phi, m_data[m_data_cnt], e3, UNKNOWN_LOCATION);
1891 44 : if (e4)
1892 14 : add_phi_arg (phi, m_data[m_data_cnt + 1], e4, UNKNOWN_LOCATION);
1893 44 : m_data_cnt += 3;
1894 44 : return t;
1895 : }
1896 : return NULL_TREE;
1897 : }
1898 :
1899 : /* Helper function for handle_stmt method, handle a BIT_FIELD_REF. */
1900 :
1901 : tree
1902 23 : bitint_large_huge::handle_bit_field_ref (tree op, tree idx)
1903 : {
1904 23 : if (tree_fits_uhwi_p (idx))
1905 : {
1906 13 : if (m_first)
1907 4 : m_data.safe_push (NULL);
1908 13 : ++m_data_cnt;
1909 13 : unsigned HOST_WIDE_INT sz = tree_to_uhwi (TYPE_SIZE (m_limb_type));
1910 13 : unsigned i = tree_to_uhwi (idx);
1911 13 : if (bitint_big_endian)
1912 0 : i = CEIL (TYPE_PRECISION (TREE_TYPE (op)), limb_prec) - 1 - i;
1913 26 : tree bfr = build3 (BIT_FIELD_REF, m_limb_type,
1914 13 : TREE_OPERAND (op, 0),
1915 13 : TYPE_SIZE (m_limb_type),
1916 13 : size_binop (PLUS_EXPR, TREE_OPERAND (op, 2),
1917 : bitsize_int (i * sz)));
1918 13 : tree r = make_ssa_name (m_limb_type);
1919 13 : gimple *g = gimple_build_assign (r, bfr);
1920 13 : insert_before (g);
1921 13 : tree type = limb_access_type (TREE_TYPE (op), idx);
1922 13 : if (!useless_type_conversion_p (type, m_limb_type))
1923 0 : r = add_cast (type, r);
1924 : return r;
1925 : }
1926 10 : tree var;
1927 10 : if (m_first)
1928 : {
1929 5 : unsigned HOST_WIDE_INT sz = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (op)));
1930 5 : machine_mode mode;
1931 5 : tree type, bfr;
1932 5 : if (bitwise_mode_for_size (sz).exists (&mode)
1933 0 : && known_eq (GET_MODE_BITSIZE (mode), sz))
1934 0 : type = bitwise_type_for_mode (mode);
1935 : else
1936 : {
1937 5 : mode = VOIDmode;
1938 5 : type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_OPERAND (op, 0)));
1939 : }
1940 5 : if (TYPE_ALIGN (type) < TYPE_ALIGN (TREE_TYPE (op)))
1941 0 : type = build_aligned_type (type, TYPE_ALIGN (TREE_TYPE (op)));
1942 5 : var = create_tmp_var (type);
1943 5 : TREE_ADDRESSABLE (var) = 1;
1944 5 : gimple *g;
1945 5 : if (mode != VOIDmode)
1946 : {
1947 0 : bfr = build3 (BIT_FIELD_REF, type, TREE_OPERAND (op, 0),
1948 0 : TYPE_SIZE (type), TREE_OPERAND (op, 2));
1949 0 : g = gimple_build_assign (make_ssa_name (type),
1950 : BIT_FIELD_REF, bfr);
1951 0 : gimple_set_location (g, m_loc);
1952 0 : gsi_insert_after (&m_init_gsi, g, GSI_NEW_STMT);
1953 0 : bfr = gimple_assign_lhs (g);
1954 : }
1955 : else
1956 5 : bfr = TREE_OPERAND (op, 0);
1957 5 : g = gimple_build_assign (var, bfr);
1958 5 : gimple_set_location (g, m_loc);
1959 5 : gsi_insert_after (&m_init_gsi, g, GSI_NEW_STMT);
1960 5 : if (mode == VOIDmode)
1961 : {
1962 5 : unsigned HOST_WIDE_INT nelts
1963 5 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (op))), limb_prec);
1964 5 : tree atype = build_array_type_nelts (m_limb_type, nelts);
1965 5 : var = build2 (MEM_REF, atype, build_fold_addr_expr (var),
1966 : build_int_cst (build_pointer_type (type),
1967 5 : tree_to_uhwi (TREE_OPERAND (op, 2))
1968 5 : / BITS_PER_UNIT));
1969 : }
1970 5 : m_data.safe_push (var);
1971 : }
1972 : else
1973 5 : var = unshare_expr (m_data[m_data_cnt]);
1974 10 : ++m_data_cnt;
1975 10 : var = limb_access (TREE_TYPE (op), var, idx, false);
1976 10 : tree r = make_ssa_name (m_limb_type);
1977 10 : gimple *g = gimple_build_assign (r, var);
1978 10 : insert_before (g);
1979 10 : return r;
1980 : }
1981 :
1982 : /* Add a new EH edge from SRC to EH_EDGE->dest, where EH_EDGE
1983 : is an older EH edge, and except for virtual PHIs duplicate the
1984 : PHI argument from the EH_EDGE to the new EH edge. */
1985 :
1986 : static void
1987 20 : add_eh_edge (basic_block src, edge eh_edge)
1988 : {
1989 20 : edge e = make_edge (src, eh_edge->dest, EDGE_EH);
1990 20 : e->probability = profile_probability::very_unlikely ();
1991 20 : for (gphi_iterator gsi = gsi_start_phis (eh_edge->dest);
1992 27 : !gsi_end_p (gsi); gsi_next (&gsi))
1993 : {
1994 7 : gphi *phi = gsi.phi ();
1995 7 : tree lhs = gimple_phi_result (phi);
1996 14 : if (virtual_operand_p (lhs))
1997 4 : continue;
1998 3 : const phi_arg_d *arg = gimple_phi_arg (phi, eh_edge->dest_idx);
1999 3 : add_phi_arg (phi, arg->def, e, arg->locus);
2000 : }
2001 20 : }
2002 :
2003 : /* Helper function for handle_stmt method, handle a load from memory. */
2004 :
2005 : tree
2006 21344 : bitint_large_huge::handle_load (gimple *stmt, tree idx)
2007 : {
2008 21344 : tree rhs1 = gimple_assign_rhs1 (stmt);
2009 21344 : tree rhs_type = TREE_TYPE (rhs1);
2010 21344 : bool eh = stmt_ends_bb_p (stmt);
2011 21344 : bool load_bitfield_p = false;
2012 21344 : edge eh_edge = NULL;
2013 21344 : gimple *g;
2014 :
2015 21344 : if (TREE_CODE (rhs1) == BIT_FIELD_REF
2016 21344 : && integer_zerop (TREE_OPERAND (rhs1, 2)))
2017 2 : rhs1 = TREE_OPERAND (rhs1, 0);
2018 :
2019 21344 : if (eh)
2020 : {
2021 10 : edge_iterator ei;
2022 10 : basic_block bb = gimple_bb (stmt);
2023 :
2024 10 : FOR_EACH_EDGE (eh_edge, ei, bb->succs)
2025 10 : if (eh_edge->flags & EDGE_EH)
2026 : break;
2027 : }
2028 :
2029 21344 : if (TREE_CODE (rhs1) == COMPONENT_REF
2030 21344 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (rhs1, 1)))
2031 : {
2032 1225 : tree fld = TREE_OPERAND (rhs1, 1);
2033 : /* For little-endian, we can allow as inputs bit-fields
2034 : which start at a limb boundary. */
2035 1225 : gcc_assert (tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld)));
2036 1225 : if (!bitint_big_endian
2037 1225 : && DECL_OFFSET_ALIGN (fld) >= TYPE_ALIGN (TREE_TYPE (rhs1))
2038 2450 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % limb_prec) == 0)
2039 : {
2040 739 : load_bitfield_p = true;
2041 751 : goto normal_load;
2042 : }
2043 : /* Even if DECL_FIELD_BIT_OFFSET (fld) is a multiple of BITS_PER_UNIT,
2044 : handle it normally for now. */
2045 486 : if (!bitint_big_endian
2046 486 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % BITS_PER_UNIT) == 0)
2047 : {
2048 12 : load_bitfield_p = true;
2049 12 : goto normal_load;
2050 : }
2051 474 : tree repr = DECL_BIT_FIELD_REPRESENTATIVE (fld);
2052 474 : poly_int64 bitoffset;
2053 474 : poly_uint64 field_offset, repr_offset;
2054 474 : bool var_field_off = false;
2055 474 : if (poly_int_tree_p (DECL_FIELD_OFFSET (fld), &field_offset)
2056 948 : && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
2057 474 : bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
2058 : else
2059 : {
2060 : bitoffset = 0;
2061 : var_field_off = true;
2062 : }
2063 474 : bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2064 474 : - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
2065 948 : tree nrhs1 = build3 (COMPONENT_REF, TREE_TYPE (repr),
2066 474 : TREE_OPERAND (rhs1, 0), repr,
2067 0 : var_field_off ? TREE_OPERAND (rhs1, 2) : NULL_TREE);
2068 474 : HOST_WIDE_INT bo = bitoffset.to_constant ();
2069 474 : unsigned bo_idx = (unsigned HOST_WIDE_INT) bo / limb_prec;
2070 474 : unsigned bo_bit = (unsigned HOST_WIDE_INT) bo % limb_prec;
2071 474 : unsigned bo_last = 0;
2072 474 : unsigned bo_shift = bo_bit;
2073 474 : unsigned nelts = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
2074 474 : if (bitint_big_endian)
2075 : {
2076 0 : bo_last = CEIL (TYPE_PRECISION (rhs_type) + bo_bit, limb_prec) - 1;
2077 0 : bo_shift = (TYPE_PRECISION (rhs_type) + bo_bit) % limb_prec;
2078 0 : if (bo_shift)
2079 0 : bo_shift = limb_prec - bo_shift;
2080 : }
2081 474 : if (m_first)
2082 : {
2083 141 : if (m_upwards && bo_shift)
2084 : {
2085 138 : gimple_stmt_iterator save_gsi = m_gsi;
2086 138 : m_gsi = m_init_gsi;
2087 138 : if (gsi_end_p (m_gsi))
2088 57 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
2089 : else
2090 81 : gsi_next (&m_gsi);
2091 138 : tree t = limb_access (NULL_TREE, nrhs1,
2092 138 : size_int (bo_idx + bo_last), true);
2093 138 : tree iv = make_ssa_name (m_limb_type);
2094 138 : g = gimple_build_assign (iv, t);
2095 138 : insert_before (g);
2096 138 : if (eh)
2097 : {
2098 2 : maybe_duplicate_eh_stmt (g, stmt);
2099 2 : if (eh_edge)
2100 : {
2101 2 : edge e = split_block (gsi_bb (m_gsi), g);
2102 2 : add_eh_edge (e->src, eh_edge);
2103 2 : m_gsi = gsi_after_labels (e->dest);
2104 2 : if (gsi_bb (save_gsi) == e->src)
2105 : {
2106 1 : if (gsi_end_p (save_gsi))
2107 0 : save_gsi = gsi_end_bb (e->dest);
2108 : else
2109 1 : save_gsi = gsi_for_stmt (gsi_stmt (save_gsi));
2110 : }
2111 2 : if (m_preheader_bb == e->src)
2112 1 : m_preheader_bb = e->dest;
2113 : }
2114 : }
2115 138 : m_init_gsi = m_gsi;
2116 138 : if (gsi_end_p (m_init_gsi))
2117 222 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
2118 : else
2119 28 : gsi_prev (&m_init_gsi);
2120 138 : m_gsi = save_gsi;
2121 138 : tree out;
2122 138 : prepare_data_in_out (iv, idx, &out);
2123 138 : out = m_data[m_data_cnt];
2124 138 : m_data.safe_push (out);
2125 138 : }
2126 : else
2127 : {
2128 3 : m_data.safe_push (NULL_TREE);
2129 3 : m_data.safe_push (NULL_TREE);
2130 3 : m_data.safe_push (NULL_TREE);
2131 : }
2132 : }
2133 :
2134 474 : tree nidx0 = NULL_TREE, nidx1 = NULL_TREE;
2135 474 : tree iv = m_data[m_data_cnt];
2136 474 : if (m_cast_conditional && iv)
2137 : {
2138 12 : gcc_assert (!m_bitfld_load);
2139 12 : m_bitfld_load = m_data_cnt;
2140 : }
2141 474 : if (tree_fits_uhwi_p (idx))
2142 : {
2143 272 : unsigned prec = TYPE_PRECISION (rhs_type);
2144 272 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
2145 272 : if (bitint_big_endian)
2146 0 : i = nelts - 1 - i;
2147 272 : gcc_assert (i * limb_prec < prec);
2148 272 : if (bo_shift)
2149 272 : nidx1 = size_int (bo_idx + (bitint_big_endian
2150 : ? bo_last - i - 1 : i + 1));
2151 272 : if ((i + 1) * limb_prec > prec)
2152 : {
2153 100 : prec %= limb_prec;
2154 100 : if (prec + bo_bit <= (unsigned) limb_prec)
2155 272 : nidx1 = NULL_TREE;
2156 : }
2157 272 : if (!iv)
2158 4 : nidx0 = size_int (bo_idx + (bitint_big_endian ? bo_last - i : i));
2159 : }
2160 : else
2161 : {
2162 202 : HOST_WIDE_INT adj = bo_idx;
2163 202 : if (bitint_big_endian)
2164 0 : adj += (HOST_WIDE_INT) bo_last + 1 - nelts;
2165 202 : if (!iv)
2166 : {
2167 4 : if (adj == 0)
2168 : nidx0 = idx;
2169 : else
2170 : {
2171 0 : nidx0 = make_ssa_name (sizetype);
2172 0 : g = gimple_build_assign (nidx0, PLUS_EXPR, idx,
2173 0 : size_int (adj));
2174 0 : insert_before (g);
2175 : }
2176 : }
2177 202 : if (bo_shift)
2178 : {
2179 202 : if (bitint_big_endian && adj == 1)
2180 : nidx1 = idx;
2181 : else
2182 : {
2183 202 : nidx1 = make_ssa_name (sizetype);
2184 202 : g = gimple_build_assign (nidx1, PLUS_EXPR, idx,
2185 404 : size_int (adj + (bitint_big_endian
2186 : ? -1 : 1)));
2187 202 : insert_before (g);
2188 : }
2189 : }
2190 : }
2191 :
2192 680 : tree iv2 = NULL_TREE;
2193 206 : if (nidx0)
2194 : {
2195 8 : tree t = limb_access (NULL_TREE, nrhs1, nidx0, true);
2196 8 : iv = make_ssa_name (m_limb_type);
2197 8 : g = gimple_build_assign (iv, t);
2198 8 : insert_before (g);
2199 8 : if (eh)
2200 : {
2201 0 : maybe_duplicate_eh_stmt (g, stmt);
2202 0 : if (eh_edge)
2203 : {
2204 0 : edge e = split_block (gsi_bb (m_gsi), g);
2205 0 : m_gsi = gsi_after_labels (e->dest);
2206 0 : add_eh_edge (e->src, eh_edge);
2207 : }
2208 : }
2209 : }
2210 474 : if (nidx1)
2211 : {
2212 385 : bool conditional = m_var_msb && !tree_fits_uhwi_p (idx);
2213 385 : unsigned prec = TYPE_PRECISION (rhs_type);
2214 385 : if (conditional)
2215 : {
2216 3 : if ((prec % limb_prec) == 0
2217 3 : || ((prec % limb_prec) + bo_bit > (unsigned) limb_prec))
2218 382 : conditional = false;
2219 : }
2220 385 : edge edge_true = NULL, edge_false = NULL;
2221 385 : if (conditional)
2222 : {
2223 6 : g = gimple_build_cond (NE_EXPR, idx,
2224 : bitint_big_endian
2225 : ? size_zero_node
2226 3 : : size_int (prec / limb_prec),
2227 : NULL_TREE, NULL_TREE);
2228 3 : if_then (g, profile_probability::likely (),
2229 : edge_true, edge_false);
2230 : }
2231 385 : tree t = limb_access (NULL_TREE, nrhs1, nidx1, true);
2232 385 : if (m_upwards_2limb
2233 283 : && !m_first
2234 184 : && !m_bitfld_load
2235 178 : && !tree_fits_uhwi_p (idx))
2236 95 : iv2 = m_data[m_data_cnt + 1];
2237 : else
2238 290 : iv2 = make_ssa_name (m_limb_type);
2239 385 : g = gimple_build_assign (iv2, t);
2240 385 : insert_before (g);
2241 385 : if (eh)
2242 : {
2243 5 : maybe_duplicate_eh_stmt (g, stmt);
2244 5 : if (eh_edge)
2245 : {
2246 5 : edge e = split_block (gsi_bb (m_gsi), g);
2247 5 : m_gsi = gsi_after_labels (e->dest);
2248 5 : add_eh_edge (e->src, eh_edge);
2249 : }
2250 : }
2251 385 : if (conditional)
2252 : {
2253 3 : tree iv3 = make_ssa_name (m_limb_type);
2254 3 : if (eh)
2255 0 : edge_true = find_edge (gsi_bb (m_gsi), edge_false->dest);
2256 3 : gphi *phi = create_phi_node (iv3, edge_true->dest);
2257 3 : add_phi_arg (phi, iv2, edge_true, UNKNOWN_LOCATION);
2258 3 : add_phi_arg (phi, build_zero_cst (m_limb_type),
2259 : edge_false, UNKNOWN_LOCATION);
2260 3 : m_gsi = gsi_after_labels (edge_true->dest);
2261 3 : iv2 = iv3;
2262 : }
2263 : }
2264 474 : if (bo_shift)
2265 : {
2266 474 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
2267 : iv, build_int_cst (unsigned_type_node,
2268 474 : bo_shift));
2269 474 : insert_before (g);
2270 474 : iv = gimple_assign_lhs (g);
2271 : }
2272 474 : if (iv2)
2273 : {
2274 385 : g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
2275 : iv2, build_int_cst (unsigned_type_node,
2276 385 : limb_prec - bo_shift));
2277 385 : insert_before (g);
2278 385 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
2279 : gimple_assign_lhs (g), iv);
2280 385 : insert_before (g);
2281 385 : iv = gimple_assign_lhs (g);
2282 385 : if (m_data[m_data_cnt])
2283 379 : m_data[m_data_cnt] = iv2;
2284 : }
2285 474 : if (tree_fits_uhwi_p (idx))
2286 : {
2287 272 : tree atype = limb_access_type (rhs_type, idx);
2288 272 : if (!useless_type_conversion_p (atype, TREE_TYPE (iv)))
2289 100 : iv = add_cast (atype, iv);
2290 : }
2291 474 : m_data_cnt += 3;
2292 474 : return iv;
2293 : }
2294 :
2295 20870 : normal_load:
2296 : /* Use write_p = true for loads with EH edges to make
2297 : sure limb_access doesn't add a cast as separate
2298 : statement after it. */
2299 20870 : rhs1 = limb_access (rhs_type, rhs1, idx, eh, !load_bitfield_p);
2300 20870 : tree ret = make_ssa_name (TREE_TYPE (rhs1));
2301 20870 : g = gimple_build_assign (ret, rhs1);
2302 20870 : insert_before (g);
2303 20870 : if (eh)
2304 : {
2305 3 : maybe_duplicate_eh_stmt (g, stmt);
2306 3 : if (eh_edge)
2307 : {
2308 3 : edge e = split_block (gsi_bb (m_gsi), g);
2309 3 : m_gsi = gsi_after_labels (e->dest);
2310 3 : add_eh_edge (e->src, eh_edge);
2311 : }
2312 3 : if (tree_fits_uhwi_p (idx))
2313 : {
2314 1 : tree atype = limb_access_type (rhs_type, idx);
2315 1 : if (!useless_type_conversion_p (atype, TREE_TYPE (rhs1)))
2316 1 : ret = add_cast (atype, ret);
2317 : }
2318 : }
2319 : return ret;
2320 : }
2321 :
2322 : /* Return a limb IDX from a mergeable statement STMT. */
2323 :
2324 : tree
2325 39172 : bitint_large_huge::handle_stmt (gimple *stmt, tree idx)
2326 : {
2327 39172 : tree lhs, rhs1, rhs2 = NULL_TREE;
2328 39172 : gimple *g;
2329 39172 : switch (gimple_code (stmt))
2330 : {
2331 39172 : case GIMPLE_ASSIGN:
2332 39172 : if (gimple_assign_load_p (stmt))
2333 21344 : return handle_load (stmt, idx);
2334 17828 : switch (gimple_assign_rhs_code (stmt))
2335 : {
2336 990 : case BIT_AND_EXPR:
2337 990 : case BIT_IOR_EXPR:
2338 990 : case BIT_XOR_EXPR:
2339 990 : rhs2 = handle_operand (gimple_assign_rhs2 (stmt), idx);
2340 : /* FALLTHRU */
2341 1410 : case BIT_NOT_EXPR:
2342 1410 : rhs1 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2343 1410 : lhs = make_ssa_name (TREE_TYPE (rhs1));
2344 1410 : g = gimple_build_assign (lhs, gimple_assign_rhs_code (stmt),
2345 : rhs1, rhs2);
2346 1410 : insert_before (g);
2347 1410 : return lhs;
2348 4019 : case PLUS_EXPR:
2349 4019 : case MINUS_EXPR:
2350 4019 : rhs1 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2351 4019 : rhs2 = handle_operand (gimple_assign_rhs2 (stmt), idx);
2352 4019 : return handle_plus_minus (gimple_assign_rhs_code (stmt),
2353 4019 : rhs1, rhs2, idx);
2354 163 : case NEGATE_EXPR:
2355 163 : rhs2 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2356 163 : rhs1 = build_zero_cst (TREE_TYPE (rhs2));
2357 163 : return handle_plus_minus (MINUS_EXPR, rhs1, rhs2, idx);
2358 152 : case LSHIFT_EXPR:
2359 152 : return handle_lshift (handle_operand (gimple_assign_rhs1 (stmt),
2360 : idx),
2361 152 : gimple_assign_rhs2 (stmt), idx);
2362 5087 : case SSA_NAME:
2363 5087 : case PAREN_EXPR:
2364 5087 : case INTEGER_CST:
2365 5087 : return handle_operand (gimple_assign_rhs1 (stmt), idx);
2366 6966 : CASE_CONVERT:
2367 6966 : return handle_cast (TREE_TYPE (gimple_assign_lhs (stmt)),
2368 6966 : gimple_assign_rhs1 (stmt), idx);
2369 8 : case VIEW_CONVERT_EXPR:
2370 8 : return handle_cast (TREE_TYPE (gimple_assign_lhs (stmt)),
2371 8 : TREE_OPERAND (gimple_assign_rhs1 (stmt), 0),
2372 8 : idx);
2373 23 : case BIT_FIELD_REF:
2374 23 : return handle_bit_field_ref (gimple_assign_rhs1 (stmt), idx);
2375 : default:
2376 : break;
2377 : }
2378 : break;
2379 : default:
2380 : break;
2381 : }
2382 0 : gcc_unreachable ();
2383 : }
2384 :
2385 : /* Return minimum precision of OP at STMT.
2386 : Positive value is minimum precision above which all bits
2387 : are zero, negative means all bits above negation of the
2388 : value are copies of the sign bit. */
2389 :
2390 : static int
2391 8446 : range_to_prec (tree op, gimple *stmt)
2392 : {
2393 8446 : int_range_max r;
2394 8446 : wide_int w;
2395 8446 : tree type = TREE_TYPE (op);
2396 8446 : unsigned int prec = TYPE_PRECISION (type);
2397 :
2398 8446 : if (!optimize
2399 7510 : || !get_range_query (cfun)->range_of_expr (r, op, stmt)
2400 12201 : || r.undefined_p ())
2401 : {
2402 4692 : if (TYPE_UNSIGNED (type))
2403 1902 : return prec;
2404 : else
2405 2790 : return MIN ((int) -prec, -2);
2406 : }
2407 :
2408 3754 : if (!TYPE_UNSIGNED (TREE_TYPE (op)))
2409 : {
2410 2339 : w = r.lower_bound ();
2411 2339 : if (wi::neg_p (w))
2412 : {
2413 1934 : int min_prec1 = wi::min_precision (w, SIGNED);
2414 1934 : w = r.upper_bound ();
2415 1934 : int min_prec2 = wi::min_precision (w, SIGNED);
2416 1934 : int min_prec = MAX (min_prec1, min_prec2);
2417 1934 : return MIN (-min_prec, -2);
2418 : }
2419 : }
2420 :
2421 1820 : w = r.upper_bound ();
2422 1820 : int min_prec = wi::min_precision (w, UNSIGNED);
2423 1820 : return MAX (min_prec, 1);
2424 8446 : }
2425 :
2426 : /* Return address of the first limb of OP and write into *PREC
2427 : its precision. If positive, the operand is zero extended
2428 : from that precision, if it is negative, the operand is sign-extended
2429 : from -*PREC. If PREC_STORED is NULL, it is the toplevel call,
2430 : otherwise *PREC_STORED is prec from the innermost call without
2431 : range optimizations (0 for uninitialized SSA_NAME). */
2432 :
2433 : tree
2434 3864 : bitint_large_huge::handle_operand_addr (tree op, gimple *stmt,
2435 : int *prec_stored, int *prec)
2436 : {
2437 3864 : wide_int w;
2438 3864 : location_t loc_save = m_loc;
2439 3864 : tree ret = NULL_TREE;
2440 3864 : int precs = 0;
2441 3899 : if ((!BITINT_TYPE_P (TREE_TYPE (op))
2442 3835 : || bitint_precision_kind (TREE_TYPE (op)) < bitint_prec_large)
2443 3988 : && TREE_CODE (op) != INTEGER_CST)
2444 : {
2445 124 : do_int:
2446 124 : *prec = range_to_prec (op, stmt);
2447 124 : bitint_prec_kind kind = bitint_prec_small;
2448 124 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (op)));
2449 124 : if (BITINT_TYPE_P (TREE_TYPE (op)))
2450 104 : kind = bitint_precision_kind (TREE_TYPE (op));
2451 104 : if (kind == bitint_prec_middle)
2452 : {
2453 12 : tree type = NULL_TREE;
2454 12 : op = maybe_cast_middle_bitint (&m_gsi, op, type);
2455 : }
2456 124 : tree op_type = TREE_TYPE (op);
2457 124 : unsigned HOST_WIDE_INT nelts
2458 124 : = CEIL (TYPE_PRECISION (op_type), limb_prec);
2459 : /* Add support for 3 or more limbs filled in from normal
2460 : integral type if this assert fails. If no target chooses
2461 : limb mode smaller than half of largest supported normal
2462 : integral type, this will not be needed. */
2463 124 : gcc_assert (nelts <= 2);
2464 124 : precs = (TYPE_UNSIGNED (op_type)
2465 124 : ? TYPE_PRECISION (op_type) : -TYPE_PRECISION (op_type));
2466 124 : if (*prec <= limb_prec && *prec >= -limb_prec)
2467 : {
2468 111 : nelts = 1;
2469 111 : if (TYPE_UNSIGNED (op_type))
2470 : {
2471 26 : if (precs > limb_prec)
2472 124 : precs = limb_prec;
2473 : }
2474 85 : else if (precs < -limb_prec)
2475 124 : precs = -limb_prec;
2476 : }
2477 124 : if (prec_stored)
2478 0 : *prec_stored = precs;
2479 124 : tree atype = build_array_type_nelts (m_limb_type, nelts);
2480 124 : tree var = create_tmp_var (atype);
2481 124 : tree t1 = op;
2482 124 : if (!useless_type_conversion_p (m_limb_type, op_type))
2483 124 : t1 = add_cast (m_limb_type, t1);
2484 124 : tree v = build4 (ARRAY_REF, m_limb_type, var,
2485 0 : bitint_big_endian && nelts > 1
2486 : ? size_one_node : size_zero_node,
2487 : NULL_TREE, NULL_TREE);
2488 124 : gimple *g = gimple_build_assign (v, t1);
2489 124 : insert_before (g);
2490 124 : if (nelts > 1)
2491 : {
2492 13 : tree lp = build_int_cst (unsigned_type_node, limb_prec);
2493 13 : g = gimple_build_assign (make_ssa_name (op_type),
2494 : RSHIFT_EXPR, op, lp);
2495 13 : insert_before (g);
2496 13 : tree t2 = gimple_assign_lhs (g);
2497 13 : t2 = add_cast (m_limb_type, t2);
2498 13 : v = build4 (ARRAY_REF, m_limb_type, var,
2499 : bitint_big_endian ? size_zero_node : size_one_node,
2500 : NULL_TREE, NULL_TREE);
2501 13 : g = gimple_build_assign (v, t2);
2502 13 : insert_before (g);
2503 : }
2504 124 : ret = build_fold_addr_expr (var);
2505 124 : if (!stmt_ends_bb_p (gsi_stmt (m_gsi)))
2506 : {
2507 123 : tree clobber = build_clobber (atype, CLOBBER_STORAGE_END);
2508 123 : g = gimple_build_assign (var, clobber);
2509 123 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
2510 : }
2511 124 : m_loc = loc_save;
2512 124 : goto do_ret;
2513 : }
2514 3788 : switch (TREE_CODE (op))
2515 : {
2516 2922 : case SSA_NAME:
2517 2922 : if (m_names == NULL
2518 2922 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (op)))
2519 : {
2520 102 : gimple *g = SSA_NAME_DEF_STMT (op);
2521 102 : m_loc = gimple_location (g);
2522 102 : if (gimple_assign_load_p (g))
2523 : {
2524 36 : *prec = range_to_prec (op, NULL);
2525 36 : precs = (TYPE_UNSIGNED (TREE_TYPE (op))
2526 36 : ? TYPE_PRECISION (TREE_TYPE (op))
2527 25 : : -TYPE_PRECISION (TREE_TYPE (op)));
2528 36 : if (prec_stored)
2529 6 : *prec_stored = precs;
2530 36 : ret = build_fold_addr_expr (gimple_assign_rhs1 (g));
2531 36 : ret = force_gimple_operand_gsi (&m_gsi, ret, true,
2532 : NULL_TREE, true, GSI_SAME_STMT);
2533 : }
2534 66 : else if (gimple_code (g) == GIMPLE_NOP)
2535 : {
2536 2 : *prec = TYPE_UNSIGNED (TREE_TYPE (op)) ? limb_prec : -limb_prec;
2537 2 : precs = *prec;
2538 2 : if (prec_stored)
2539 1 : *prec_stored = 0;
2540 2 : tree var = create_tmp_var (m_limb_type);
2541 2 : TREE_ADDRESSABLE (var) = 1;
2542 2 : ret = build_fold_addr_expr (var);
2543 2 : if (!stmt_ends_bb_p (gsi_stmt (m_gsi)))
2544 : {
2545 2 : tree clobber = build_clobber (m_limb_type,
2546 : CLOBBER_STORAGE_END);
2547 2 : g = gimple_build_assign (var, clobber);
2548 2 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
2549 : }
2550 : }
2551 : else
2552 : {
2553 64 : gcc_assert (gimple_assign_cast_p (g));
2554 64 : tree rhs1 = gimple_assign_rhs1 (g);
2555 64 : bitint_prec_kind kind = bitint_prec_small;
2556 64 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
2557 1 : rhs1 = TREE_OPERAND (rhs1, 0);
2558 64 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (rhs1)));
2559 64 : if (BITINT_TYPE_P (TREE_TYPE (rhs1)))
2560 49 : kind = bitint_precision_kind (TREE_TYPE (rhs1));
2561 49 : if (kind >= bitint_prec_large)
2562 : {
2563 16 : tree lhs_type = TREE_TYPE (op);
2564 16 : tree rhs_type = TREE_TYPE (rhs1);
2565 16 : int prec_stored_val = 0;
2566 16 : ret = handle_operand_addr (rhs1, g, &prec_stored_val, prec);
2567 16 : precs = prec_stored_val;
2568 16 : if (prec_stored)
2569 0 : *prec_stored = prec_stored_val;
2570 16 : if (precs == 0)
2571 : {
2572 1 : gcc_assert (*prec == limb_prec || *prec == -limb_prec);
2573 : precs = *prec;
2574 : }
2575 16 : if (TYPE_PRECISION (lhs_type) > TYPE_PRECISION (rhs_type))
2576 : {
2577 4 : if (TYPE_UNSIGNED (lhs_type)
2578 4 : && !TYPE_UNSIGNED (rhs_type))
2579 1 : gcc_assert (*prec >= 0 || prec_stored == NULL);
2580 : }
2581 : else
2582 : {
2583 12 : if (prec_stored_val == 0)
2584 : /* Non-widening cast of uninitialized value. */;
2585 11 : else if (*prec > 0 && *prec < TYPE_PRECISION (lhs_type))
2586 : ;
2587 11 : else if (TYPE_UNSIGNED (lhs_type))
2588 : {
2589 8 : gcc_assert (*prec > 0
2590 : || prec_stored_val > 0
2591 : || (-prec_stored_val
2592 : >= TYPE_PRECISION (lhs_type)));
2593 8 : *prec = TYPE_PRECISION (lhs_type);
2594 : }
2595 3 : else if (*prec < 0 && -*prec < TYPE_PRECISION (lhs_type))
2596 : ;
2597 : else
2598 3 : *prec = -TYPE_PRECISION (lhs_type);
2599 : }
2600 : }
2601 : else
2602 : {
2603 48 : op = rhs1;
2604 48 : stmt = g;
2605 48 : goto do_int;
2606 : }
2607 : }
2608 54 : m_loc = loc_save;
2609 54 : goto do_ret;
2610 : }
2611 : else
2612 : {
2613 2820 : int p = var_to_partition (m_map, op);
2614 2820 : gcc_assert (m_vars[p] != NULL_TREE);
2615 2820 : *prec = range_to_prec (op, stmt);
2616 2820 : precs = (TYPE_UNSIGNED (TREE_TYPE (op))
2617 2820 : ? TYPE_PRECISION (TREE_TYPE (op))
2618 1669 : : -TYPE_PRECISION (TREE_TYPE (op)));
2619 2820 : if (prec_stored)
2620 9 : *prec_stored = precs;
2621 2820 : ret = build_fold_addr_expr (m_vars[p]);
2622 2820 : goto do_ret;
2623 : }
2624 866 : case INTEGER_CST:
2625 866 : unsigned int min_prec, mp;
2626 866 : tree type;
2627 866 : w = wi::to_wide (op);
2628 866 : if (tree_int_cst_sgn (op) >= 0)
2629 : {
2630 651 : min_prec = wi::min_precision (w, UNSIGNED);
2631 651 : *prec = MAX (min_prec, 1);
2632 : }
2633 : else
2634 : {
2635 215 : min_prec = wi::min_precision (w, SIGNED);
2636 215 : *prec = MIN ((int) -min_prec, -2);
2637 : }
2638 866 : mp = CEIL (min_prec, limb_prec) * limb_prec;
2639 866 : if (mp == 0)
2640 : mp = 1;
2641 866 : if (mp >= (unsigned) TYPE_PRECISION (TREE_TYPE (op))
2642 866 : && (BITINT_TYPE_P (TREE_TYPE (op))
2643 24 : || TYPE_PRECISION (TREE_TYPE (op)) <= limb_prec))
2644 316 : type = TREE_TYPE (op);
2645 : else
2646 550 : type = build_bitint_type (mp, 1);
2647 20 : if (!BITINT_TYPE_P (type)
2648 866 : || bitint_precision_kind (type) == bitint_prec_small)
2649 : {
2650 587 : if (TYPE_PRECISION (type) <= limb_prec)
2651 587 : type = m_limb_type;
2652 : else
2653 : {
2654 0 : while (bitint_precision_kind (mp) == bitint_prec_small)
2655 0 : mp += limb_prec;
2656 : /* This case is for targets which e.g. have 64-bit
2657 : limb but categorize up to 128-bits _BitInts as
2658 : small. We could use type of m_limb_type[2] and
2659 : similar instead to save space. */
2660 0 : type = build_bitint_type (mp, 1);
2661 : }
2662 : }
2663 866 : if (tree_int_cst_sgn (op) >= 0)
2664 651 : precs = MAX (TYPE_PRECISION (type), 1);
2665 : else
2666 215 : precs = MIN ((int) -TYPE_PRECISION (type), -2);
2667 866 : if (prec_stored)
2668 0 : *prec_stored = precs;
2669 866 : op = tree_output_constant_def (fold_convert (type, op));
2670 866 : ret = build_fold_addr_expr (op);
2671 866 : goto do_ret;
2672 0 : default:
2673 0 : gcc_unreachable ();
2674 : }
2675 3864 : do_ret:
2676 3864 : if (bitint_big_endian && prec_stored == NULL)
2677 : {
2678 0 : int p1 = *prec < 0 ? -*prec : *prec;
2679 0 : int p2 = precs < 0 ? -precs : precs;
2680 0 : int c1 = CEIL (p1, limb_prec);
2681 0 : int c2 = CEIL (p2, limb_prec);
2682 0 : gcc_assert (c1 <= c2);
2683 0 : if (c1 != c2)
2684 : {
2685 0 : gimple *g
2686 0 : = gimple_build_assign (make_ssa_name (TREE_TYPE (ret)),
2687 : POINTER_PLUS_EXPR, ret,
2688 0 : size_int ((c2 - c1) * m_limb_size));
2689 0 : insert_before (g);
2690 0 : ret = gimple_assign_lhs (g);
2691 : }
2692 : }
2693 3864 : return ret;
2694 3864 : }
2695 :
2696 : /* Helper function, create a loop before the current location,
2697 : start with sizetype INIT value from the preheader edge. Return
2698 : a PHI result and set *IDX_NEXT to SSA_NAME it creates and uses
2699 : from the latch edge. */
2700 :
2701 : tree
2702 14670 : bitint_large_huge::create_loop (tree init, tree *idx_next)
2703 : {
2704 14670 : if (!gsi_end_p (m_gsi))
2705 15772 : gsi_prev (&m_gsi);
2706 : else
2707 4336 : m_gsi = gsi_last_bb (gsi_bb (m_gsi));
2708 14670 : edge e1 = split_block (gsi_bb (m_gsi), gsi_stmt (m_gsi));
2709 14670 : edge e2 = split_block (e1->dest, (gimple *) NULL);
2710 14670 : edge e3 = make_edge (e1->dest, e1->dest, EDGE_TRUE_VALUE);
2711 14670 : e3->probability = profile_probability::very_unlikely ();
2712 14670 : e2->flags = EDGE_FALSE_VALUE;
2713 14670 : e2->probability = e3->probability.invert ();
2714 14670 : tree idx = make_ssa_name (sizetype);
2715 14670 : gphi *phi = create_phi_node (idx, e1->dest);
2716 14670 : add_phi_arg (phi, init, e1, UNKNOWN_LOCATION);
2717 14670 : *idx_next = make_ssa_name (sizetype);
2718 14670 : add_phi_arg (phi, *idx_next, e3, UNKNOWN_LOCATION);
2719 14670 : m_gsi = gsi_after_labels (e1->dest);
2720 14670 : m_bb = e1->dest;
2721 14670 : m_preheader_bb = e1->src;
2722 14670 : class loop *loop = alloc_loop ();
2723 14670 : loop->header = e1->dest;
2724 14670 : add_loop (loop, e1->src->loop_father);
2725 14670 : return idx;
2726 : }
2727 :
2728 : /* Lower large/huge _BitInt statement mergeable or similar STMT which can be
2729 : lowered using iteration from the least significant limb up to the most
2730 : significant limb. For large _BitInt it is emitted as straight line code
2731 : before current location, for huge _BitInt as a loop handling two limbs
2732 : at once, followed by handling up to limbs in straight line code (at most
2733 : one full and one partial limb). It can also handle EQ_EXPR/NE_EXPR
2734 : comparisons, in that case CMP_CODE should be the comparison code and
2735 : CMP_OP1/CMP_OP2 the comparison operands. */
2736 :
2737 : tree
2738 23377 : bitint_large_huge::lower_mergeable_stmt (gimple *stmt, tree_code &cmp_code,
2739 : tree cmp_op1, tree cmp_op2)
2740 : {
2741 23377 : bool eq_p = cmp_code != ERROR_MARK;
2742 23377 : tree type;
2743 23377 : if (eq_p)
2744 6550 : type = TREE_TYPE (cmp_op1);
2745 : else
2746 16827 : type = TREE_TYPE (gimple_assign_lhs (stmt));
2747 23377 : gcc_assert (BITINT_TYPE_P (type));
2748 23377 : bitint_prec_kind kind = bitint_precision_kind (type);
2749 23377 : gcc_assert (kind >= bitint_prec_large);
2750 23377 : gimple *g;
2751 23377 : tree lhs = gimple_get_lhs (stmt);
2752 23377 : tree rhs1, lhs_type = lhs ? TREE_TYPE (lhs) : NULL_TREE;
2753 17267 : if (lhs
2754 17267 : && TREE_CODE (lhs) == SSA_NAME
2755 8745 : && BITINT_TYPE_P (TREE_TYPE (lhs))
2756 8305 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large)
2757 : {
2758 8305 : int p = var_to_partition (m_map, lhs);
2759 8305 : gcc_assert (m_vars[p] != NULL_TREE);
2760 8305 : m_lhs = lhs = m_vars[p];
2761 : }
2762 23377 : unsigned cnt, rem = 0, end = 0, prec = TYPE_PRECISION (type);
2763 23377 : bool sext = false;
2764 23377 : tree ext = NULL_TREE, store_operand = NULL_TREE;
2765 23377 : bool eh = false;
2766 23377 : basic_block eh_pad = NULL;
2767 23377 : tree nlhs = NULL_TREE;
2768 23377 : unsigned HOST_WIDE_INT bo_idx = 0;
2769 23377 : unsigned HOST_WIDE_INT bo_bit = 0;
2770 23377 : unsigned bo_shift = 0;
2771 23377 : unsigned bo_last = 0;
2772 23377 : bool bo_be_p = false;
2773 23377 : tree bf_cur = NULL_TREE, bf_next = NULL_TREE;
2774 23377 : if (gimple_store_p (stmt))
2775 : {
2776 8522 : store_operand = gimple_assign_rhs1 (stmt);
2777 8522 : eh = stmt_ends_bb_p (stmt);
2778 8522 : if (eh)
2779 : {
2780 2 : edge e;
2781 2 : edge_iterator ei;
2782 2 : basic_block bb = gimple_bb (stmt);
2783 :
2784 2 : FOR_EACH_EDGE (e, ei, bb->succs)
2785 2 : if (e->flags & EDGE_EH)
2786 : {
2787 2 : eh_pad = e->dest;
2788 2 : break;
2789 : }
2790 : }
2791 8522 : if (TREE_CODE (lhs) == COMPONENT_REF
2792 8522 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
2793 : {
2794 162 : tree fld = TREE_OPERAND (lhs, 1);
2795 162 : gcc_assert (tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld)));
2796 162 : tree repr = DECL_BIT_FIELD_REPRESENTATIVE (fld);
2797 162 : poly_int64 bitoffset;
2798 162 : poly_uint64 field_offset, repr_offset;
2799 162 : if (!bitint_big_endian
2800 162 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2801 162 : % BITS_PER_UNIT) == 0)
2802 : nlhs = lhs;
2803 : else
2804 : {
2805 143 : bool var_field_off = false;
2806 143 : if (poly_int_tree_p (DECL_FIELD_OFFSET (fld), &field_offset)
2807 286 : && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
2808 143 : bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
2809 : else
2810 : {
2811 : bitoffset = 0;
2812 : var_field_off = true;
2813 : }
2814 143 : bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2815 143 : - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
2816 286 : nlhs = build3 (COMPONENT_REF, TREE_TYPE (repr),
2817 143 : TREE_OPERAND (lhs, 0), repr,
2818 : var_field_off
2819 0 : ? TREE_OPERAND (lhs, 2) : NULL_TREE);
2820 143 : HOST_WIDE_INT bo = bitoffset.to_constant ();
2821 143 : bo_idx = (unsigned HOST_WIDE_INT) bo / limb_prec;
2822 143 : bo_bit = (unsigned HOST_WIDE_INT) bo % limb_prec;
2823 143 : bo_shift = bo_bit;
2824 143 : if (bitint_big_endian)
2825 : {
2826 0 : bo_last = CEIL (prec + bo_bit, limb_prec) - 1;
2827 0 : bo_shift = (prec + bo_bit) % limb_prec;
2828 0 : bo_be_p = true;
2829 0 : if (bo_shift)
2830 0 : bo_shift = limb_prec - bo_shift;
2831 : }
2832 : }
2833 : }
2834 : }
2835 8522 : if ((store_operand
2836 8522 : && TREE_CODE (store_operand) == SSA_NAME
2837 7544 : && (m_names == NULL
2838 7514 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (store_operand)))
2839 1154 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (store_operand)))
2840 31345 : || gimple_assign_cast_p (stmt))
2841 : {
2842 2155 : rhs1 = gimple_assign_rhs1 (store_operand
2843 554 : ? SSA_NAME_DEF_STMT (store_operand)
2844 : : stmt);
2845 1601 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
2846 2 : rhs1 = TREE_OPERAND (rhs1, 0);
2847 : /* Optimize mergeable ops ending with widening cast to _BitInt
2848 : (or followed by store). We can lower just the limbs of the
2849 : cast operand and widen afterwards. */
2850 1601 : if (TREE_CODE (rhs1) == SSA_NAME
2851 1601 : && (m_names == NULL
2852 1595 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))
2853 617 : && BITINT_TYPE_P (TREE_TYPE (rhs1))
2854 478 : && bitint_precision_kind (TREE_TYPE (rhs1)) >= bitint_prec_large
2855 2030 : && (CEIL ((unsigned) TYPE_PRECISION (TREE_TYPE (rhs1)),
2856 429 : limb_prec) < CEIL (prec, limb_prec)
2857 306 : || (kind == bitint_prec_huge
2858 262 : && TYPE_PRECISION (TREE_TYPE (rhs1)) < prec)))
2859 : {
2860 129 : store_operand = rhs1;
2861 129 : prec = TYPE_PRECISION (TREE_TYPE (rhs1));
2862 129 : kind = bitint_precision_kind (TREE_TYPE (rhs1));
2863 129 : if (!TYPE_UNSIGNED (TREE_TYPE (rhs1)))
2864 53 : sext = true;
2865 : }
2866 : }
2867 23377 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
2868 23377 : if (kind == bitint_prec_large)
2869 12201 : cnt = CEIL (prec, limb_prec);
2870 : else
2871 : {
2872 11176 : rem = (prec % (2 * limb_prec));
2873 11176 : end = (prec - rem) / limb_prec;
2874 11176 : cnt = 2 + CEIL (rem, limb_prec);
2875 11176 : idx = idx_first = create_loop (bitint_big_endian
2876 11176 : ? size_int (cnt - 2 + end - 1)
2877 : : size_zero_node, &idx_next);
2878 : }
2879 :
2880 23377 : basic_block edge_bb = NULL;
2881 23377 : if (eq_p)
2882 : {
2883 6550 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
2884 6550 : gsi_prev (&gsi);
2885 6550 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
2886 6550 : edge_bb = e->src;
2887 6550 : if (kind == bitint_prec_large)
2888 7186 : m_gsi = gsi_end_bb (edge_bb);
2889 : }
2890 : else
2891 16827 : m_after_stmt = stmt;
2892 23377 : if (kind != bitint_prec_large)
2893 11176 : m_upwards_2limb = end;
2894 23377 : m_upwards = true;
2895 :
2896 23377 : bool separate_ext
2897 23377 : = (prec != (unsigned) TYPE_PRECISION (type)
2898 23377 : && (CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)
2899 129 : > CEIL (prec, limb_prec)));
2900 23377 : bool zero_ms_limb = false;
2901 23377 : if (bitint_extended == bitint_ext_full
2902 0 : && !eq_p
2903 0 : && !nlhs
2904 0 : && abi_limb_prec > limb_prec
2905 23377 : && ((CEIL ((unsigned) TYPE_PRECISION (type), abi_limb_prec)
2906 0 : * abi_limb_prec / limb_prec)
2907 0 : > CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)))
2908 : {
2909 0 : if (prec == (unsigned) TYPE_PRECISION (type))
2910 : {
2911 0 : sext = !TYPE_UNSIGNED (type);
2912 0 : separate_ext = true;
2913 : }
2914 0 : else if (TYPE_UNSIGNED (type) && sext)
2915 : zero_ms_limb = true;
2916 : else
2917 : separate_ext = true;
2918 : }
2919 23500 : unsigned dst_idx_off = 0;
2920 23377 : if (separate_ext && bitint_big_endian)
2921 0 : dst_idx_off = (CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)
2922 0 : - CEIL (prec, limb_prec));
2923 :
2924 93726 : for (unsigned i = 0; i < cnt; i++)
2925 : {
2926 70349 : m_data_cnt = 0;
2927 70349 : if (kind == bitint_prec_large)
2928 37214 : idx = size_int (bitint_big_endian ? cnt - 1 - i : i);
2929 33135 : else if (i >= 2)
2930 10783 : idx = size_int (bitint_big_endian ? cnt - 1 - i : end + (i > 2));
2931 70349 : if (eq_p)
2932 : {
2933 19841 : rhs1 = handle_operand (cmp_op1, idx);
2934 19841 : tree rhs2 = handle_operand (cmp_op2, idx);
2935 19841 : g = gimple_build_cond (NE_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
2936 19841 : insert_before (g);
2937 19841 : edge e1 = split_block (gsi_bb (m_gsi), g);
2938 19841 : e1->flags = EDGE_FALSE_VALUE;
2939 19841 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
2940 19841 : e1->probability = profile_probability::unlikely ();
2941 19841 : e2->probability = e1->probability.invert ();
2942 19841 : if (i == 0)
2943 6550 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
2944 19841 : m_gsi = gsi_after_labels (e1->dest);
2945 : }
2946 : else
2947 : {
2948 50508 : if (store_operand)
2949 25796 : rhs1 = handle_operand (store_operand, idx);
2950 : else
2951 24712 : rhs1 = handle_stmt (stmt, idx);
2952 50508 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
2953 12050 : rhs1 = add_cast (m_limb_type, rhs1);
2954 50508 : if (sext && i == cnt - 1)
2955 50508 : ext = rhs1;
2956 50508 : tree nidx = idx;
2957 50508 : HOST_WIDE_INT adj = bo_idx;
2958 50508 : if (bo_be_p)
2959 0 : adj += bo_last - (CEIL (prec, limb_prec) - 1);
2960 : else
2961 50508 : adj += dst_idx_off;
2962 50508 : if (adj)
2963 : {
2964 220 : if (tree_fits_uhwi_p (idx))
2965 116 : nidx = size_int (tree_to_uhwi (idx) + adj);
2966 : else
2967 : {
2968 104 : nidx = make_ssa_name (sizetype);
2969 104 : g = gimple_build_assign (nidx, PLUS_EXPR, idx,
2970 104 : size_int (adj));
2971 104 : insert_before (g);
2972 : }
2973 : }
2974 50508 : bool done = false;
2975 50508 : basic_block new_bb = NULL;
2976 : /* Handle stores into bit-fields. */
2977 50508 : if (bo_shift)
2978 : {
2979 455 : if (i == 0)
2980 : {
2981 143 : edge e2 = NULL;
2982 143 : if (kind != bitint_prec_large)
2983 : {
2984 112 : prepare_data_in_out (build_zero_cst (m_limb_type),
2985 : idx, &bf_next);
2986 112 : bf_next = m_data.pop ();
2987 112 : bf_cur = m_data.pop ();
2988 112 : g = gimple_build_cond (EQ_EXPR, idx,
2989 : bitint_big_endian
2990 0 : ? size_int (CEIL (prec,
2991 : limb_prec) - 1)
2992 : : size_zero_node,
2993 : NULL_TREE, NULL_TREE);
2994 112 : edge edge_true;
2995 112 : if_then_else (g, profile_probability::unlikely (),
2996 : edge_true, e2);
2997 112 : new_bb = e2->dest;
2998 : }
2999 143 : tree ftype
3000 143 : = build_nonstandard_integer_type (limb_prec - bo_shift, 1);
3001 286 : tree bfr = build_bit_field_ref (ftype, unshare_expr (nlhs),
3002 143 : limb_prec - bo_shift,
3003 : bitint_big_endian
3004 0 : ? (bo_idx + bo_last)
3005 0 : * limb_prec
3006 143 : : bo_idx * limb_prec
3007 : + bo_bit);
3008 143 : tree t = add_cast (ftype, rhs1);
3009 143 : g = gimple_build_assign (bfr, t);
3010 143 : insert_before (g);
3011 143 : if (eh)
3012 : {
3013 0 : maybe_duplicate_eh_stmt (g, stmt);
3014 0 : if (eh_pad)
3015 : {
3016 0 : edge e = split_block (gsi_bb (m_gsi), g);
3017 0 : m_gsi = gsi_after_labels (e->dest);
3018 0 : add_eh_edge (e->src,
3019 : find_edge (gimple_bb (stmt), eh_pad));
3020 : }
3021 : }
3022 143 : if (kind == bitint_prec_large)
3023 : {
3024 : bf_cur = rhs1;
3025 : done = true;
3026 : }
3027 112 : else if (e2)
3028 112 : m_gsi = gsi_after_labels (e2->src);
3029 : }
3030 112 : if (!done)
3031 : {
3032 424 : tree t1 = make_ssa_name (m_limb_type);
3033 424 : tree t2 = make_ssa_name (m_limb_type);
3034 424 : tree t3 = make_ssa_name (m_limb_type);
3035 424 : g = gimple_build_assign (t1, RSHIFT_EXPR, bf_cur,
3036 : build_int_cst (unsigned_type_node,
3037 424 : limb_prec
3038 424 : - bo_shift));
3039 424 : insert_before (g);
3040 424 : g = gimple_build_assign (t2, LSHIFT_EXPR, rhs1,
3041 : build_int_cst (unsigned_type_node,
3042 424 : bo_shift));
3043 424 : insert_before (g);
3044 424 : bf_cur = rhs1;
3045 424 : g = gimple_build_assign (t3, BIT_IOR_EXPR, t1, t2);
3046 424 : insert_before (g);
3047 424 : rhs1 = t3;
3048 424 : if (bf_next && i == 1)
3049 : {
3050 112 : g = gimple_build_assign (bf_next, bf_cur);
3051 112 : insert_before (g);
3052 : }
3053 : }
3054 : }
3055 31 : if (!done)
3056 : {
3057 : /* Handle bit-field access to partial last limb if needed. */
3058 50477 : if (nlhs
3059 480 : && i == cnt - 1
3060 162 : && !separate_ext
3061 138 : && tree_fits_uhwi_p (idx))
3062 : {
3063 123 : unsigned int tprec = TYPE_PRECISION (type);
3064 123 : unsigned int rprec = (tprec - 1) % limb_prec + 1;
3065 123 : if (rprec + bo_shift < (unsigned) limb_prec)
3066 : {
3067 46 : tree ftype
3068 46 : = build_nonstandard_integer_type (rprec + bo_shift, 1);
3069 46 : tree bfr
3070 92 : = build_bit_field_ref (ftype, unshare_expr (nlhs),
3071 : rprec + bo_shift,
3072 : bitint_big_endian
3073 0 : ? bo_idx * limb_prec + bo_bit
3074 46 : : (bo_idx + tprec / limb_prec)
3075 46 : * limb_prec);
3076 46 : tree t = add_cast (ftype, rhs1);
3077 46 : g = gimple_build_assign (bfr, t);
3078 46 : done = true;
3079 46 : bf_cur = NULL_TREE;
3080 : }
3081 77 : else if (rprec + bo_shift == (unsigned) limb_prec)
3082 50431 : bf_cur = NULL_TREE;
3083 : }
3084 : /* Otherwise, stores to any other lhs. */
3085 46 : if (!done)
3086 : {
3087 50865 : tree l = limb_access (nlhs ? NULL_TREE : lhs_type,
3088 : nlhs ? nlhs : lhs, nidx, true);
3089 50431 : g = gimple_build_assign (l, rhs1);
3090 : }
3091 50477 : insert_before (g);
3092 50477 : if (eh)
3093 : {
3094 6 : maybe_duplicate_eh_stmt (g, stmt);
3095 6 : if (eh_pad)
3096 : {
3097 6 : edge e = split_block (gsi_bb (m_gsi), g);
3098 6 : m_gsi = gsi_after_labels (e->dest);
3099 6 : add_eh_edge (e->src,
3100 : find_edge (gimple_bb (stmt), eh_pad));
3101 : }
3102 : }
3103 50477 : if (new_bb)
3104 112 : m_gsi = gsi_after_labels (new_bb);
3105 : }
3106 : }
3107 70349 : m_first = false;
3108 70349 : if (kind == bitint_prec_huge && i <= 1)
3109 : {
3110 22352 : if (i == 0)
3111 : {
3112 11176 : idx = make_ssa_name (sizetype);
3113 11176 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
3114 : bitint_big_endian
3115 0 : ? size_int (-1) : size_one_node);
3116 11176 : insert_before (g);
3117 : }
3118 : else
3119 : {
3120 11176 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
3121 22352 : size_int (bitint_big_endian ? -2 : 2));
3122 11176 : insert_before (g);
3123 11176 : if (bitint_big_endian)
3124 0 : g = gimple_build_cond (NE_EXPR, idx_first, size_int (cnt - 1),
3125 : NULL_TREE, NULL_TREE);
3126 : else
3127 11176 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (end),
3128 : NULL_TREE, NULL_TREE);
3129 11176 : insert_before (g);
3130 11176 : if (eq_p)
3131 2957 : m_gsi = gsi_after_labels (edge_bb);
3132 : else
3133 8219 : m_gsi = gsi_for_stmt (stmt);
3134 11176 : m_bb = NULL;
3135 : }
3136 : }
3137 : }
3138 :
3139 23377 : if (separate_ext)
3140 : {
3141 123 : if (sext)
3142 : {
3143 50 : ext = add_cast (signed_type_for (m_limb_type), ext);
3144 100 : tree lpm1 = build_int_cst (unsigned_type_node,
3145 50 : limb_prec - 1);
3146 50 : tree n = make_ssa_name (TREE_TYPE (ext));
3147 50 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
3148 50 : insert_before (g);
3149 50 : ext = add_cast (m_limb_type, n);
3150 : }
3151 : else
3152 73 : ext = build_zero_cst (m_limb_type);
3153 123 : kind = bitint_precision_kind (type);
3154 123 : unsigned start = CEIL (prec, limb_prec);
3155 123 : prec = TYPE_PRECISION (type);
3156 123 : if (bitint_extended == bitint_ext_full
3157 0 : && !nlhs
3158 0 : && !zero_ms_limb
3159 0 : && abi_limb_prec > limb_prec)
3160 : {
3161 0 : prec = CEIL (prec, abi_limb_prec) * abi_limb_prec;
3162 0 : kind = bitint_precision_kind (prec);
3163 : }
3164 123 : unsigned total = CEIL (prec, limb_prec);
3165 123 : idx = idx_first = idx_next = NULL_TREE;
3166 123 : if (prec <= (start + 2 + (bo_shift != 0)) * limb_prec)
3167 : kind = bitint_prec_large;
3168 78 : if (kind == bitint_prec_large)
3169 45 : cnt = total - start;
3170 : else
3171 : {
3172 78 : rem = prec % limb_prec;
3173 78 : end = (prec - rem) / limb_prec;
3174 148 : cnt = (bo_shift != 0) + 1 + (rem != 0);
3175 : }
3176 123 : if (bitint_big_endian && bo_shift != 0 && (prec % limb_prec) == 0)
3177 0 : ++total;
3178 317 : for (unsigned i = 0; i < cnt; i++)
3179 : {
3180 194 : if (kind == bitint_prec_large || (i == 0 && bo_shift != 0))
3181 62 : idx = size_int (bo_idx
3182 : + (bitint_big_endian
3183 : ? total - 1 - start - i : start + i));
3184 132 : else if (i == cnt - 1 && rem != 0)
3185 108 : idx = size_int (bo_idx + (bitint_big_endian ? 0 : end));
3186 78 : else if (i == (bo_shift != 0))
3187 78 : idx = create_loop (size_int (bo_idx
3188 : + (bitint_big_endian
3189 : ? total - 1 - start - i
3190 : : start + i)), &idx_next);
3191 194 : rhs1 = ext;
3192 194 : if (bf_cur != NULL_TREE && bf_cur != ext)
3193 : {
3194 20 : tree t1 = make_ssa_name (m_limb_type);
3195 20 : g = gimple_build_assign (t1, RSHIFT_EXPR, bf_cur,
3196 : build_int_cst (unsigned_type_node,
3197 20 : limb_prec - bo_shift));
3198 20 : insert_before (g);
3199 20 : if (integer_zerop (ext))
3200 : rhs1 = t1;
3201 : else
3202 : {
3203 10 : tree t2 = make_ssa_name (m_limb_type);
3204 10 : rhs1 = make_ssa_name (m_limb_type);
3205 10 : g = gimple_build_assign (t2, LSHIFT_EXPR, ext,
3206 : build_int_cst (unsigned_type_node,
3207 10 : bo_shift));
3208 10 : insert_before (g);
3209 10 : g = gimple_build_assign (rhs1, BIT_IOR_EXPR, t1, t2);
3210 10 : insert_before (g);
3211 : }
3212 : bf_cur = ext;
3213 : }
3214 194 : bool done = false;
3215 : /* Handle bit-field access to partial last limb if needed. */
3216 194 : if (nlhs && i == cnt - 1)
3217 : {
3218 24 : unsigned int tprec = TYPE_PRECISION (type);
3219 24 : unsigned int rprec = (tprec - 1) % limb_prec + 1;
3220 24 : if (rprec + bo_shift < (unsigned) limb_prec)
3221 : {
3222 12 : tree ftype
3223 12 : = build_nonstandard_integer_type (rprec + bo_shift, 1);
3224 12 : tree bfr
3225 24 : = build_bit_field_ref (ftype, unshare_expr (nlhs),
3226 : rprec + bo_shift,
3227 : bitint_big_endian
3228 0 : ? bo_idx * limb_prec + bo_bit
3229 12 : : (bo_idx + tprec / limb_prec)
3230 12 : * limb_prec);
3231 12 : tree t = add_cast (ftype, rhs1);
3232 12 : g = gimple_build_assign (bfr, t);
3233 12 : done = true;
3234 12 : bf_cur = NULL_TREE;
3235 : }
3236 12 : else if (rprec + bo_shift == (unsigned) limb_prec)
3237 : bf_cur = NULL_TREE;
3238 : }
3239 : /* Otherwise, stores to any other lhs. */
3240 12 : if (!done)
3241 : {
3242 212 : tree l = limb_access (nlhs ? NULL_TREE : lhs_type,
3243 : nlhs ? nlhs : lhs, idx, true);
3244 :
3245 182 : if (bitint_extended
3246 0 : && sext
3247 0 : && TYPE_UNSIGNED (lhs_type)
3248 0 : && tree_fits_uhwi_p (idx)
3249 182 : && !nlhs)
3250 : {
3251 0 : rhs1 = add_cast (limb_access_type (lhs_type, idx), rhs1);
3252 0 : rhs1 = add_cast (TREE_TYPE (l), rhs1);
3253 : }
3254 :
3255 182 : g = gimple_build_assign (l, rhs1);
3256 : }
3257 194 : insert_before (g);
3258 194 : if (eh)
3259 : {
3260 0 : maybe_duplicate_eh_stmt (g, stmt);
3261 0 : if (eh_pad)
3262 : {
3263 0 : edge e = split_block (gsi_bb (m_gsi), g);
3264 0 : m_gsi = gsi_after_labels (e->dest);
3265 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3266 : }
3267 : }
3268 194 : if (kind == bitint_prec_huge && i == (bo_shift != 0))
3269 : {
3270 78 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3271 : bitint_big_endian
3272 0 : ? size_int (-1) : size_one_node);
3273 78 : insert_before (g);
3274 78 : if (bitint_big_endian && rem != 0)
3275 0 : g = gimple_build_cond (NE_EXPR, idx,
3276 0 : size_int (bo_idx + 1),
3277 : NULL_TREE, NULL_TREE);
3278 : else
3279 78 : g = gimple_build_cond (NE_EXPR, idx_next,
3280 156 : size_int (bo_idx
3281 : + (bitint_big_endian
3282 : ? 0 : end)),
3283 : NULL_TREE, NULL_TREE);
3284 78 : insert_before (g);
3285 78 : m_gsi = gsi_for_stmt (stmt);
3286 78 : m_bb = NULL;
3287 : }
3288 : }
3289 : }
3290 23377 : if (bf_cur != NULL_TREE)
3291 : {
3292 72 : unsigned int tprec = TYPE_PRECISION (type);
3293 72 : unsigned int rprec = (tprec + bo_shift) % limb_prec;
3294 72 : tree ftype = build_nonstandard_integer_type (rprec, 1);
3295 144 : tree bfr = build_bit_field_ref (ftype, unshare_expr (nlhs),
3296 : rprec,
3297 : bitint_big_endian
3298 0 : ? bo_idx * limb_prec + bo_bit
3299 72 : : (bo_idx + (tprec + bo_bit) / limb_prec)
3300 : * limb_prec);
3301 72 : rhs1 = bf_cur;
3302 72 : if (bf_cur != ext)
3303 : {
3304 64 : rhs1 = make_ssa_name (TREE_TYPE (rhs1));
3305 64 : g = gimple_build_assign (rhs1, RSHIFT_EXPR, bf_cur,
3306 : build_int_cst (unsigned_type_node,
3307 64 : limb_prec - bo_shift));
3308 64 : insert_before (g);
3309 : }
3310 72 : rhs1 = add_cast (ftype, rhs1);
3311 72 : g = gimple_build_assign (bfr, rhs1);
3312 72 : insert_before (g);
3313 72 : if (eh)
3314 : {
3315 0 : maybe_duplicate_eh_stmt (g, stmt);
3316 0 : if (eh_pad)
3317 : {
3318 0 : edge e = split_block (gsi_bb (m_gsi), g);
3319 0 : m_gsi = gsi_after_labels (e->dest);
3320 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3321 : }
3322 : }
3323 : }
3324 23377 : if (zero_ms_limb)
3325 : {
3326 0 : tree p2 = build_int_cst (sizetype,
3327 0 : CEIL ((unsigned) TYPE_PRECISION (type),
3328 : abi_limb_prec)
3329 0 : * abi_limb_prec / limb_prec - 1);
3330 0 : tree l = limb_access (lhs_type, lhs, p2, true);
3331 0 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
3332 0 : insert_before (g);
3333 0 : if (eh)
3334 : {
3335 0 : maybe_duplicate_eh_stmt (g, stmt);
3336 0 : if (eh_pad)
3337 : {
3338 0 : edge e = split_block (gsi_bb (m_gsi), g);
3339 0 : m_gsi = gsi_after_labels (e->dest);
3340 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3341 : }
3342 : }
3343 : }
3344 :
3345 23377 : if (gimple_store_p (stmt))
3346 : {
3347 8522 : unlink_stmt_vdef (stmt);
3348 17044 : release_ssa_name (gimple_vdef (stmt));
3349 8522 : gsi_remove (&m_gsi, true);
3350 : }
3351 23377 : if (eq_p)
3352 : {
3353 6550 : lhs = make_ssa_name (boolean_type_node);
3354 6550 : basic_block bb = gimple_bb (stmt);
3355 6550 : gphi *phi = create_phi_node (lhs, bb);
3356 6550 : edge e = find_edge (gsi_bb (m_gsi), bb);
3357 6550 : unsigned int n = EDGE_COUNT (bb->preds);
3358 32941 : for (unsigned int i = 0; i < n; i++)
3359 : {
3360 26391 : edge e2 = EDGE_PRED (bb, i);
3361 26391 : add_phi_arg (phi, e == e2 ? boolean_true_node : boolean_false_node,
3362 : e2, UNKNOWN_LOCATION);
3363 : }
3364 6550 : cmp_code = cmp_code == EQ_EXPR ? NE_EXPR : EQ_EXPR;
3365 6550 : return lhs;
3366 : }
3367 : else
3368 : return NULL_TREE;
3369 : }
3370 :
3371 : /* Handle a large/huge _BitInt comparison statement STMT other than
3372 : EQ_EXPR/NE_EXPR. CMP_CODE, CMP_OP1 and CMP_OP2 meaning is like in
3373 : lower_mergeable_stmt. The {GT,GE,LT,LE}_EXPR comparisons are
3374 : lowered by iteration from the most significant limb downwards to
3375 : the least significant one, for large _BitInt in straight line code,
3376 : otherwise with most significant limb handled in
3377 : straight line code followed by a loop handling one limb at a time.
3378 : Comparisons with unsigned huge _BitInt with precisions which are
3379 : multiples of limb precision can use just the loop and don't need to
3380 : handle most significant limb before the loop. The loop or straight
3381 : line code jumps to final basic block if a particular pair of limbs
3382 : is not equal. */
3383 :
3384 : tree
3385 722 : bitint_large_huge::lower_comparison_stmt (gimple *stmt, tree_code &cmp_code,
3386 : tree cmp_op1, tree cmp_op2)
3387 : {
3388 722 : tree type = TREE_TYPE (cmp_op1);
3389 722 : gcc_assert (BITINT_TYPE_P (type));
3390 722 : bitint_prec_kind kind = bitint_precision_kind (type);
3391 722 : gcc_assert (kind >= bitint_prec_large);
3392 722 : gimple *g;
3393 722 : if (!TYPE_UNSIGNED (type)
3394 441 : && integer_zerop (cmp_op2)
3395 750 : && (cmp_code == GE_EXPR || cmp_code == LT_EXPR))
3396 : {
3397 28 : unsigned end = CEIL ((unsigned) TYPE_PRECISION (type), limb_prec) - 1;
3398 56 : tree idx = size_int (bitint_big_endian ? 0 : end);
3399 28 : m_data_cnt = 0;
3400 28 : tree rhs1 = handle_operand (cmp_op1, idx);
3401 28 : if (TYPE_UNSIGNED (TREE_TYPE (rhs1)))
3402 : {
3403 24 : tree stype = signed_type_for (TREE_TYPE (rhs1));
3404 24 : rhs1 = add_cast (stype, rhs1);
3405 : }
3406 28 : tree lhs = make_ssa_name (boolean_type_node);
3407 28 : g = gimple_build_assign (lhs, cmp_code, rhs1,
3408 28 : build_zero_cst (TREE_TYPE (rhs1)));
3409 28 : insert_before (g);
3410 28 : cmp_code = NE_EXPR;
3411 28 : return lhs;
3412 : }
3413 :
3414 694 : unsigned cnt, rem = 0, end = 0;
3415 694 : tree idx = NULL_TREE, idx_next = NULL_TREE;
3416 694 : if (kind == bitint_prec_large)
3417 377 : cnt = CEIL ((unsigned) TYPE_PRECISION (type), limb_prec);
3418 : else
3419 : {
3420 317 : rem = ((unsigned) TYPE_PRECISION (type) % limb_prec);
3421 317 : if (rem == 0 && !TYPE_UNSIGNED (type))
3422 : rem = limb_prec;
3423 317 : end = ((unsigned) TYPE_PRECISION (type) - rem) / limb_prec;
3424 317 : cnt = 1 + (rem != 0);
3425 : }
3426 :
3427 694 : basic_block edge_bb = NULL;
3428 694 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
3429 694 : gsi_prev (&gsi);
3430 694 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
3431 694 : edge_bb = e->src;
3432 694 : m_gsi = gsi_end_bb (edge_bb);
3433 :
3434 694 : edge *edges = XALLOCAVEC (edge, cnt * 2);
3435 2519 : for (unsigned i = 0; i < cnt; i++)
3436 : {
3437 1825 : m_data_cnt = 0;
3438 1825 : if (kind == bitint_prec_large)
3439 1246 : idx = size_int (bitint_big_endian ? i : cnt - i - 1);
3440 579 : else if (i == cnt - 1)
3441 317 : idx = create_loop (size_int (bitint_big_endian ? cnt - 1 : end - 1),
3442 : &idx_next);
3443 : else
3444 524 : idx = size_int (bitint_big_endian ? 0 : end);
3445 1825 : tree rhs1 = handle_operand (cmp_op1, idx);
3446 1825 : tree rhs2 = handle_operand (cmp_op2, idx);
3447 1825 : if (i == 0
3448 694 : && !TYPE_UNSIGNED (type)
3449 2238 : && TYPE_UNSIGNED (TREE_TYPE (rhs1)))
3450 : {
3451 112 : tree stype = signed_type_for (TREE_TYPE (rhs1));
3452 112 : rhs1 = add_cast (stype, rhs1);
3453 112 : rhs2 = add_cast (stype, rhs2);
3454 : }
3455 1825 : g = gimple_build_cond (GT_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
3456 1825 : insert_before (g);
3457 1825 : edge e1 = split_block (gsi_bb (m_gsi), g);
3458 1825 : e1->flags = EDGE_FALSE_VALUE;
3459 1825 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
3460 1825 : e1->probability = profile_probability::likely ();
3461 1825 : e2->probability = e1->probability.invert ();
3462 1825 : if (i == 0)
3463 694 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
3464 1825 : m_gsi = gsi_after_labels (e1->dest);
3465 1825 : edges[2 * i] = e2;
3466 1825 : g = gimple_build_cond (LT_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
3467 1825 : insert_before (g);
3468 1825 : e1 = split_block (gsi_bb (m_gsi), g);
3469 1825 : e1->flags = EDGE_FALSE_VALUE;
3470 1825 : e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
3471 1825 : e1->probability = profile_probability::unlikely ();
3472 1825 : e2->probability = e1->probability.invert ();
3473 1825 : m_gsi = gsi_after_labels (e1->dest);
3474 1825 : edges[2 * i + 1] = e2;
3475 1825 : m_first = false;
3476 1825 : if (kind == bitint_prec_huge && i == cnt - 1)
3477 : {
3478 634 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3479 : bitint_big_endian ? size_one_node
3480 317 : : size_int (-1));
3481 317 : insert_before (g);
3482 317 : g = gimple_build_cond (NE_EXPR, idx,
3483 : bitint_big_endian
3484 0 : ? size_int (end - 1 + (cnt != 1))
3485 : : size_zero_node,
3486 : NULL_TREE, NULL_TREE);
3487 317 : insert_before (g);
3488 317 : edge true_edge, false_edge;
3489 317 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
3490 : &true_edge, &false_edge);
3491 317 : m_gsi = gsi_after_labels (false_edge->dest);
3492 317 : m_bb = NULL;
3493 : }
3494 : }
3495 :
3496 694 : tree lhs = make_ssa_name (boolean_type_node);
3497 694 : basic_block bb = gimple_bb (stmt);
3498 694 : gphi *phi = create_phi_node (lhs, bb);
3499 5038 : for (unsigned int i = 0; i < cnt * 2; i++)
3500 : {
3501 3650 : tree val = ((cmp_code == GT_EXPR || cmp_code == GE_EXPR)
3502 3650 : ^ (i & 1)) ? boolean_true_node : boolean_false_node;
3503 3650 : add_phi_arg (phi, val, edges[i], UNKNOWN_LOCATION);
3504 : }
3505 694 : add_phi_arg (phi, (cmp_code == GE_EXPR || cmp_code == LE_EXPR)
3506 : ? boolean_true_node : boolean_false_node,
3507 : find_edge (gsi_bb (m_gsi), bb), UNKNOWN_LOCATION);
3508 694 : cmp_code = NE_EXPR;
3509 694 : return lhs;
3510 : }
3511 :
3512 : /* Lower large/huge _BitInt left and right shift except for left
3513 : shift by < limb_prec constant. */
3514 :
3515 : void
3516 584 : bitint_large_huge::lower_shift_stmt (tree obj, gimple *stmt)
3517 : {
3518 584 : tree rhs1 = gimple_assign_rhs1 (stmt);
3519 584 : tree lhs = gimple_assign_lhs (stmt);
3520 584 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
3521 584 : tree type = TREE_TYPE (rhs1);
3522 584 : gimple *final_stmt = gsi_stmt (m_gsi);
3523 584 : gcc_assert (BITINT_TYPE_P (type)
3524 : && bitint_precision_kind (type) >= bitint_prec_large);
3525 584 : int prec = TYPE_PRECISION (type);
3526 584 : tree n = gimple_assign_rhs2 (stmt), n1, n2, n3, n4;
3527 584 : gimple *g;
3528 584 : if (obj == NULL_TREE)
3529 : {
3530 465 : int part = var_to_partition (m_map, lhs);
3531 465 : gcc_assert (m_vars[part] != NULL_TREE);
3532 : obj = m_vars[part];
3533 : }
3534 : /* Preparation code common for both left and right shifts.
3535 : unsigned n1 = n % limb_prec;
3536 : size_t n2 = n / limb_prec;
3537 : size_t n3 = n1 != 0;
3538 : unsigned n4 = (limb_prec - n1) % limb_prec;
3539 : (for power of 2 limb_prec n4 can be -n1 & limb_prec). */
3540 584 : if (TREE_CODE (n) == INTEGER_CST)
3541 : {
3542 249 : tree lp = build_int_cst (TREE_TYPE (n), limb_prec);
3543 249 : n1 = int_const_binop (TRUNC_MOD_EXPR, n, lp);
3544 249 : n2 = fold_convert (sizetype, int_const_binop (TRUNC_DIV_EXPR, n, lp));
3545 249 : n3 = size_int (!integer_zerop (n1));
3546 249 : n4 = int_const_binop (TRUNC_MOD_EXPR,
3547 249 : int_const_binop (MINUS_EXPR, lp, n1), lp);
3548 : }
3549 : else
3550 : {
3551 335 : n1 = make_ssa_name (TREE_TYPE (n));
3552 335 : n2 = make_ssa_name (sizetype);
3553 335 : n3 = make_ssa_name (sizetype);
3554 335 : n4 = make_ssa_name (TREE_TYPE (n));
3555 335 : if (pow2p_hwi (limb_prec))
3556 : {
3557 335 : tree lpm1 = build_int_cst (TREE_TYPE (n), limb_prec - 1);
3558 335 : g = gimple_build_assign (n1, BIT_AND_EXPR, n, lpm1);
3559 335 : insert_before (g);
3560 989 : g = gimple_build_assign (useless_type_conversion_p (sizetype,
3561 335 : TREE_TYPE (n))
3562 319 : ? n2 : make_ssa_name (TREE_TYPE (n)),
3563 : RSHIFT_EXPR, n,
3564 335 : build_int_cst (TREE_TYPE (n),
3565 670 : exact_log2 (limb_prec)));
3566 335 : insert_before (g);
3567 335 : if (gimple_assign_lhs (g) != n2)
3568 : {
3569 319 : g = gimple_build_assign (n2, NOP_EXPR, gimple_assign_lhs (g));
3570 319 : insert_before (g);
3571 : }
3572 335 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (n)),
3573 : NEGATE_EXPR, n1);
3574 335 : insert_before (g);
3575 335 : g = gimple_build_assign (n4, BIT_AND_EXPR, gimple_assign_lhs (g),
3576 : lpm1);
3577 335 : insert_before (g);
3578 : }
3579 : else
3580 : {
3581 0 : tree lp = build_int_cst (TREE_TYPE (n), limb_prec);
3582 0 : g = gimple_build_assign (n1, TRUNC_MOD_EXPR, n, lp);
3583 0 : insert_before (g);
3584 0 : g = gimple_build_assign (useless_type_conversion_p (sizetype,
3585 0 : TREE_TYPE (n))
3586 0 : ? n2 : make_ssa_name (TREE_TYPE (n)),
3587 : TRUNC_DIV_EXPR, n, lp);
3588 0 : insert_before (g);
3589 0 : if (gimple_assign_lhs (g) != n2)
3590 : {
3591 0 : g = gimple_build_assign (n2, NOP_EXPR, gimple_assign_lhs (g));
3592 0 : insert_before (g);
3593 : }
3594 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (n)),
3595 : MINUS_EXPR, lp, n1);
3596 0 : insert_before (g);
3597 0 : g = gimple_build_assign (n4, TRUNC_MOD_EXPR, gimple_assign_lhs (g),
3598 : lp);
3599 0 : insert_before (g);
3600 : }
3601 335 : g = gimple_build_assign (make_ssa_name (boolean_type_node), NE_EXPR, n1,
3602 335 : build_zero_cst (TREE_TYPE (n)));
3603 335 : insert_before (g);
3604 335 : g = gimple_build_assign (n3, NOP_EXPR, gimple_assign_lhs (g));
3605 335 : insert_before (g);
3606 : }
3607 1168 : tree p = build_int_cst (sizetype,
3608 584 : prec / limb_prec - (prec % limb_prec == 0));
3609 584 : if (rhs_code == RSHIFT_EXPR)
3610 : {
3611 : /* Lower
3612 : dst = src >> n;
3613 : as
3614 : unsigned n1 = n % limb_prec;
3615 : size_t n2 = n / limb_prec;
3616 : size_t n3 = n1 != 0;
3617 : unsigned n4 = (limb_prec - n1) % limb_prec;
3618 : size_t idx;
3619 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
3620 : int signed_p = (typeof (src) -1) < 0;
3621 : for (idx = n2; idx < ((!signed_p && (prec % limb_prec == 0))
3622 : ? p : p - n3); ++idx)
3623 : dst[idx - n2] = (src[idx] >> n1) | (src[idx + n3] << n4);
3624 : limb_type ext;
3625 : if (prec % limb_prec == 0)
3626 : ext = src[p];
3627 : else if (signed_p)
3628 : ext = ((signed limb_type) (src[p] << (limb_prec
3629 : - (prec % limb_prec))))
3630 : >> (limb_prec - (prec % limb_prec));
3631 : else
3632 : ext = src[p] & (((limb_type) 1 << (prec % limb_prec)) - 1);
3633 : if (!signed_p && (prec % limb_prec == 0))
3634 : ;
3635 : else if (idx < prec / 64)
3636 : {
3637 : dst[idx - n2] = (src[idx] >> n1) | (ext << n4);
3638 : ++idx;
3639 : }
3640 : idx -= n2;
3641 : if (signed_p)
3642 : {
3643 : dst[idx] = ((signed limb_type) ext) >> n1;
3644 : ext = ((signed limb_type) ext) >> (limb_prec - 1);
3645 : }
3646 : else
3647 : {
3648 : dst[idx] = ext >> n1;
3649 : ext = 0;
3650 : }
3651 : for (++idx; idx <= p; ++idx)
3652 : dst[idx] = ext; */
3653 387 : tree pmn3;
3654 387 : if (TYPE_UNSIGNED (type) && prec % limb_prec == 0)
3655 100 : pmn3 = bitint_big_endian ? size_zero_node : p;
3656 287 : else if (bitint_big_endian)
3657 : pmn3 = n3;
3658 287 : else if (TREE_CODE (n3) == INTEGER_CST)
3659 114 : pmn3 = int_const_binop (MINUS_EXPR, p, n3);
3660 : else
3661 : {
3662 173 : pmn3 = make_ssa_name (sizetype);
3663 173 : g = gimple_build_assign (pmn3, MINUS_EXPR, p, n3);
3664 173 : insert_before (g);
3665 : }
3666 387 : tree pmn2 = NULL_TREE;
3667 387 : if (bitint_big_endian)
3668 : {
3669 0 : if (TREE_CODE (n2) == INTEGER_CST)
3670 0 : pmn2 = int_const_binop (MINUS_EXPR, p, n2);
3671 : else
3672 : {
3673 0 : pmn2 = make_ssa_name (sizetype);
3674 0 : g = gimple_build_assign (pmn2, MINUS_EXPR, p, n2);
3675 0 : insert_before (g);
3676 : }
3677 0 : g = gimple_build_cond (GT_EXPR, pmn2, pmn3, NULL_TREE, NULL_TREE);
3678 : }
3679 : else
3680 387 : g = gimple_build_cond (LT_EXPR, n2, pmn3, NULL_TREE, NULL_TREE);
3681 387 : edge edge_true, edge_false;
3682 387 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3683 387 : tree idx_next;
3684 774 : tree idx = create_loop (bitint_big_endian ? pmn2 : n2, &idx_next);
3685 387 : tree idxmn2 = make_ssa_name (sizetype);
3686 387 : tree idxpn3 = make_ssa_name (sizetype);
3687 774 : g = gimple_build_assign (idxmn2,
3688 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3689 : idx, n2);
3690 387 : insert_before (g);
3691 774 : g = gimple_build_assign (idxpn3,
3692 : bitint_big_endian ? MINUS_EXPR : PLUS_EXPR,
3693 : idx, n3);
3694 387 : insert_before (g);
3695 387 : m_data_cnt = 0;
3696 387 : tree t1 = handle_operand (rhs1, idx);
3697 387 : m_first = false;
3698 387 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3699 : RSHIFT_EXPR, t1, n1);
3700 387 : insert_before (g);
3701 387 : t1 = gimple_assign_lhs (g);
3702 387 : if (!integer_zerop (n3))
3703 : {
3704 299 : m_data_cnt = 0;
3705 299 : tree t2 = handle_operand (rhs1, idxpn3);
3706 299 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3707 : LSHIFT_EXPR, t2, n4);
3708 299 : insert_before (g);
3709 299 : t2 = gimple_assign_lhs (g);
3710 299 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3711 : BIT_IOR_EXPR, t1, t2);
3712 299 : insert_before (g);
3713 299 : t1 = gimple_assign_lhs (g);
3714 : }
3715 387 : tree l = limb_access (TREE_TYPE (lhs), obj, idxmn2, true);
3716 387 : g = gimple_build_assign (l, t1);
3717 387 : insert_before (g);
3718 387 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3719 0 : bitint_big_endian ? size_int (-1)
3720 : : size_one_node);
3721 387 : insert_before (g);
3722 774 : g = gimple_build_cond (bitint_big_endian ? GT_EXPR : LT_EXPR,
3723 : idx_next, pmn3, NULL_TREE, NULL_TREE);
3724 387 : insert_before (g);
3725 387 : idx = make_ssa_name (sizetype);
3726 387 : m_gsi = gsi_for_stmt (final_stmt);
3727 387 : gphi *phi = create_phi_node (idx, gsi_bb (m_gsi));
3728 387 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3729 387 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3730 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3731 774 : add_phi_arg (phi, bitint_big_endian ? pmn2 : n2, edge_false,
3732 : UNKNOWN_LOCATION);
3733 387 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3734 387 : m_data_cnt = 0;
3735 387 : tree ms = handle_operand (rhs1, bitint_big_endian ? size_zero_node : p);
3736 387 : tree ext = ms;
3737 387 : if (!types_compatible_p (TREE_TYPE (ms), m_limb_type))
3738 230 : ext = add_cast (m_limb_type, ms);
3739 587 : if (!(TYPE_UNSIGNED (type) && prec % limb_prec == 0)
3740 487 : && !integer_zerop (n3))
3741 : {
3742 249 : if (bitint_big_endian)
3743 0 : g = gimple_build_cond (GT_EXPR, idx, size_zero_node,
3744 : NULL_TREE, NULL_TREE);
3745 : else
3746 249 : g = gimple_build_cond (LT_EXPR, idx, p, NULL_TREE, NULL_TREE);
3747 249 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3748 249 : m_data_cnt = 0;
3749 249 : t1 = handle_operand (rhs1, idx);
3750 249 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3751 : RSHIFT_EXPR, t1, n1);
3752 249 : insert_before (g);
3753 249 : t1 = gimple_assign_lhs (g);
3754 249 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3755 : LSHIFT_EXPR, ext, n4);
3756 249 : insert_before (g);
3757 249 : tree t2 = gimple_assign_lhs (g);
3758 249 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3759 : BIT_IOR_EXPR, t1, t2);
3760 249 : insert_before (g);
3761 249 : t1 = gimple_assign_lhs (g);
3762 249 : idxmn2 = make_ssa_name (sizetype);
3763 498 : g = gimple_build_assign (idxmn2, bitint_big_endian
3764 : ? PLUS_EXPR : MINUS_EXPR, idx, n2);
3765 249 : insert_before (g);
3766 249 : l = limb_access (TREE_TYPE (lhs), obj, idxmn2, true);
3767 249 : g = gimple_build_assign (l, t1);
3768 249 : insert_before (g);
3769 249 : idx_next = make_ssa_name (sizetype);
3770 249 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3771 : bitint_big_endian
3772 0 : ? size_int (-1) : size_one_node);
3773 249 : insert_before (g);
3774 249 : m_gsi = gsi_for_stmt (final_stmt);
3775 249 : tree nidx = make_ssa_name (sizetype);
3776 249 : phi = create_phi_node (nidx, gsi_bb (m_gsi));
3777 249 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3778 249 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3779 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3780 249 : add_phi_arg (phi, idx, edge_false, UNKNOWN_LOCATION);
3781 249 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3782 249 : idx = nidx;
3783 : }
3784 774 : g = gimple_build_assign (make_ssa_name (sizetype),
3785 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3786 : idx, n2);
3787 387 : insert_before (g);
3788 387 : idx = gimple_assign_lhs (g);
3789 387 : tree sext = ext;
3790 387 : if (!TYPE_UNSIGNED (type))
3791 187 : sext = add_cast (signed_type_for (m_limb_type), ext);
3792 387 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (sext)),
3793 : RSHIFT_EXPR, sext, n1);
3794 387 : insert_before (g);
3795 387 : t1 = gimple_assign_lhs (g);
3796 387 : if (!TYPE_UNSIGNED (type))
3797 : {
3798 187 : t1 = add_cast (m_limb_type, t1);
3799 187 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (sext)),
3800 : RSHIFT_EXPR, sext,
3801 187 : build_int_cst (TREE_TYPE (n),
3802 187 : limb_prec - 1));
3803 187 : insert_before (g);
3804 187 : ext = add_cast (m_limb_type, gimple_assign_lhs (g));
3805 : }
3806 : else
3807 200 : ext = build_zero_cst (m_limb_type);
3808 387 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3809 387 : g = gimple_build_assign (l, t1);
3810 387 : insert_before (g);
3811 387 : g = gimple_build_assign (make_ssa_name (sizetype), PLUS_EXPR, idx,
3812 : bitint_big_endian
3813 0 : ? size_int (-1) : size_one_node);
3814 387 : insert_before (g);
3815 387 : tree p2 = p;
3816 387 : if (bitint_big_endian)
3817 : {
3818 0 : tree new_idx = gimple_assign_lhs (g);
3819 0 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
3820 : NULL_TREE, NULL_TREE);
3821 0 : idx = new_idx;
3822 : }
3823 : else
3824 : {
3825 387 : if (bitint_extended == bitint_ext_full
3826 0 : && abi_limb_prec > limb_prec)
3827 0 : p2 = build_int_cst (sizetype,
3828 0 : CEIL (prec, abi_limb_prec)
3829 0 : * abi_limb_prec / limb_prec - 1);
3830 387 : idx = gimple_assign_lhs (g);
3831 387 : g = gimple_build_cond (LE_EXPR, idx, p2, NULL_TREE, NULL_TREE);
3832 : }
3833 387 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3834 387 : idx = create_loop (idx, &idx_next);
3835 387 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3836 387 : g = gimple_build_assign (l, ext);
3837 387 : insert_before (g);
3838 387 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3839 : bitint_big_endian
3840 0 : ? size_int (-1) : size_one_node);
3841 387 : insert_before (g);
3842 387 : if (bitint_big_endian)
3843 0 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
3844 : NULL_TREE, NULL_TREE);
3845 : else
3846 387 : g = gimple_build_cond (LE_EXPR, idx_next, p2, NULL_TREE, NULL_TREE);
3847 387 : insert_before (g);
3848 : }
3849 : else
3850 : {
3851 : /* Lower
3852 : dst = src << n;
3853 : as
3854 : unsigned n1 = n % limb_prec;
3855 : size_t n2 = n / limb_prec;
3856 : size_t n3 = n1 != 0;
3857 : unsigned n4 = (limb_prec - n1) % limb_prec;
3858 : size_t idx;
3859 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
3860 : for (idx = p; (ssize_t) idx >= (ssize_t) (n2 + n3); --idx)
3861 : dst[idx] = (src[idx - n2] << n1) | (src[idx - n2 - n3] >> n4);
3862 : if (n1)
3863 : {
3864 : dst[idx] = src[idx - n2] << n1;
3865 : --idx;
3866 : }
3867 : for (; (ssize_t) idx >= 0; --idx)
3868 : dst[idx] = 0; */
3869 197 : tree n2pn3;
3870 197 : if (TREE_CODE (n2) == INTEGER_CST && TREE_CODE (n3) == INTEGER_CST)
3871 71 : n2pn3 = int_const_binop (PLUS_EXPR, n2, n3);
3872 : else
3873 : {
3874 126 : n2pn3 = make_ssa_name (sizetype);
3875 126 : g = gimple_build_assign (n2pn3, PLUS_EXPR, n2, n3);
3876 126 : insert_before (g);
3877 : }
3878 197 : if (bitint_big_endian)
3879 : {
3880 0 : if (TREE_CODE (n2pn3) == INTEGER_CST)
3881 0 : n2pn3 = int_const_binop (MINUS_EXPR, p, n2pn3);
3882 : else
3883 : {
3884 0 : g = gimple_build_assign (make_ssa_name (sizetype),
3885 : MINUS_EXPR, p, n2pn3);
3886 0 : insert_before (g);
3887 0 : n2pn3 = gimple_assign_lhs (g);
3888 : }
3889 : }
3890 : /* For LSHIFT_EXPR, we can use handle_operand with non-INTEGER_CST
3891 : idx even to access the most significant partial limb. */
3892 197 : m_var_msb = true;
3893 197 : if (integer_zerop (n3))
3894 : /* For n3 == 0 p >= n2 + n3 is always true for all valid shift
3895 : counts. Emit if (true) condition that can be optimized later. */
3896 45 : g = gimple_build_cond (NE_EXPR, boolean_true_node, boolean_false_node,
3897 : NULL_TREE, NULL_TREE);
3898 152 : else if (bitint_big_endian)
3899 0 : g = gimple_build_cond (NE_EXPR, n2pn3, size_int (-1), NULL_TREE,
3900 : NULL_TREE);
3901 : else
3902 152 : g = gimple_build_cond (LE_EXPR, n2pn3, p, NULL_TREE, NULL_TREE);
3903 197 : edge edge_true, edge_false;
3904 197 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3905 197 : tree idx_next;
3906 197 : tree idx = create_loop (bitint_big_endian ? size_zero_node : p,
3907 : &idx_next);
3908 197 : tree idxmn2 = make_ssa_name (sizetype);
3909 197 : tree idxmn2mn3 = make_ssa_name (sizetype);
3910 394 : g = gimple_build_assign (idxmn2,
3911 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3912 : idx, n2);
3913 197 : insert_before (g);
3914 394 : g = gimple_build_assign (idxmn2mn3,
3915 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3916 : idxmn2, n3);
3917 197 : insert_before (g);
3918 197 : m_data_cnt = 0;
3919 197 : tree t1 = handle_operand (rhs1, idxmn2);
3920 197 : m_first = false;
3921 197 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3922 : LSHIFT_EXPR, t1, n1);
3923 197 : insert_before (g);
3924 197 : t1 = gimple_assign_lhs (g);
3925 197 : if (!integer_zerop (n3))
3926 : {
3927 152 : m_data_cnt = 0;
3928 152 : tree t2 = handle_operand (rhs1, idxmn2mn3);
3929 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3930 : RSHIFT_EXPR, t2, n4);
3931 152 : insert_before (g);
3932 152 : t2 = gimple_assign_lhs (g);
3933 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3934 : BIT_IOR_EXPR, t1, t2);
3935 152 : insert_before (g);
3936 152 : t1 = gimple_assign_lhs (g);
3937 : }
3938 197 : tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3939 197 : g = gimple_build_assign (l, t1);
3940 197 : insert_before (g);
3941 394 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3942 : bitint_big_endian
3943 197 : ? size_one_node : size_int (-1));
3944 197 : insert_before (g);
3945 197 : tree sn2pn3 = add_cast (ssizetype, n2pn3);
3946 394 : g = gimple_build_cond (bitint_big_endian ? LE_EXPR : GE_EXPR,
3947 : add_cast (ssizetype, idx_next), sn2pn3,
3948 : NULL_TREE, NULL_TREE);
3949 197 : insert_before (g);
3950 197 : idx = make_ssa_name (sizetype);
3951 197 : m_gsi = gsi_for_stmt (final_stmt);
3952 197 : gphi *phi = create_phi_node (idx, gsi_bb (m_gsi));
3953 197 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3954 197 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3955 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3956 197 : add_phi_arg (phi, bitint_big_endian ? size_zero_node : p,
3957 : edge_false, UNKNOWN_LOCATION);
3958 197 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3959 197 : m_data_cnt = 0;
3960 197 : if (!integer_zerop (n3))
3961 : {
3962 152 : g = gimple_build_cond (NE_EXPR, n3, size_zero_node,
3963 : NULL_TREE, NULL_TREE);
3964 152 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3965 152 : idxmn2 = make_ssa_name (sizetype);
3966 304 : g = gimple_build_assign (idxmn2,
3967 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3968 : idx, n2);
3969 152 : insert_before (g);
3970 152 : m_data_cnt = 0;
3971 152 : t1 = handle_operand (rhs1, idxmn2);
3972 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3973 : LSHIFT_EXPR, t1, n1);
3974 152 : insert_before (g);
3975 152 : t1 = gimple_assign_lhs (g);
3976 152 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3977 152 : g = gimple_build_assign (l, t1);
3978 152 : insert_before (g);
3979 152 : idx_next = make_ssa_name (sizetype);
3980 304 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3981 : bitint_big_endian
3982 152 : ? size_one_node : size_int (-1));
3983 152 : insert_before (g);
3984 152 : m_gsi = gsi_for_stmt (final_stmt);
3985 152 : tree nidx = make_ssa_name (sizetype);
3986 152 : phi = create_phi_node (nidx, gsi_bb (m_gsi));
3987 152 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3988 152 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3989 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3990 152 : add_phi_arg (phi, idx, edge_false, UNKNOWN_LOCATION);
3991 152 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3992 152 : idx = nidx;
3993 : }
3994 197 : if (bitint_big_endian)
3995 0 : g = gimple_build_cond (LE_EXPR, idx, p, NULL_TREE, NULL_TREE);
3996 : else
3997 197 : g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx),
3998 : ssize_int (0), NULL_TREE, NULL_TREE);
3999 197 : if_then (g, profile_probability::likely (), edge_true, edge_false);
4000 197 : idx = create_loop (idx, &idx_next);
4001 197 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
4002 197 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
4003 197 : insert_before (g);
4004 394 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
4005 : bitint_big_endian
4006 197 : ? size_one_node : size_int (-1));
4007 197 : insert_before (g);
4008 197 : if (bitint_big_endian)
4009 0 : g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
4010 : else
4011 197 : g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx_next),
4012 : ssize_int (0), NULL_TREE, NULL_TREE);
4013 197 : insert_before (g);
4014 197 : if (bitint_extended && prec % limb_prec != 0)
4015 : {
4016 : /* The most significant limb has been updated either in the
4017 : loop or in the if after it. To simplify the code, just
4018 : read it back from memory and extend. */
4019 0 : m_gsi = gsi_after_labels (edge_false->dest);
4020 0 : idx = bitint_big_endian ? size_zero_node : p;
4021 0 : tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
4022 0 : tree type = limb_access_type (TREE_TYPE (lhs), idx);
4023 0 : tree v = make_ssa_name (m_limb_type);
4024 0 : g = gimple_build_assign (v, l);
4025 0 : insert_before (g);
4026 0 : v = add_cast (type, v);
4027 0 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
4028 0 : v = add_cast (m_limb_type, v);
4029 0 : g = gimple_build_assign (l, v);
4030 0 : insert_before (g);
4031 0 : if (bitint_extended == bitint_ext_full
4032 0 : && abi_limb_prec > limb_prec
4033 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4034 0 : > CEIL (prec, limb_prec) * limb_prec))
4035 : {
4036 0 : tree p2 = build_int_cst (sizetype,
4037 : CEIL (prec, abi_limb_prec)
4038 0 : * abi_limb_prec / limb_prec - 1);
4039 0 : if (TYPE_UNSIGNED (TREE_TYPE (lhs)))
4040 0 : v = build_zero_cst (m_limb_type);
4041 : else
4042 : {
4043 0 : v = add_cast (signed_type_for (m_limb_type), v);
4044 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (v)),
4045 : RSHIFT_EXPR, v,
4046 : build_int_cst (unsigned_type_node,
4047 0 : limb_prec - 1));
4048 0 : insert_before (g);
4049 0 : v = add_cast (m_limb_type, gimple_assign_lhs (g));
4050 : }
4051 0 : l = limb_access (TREE_TYPE (lhs), obj, p2, true);
4052 0 : g = gimple_build_assign (l, v);
4053 0 : insert_before (g);
4054 : }
4055 : }
4056 197 : else if (bitint_extended == bitint_ext_full
4057 0 : && abi_limb_prec > limb_prec
4058 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4059 0 : > CEIL (prec, limb_prec) * limb_prec))
4060 : {
4061 0 : m_gsi = gsi_after_labels (edge_false->dest);
4062 0 : tree p2 = build_int_cst (sizetype,
4063 : CEIL (prec, abi_limb_prec)
4064 0 : * abi_limb_prec / limb_prec - 1);
4065 0 : tree v;
4066 0 : if (TYPE_UNSIGNED (TREE_TYPE (lhs)))
4067 0 : v = build_zero_cst (m_limb_type);
4068 : else
4069 : {
4070 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p, true);
4071 0 : v = make_ssa_name (m_limb_type);
4072 0 : g = gimple_build_assign (v, l);
4073 0 : insert_before (g);
4074 0 : v = add_cast (signed_type_for (m_limb_type), v);
4075 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (v)),
4076 : RSHIFT_EXPR, v,
4077 : build_int_cst (unsigned_type_node,
4078 0 : limb_prec - 1));
4079 0 : insert_before (g);
4080 0 : v = add_cast (m_limb_type, gimple_assign_lhs (g));
4081 : }
4082 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
4083 0 : g = gimple_build_assign (l, v);
4084 0 : insert_before (g);
4085 : }
4086 : }
4087 584 : }
4088 :
4089 : /* Lower large/huge _BitInt multiplication or division. */
4090 :
4091 : void
4092 371 : bitint_large_huge::lower_muldiv_stmt (tree obj, gimple *stmt)
4093 : {
4094 371 : tree rhs1 = gimple_assign_rhs1 (stmt);
4095 371 : tree rhs2 = gimple_assign_rhs2 (stmt);
4096 371 : tree lhs = gimple_assign_lhs (stmt);
4097 371 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
4098 371 : tree type = TREE_TYPE (rhs1);
4099 371 : gcc_assert (BITINT_TYPE_P (type)
4100 : && bitint_precision_kind (type) >= bitint_prec_large);
4101 371 : int prec = TYPE_PRECISION (type), prec1, prec2;
4102 371 : bool ext_ms_limb = false;
4103 371 : bool do_ext = false;
4104 371 : rhs1 = handle_operand_addr (rhs1, stmt, NULL, &prec1);
4105 371 : rhs2 = handle_operand_addr (rhs2, stmt, NULL, &prec2);
4106 371 : if (obj == NULL_TREE)
4107 : {
4108 166 : int part = var_to_partition (m_map, lhs);
4109 166 : gcc_assert (m_vars[part] != NULL_TREE);
4110 166 : obj = m_vars[part];
4111 166 : lhs = build_fold_addr_expr (obj);
4112 : }
4113 : else
4114 : {
4115 205 : lhs = build_fold_addr_expr (obj);
4116 205 : lhs = force_gimple_operand_gsi (&m_gsi, lhs, true,
4117 : NULL_TREE, true, GSI_SAME_STMT);
4118 : }
4119 371 : if (bitint_extended && TYPE_OVERFLOW_WRAPS (type))
4120 : {
4121 0 : if (rhs_code == MULT_EXPR)
4122 : do_ext = true;
4123 : /* For signed division with -fwrapv, minimum negative / -1 needs
4124 : is minimum negative and the padding bits above it should be all
4125 : set. */
4126 0 : else if (!TYPE_UNSIGNED (type)
4127 0 : && (rhs_code == TRUNC_DIV_EXPR || rhs_code == EXACT_DIV_EXPR))
4128 371 : do_ext = true;
4129 : }
4130 371 : if (bitint_extended == bitint_ext_full
4131 0 : && abi_limb_prec > limb_prec
4132 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4133 0 : > CEIL (prec, limb_prec) * limb_prec))
4134 : {
4135 : /* unsigned multiplication needs to wrap around, so we can't
4136 : increase prec. Similarly for -fwrapv. */
4137 0 : if (do_ext)
4138 : ext_ms_limb = true;
4139 : else
4140 0 : prec = CEIL (prec, abi_limb_prec) * abi_limb_prec;
4141 : }
4142 371 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
4143 371 : gimple *g;
4144 371 : switch (rhs_code)
4145 : {
4146 208 : case MULT_EXPR:
4147 208 : g = gimple_build_call_internal (IFN_MULBITINT, 6,
4148 208 : lhs, build_int_cst (sitype, prec),
4149 208 : rhs1, build_int_cst (sitype, prec1),
4150 208 : rhs2, build_int_cst (sitype, prec2));
4151 208 : insert_before (g);
4152 208 : break;
4153 103 : case TRUNC_DIV_EXPR:
4154 103 : case EXACT_DIV_EXPR:
4155 103 : g = gimple_build_call_internal (IFN_DIVMODBITINT, 8,
4156 103 : lhs, build_int_cst (sitype, prec),
4157 : null_pointer_node,
4158 : build_int_cst (sitype, 0),
4159 103 : rhs1, build_int_cst (sitype, prec1),
4160 103 : rhs2, build_int_cst (sitype, prec2));
4161 103 : if (!stmt_ends_bb_p (stmt))
4162 102 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4163 103 : insert_before (g);
4164 103 : break;
4165 60 : case TRUNC_MOD_EXPR:
4166 60 : g = gimple_build_call_internal (IFN_DIVMODBITINT, 8, null_pointer_node,
4167 : build_int_cst (sitype, 0),
4168 60 : lhs, build_int_cst (sitype, prec),
4169 60 : rhs1, build_int_cst (sitype, prec1),
4170 60 : rhs2, build_int_cst (sitype, prec2));
4171 60 : if (!stmt_ends_bb_p (stmt))
4172 57 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4173 60 : insert_before (g);
4174 60 : break;
4175 0 : default:
4176 0 : gcc_unreachable ();
4177 : }
4178 371 : if (stmt_ends_bb_p (stmt))
4179 : {
4180 4 : maybe_duplicate_eh_stmt (g, stmt);
4181 4 : edge e1;
4182 4 : edge_iterator ei;
4183 4 : basic_block bb = gimple_bb (stmt);
4184 :
4185 4 : FOR_EACH_EDGE (e1, ei, bb->succs)
4186 4 : if (e1->flags & EDGE_EH)
4187 : break;
4188 4 : if (e1)
4189 : {
4190 4 : edge e2 = split_block (gsi_bb (m_gsi), g);
4191 4 : m_gsi = gsi_after_labels (e2->dest);
4192 4 : add_eh_edge (e2->src, e1);
4193 : }
4194 : }
4195 371 : if (do_ext
4196 371 : && ((prec % limb_prec) != 0 || (ext_ms_limb && !TYPE_UNSIGNED (type))))
4197 : {
4198 : /* Unsigned multiplication wraps, but libgcc function will return the
4199 : bits beyond prec within the top limb as another limb of the full
4200 : multiplication. So, clear the padding bits here. */
4201 0 : tree idx = size_int (bitint_big_endian ? 0 : prec / limb_prec);
4202 0 : tree l = limb_access (type, obj, idx, true);
4203 0 : tree ctype = limb_access_type (type, idx);
4204 0 : tree v = make_ssa_name (m_limb_type);
4205 0 : g = gimple_build_assign (v, l);
4206 0 : insert_before (g);
4207 0 : tree v2 = v;
4208 0 : if ((prec % limb_prec) != 0)
4209 : {
4210 0 : v = add_cast (ctype, v);
4211 0 : l = limb_access (type, obj, idx, true);
4212 0 : v = add_cast (m_limb_type, v);
4213 0 : v2 = v;
4214 0 : g = gimple_build_assign (l, v);
4215 0 : insert_before (g);
4216 : }
4217 0 : if (ext_ms_limb && !TYPE_UNSIGNED (type))
4218 : {
4219 0 : v2 = add_cast (signed_type_for (m_limb_type), v2);
4220 0 : tree lpm1 = build_int_cst (unsigned_type_node, limb_prec - 1);
4221 0 : v = make_ssa_name (TREE_TYPE (v2));
4222 0 : g = gimple_build_assign (v, RSHIFT_EXPR, v2, lpm1);
4223 0 : insert_before (g);
4224 0 : unsigned int i
4225 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4226 0 : v = add_cast (m_limb_type, v);
4227 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1),
4228 : true), v);
4229 0 : insert_before (g);
4230 0 : ext_ms_limb = false;
4231 : }
4232 : }
4233 371 : if (ext_ms_limb)
4234 : {
4235 0 : unsigned int i = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4236 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1), true),
4237 : build_zero_cst (m_limb_type));
4238 0 : insert_before (g);
4239 : }
4240 371 : }
4241 :
4242 : /* Lower large/huge _BitInt conversion to/from floating point. */
4243 :
4244 : void
4245 327 : bitint_large_huge::lower_float_conv_stmt (tree obj, gimple *stmt)
4246 : {
4247 327 : tree rhs1 = gimple_assign_rhs1 (stmt);
4248 327 : tree lhs = gimple_assign_lhs (stmt);
4249 327 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
4250 327 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
4251 327 : gimple *g;
4252 327 : if (rhs_code == FIX_TRUNC_EXPR)
4253 : {
4254 179 : tree type = TREE_TYPE (lhs);
4255 179 : int prec = TYPE_PRECISION (type);
4256 179 : bool extend_ms_limb = false;
4257 179 : if (bitint_extended == bitint_ext_full
4258 0 : && abi_limb_prec > limb_prec
4259 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4260 0 : > CEIL (prec, limb_prec) * limb_prec))
4261 179 : extend_ms_limb = true;
4262 179 : if (!TYPE_UNSIGNED (type))
4263 93 : prec = -prec;
4264 179 : if (obj == NULL_TREE)
4265 : {
4266 135 : int part = var_to_partition (m_map, lhs);
4267 135 : gcc_assert (m_vars[part] != NULL_TREE);
4268 135 : obj = m_vars[part];
4269 135 : lhs = build_fold_addr_expr (obj);
4270 : }
4271 : else
4272 : {
4273 44 : lhs = build_fold_addr_expr (obj);
4274 44 : lhs = force_gimple_operand_gsi (&m_gsi, lhs, true,
4275 : NULL_TREE, true, GSI_SAME_STMT);
4276 : }
4277 179 : scalar_mode from_mode
4278 179 : = as_a <scalar_mode> (TYPE_MODE (TREE_TYPE (rhs1)));
4279 : #ifdef HAVE_SFmode
4280 : /* IEEE single is a full superset of both IEEE half and
4281 : bfloat formats, convert to float first and then to _BitInt
4282 : to avoid the need of another 2 library routines. */
4283 179 : if ((REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format
4284 179 : || REAL_MODE_FORMAT (from_mode) == &ieee_half_format)
4285 191 : && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
4286 : {
4287 12 : tree type = lang_hooks.types.type_for_mode (SFmode, 0);
4288 12 : if (type)
4289 12 : rhs1 = add_cast (type, rhs1);
4290 : }
4291 : #endif
4292 179 : g = gimple_build_call_internal (IFN_FLOATTOBITINT, 3,
4293 179 : lhs, build_int_cst (sitype, prec),
4294 : rhs1);
4295 179 : insert_before (g);
4296 179 : if (extend_ms_limb)
4297 : {
4298 0 : unsigned int i
4299 0 : = (CEIL (prec < 0 ? -prec : prec, abi_limb_prec)
4300 0 : * abi_limb_prec / limb_prec);
4301 0 : tree val;
4302 0 : if (prec < 0)
4303 : {
4304 0 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4305 : limb_access (type, obj,
4306 0 : size_int (i - 2),
4307 : true));
4308 0 : insert_before (g);
4309 0 : val = add_cast (signed_type_for (m_limb_type),
4310 : gimple_assign_lhs (g));
4311 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (val)),
4312 : RSHIFT_EXPR, val,
4313 : build_int_cst (unsigned_type_node,
4314 0 : limb_prec - 1));
4315 0 : insert_before (g);
4316 0 : val = add_cast (m_limb_type, gimple_assign_lhs (g));
4317 : }
4318 : else
4319 0 : val = build_zero_cst (m_limb_type);
4320 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1),
4321 : true), val);
4322 0 : insert_before (g);
4323 : }
4324 : }
4325 : else
4326 : {
4327 148 : int prec;
4328 148 : rhs1 = handle_operand_addr (rhs1, stmt, NULL, &prec);
4329 148 : g = gimple_build_call_internal (IFN_BITINTTOFLOAT, 2,
4330 148 : rhs1, build_int_cst (sitype, prec));
4331 148 : gimple_call_set_lhs (g, lhs);
4332 148 : if (!stmt_ends_bb_p (stmt))
4333 147 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4334 148 : gsi_replace (&m_gsi, g, true);
4335 : }
4336 327 : }
4337 :
4338 : /* Helper method for lower_addsub_overflow and lower_mul_overflow.
4339 : If check_zero is true, caller wants to check if all bits in [start, end)
4340 : are zero, otherwise if bits in [start, end) are either all zero or
4341 : all ones. L is the limb with index LIMB, START and END are measured
4342 : in bits. */
4343 :
4344 : tree
4345 6453 : bitint_large_huge::arith_overflow_extract_bits (unsigned int start,
4346 : unsigned int end, tree l,
4347 : unsigned int limb,
4348 : bool check_zero)
4349 : {
4350 6453 : unsigned startlimb = start / limb_prec;
4351 6453 : unsigned endlimb = (end - 1) / limb_prec;
4352 6453 : gimple *g;
4353 :
4354 6453 : if ((start % limb_prec) == 0 && (end % limb_prec) == 0)
4355 : return l;
4356 6184 : if (startlimb == endlimb && limb == startlimb)
4357 : {
4358 2039 : if (check_zero)
4359 : {
4360 1486 : wide_int w = wi::shifted_mask (start % limb_prec,
4361 1486 : end - start, false, limb_prec);
4362 2972 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4363 : BIT_AND_EXPR, l,
4364 1486 : wide_int_to_tree (m_limb_type, w));
4365 1486 : insert_before (g);
4366 1486 : return gimple_assign_lhs (g);
4367 1486 : }
4368 553 : unsigned int shift = start % limb_prec;
4369 553 : if ((end % limb_prec) != 0)
4370 : {
4371 336 : unsigned int lshift = (-end) % limb_prec;
4372 336 : shift += lshift;
4373 336 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4374 : LSHIFT_EXPR, l,
4375 : build_int_cst (unsigned_type_node,
4376 336 : lshift));
4377 336 : insert_before (g);
4378 336 : l = gimple_assign_lhs (g);
4379 : }
4380 553 : l = add_cast (signed_type_for (m_limb_type), l);
4381 553 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4382 : RSHIFT_EXPR, l,
4383 553 : build_int_cst (unsigned_type_node, shift));
4384 553 : insert_before (g);
4385 553 : return add_cast (m_limb_type, gimple_assign_lhs (g));
4386 : }
4387 4145 : else if (limb == startlimb)
4388 : {
4389 1990 : if ((start % limb_prec) == 0)
4390 : return l;
4391 1876 : if (!check_zero)
4392 950 : l = add_cast (signed_type_for (m_limb_type), l);
4393 1876 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4394 : RSHIFT_EXPR, l,
4395 : build_int_cst (unsigned_type_node,
4396 1876 : start % limb_prec));
4397 1876 : insert_before (g);
4398 1876 : l = gimple_assign_lhs (g);
4399 1876 : if (!check_zero)
4400 950 : l = add_cast (m_limb_type, l);
4401 : return l;
4402 : }
4403 2155 : else if (limb == endlimb)
4404 : {
4405 1704 : if ((end % limb_prec) == 0)
4406 : return l;
4407 1687 : if (check_zero)
4408 : {
4409 880 : wide_int w = wi::mask (end % limb_prec, false, limb_prec);
4410 1760 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4411 : BIT_AND_EXPR, l,
4412 880 : wide_int_to_tree (m_limb_type, w));
4413 880 : insert_before (g);
4414 880 : return gimple_assign_lhs (g);
4415 880 : }
4416 807 : unsigned int shift = (-end) % limb_prec;
4417 807 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4418 : LSHIFT_EXPR, l,
4419 807 : build_int_cst (unsigned_type_node, shift));
4420 807 : insert_before (g);
4421 807 : l = add_cast (signed_type_for (m_limb_type), gimple_assign_lhs (g));
4422 807 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4423 : RSHIFT_EXPR, l,
4424 807 : build_int_cst (unsigned_type_node, shift));
4425 807 : insert_before (g);
4426 807 : return add_cast (m_limb_type, gimple_assign_lhs (g));
4427 : }
4428 : return l;
4429 : }
4430 :
4431 : /* Helper method for lower_addsub_overflow and lower_mul_overflow. Store
4432 : result including overflow flag into the right locations. */
4433 :
4434 : void
4435 4212 : bitint_large_huge::finish_arith_overflow (tree var, tree obj, tree type,
4436 : tree ovf, tree lhs, tree orig_obj,
4437 : gimple *stmt, unsigned nelts,
4438 : tree_code code)
4439 : {
4440 4212 : gimple *g;
4441 :
4442 4212 : if (obj == NULL_TREE
4443 4212 : && (!BITINT_TYPE_P (type)
4444 231 : || bitint_precision_kind (type) < bitint_prec_large))
4445 : {
4446 : /* Add support for 3 or more limbs filled in from normal integral
4447 : type if this assert fails. If no target chooses limb mode smaller
4448 : than half of largest supported normal integral type, this will not
4449 : be needed. */
4450 250 : gcc_assert (TYPE_PRECISION (type) <= 2 * limb_prec);
4451 250 : tree lhs_type = type;
4452 110 : if (BITINT_TYPE_P (type)
4453 250 : && bitint_precision_kind (type) == bitint_prec_middle)
4454 46 : lhs_type = build_nonstandard_integer_type (TYPE_PRECISION (type),
4455 46 : TYPE_UNSIGNED (type));
4456 250 : tree r1 = limb_access (NULL_TREE, var,
4457 : bitint_big_endian
4458 0 : ? size_int (nelts - 1) : size_zero_node, true);
4459 250 : g = gimple_build_assign (make_ssa_name (m_limb_type), r1);
4460 250 : insert_before (g);
4461 250 : r1 = gimple_assign_lhs (g);
4462 250 : if (!useless_type_conversion_p (lhs_type, TREE_TYPE (r1)))
4463 250 : r1 = add_cast (lhs_type, r1);
4464 250 : if (TYPE_PRECISION (lhs_type) > limb_prec)
4465 : {
4466 90 : tree r2 = limb_access (NULL_TREE, var,
4467 : bitint_big_endian
4468 0 : ? size_int (nelts - 2) : size_one_node, true);
4469 90 : g = gimple_build_assign (make_ssa_name (m_limb_type), r2);
4470 90 : insert_before (g);
4471 90 : r2 = gimple_assign_lhs (g);
4472 90 : r2 = add_cast (lhs_type, r2);
4473 90 : g = gimple_build_assign (make_ssa_name (lhs_type), LSHIFT_EXPR, r2,
4474 : build_int_cst (unsigned_type_node,
4475 90 : limb_prec));
4476 90 : insert_before (g);
4477 90 : g = gimple_build_assign (make_ssa_name (lhs_type), BIT_IOR_EXPR, r1,
4478 : gimple_assign_lhs (g));
4479 90 : insert_before (g);
4480 90 : r1 = gimple_assign_lhs (g);
4481 : }
4482 250 : if (lhs_type != type)
4483 46 : r1 = add_cast (type, r1);
4484 250 : ovf = add_cast (lhs_type, ovf);
4485 250 : if (lhs_type != type)
4486 46 : ovf = add_cast (type, ovf);
4487 250 : g = gimple_build_assign (lhs, COMPLEX_EXPR, r1, ovf);
4488 250 : m_gsi = gsi_for_stmt (stmt);
4489 250 : gsi_replace (&m_gsi, g, true);
4490 : }
4491 : else
4492 : {
4493 3962 : unsigned HOST_WIDE_INT obj_nelts = 0;
4494 3962 : tree atype = NULL_TREE;
4495 3962 : if (obj)
4496 : {
4497 3871 : obj_nelts = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))) / limb_prec;
4498 3871 : if (orig_obj == NULL_TREE)
4499 2277 : obj_nelts >>= 1;
4500 3871 : atype = build_array_type_nelts (m_limb_type, obj_nelts);
4501 : }
4502 3962 : if (bitint_extended && (var || obj))
4503 : {
4504 0 : unsigned prec = TYPE_PRECISION (type);
4505 0 : unsigned prec_limbs = CEIL (prec, limb_prec);
4506 0 : bool ext_ms_limb
4507 : = (bitint_extended == bitint_ext_full
4508 0 : && abi_limb_prec > limb_prec
4509 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4510 0 : > CEIL (prec, limb_prec) * limb_prec));
4511 : /* For .{ADD,SUB}_OVERFLOW the partial limb if any is
4512 : already extended in lower_addsub_overflow. */
4513 0 : if ((code == MULT_EXPR && (prec % limb_prec) != 0)
4514 0 : || (ext_ms_limb && !TYPE_UNSIGNED (type)))
4515 : {
4516 0 : tree plm1idx = size_int (bitint_big_endian
4517 : ? nelts - obj_nelts : prec_limbs - 1);
4518 0 : tree plm1type
4519 0 : = limb_access_type (type, bitint_big_endian
4520 : ? size_zero_node : plm1idx);
4521 0 : tree l = limb_access (bitint_big_endian ? NULL_TREE : type,
4522 : var ? var : obj, plm1idx, true);
4523 0 : tree rhs = make_ssa_name (TREE_TYPE (l));
4524 0 : g = gimple_build_assign (rhs, l);
4525 0 : insert_before (g);
4526 0 : if (code == MULT_EXPR && (prec % limb_prec) != 0)
4527 : {
4528 0 : if (!useless_type_conversion_p (plm1type, TREE_TYPE (rhs)))
4529 0 : rhs = add_cast (plm1type, rhs);
4530 0 : if (!useless_type_conversion_p (TREE_TYPE (l),
4531 0 : TREE_TYPE (rhs)))
4532 0 : rhs = add_cast (TREE_TYPE (l), rhs);
4533 0 : l = limb_access (bitint_big_endian ? NULL_TREE : type,
4534 : var ? var : obj, plm1idx, true);
4535 0 : g = gimple_build_assign (l, rhs);
4536 0 : insert_before (g);
4537 : }
4538 0 : if (ext_ms_limb && !TYPE_UNSIGNED (type))
4539 : {
4540 0 : rhs = add_cast (signed_type_for (m_limb_type), rhs);
4541 0 : tree lpm1 = build_int_cst (unsigned_type_node,
4542 0 : limb_prec - 1);
4543 0 : tree v = make_ssa_name (TREE_TYPE (rhs));
4544 0 : g = gimple_build_assign (v, RSHIFT_EXPR, rhs, lpm1);
4545 0 : insert_before (g);
4546 0 : unsigned int i
4547 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4548 0 : v = add_cast (m_limb_type, v);
4549 0 : g = gimple_build_assign (limb_access (type, var ? var : obj,
4550 0 : size_int (i - 1),
4551 : true), v);
4552 0 : insert_before (g);
4553 0 : ext_ms_limb = false;
4554 : }
4555 : }
4556 0 : if (ext_ms_limb)
4557 : {
4558 0 : unsigned int i
4559 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4560 0 : g = gimple_build_assign (limb_access (type, var ? var : obj,
4561 0 : size_int (i - 1), true),
4562 : build_zero_cst (m_limb_type));
4563 0 : insert_before (g);
4564 : }
4565 : }
4566 3962 : if (var && obj)
4567 : {
4568 519 : tree v1, v2;
4569 519 : tree off;
4570 519 : if (orig_obj == NULL_TREE)
4571 : {
4572 32 : off = build_zero_cst (build_pointer_type (TREE_TYPE (obj)));
4573 32 : v1 = build2 (MEM_REF, atype,
4574 : build_fold_addr_expr (unshare_expr (obj)), off);
4575 : }
4576 487 : else if (!useless_type_conversion_p (atype, TREE_TYPE (obj)))
4577 9 : v1 = build1 (VIEW_CONVERT_EXPR, atype, unshare_expr (obj));
4578 : else
4579 478 : v1 = unshare_expr (obj);
4580 519 : off = build_int_cst (build_pointer_type (TREE_TYPE (var)),
4581 : bitint_big_endian
4582 519 : ? (nelts - obj_nelts) * m_limb_size : 0);
4583 519 : v2 = build2 (MEM_REF, atype, build_fold_addr_expr (var), off);
4584 519 : g = gimple_build_assign (v1, v2);
4585 519 : insert_before (g);
4586 : }
4587 3443 : else if (obj && bitint_big_endian && nelts != obj_nelts)
4588 : {
4589 0 : gcc_assert (nelts > obj_nelts);
4590 0 : tree fn = builtin_decl_implicit (BUILT_IN_MEMMOVE);
4591 0 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (obj)),
4592 0 : (nelts - obj_nelts) * m_limb_size);
4593 0 : tree src = build2 (MEM_REF, atype,
4594 : build_fold_addr_expr (unshare_expr (obj)), off);
4595 0 : g = gimple_build_call (fn, 3,
4596 : build_fold_addr_expr (unshare_expr (obj)),
4597 : build_fold_addr_expr (src),
4598 : build_int_cst (size_type_node,
4599 0 : obj_nelts * m_limb_size));
4600 0 : insert_before (g);
4601 : }
4602 3962 : if (orig_obj == NULL_TREE && obj)
4603 : {
4604 2277 : ovf = add_cast (m_limb_type, ovf);
4605 2277 : tree l = limb_access (NULL_TREE, obj,
4606 2277 : size_int (bitint_big_endian
4607 : ? obj_nelts * 2 - 1 : obj_nelts),
4608 : true);
4609 2277 : g = gimple_build_assign (l, ovf);
4610 2277 : insert_before (g);
4611 2277 : if (obj_nelts > 1)
4612 : {
4613 2277 : atype = build_array_type_nelts (m_limb_type, obj_nelts - 1);
4614 2277 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (obj)),
4615 2277 : (obj_nelts + !bitint_big_endian)
4616 2277 : * m_limb_size);
4617 2277 : tree v1 = build2 (MEM_REF, atype,
4618 : build_fold_addr_expr (unshare_expr (obj)),
4619 : off);
4620 2277 : g = gimple_build_assign (v1, build_zero_cst (atype));
4621 2277 : insert_before (g);
4622 : }
4623 : }
4624 1685 : else if (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE)
4625 : {
4626 1640 : imm_use_iterator ui;
4627 1640 : use_operand_p use_p;
4628 1640 : FOR_EACH_IMM_USE_FAST (use_p, ui, lhs)
4629 : {
4630 1640 : g = USE_STMT (use_p);
4631 1640 : if (!is_gimple_assign (g)
4632 1640 : || gimple_assign_rhs_code (g) != IMAGPART_EXPR)
4633 0 : continue;
4634 1640 : tree lhs2 = gimple_assign_lhs (g);
4635 1640 : gimple *use_stmt;
4636 1640 : single_imm_use (lhs2, &use_p, &use_stmt);
4637 1640 : lhs2 = gimple_assign_lhs (use_stmt);
4638 1640 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
4639 1640 : if (useless_type_conversion_p (TREE_TYPE (lhs2), TREE_TYPE (ovf)))
4640 1619 : g = gimple_build_assign (lhs2, ovf);
4641 : else
4642 21 : g = gimple_build_assign (lhs2, NOP_EXPR, ovf);
4643 1640 : gsi_replace (&gsi, g, true);
4644 1640 : if (gsi_stmt (m_gsi) == use_stmt)
4645 91 : m_gsi = gsi_for_stmt (g);
4646 1640 : break;
4647 1640 : }
4648 : }
4649 45 : else if (ovf != boolean_false_node)
4650 : {
4651 45 : g = gimple_build_cond (NE_EXPR, ovf, boolean_false_node,
4652 : NULL_TREE, NULL_TREE);
4653 45 : edge edge_true, edge_false;
4654 45 : if_then (g, profile_probability::very_unlikely (),
4655 : edge_true, edge_false);
4656 45 : tree zero = build_zero_cst (TREE_TYPE (lhs));
4657 45 : tree fn = ubsan_build_overflow_builtin (code, m_loc,
4658 45 : TREE_TYPE (lhs),
4659 : zero, zero, NULL);
4660 45 : force_gimple_operand_gsi (&m_gsi, fn, true, NULL_TREE,
4661 : true, GSI_SAME_STMT);
4662 45 : m_gsi = gsi_after_labels (edge_true->dest);
4663 : }
4664 : }
4665 4212 : if (var)
4666 : {
4667 781 : tree clobber = build_clobber (TREE_TYPE (var), CLOBBER_STORAGE_END);
4668 781 : g = gimple_build_assign (var, clobber);
4669 781 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
4670 : }
4671 4212 : }
4672 :
4673 : /* Helper function for lower_addsub_overflow and lower_mul_overflow.
4674 : Given precisions of result TYPE (PREC), argument 0 precision PREC0,
4675 : argument 1 precision PREC1 and minimum precision for the result
4676 : PREC2, compute *START, *END, *CHECK_ZERO and return OVF. */
4677 :
4678 : static tree
4679 4212 : arith_overflow (tree_code code, tree type, int prec, int prec0, int prec1,
4680 : int prec2, unsigned *start, unsigned *end, bool *check_zero)
4681 : {
4682 4212 : *start = 0;
4683 4212 : *end = 0;
4684 4212 : *check_zero = true;
4685 : /* Ignore this special rule for subtraction, even if both
4686 : prec0 >= 0 and prec1 >= 0, their subtraction can be negative
4687 : in infinite precision. */
4688 4212 : if (code != MINUS_EXPR && prec0 >= 0 && prec1 >= 0)
4689 : {
4690 : /* Result in [0, prec2) is unsigned, if prec > prec2,
4691 : all bits above it will be zero. */
4692 681 : if ((prec - !TYPE_UNSIGNED (type)) >= prec2)
4693 0 : return boolean_false_node;
4694 : else
4695 : {
4696 : /* ovf if any of bits in [start, end) is non-zero. */
4697 681 : *start = prec - !TYPE_UNSIGNED (type);
4698 681 : *end = prec2;
4699 : }
4700 : }
4701 3531 : else if (TYPE_UNSIGNED (type))
4702 : {
4703 : /* If result in [0, prec2) is signed and if prec > prec2,
4704 : all bits above it will be sign bit copies. */
4705 1950 : if (prec >= prec2)
4706 : {
4707 : /* ovf if bit prec - 1 is non-zero. */
4708 184 : *start = prec - 1;
4709 184 : *end = prec;
4710 : }
4711 : else
4712 : {
4713 : /* ovf if any of bits in [start, end) is non-zero. */
4714 1766 : *start = prec;
4715 1766 : *end = prec2;
4716 : }
4717 : }
4718 1581 : else if (prec >= prec2)
4719 2 : return boolean_false_node;
4720 : else
4721 : {
4722 : /* ovf if [start, end) bits aren't all zeros or all ones. */
4723 1579 : *start = prec - 1;
4724 1579 : *end = prec2;
4725 1579 : *check_zero = false;
4726 : }
4727 : return NULL_TREE;
4728 : }
4729 :
4730 : /* Lower a .{ADD,SUB}_OVERFLOW call with at least one large/huge _BitInt
4731 : argument or return type _Complex large/huge _BitInt. */
4732 :
4733 : void
4734 2733 : bitint_large_huge::lower_addsub_overflow (tree obj, gimple *stmt)
4735 : {
4736 2733 : tree arg0 = gimple_call_arg (stmt, 0);
4737 2733 : tree arg1 = gimple_call_arg (stmt, 1);
4738 2733 : tree lhs = gimple_call_lhs (stmt);
4739 2733 : gimple *g;
4740 :
4741 2733 : if (!lhs)
4742 : {
4743 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
4744 0 : gsi_remove (&gsi, true);
4745 0 : return;
4746 : }
4747 2733 : gimple *final_stmt = gsi_stmt (m_gsi);
4748 2733 : tree type = TREE_TYPE (lhs);
4749 2733 : if (TREE_CODE (type) == COMPLEX_TYPE)
4750 2703 : type = TREE_TYPE (type);
4751 2733 : int prec = TYPE_PRECISION (type);
4752 2733 : int prec0 = range_to_prec (arg0, stmt);
4753 2733 : int prec1 = range_to_prec (arg1, stmt);
4754 : /* If PREC0 >= 0 && PREC1 >= 0 and CODE is not MINUS_EXPR, PREC2 is
4755 : the minimum unsigned precision of any possible operation's
4756 : result, otherwise it is minimum signed precision.
4757 : Some examples:
4758 : If PREC0 or PREC1 is 8, it means that argument is [0, 0xff],
4759 : if PREC0 or PREC1 is 10, it means that argument is [0, 0x3ff],
4760 : if PREC0 or PREC1 is -8, it means that argument is [-0x80, 0x7f],
4761 : if PREC0 or PREC1 is -10, it means that argument is [-0x200, 0x1ff].
4762 : PREC0 CODE PREC1 RESULT PREC2 SIGNED vs. UNSIGNED
4763 : 8 + 8 [0, 0x1fe] 9 UNSIGNED
4764 : 8 + 10 [0, 0x4fe] 11 UNSIGNED
4765 : -8 + -8 [-0x100, 0xfe] 9 SIGNED
4766 : -8 + -10 [-0x280, 0x27e] 11 SIGNED
4767 : 8 + -8 [-0x80, 0x17e] 10 SIGNED
4768 : 8 + -10 [-0x200, 0x2fe] 11 SIGNED
4769 : 10 + -8 [-0x80, 0x47e] 12 SIGNED
4770 : 8 - 8 [-0xff, 0xff] 9 SIGNED
4771 : 8 - 10 [-0x3ff, 0xff] 11 SIGNED
4772 : 10 - 8 [-0xff, 0x3ff] 11 SIGNED
4773 : -8 - -8 [-0xff, 0xff] 9 SIGNED
4774 : -8 - -10 [-0x27f, 0x27f] 11 SIGNED
4775 : -10 - -8 [-0x27f, 0x27f] 11 SIGNED
4776 : 8 - -8 [-0x7f, 0x17f] 10 SIGNED
4777 : 8 - -10 [-0x1ff, 0x2ff] 11 SIGNED
4778 : 10 - -8 [-0x7f, 0x47f] 12 SIGNED
4779 : -8 - 8 [-0x17f, 0x7f] 10 SIGNED
4780 : -8 - 10 [-0x47f, 0x7f] 12 SIGNED
4781 : -10 - 8 [-0x2ff, 0x1ff] 11 SIGNED */
4782 2733 : int prec2 = MAX (prec0 < 0 ? -prec0 : prec0,
4783 : prec1 < 0 ? -prec1 : prec1);
4784 : /* If operands are either both signed or both unsigned,
4785 : we need just one additional bit. */
4786 3777 : prec2 = (((prec0 < 0) == (prec1 < 0)
4787 : /* If one operand is signed and one unsigned and
4788 : the signed one has larger precision, we need
4789 : just one extra bit, otherwise two. */
4790 707 : || (prec0 < 0 ? (prec2 == -prec0 && prec2 != prec1)
4791 337 : : (prec2 == -prec1 && prec2 != prec0)))
4792 2733 : ? prec2 + 1 : prec2 + 2);
4793 2733 : int prec3 = MAX (prec0 < 0 ? -prec0 : prec0,
4794 : prec1 < 0 ? -prec1 : prec1);
4795 2733 : prec3 = MAX (prec3, prec);
4796 2733 : tree var = NULL_TREE;
4797 2733 : tree orig_obj = obj;
4798 2733 : if (obj == NULL_TREE
4799 1753 : && BITINT_TYPE_P (type)
4800 1652 : && bitint_precision_kind (type) >= bitint_prec_large
4801 1536 : && m_names
4802 4245 : && bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
4803 : {
4804 1457 : int part = var_to_partition (m_map, lhs);
4805 1457 : gcc_assert (m_vars[part] != NULL_TREE);
4806 1457 : obj = m_vars[part];
4807 1457 : if (TREE_TYPE (lhs) == type)
4808 14 : orig_obj = obj;
4809 : }
4810 101 : if (!BITINT_TYPE_P (type)
4811 2733 : || bitint_precision_kind (type) < bitint_prec_large)
4812 : {
4813 217 : unsigned HOST_WIDE_INT nelts = CEIL (prec, limb_prec);
4814 217 : tree atype = build_array_type_nelts (m_limb_type, nelts);
4815 217 : var = create_tmp_var (atype);
4816 : }
4817 :
4818 2733 : enum tree_code code;
4819 2733 : switch (gimple_call_internal_fn (stmt))
4820 : {
4821 : case IFN_ADD_OVERFLOW:
4822 : case IFN_UBSAN_CHECK_ADD:
4823 : code = PLUS_EXPR;
4824 : break;
4825 1399 : case IFN_SUB_OVERFLOW:
4826 1399 : case IFN_UBSAN_CHECK_SUB:
4827 1399 : code = MINUS_EXPR;
4828 1399 : break;
4829 0 : default:
4830 0 : gcc_unreachable ();
4831 : }
4832 2733 : unsigned start, end;
4833 2733 : bool check_zero;
4834 2733 : tree ovf = arith_overflow (code, type, prec, prec0, prec1, prec2,
4835 : &start, &end, &check_zero);
4836 :
4837 2733 : unsigned startlimb, endlimb;
4838 2733 : if (ovf)
4839 : {
4840 : startlimb = ~0U;
4841 : endlimb = ~0U;
4842 : }
4843 : else
4844 : {
4845 2733 : startlimb = start / limb_prec;
4846 2733 : endlimb = (end - 1) / limb_prec;
4847 : }
4848 :
4849 2733 : int prec4 = ovf != NULL_TREE ? prec : prec3;
4850 2733 : bitint_prec_kind kind = bitint_precision_kind (prec4);
4851 2733 : unsigned cnt, rem = 0, fin = 0, nelts;
4852 2733 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
4853 5466 : bool last_ovf = (ovf == NULL_TREE
4854 2733 : && CEIL (prec2, limb_prec) > CEIL (prec3, limb_prec));
4855 2733 : if (kind != bitint_prec_huge)
4856 1539 : nelts = cnt = CEIL (prec4, limb_prec) + last_ovf;
4857 : else
4858 : {
4859 1194 : rem = prec4 % (2 * limb_prec);
4860 1194 : fin = (prec4 - rem) / limb_prec;
4861 1194 : cnt = 2 + CEIL (rem, limb_prec) + last_ovf;
4862 1194 : nelts = fin + cnt - 2;
4863 1194 : idx = idx_first = create_loop (bitint_big_endian
4864 1194 : ? size_int (nelts - 1) : size_zero_node,
4865 : &idx_next);
4866 : }
4867 :
4868 2733 : if (kind == bitint_prec_huge)
4869 1194 : m_upwards_2limb = fin;
4870 2733 : m_upwards = true;
4871 :
4872 2733 : tree type0 = TREE_TYPE (arg0);
4873 2733 : tree type1 = TREE_TYPE (arg1);
4874 2733 : int prec5 = prec3;
4875 2733 : if (bitint_precision_kind (prec5) < bitint_prec_large)
4876 10 : prec5 = MAX (TYPE_PRECISION (type0), TYPE_PRECISION (type1));
4877 2733 : if (TYPE_PRECISION (type0) < prec5)
4878 : {
4879 146 : type0 = build_bitint_type (prec5, TYPE_UNSIGNED (type0));
4880 146 : if (TREE_CODE (arg0) == INTEGER_CST)
4881 27 : arg0 = fold_convert (type0, arg0);
4882 : }
4883 2733 : if (TYPE_PRECISION (type1) < prec5)
4884 : {
4885 156 : type1 = build_bitint_type (prec5, TYPE_UNSIGNED (type1));
4886 156 : if (TREE_CODE (arg1) == INTEGER_CST)
4887 76 : arg1 = fold_convert (type1, arg1);
4888 : }
4889 2733 : unsigned int data_cnt = 0;
4890 2733 : tree last_rhs1 = NULL_TREE, last_rhs2 = NULL_TREE;
4891 2733 : tree cmp = build_zero_cst (m_limb_type);
4892 2733 : unsigned prec_limbs = CEIL ((unsigned) prec, limb_prec);
4893 2733 : tree ovf_out = NULL_TREE, cmp_out = NULL_TREE;
4894 11904 : for (unsigned i = 0; i < cnt; i++)
4895 : {
4896 9171 : m_data_cnt = 0;
4897 9171 : tree rhs1, rhs2;
4898 9171 : if (kind != bitint_prec_huge)
4899 5303 : idx = size_int (bitint_big_endian ? nelts - 1 - i : i);
4900 3868 : else if (i >= 2)
4901 1480 : idx = size_int (bitint_big_endian ? nelts + 1 - fin - i : fin + i - 2);
4902 9171 : if (!last_ovf || i < cnt - 1)
4903 : {
4904 8225 : tree idx0 = idx, idx1 = idx;
4905 8225 : if (bitint_big_endian
4906 8225 : && CEIL ((unsigned) TYPE_PRECISION (type0), limb_prec) != nelts)
4907 : {
4908 0 : HOST_WIDE_INT diff
4909 0 : = ((HOST_WIDE_INT) CEIL (TYPE_PRECISION (type0), limb_prec)
4910 0 : - (HOST_WIDE_INT) nelts);
4911 0 : if (tree_fits_uhwi_p (idx))
4912 0 : idx0 = size_int (tree_to_uhwi (idx) + diff);
4913 : else
4914 : {
4915 0 : idx0 = make_ssa_name (sizetype);
4916 0 : g = gimple_build_assign (idx0, PLUS_EXPR, idx,
4917 0 : size_int (diff));
4918 0 : insert_before (g);
4919 : }
4920 : }
4921 8225 : if (type0 != TREE_TYPE (arg0))
4922 334 : rhs1 = handle_cast (type0, arg0, idx0);
4923 : else
4924 7891 : rhs1 = handle_operand (arg0, idx0);
4925 8225 : if (bitint_big_endian
4926 8225 : && CEIL ((unsigned) TYPE_PRECISION (type1), limb_prec) != nelts)
4927 : {
4928 0 : HOST_WIDE_INT diff
4929 0 : = ((HOST_WIDE_INT) CEIL (TYPE_PRECISION (type1), limb_prec)
4930 0 : - (HOST_WIDE_INT) nelts);
4931 0 : if (tree_fits_uhwi_p (idx))
4932 0 : idx1 = size_int (tree_to_uhwi (idx) + diff);
4933 : else
4934 : {
4935 0 : idx1 = make_ssa_name (sizetype);
4936 0 : g = gimple_build_assign (idx1, PLUS_EXPR, idx,
4937 0 : size_int (diff));
4938 0 : insert_before (g);
4939 : }
4940 : }
4941 8225 : if (type1 != TREE_TYPE (arg1))
4942 250 : rhs2 = handle_cast (type1, arg1, idx1);
4943 : else
4944 7975 : rhs2 = handle_operand (arg1, idx1);
4945 8225 : if (i == 0)
4946 2733 : data_cnt = m_data_cnt;
4947 8225 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
4948 1983 : rhs1 = add_cast (m_limb_type, rhs1);
4949 8225 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs2)))
4950 1983 : rhs2 = add_cast (m_limb_type, rhs2);
4951 : last_rhs1 = rhs1;
4952 : last_rhs2 = rhs2;
4953 : }
4954 : else
4955 : {
4956 946 : m_data_cnt = data_cnt;
4957 946 : if (TYPE_UNSIGNED (type0) || prec0 >= 0)
4958 421 : rhs1 = build_zero_cst (m_limb_type);
4959 : else
4960 : {
4961 525 : rhs1 = add_cast (signed_type_for (m_limb_type), last_rhs1);
4962 525 : if (TREE_CODE (rhs1) == INTEGER_CST)
4963 52 : rhs1 = build_int_cst (m_limb_type,
4964 74 : tree_int_cst_sgn (rhs1) < 0 ? -1 : 0);
4965 : else
4966 : {
4967 946 : tree lpm1 = build_int_cst (unsigned_type_node,
4968 473 : limb_prec - 1);
4969 473 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
4970 : RSHIFT_EXPR, rhs1, lpm1);
4971 473 : insert_before (g);
4972 473 : rhs1 = add_cast (m_limb_type, gimple_assign_lhs (g));
4973 : }
4974 : }
4975 946 : if (TYPE_UNSIGNED (type1) || prec1 >= 0)
4976 543 : rhs2 = build_zero_cst (m_limb_type);
4977 : else
4978 : {
4979 403 : rhs2 = add_cast (signed_type_for (m_limb_type), last_rhs2);
4980 403 : if (TREE_CODE (rhs2) == INTEGER_CST)
4981 114 : rhs2 = build_int_cst (m_limb_type,
4982 153 : tree_int_cst_sgn (rhs2) < 0 ? -1 : 0);
4983 : else
4984 : {
4985 578 : tree lpm1 = build_int_cst (unsigned_type_node,
4986 289 : limb_prec - 1);
4987 289 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs2)),
4988 : RSHIFT_EXPR, rhs2, lpm1);
4989 289 : insert_before (g);
4990 289 : rhs2 = add_cast (m_limb_type, gimple_assign_lhs (g));
4991 : }
4992 : }
4993 : }
4994 9171 : tree rhs = handle_plus_minus (code, rhs1, rhs2, idx);
4995 9171 : if (ovf != boolean_false_node)
4996 : {
4997 9171 : if (tree_fits_uhwi_p (idx))
4998 : {
4999 6783 : unsigned limb = tree_to_uhwi (idx);
5000 6783 : if (bitint_big_endian)
5001 0 : limb = nelts - 1 - limb;
5002 6783 : if (limb >= startlimb && limb <= endlimb)
5003 : {
5004 3370 : tree l = arith_overflow_extract_bits (start, end, rhs,
5005 : limb, check_zero);
5006 3370 : tree this_ovf = make_ssa_name (boolean_type_node);
5007 3370 : if (ovf == NULL_TREE && !check_zero)
5008 : {
5009 907 : cmp = l;
5010 907 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5011 : PLUS_EXPR, l,
5012 : build_int_cst (m_limb_type, 1));
5013 907 : insert_before (g);
5014 907 : g = gimple_build_assign (this_ovf, GT_EXPR,
5015 : gimple_assign_lhs (g),
5016 : build_int_cst (m_limb_type, 1));
5017 : }
5018 : else
5019 2463 : g = gimple_build_assign (this_ovf, NE_EXPR, l, cmp);
5020 3370 : insert_before (g);
5021 3370 : if (ovf == NULL_TREE)
5022 : ovf = this_ovf;
5023 : else
5024 : {
5025 1064 : tree b = make_ssa_name (boolean_type_node);
5026 1064 : g = gimple_build_assign (b, BIT_IOR_EXPR, ovf, this_ovf);
5027 1064 : insert_before (g);
5028 1064 : ovf = b;
5029 : }
5030 : }
5031 : }
5032 2388 : else if (startlimb < fin)
5033 : {
5034 854 : if (m_first && startlimb + 2 < fin)
5035 : {
5036 324 : tree data_out;
5037 324 : ovf = prepare_data_in_out (boolean_false_node, idx, &data_out);
5038 324 : ovf_out = m_data.pop ();
5039 324 : m_data.pop ();
5040 324 : if (!check_zero)
5041 : {
5042 169 : cmp = prepare_data_in_out (cmp, idx, &data_out);
5043 169 : cmp_out = m_data.pop ();
5044 169 : m_data.pop ();
5045 : }
5046 : }
5047 854 : if (i != 0 || startlimb != fin - 1)
5048 : {
5049 839 : tree_code cmp_code;
5050 839 : bool single_comparison
5051 839 : = (startlimb + 2 >= fin || (startlimb & 1) != (i & 1));
5052 : if (!single_comparison)
5053 : cmp_code = GE_EXPR;
5054 515 : else if ((startlimb & 1) == (i & 1))
5055 : cmp_code = EQ_EXPR;
5056 : else
5057 412 : cmp_code = GT_EXPR;
5058 839 : if (bitint_big_endian)
5059 0 : g = gimple_build_cond (swap_tree_comparison (cmp_code),
5060 0 : idx, size_int (nelts - 1
5061 : - startlimb),
5062 : NULL_TREE, NULL_TREE);
5063 : else
5064 839 : g = gimple_build_cond (cmp_code, idx, size_int (startlimb),
5065 : NULL_TREE, NULL_TREE);
5066 839 : edge edge_true_true, edge_true_false, edge_false;
5067 839 : gimple *g2 = NULL;
5068 839 : if (!single_comparison)
5069 324 : g2 = gimple_build_cond (NE_EXPR, idx,
5070 324 : size_int (bitint_big_endian
5071 : ? nelts - 1 - startlimb
5072 : : startlimb),
5073 : NULL_TREE, NULL_TREE);
5074 839 : if_then_if_then_else (g, g2, profile_probability::likely (),
5075 : profile_probability::likely (),
5076 : edge_true_true, edge_true_false,
5077 : edge_false);
5078 839 : unsigned tidx = startlimb + (cmp_code == GT_EXPR);
5079 839 : tree l = arith_overflow_extract_bits (start, end, rhs, tidx,
5080 : check_zero);
5081 839 : tree this_ovf = make_ssa_name (boolean_type_node);
5082 839 : if (cmp_code != GT_EXPR && !check_zero)
5083 : {
5084 173 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5085 : PLUS_EXPR, l,
5086 : build_int_cst (m_limb_type, 1));
5087 173 : insert_before (g);
5088 173 : g = gimple_build_assign (this_ovf, GT_EXPR,
5089 : gimple_assign_lhs (g),
5090 : build_int_cst (m_limb_type, 1));
5091 : }
5092 : else
5093 666 : g = gimple_build_assign (this_ovf, NE_EXPR, l, cmp);
5094 839 : insert_before (g);
5095 839 : if (cmp_code == GT_EXPR)
5096 : {
5097 412 : tree t = make_ssa_name (boolean_type_node);
5098 412 : g = gimple_build_assign (t, BIT_IOR_EXPR, ovf, this_ovf);
5099 412 : insert_before (g);
5100 412 : this_ovf = t;
5101 : }
5102 839 : tree this_ovf2 = NULL_TREE;
5103 839 : if (!single_comparison)
5104 : {
5105 324 : m_gsi = gsi_after_labels (edge_true_true->src);
5106 324 : tree t = make_ssa_name (boolean_type_node);
5107 324 : g = gimple_build_assign (t, NE_EXPR, rhs, cmp);
5108 324 : insert_before (g);
5109 324 : this_ovf2 = make_ssa_name (boolean_type_node);
5110 324 : g = gimple_build_assign (this_ovf2, BIT_IOR_EXPR,
5111 : ovf, t);
5112 324 : insert_before (g);
5113 : }
5114 839 : m_gsi = gsi_after_labels (edge_true_false->dest);
5115 839 : tree t;
5116 839 : if (i == 1 && ovf_out)
5117 : t = ovf_out;
5118 : else
5119 515 : t = make_ssa_name (boolean_type_node);
5120 839 : gphi *phi = create_phi_node (t, edge_true_false->dest);
5121 839 : add_phi_arg (phi, this_ovf, edge_true_false,
5122 : UNKNOWN_LOCATION);
5123 839 : add_phi_arg (phi, ovf ? ovf
5124 : : boolean_false_node, edge_false,
5125 : UNKNOWN_LOCATION);
5126 839 : if (edge_true_true)
5127 324 : add_phi_arg (phi, this_ovf2, edge_true_true,
5128 : UNKNOWN_LOCATION);
5129 839 : ovf = t;
5130 839 : if (!check_zero && cmp_code != GT_EXPR)
5131 : {
5132 173 : t = cmp_out ? cmp_out : make_ssa_name (m_limb_type);
5133 173 : phi = create_phi_node (t, edge_true_false->dest);
5134 173 : add_phi_arg (phi, l, edge_true_false, UNKNOWN_LOCATION);
5135 173 : add_phi_arg (phi, cmp, edge_false, UNKNOWN_LOCATION);
5136 173 : if (edge_true_true)
5137 169 : add_phi_arg (phi, cmp, edge_true_true,
5138 : UNKNOWN_LOCATION);
5139 : cmp = t;
5140 : }
5141 : }
5142 : }
5143 : }
5144 :
5145 9171 : if (var || obj)
5146 : {
5147 8943 : if (tree_fits_uhwi_p (idx)
5148 6673 : && (bitint_big_endian
5149 6673 : ? nelts - 1 - tree_to_uhwi (idx)
5150 6673 : : tree_to_uhwi (idx)) >= prec_limbs)
5151 : ;
5152 7555 : else if (!tree_fits_uhwi_p (idx)
5153 2270 : && (unsigned) prec < (fin - (i == 0)) * limb_prec)
5154 : {
5155 1458 : bool single_comparison
5156 729 : = (((unsigned) prec % limb_prec) == 0
5157 571 : || prec_limbs + 1 >= fin
5158 1231 : || (prec_limbs & 1) == (i & 1));
5159 729 : if (bitint_big_endian)
5160 0 : g = gimple_build_cond (GE_EXPR, idx,
5161 0 : size_int (nelts - prec_limbs),
5162 : NULL_TREE, NULL_TREE);
5163 : else
5164 729 : g = gimple_build_cond (LE_EXPR, idx, size_int (prec_limbs - 1),
5165 : NULL_TREE, NULL_TREE);
5166 729 : gimple *g2 = NULL;
5167 729 : if (!single_comparison)
5168 251 : g2 = gimple_build_cond (EQ_EXPR, idx,
5169 251 : size_int (bitint_big_endian
5170 : ? nelts - prec_limbs
5171 : : prec_limbs - 1),
5172 : NULL_TREE, NULL_TREE);
5173 729 : edge edge_true_true, edge_true_false, edge_false;
5174 729 : if_then_if_then_else (g, g2, profile_probability::likely (),
5175 : profile_probability::unlikely (),
5176 : edge_true_true, edge_true_false,
5177 : edge_false);
5178 729 : tree idxl = idx;
5179 729 : if (bitint_big_endian && prec_limbs != nelts)
5180 : {
5181 0 : HOST_WIDE_INT diff = ((HOST_WIDE_INT) prec_limbs
5182 0 : - (HOST_WIDE_INT) nelts);
5183 0 : if (tree_fits_uhwi_p (idx))
5184 0 : idxl = size_int (tree_to_uhwi (idx) + diff);
5185 : else
5186 : {
5187 0 : idxl = make_ssa_name (sizetype);
5188 0 : g = gimple_build_assign (idxl, PLUS_EXPR, idx,
5189 0 : size_int (diff));
5190 0 : insert_before (g);
5191 : }
5192 : }
5193 1082 : tree l = limb_access (type, var ? var : obj, idxl, true);
5194 729 : g = gimple_build_assign (l, rhs);
5195 729 : insert_before (g);
5196 729 : if (!single_comparison)
5197 : {
5198 251 : m_gsi = gsi_after_labels (edge_true_true->src);
5199 251 : tree plm1idx = size_int (bitint_big_endian
5200 : ? 0 : prec_limbs - 1);
5201 251 : tree plm1type = limb_access_type (type, plm1idx);
5202 251 : l = limb_access (type, var ? var : obj, plm1idx, true);
5203 251 : if (!useless_type_conversion_p (plm1type, TREE_TYPE (rhs)))
5204 251 : rhs = add_cast (plm1type, rhs);
5205 251 : if (!useless_type_conversion_p (TREE_TYPE (l),
5206 251 : TREE_TYPE (rhs)))
5207 251 : rhs = add_cast (TREE_TYPE (l), rhs);
5208 251 : g = gimple_build_assign (l, rhs);
5209 251 : insert_before (g);
5210 : }
5211 729 : m_gsi = gsi_after_labels (edge_true_false->dest);
5212 729 : }
5213 : else
5214 : {
5215 6826 : tree idxl = idx;
5216 6826 : if (bitint_big_endian && prec_limbs != nelts)
5217 : {
5218 0 : HOST_WIDE_INT diff = ((HOST_WIDE_INT) prec_limbs
5219 0 : - (HOST_WIDE_INT) nelts);
5220 0 : if (tree_fits_uhwi_p (idx))
5221 0 : idxl = size_int (tree_to_uhwi (idx) + diff);
5222 : else
5223 : {
5224 0 : idxl = make_ssa_name (sizetype);
5225 0 : g = gimple_build_assign (idxl, PLUS_EXPR, idx,
5226 0 : size_int (diff));
5227 0 : insert_before (g);
5228 : }
5229 : }
5230 13623 : tree l = limb_access (type, var ? var : obj, idxl, true);
5231 6826 : if (bitint_extended && tree_fits_uhwi_p (idxl))
5232 : {
5233 0 : tree atype = limb_access_type (type, idxl);
5234 0 : if (!useless_type_conversion_p (atype, TREE_TYPE (rhs)))
5235 0 : rhs = add_cast (atype, rhs);
5236 : }
5237 6826 : if (!useless_type_conversion_p (TREE_TYPE (l), TREE_TYPE (rhs)))
5238 0 : rhs = add_cast (TREE_TYPE (l), rhs);
5239 6826 : g = gimple_build_assign (l, rhs);
5240 6826 : insert_before (g);
5241 : }
5242 : }
5243 9171 : m_first = false;
5244 9171 : if (kind == bitint_prec_huge && i <= 1)
5245 : {
5246 2388 : if (i == 0)
5247 : {
5248 1194 : idx = make_ssa_name (sizetype);
5249 1194 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
5250 : bitint_big_endian
5251 0 : ? size_int (-1) : size_one_node);
5252 1194 : insert_before (g);
5253 : }
5254 : else
5255 : {
5256 1194 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
5257 2388 : size_int (bitint_big_endian ? -2 : 2));
5258 1194 : insert_before (g);
5259 1194 : if (bitint_big_endian)
5260 0 : g = gimple_build_cond (NE_EXPR, idx_first,
5261 0 : size_int (nelts + 1 - fin),
5262 : NULL_TREE, NULL_TREE);
5263 : else
5264 1194 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (fin),
5265 : NULL_TREE, NULL_TREE);
5266 1194 : insert_before (g);
5267 1194 : m_gsi = gsi_for_stmt (final_stmt);
5268 1194 : m_bb = NULL;
5269 : }
5270 : }
5271 : }
5272 :
5273 2733 : finish_arith_overflow (var, obj, type, ovf, lhs, orig_obj, stmt,
5274 : prec_limbs, code);
5275 : }
5276 :
5277 : /* Lower a .MUL_OVERFLOW call with at least one large/huge _BitInt
5278 : argument or return type _Complex large/huge _BitInt. */
5279 :
5280 : void
5281 1479 : bitint_large_huge::lower_mul_overflow (tree obj, gimple *stmt)
5282 : {
5283 1479 : tree arg0 = gimple_call_arg (stmt, 0);
5284 1479 : tree arg1 = gimple_call_arg (stmt, 1);
5285 1479 : tree lhs = gimple_call_lhs (stmt);
5286 1479 : if (!lhs)
5287 : {
5288 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5289 0 : gsi_remove (&gsi, true);
5290 0 : return;
5291 : }
5292 1479 : gimple *final_stmt = gsi_stmt (m_gsi);
5293 1479 : tree type = TREE_TYPE (lhs);
5294 1479 : if (TREE_CODE (type) == COMPLEX_TYPE)
5295 1464 : type = TREE_TYPE (type);
5296 1479 : int prec = TYPE_PRECISION (type), prec0, prec1;
5297 1479 : arg0 = handle_operand_addr (arg0, stmt, NULL, &prec0);
5298 1479 : arg1 = handle_operand_addr (arg1, stmt, NULL, &prec1);
5299 1479 : int prec2 = ((prec0 < 0 ? -prec0 : prec0)
5300 1479 : + (prec1 < 0 ? -prec1 : prec1));
5301 1479 : if (prec0 == 1 || prec1 == 1)
5302 35 : --prec2;
5303 1479 : tree var = NULL_TREE;
5304 1479 : tree orig_obj = obj;
5305 1479 : bool force_var = false;
5306 1479 : if (obj == NULL_TREE
5307 886 : && BITINT_TYPE_P (type)
5308 877 : && bitint_precision_kind (type) >= bitint_prec_large
5309 853 : && m_names
5310 2320 : && bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
5311 : {
5312 841 : int part = var_to_partition (m_map, lhs);
5313 841 : gcc_assert (m_vars[part] != NULL_TREE);
5314 841 : obj = m_vars[part];
5315 841 : if (TREE_TYPE (lhs) == type)
5316 7 : orig_obj = obj;
5317 : }
5318 638 : else if (obj != NULL_TREE && DECL_P (obj))
5319 : {
5320 1755 : for (int i = 0; i < 2; ++i)
5321 : {
5322 1170 : tree arg = i ? arg1 : arg0;
5323 1170 : if (TREE_CODE (arg) == ADDR_EXPR)
5324 1170 : arg = TREE_OPERAND (arg, 0);
5325 1170 : if (get_base_address (arg) == obj)
5326 : {
5327 : force_var = true;
5328 : break;
5329 : }
5330 : }
5331 : }
5332 1479 : if (obj == NULL_TREE
5333 1479 : || force_var
5334 1434 : || !BITINT_TYPE_P (type)
5335 1434 : || bitint_precision_kind (type) < bitint_prec_large
5336 3747 : || prec2 > (CEIL (prec, limb_prec) * limb_prec * (orig_obj ? 1 : 2)))
5337 : {
5338 564 : unsigned HOST_WIDE_INT nelts = CEIL (MAX (prec, prec2), limb_prec);
5339 564 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5340 564 : var = create_tmp_var (atype);
5341 : }
5342 1479 : tree addr = build_fold_addr_expr (var ? var : obj);
5343 1479 : addr = force_gimple_operand_gsi (&m_gsi, addr, true,
5344 : NULL_TREE, true, GSI_SAME_STMT);
5345 1479 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
5346 1479 : gimple *g
5347 1479 : = gimple_build_call_internal (IFN_MULBITINT, 6,
5348 : addr, build_int_cst (sitype,
5349 1479 : MAX (prec2, prec)),
5350 1479 : arg0, build_int_cst (sitype, prec0),
5351 1479 : arg1, build_int_cst (sitype, prec1));
5352 1479 : insert_before (g);
5353 :
5354 1479 : unsigned start, end;
5355 1479 : bool check_zero;
5356 1479 : tree ovf = arith_overflow (MULT_EXPR, type, prec, prec0, prec1, prec2,
5357 : &start, &end, &check_zero);
5358 1479 : if (ovf == NULL_TREE)
5359 : {
5360 1477 : unsigned startlimb = start / limb_prec;
5361 1477 : unsigned endlimb = (end - 1) / limb_prec;
5362 1477 : unsigned nelts = CEIL (MAX (prec, prec2), limb_prec);
5363 1477 : unsigned cnt;
5364 1477 : bool use_loop = false;
5365 1477 : if (startlimb == endlimb)
5366 : cnt = 1;
5367 1190 : else if (startlimb + 1 == endlimb)
5368 : cnt = 2;
5369 1013 : else if ((end % limb_prec) == 0)
5370 : {
5371 : cnt = 2;
5372 : use_loop = true;
5373 : }
5374 : else
5375 : {
5376 767 : cnt = 3;
5377 767 : use_loop = startlimb + 2 < endlimb;
5378 : }
5379 767 : if (cnt == 1)
5380 : {
5381 507 : tree l = limb_access (NULL_TREE, var ? var : obj,
5382 287 : size_int (bitint_big_endian
5383 : ? nelts - 1 - startlimb
5384 : : startlimb), true);
5385 287 : g = gimple_build_assign (make_ssa_name (m_limb_type), l);
5386 287 : insert_before (g);
5387 287 : l = arith_overflow_extract_bits (start, end, gimple_assign_lhs (g),
5388 : startlimb, check_zero);
5389 287 : ovf = make_ssa_name (boolean_type_node);
5390 287 : if (check_zero)
5391 247 : g = gimple_build_assign (ovf, NE_EXPR, l,
5392 : build_zero_cst (m_limb_type));
5393 : else
5394 : {
5395 40 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5396 : PLUS_EXPR, l,
5397 : build_int_cst (m_limb_type, 1));
5398 40 : insert_before (g);
5399 40 : g = gimple_build_assign (ovf, GT_EXPR, gimple_assign_lhs (g),
5400 : build_int_cst (m_limb_type, 1));
5401 : }
5402 287 : insert_before (g);
5403 : }
5404 : else
5405 : {
5406 1190 : basic_block edge_bb = NULL;
5407 1190 : gimple_stmt_iterator gsi = m_gsi;
5408 1190 : gsi_prev (&gsi);
5409 1190 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5410 1190 : edge_bb = e->src;
5411 1190 : m_gsi = gsi_end_bb (edge_bb);
5412 :
5413 1190 : tree cmp = build_zero_cst (m_limb_type);
5414 5527 : for (unsigned i = 0; i < cnt; i++)
5415 : {
5416 3147 : tree idx, idx_next = NULL_TREE;
5417 3147 : if (i == 0)
5418 1190 : idx = size_int (bitint_big_endian
5419 : ? nelts - 1 - startlimb : startlimb);
5420 1957 : else if (i == 2)
5421 767 : idx = size_int (bitint_big_endian
5422 : ? nelts - 1 - endlimb : endlimb);
5423 1190 : else if (use_loop)
5424 665 : idx = create_loop (size_int (bitint_big_endian
5425 : ? nelts - startlimb - 2
5426 : : startlimb + 1), &idx_next);
5427 : else
5428 525 : idx = size_int (bitint_big_endian
5429 : ? nelts - startlimb - 2 : startlimb + 1);
5430 4988 : tree l = limb_access (NULL_TREE, var ? var : obj, idx, true);
5431 3147 : g = gimple_build_assign (make_ssa_name (m_limb_type), l);
5432 3147 : insert_before (g);
5433 3147 : l = gimple_assign_lhs (g);
5434 3147 : if (i == 0 || i == 2)
5435 2724 : l = arith_overflow_extract_bits (start, end, l,
5436 : i == 0 ? startlimb : endlimb,
5437 : check_zero);
5438 1957 : if (i == 0 && !check_zero)
5439 : {
5440 459 : cmp = l;
5441 459 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5442 : PLUS_EXPR, l,
5443 : build_int_cst (m_limb_type, 1));
5444 459 : insert_before (g);
5445 459 : g = gimple_build_cond (GT_EXPR, gimple_assign_lhs (g),
5446 : build_int_cst (m_limb_type, 1),
5447 : NULL_TREE, NULL_TREE);
5448 : }
5449 : else
5450 2688 : g = gimple_build_cond (NE_EXPR, l, cmp, NULL_TREE, NULL_TREE);
5451 3147 : insert_before (g);
5452 3147 : edge e1 = split_block (gsi_bb (m_gsi), g);
5453 3147 : e1->flags = EDGE_FALSE_VALUE;
5454 3147 : edge e2 = make_edge (e1->src, gimple_bb (final_stmt),
5455 : EDGE_TRUE_VALUE);
5456 3147 : e1->probability = profile_probability::likely ();
5457 3147 : e2->probability = e1->probability.invert ();
5458 3147 : if (i == 0)
5459 1190 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5460 3147 : m_gsi = gsi_after_labels (e1->dest);
5461 3147 : if (i == 1 && use_loop)
5462 : {
5463 665 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
5464 : bitint_big_endian
5465 0 : ? size_int (-1) : size_one_node);
5466 665 : insert_before (g);
5467 665 : if (bitint_big_endian)
5468 0 : g = gimple_build_cond (NE_EXPR, idx,
5469 0 : size_int (nelts - endlimb
5470 : - (cnt == 2)),
5471 : NULL_TREE, NULL_TREE);
5472 : else
5473 665 : g = gimple_build_cond (NE_EXPR, idx_next,
5474 665 : size_int (endlimb + (cnt == 2)),
5475 : NULL_TREE, NULL_TREE);
5476 665 : insert_before (g);
5477 665 : edge true_edge, false_edge;
5478 665 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
5479 : &true_edge,
5480 : &false_edge);
5481 665 : m_gsi = gsi_after_labels (false_edge->dest);
5482 665 : m_bb = NULL;
5483 : }
5484 : }
5485 :
5486 1190 : ovf = make_ssa_name (boolean_type_node);
5487 1190 : basic_block bb = gimple_bb (final_stmt);
5488 1190 : gphi *phi = create_phi_node (ovf, bb);
5489 1190 : edge e1 = find_edge (gsi_bb (m_gsi), bb);
5490 1190 : edge_iterator ei;
5491 5527 : FOR_EACH_EDGE (e, ei, bb->preds)
5492 : {
5493 4337 : tree val = e == e1 ? boolean_false_node : boolean_true_node;
5494 4337 : add_phi_arg (phi, val, e, UNKNOWN_LOCATION);
5495 : }
5496 1190 : m_gsi = gsi_for_stmt (final_stmt);
5497 : }
5498 : }
5499 :
5500 1479 : finish_arith_overflow (var, obj, type, ovf, lhs, orig_obj, stmt,
5501 1479 : CEIL (MAX (prec, prec2), limb_prec), MULT_EXPR);
5502 : }
5503 :
5504 : /* Lower REALPART_EXPR or IMAGPART_EXPR stmt extracting part of result from
5505 : .{ADD,SUB,MUL}_OVERFLOW call. */
5506 :
5507 : void
5508 6002 : bitint_large_huge::lower_cplxpart_stmt (tree obj, gimple *stmt)
5509 : {
5510 6002 : tree rhs1 = gimple_assign_rhs1 (stmt);
5511 6002 : rhs1 = TREE_OPERAND (rhs1, 0);
5512 6002 : if (obj == NULL_TREE)
5513 : {
5514 5929 : int part = var_to_partition (m_map, gimple_assign_lhs (stmt));
5515 5929 : gcc_assert (m_vars[part] != NULL_TREE);
5516 : obj = m_vars[part];
5517 : }
5518 6002 : if (TREE_CODE (rhs1) == SSA_NAME
5519 6002 : && (m_names == NULL
5520 6001 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
5521 : {
5522 1549 : lower_call (obj, SSA_NAME_DEF_STMT (rhs1));
5523 1549 : return;
5524 : }
5525 4453 : int part = var_to_partition (m_map, rhs1);
5526 4453 : gcc_assert (m_vars[part] != NULL_TREE);
5527 4453 : tree var = m_vars[part];
5528 4453 : unsigned HOST_WIDE_INT nelts
5529 4453 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))) / limb_prec;
5530 4453 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5531 4453 : if (!useless_type_conversion_p (atype, TREE_TYPE (obj)))
5532 102 : obj = build1 (VIEW_CONVERT_EXPR, atype, obj);
5533 4453 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (var)),
5534 4453 : gimple_assign_rhs_code (stmt) == REALPART_EXPR
5535 4453 : ? 0 : nelts * m_limb_size);
5536 4453 : tree v2 = build2 (MEM_REF, atype, build_fold_addr_expr (var), off);
5537 4453 : gimple *g = gimple_build_assign (obj, v2);
5538 4453 : insert_before (g);
5539 : }
5540 :
5541 : /* Lower COMPLEX_EXPR stmt. */
5542 :
5543 : void
5544 18 : bitint_large_huge::lower_complexexpr_stmt (gimple *stmt)
5545 : {
5546 18 : tree lhs = gimple_assign_lhs (stmt);
5547 18 : tree rhs1 = gimple_assign_rhs1 (stmt);
5548 18 : tree rhs2 = gimple_assign_rhs2 (stmt);
5549 18 : int part = var_to_partition (m_map, lhs);
5550 18 : gcc_assert (m_vars[part] != NULL_TREE);
5551 18 : lhs = m_vars[part];
5552 18 : unsigned HOST_WIDE_INT nelts
5553 18 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (rhs1))) / limb_prec;
5554 18 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5555 18 : tree zero = build_zero_cst (build_pointer_type (TREE_TYPE (lhs)));
5556 18 : tree v1 = build2 (MEM_REF, atype, build_fold_addr_expr (lhs), zero);
5557 18 : tree v2;
5558 18 : if (TREE_CODE (rhs1) == SSA_NAME)
5559 : {
5560 18 : part = var_to_partition (m_map, rhs1);
5561 18 : gcc_assert (m_vars[part] != NULL_TREE);
5562 : v2 = m_vars[part];
5563 : }
5564 0 : else if (integer_zerop (rhs1))
5565 0 : v2 = build_zero_cst (atype);
5566 : else
5567 0 : v2 = tree_output_constant_def (rhs1);
5568 18 : if (!useless_type_conversion_p (atype, TREE_TYPE (v2)))
5569 18 : v2 = build1 (VIEW_CONVERT_EXPR, atype, v2);
5570 18 : gimple *g = gimple_build_assign (v1, v2);
5571 18 : insert_before (g);
5572 18 : tree off = fold_convert (build_pointer_type (TREE_TYPE (lhs)),
5573 : TYPE_SIZE_UNIT (atype));
5574 18 : v1 = build2 (MEM_REF, atype, build_fold_addr_expr (lhs), off);
5575 18 : if (TREE_CODE (rhs2) == SSA_NAME)
5576 : {
5577 0 : part = var_to_partition (m_map, rhs2);
5578 0 : gcc_assert (m_vars[part] != NULL_TREE);
5579 : v2 = m_vars[part];
5580 : }
5581 18 : else if (integer_zerop (rhs2))
5582 18 : v2 = build_zero_cst (atype);
5583 : else
5584 0 : v2 = tree_output_constant_def (rhs2);
5585 18 : if (!useless_type_conversion_p (atype, TREE_TYPE (v2)))
5586 0 : v2 = build1 (VIEW_CONVERT_EXPR, atype, v2);
5587 18 : g = gimple_build_assign (v1, v2);
5588 18 : insert_before (g);
5589 18 : }
5590 :
5591 : /* Lower a .{CLZ,CTZ,CLRSB,FFS,PARITY,POPCOUNT} call with one large/huge _BitInt
5592 : argument. */
5593 :
5594 : void
5595 101 : bitint_large_huge::lower_bit_query (gimple *stmt)
5596 : {
5597 101 : tree arg0 = gimple_call_arg (stmt, 0);
5598 101 : tree arg1 = (gimple_call_num_args (stmt) == 2
5599 101 : ? gimple_call_arg (stmt, 1) : NULL_TREE);
5600 101 : tree lhs = gimple_call_lhs (stmt);
5601 101 : gimple *g;
5602 :
5603 101 : if (!lhs)
5604 : {
5605 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5606 0 : gsi_remove (&gsi, true);
5607 0 : return;
5608 : }
5609 101 : tree type = TREE_TYPE (arg0);
5610 101 : gcc_assert (BITINT_TYPE_P (type));
5611 101 : bitint_prec_kind kind = bitint_precision_kind (type);
5612 101 : gcc_assert (kind >= bitint_prec_large);
5613 101 : enum internal_fn ifn = gimple_call_internal_fn (stmt);
5614 101 : enum built_in_function fcode = END_BUILTINS;
5615 101 : gcc_assert (TYPE_PRECISION (unsigned_type_node) == limb_prec
5616 : || TYPE_PRECISION (long_unsigned_type_node) == limb_prec
5617 : || TYPE_PRECISION (long_long_unsigned_type_node) == limb_prec);
5618 101 : switch (ifn)
5619 : {
5620 26 : case IFN_CLZ:
5621 26 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5622 : fcode = BUILT_IN_CLZ;
5623 26 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5624 : fcode = BUILT_IN_CLZL;
5625 : else
5626 0 : fcode = BUILT_IN_CLZLL;
5627 : break;
5628 11 : case IFN_FFS:
5629 : /* .FFS (X) is .CTZ (X, -1) + 1, though under the hood
5630 : we don't add the addend at the end. */
5631 11 : arg1 = integer_zero_node;
5632 : /* FALLTHRU */
5633 39 : case IFN_CTZ:
5634 39 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5635 : fcode = BUILT_IN_CTZ;
5636 39 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5637 : fcode = BUILT_IN_CTZL;
5638 : else
5639 0 : fcode = BUILT_IN_CTZLL;
5640 39 : m_upwards = true;
5641 39 : break;
5642 9 : case IFN_CLRSB:
5643 9 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5644 : fcode = BUILT_IN_CLRSB;
5645 9 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5646 : fcode = BUILT_IN_CLRSBL;
5647 : else
5648 0 : fcode = BUILT_IN_CLRSBLL;
5649 : break;
5650 16 : case IFN_PARITY:
5651 16 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5652 : fcode = BUILT_IN_PARITY;
5653 16 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5654 : fcode = BUILT_IN_PARITYL;
5655 : else
5656 0 : fcode = BUILT_IN_PARITYLL;
5657 16 : m_upwards = true;
5658 16 : break;
5659 11 : case IFN_POPCOUNT:
5660 11 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5661 : fcode = BUILT_IN_POPCOUNT;
5662 11 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5663 : fcode = BUILT_IN_POPCOUNTL;
5664 : else
5665 0 : fcode = BUILT_IN_POPCOUNTLL;
5666 11 : m_upwards = true;
5667 11 : break;
5668 0 : default:
5669 0 : gcc_unreachable ();
5670 : }
5671 101 : tree fndecl = builtin_decl_explicit (fcode), res = NULL_TREE;
5672 101 : unsigned cnt = 0, rem = 0, end = 0, prec = TYPE_PRECISION (type);
5673 101 : unsigned nelts = CEIL (prec, limb_prec);
5674 101 : struct bq_details { edge e; tree val, addend; } *bqp = NULL;
5675 101 : basic_block edge_bb = NULL;
5676 101 : if (m_upwards)
5677 : {
5678 66 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
5679 66 : if (kind == bitint_prec_large)
5680 : cnt = nelts;
5681 : else
5682 : {
5683 39 : rem = (prec % (2 * limb_prec));
5684 39 : end = (prec - rem) / limb_prec;
5685 39 : cnt = 2 + CEIL (rem, limb_prec);
5686 39 : idx = idx_first = create_loop (bitint_big_endian
5687 39 : ? size_int (nelts - 1)
5688 : : size_zero_node, &idx_next);
5689 : }
5690 :
5691 66 : if (ifn == IFN_CTZ || ifn == IFN_FFS)
5692 : {
5693 39 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5694 39 : gsi_prev (&gsi);
5695 39 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5696 39 : edge_bb = e->src;
5697 39 : if (kind == bitint_prec_large)
5698 32 : m_gsi = gsi_end_bb (edge_bb);
5699 39 : bqp = XALLOCAVEC (struct bq_details, cnt);
5700 : }
5701 : else
5702 27 : m_after_stmt = stmt;
5703 66 : if (kind != bitint_prec_large)
5704 39 : m_upwards_2limb = end;
5705 :
5706 244 : for (unsigned i = 0; i < cnt; i++)
5707 : {
5708 178 : m_data_cnt = 0;
5709 178 : if (kind == bitint_prec_large)
5710 83 : idx = size_int (bitint_big_endian ? nelts - 1 - i : i);
5711 95 : else if (i >= 2)
5712 17 : idx = size_int (bitint_big_endian
5713 : ? nelts - 1 - end - (i > 2) : end + (i > 2));
5714 :
5715 178 : tree rhs1 = handle_operand (arg0, idx);
5716 178 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
5717 : {
5718 32 : if (!TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5719 5 : rhs1 = add_cast (unsigned_type_for (TREE_TYPE (rhs1)), rhs1);
5720 32 : rhs1 = add_cast (m_limb_type, rhs1);
5721 : }
5722 :
5723 178 : tree in, out, tem;
5724 178 : if (ifn == IFN_PARITY)
5725 43 : in = prepare_data_in_out (build_zero_cst (m_limb_type), idx, &out);
5726 135 : else if (ifn == IFN_FFS)
5727 29 : in = prepare_data_in_out (integer_one_node, idx, &out);
5728 : else
5729 106 : in = prepare_data_in_out (integer_zero_node, idx, &out);
5730 :
5731 178 : switch (ifn)
5732 : {
5733 104 : case IFN_CTZ:
5734 104 : case IFN_FFS:
5735 104 : g = gimple_build_cond (NE_EXPR, rhs1,
5736 : build_zero_cst (m_limb_type),
5737 : NULL_TREE, NULL_TREE);
5738 104 : insert_before (g);
5739 104 : edge e1, e2;
5740 104 : e1 = split_block (gsi_bb (m_gsi), g);
5741 104 : e1->flags = EDGE_FALSE_VALUE;
5742 104 : e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
5743 104 : e1->probability = profile_probability::unlikely ();
5744 104 : e2->probability = e1->probability.invert ();
5745 104 : if (i == 0)
5746 39 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5747 104 : m_gsi = gsi_after_labels (e1->dest);
5748 104 : bqp[i].e = e2;
5749 104 : bqp[i].val = rhs1;
5750 104 : if (tree_fits_uhwi_p (idx))
5751 58 : bqp[i].addend
5752 58 : = build_int_cst (integer_type_node,
5753 : (bitint_big_endian
5754 58 : ? nelts - 1 - tree_to_uhwi (idx)
5755 58 : : tree_to_uhwi (idx)) * limb_prec
5756 58 : + (ifn == IFN_FFS));
5757 : else
5758 : {
5759 46 : bqp[i].addend = in;
5760 46 : if (i == 1)
5761 23 : res = out;
5762 : else
5763 23 : res = make_ssa_name (integer_type_node);
5764 46 : g = gimple_build_assign (res, PLUS_EXPR, in,
5765 : build_int_cst (integer_type_node,
5766 46 : limb_prec));
5767 46 : insert_before (g);
5768 46 : m_data[m_data_cnt] = res;
5769 : }
5770 : break;
5771 43 : case IFN_PARITY:
5772 43 : if (!integer_zerop (in))
5773 : {
5774 37 : if (kind == bitint_prec_huge && i == 1)
5775 10 : res = out;
5776 : else
5777 27 : res = make_ssa_name (m_limb_type);
5778 37 : g = gimple_build_assign (res, BIT_XOR_EXPR, in, rhs1);
5779 37 : insert_before (g);
5780 : }
5781 : else
5782 : res = rhs1;
5783 43 : m_data[m_data_cnt] = res;
5784 43 : break;
5785 31 : case IFN_POPCOUNT:
5786 31 : g = gimple_build_call (fndecl, 1, rhs1);
5787 31 : tem = make_ssa_name (integer_type_node);
5788 31 : gimple_call_set_lhs (g, tem);
5789 31 : insert_before (g);
5790 31 : if (!integer_zerop (in))
5791 : {
5792 26 : if (kind == bitint_prec_huge && i == 1)
5793 6 : res = out;
5794 : else
5795 20 : res = make_ssa_name (integer_type_node);
5796 26 : g = gimple_build_assign (res, PLUS_EXPR, in, tem);
5797 26 : insert_before (g);
5798 : }
5799 : else
5800 : res = tem;
5801 31 : m_data[m_data_cnt] = res;
5802 31 : break;
5803 0 : default:
5804 0 : gcc_unreachable ();
5805 : }
5806 :
5807 178 : m_first = false;
5808 178 : if (kind == bitint_prec_huge && i <= 1)
5809 : {
5810 78 : if (i == 0)
5811 : {
5812 39 : idx = make_ssa_name (sizetype);
5813 39 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
5814 : bitint_big_endian
5815 0 : ? size_int (-1) : size_one_node);
5816 39 : insert_before (g);
5817 : }
5818 : else
5819 : {
5820 39 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
5821 78 : size_int (bitint_big_endian
5822 : ? -2 : 2));
5823 39 : insert_before (g);
5824 39 : if (bitint_big_endian)
5825 0 : g = gimple_build_cond (NE_EXPR, idx_first,
5826 0 : size_int (cnt - 1),
5827 : NULL_TREE, NULL_TREE);
5828 : else
5829 39 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (end),
5830 : NULL_TREE, NULL_TREE);
5831 39 : insert_before (g);
5832 39 : if (ifn == IFN_CTZ || ifn == IFN_FFS)
5833 23 : m_gsi = gsi_after_labels (edge_bb);
5834 : else
5835 16 : m_gsi = gsi_for_stmt (stmt);
5836 39 : m_bb = NULL;
5837 : }
5838 : }
5839 : }
5840 : }
5841 : else
5842 : {
5843 35 : tree idx = NULL_TREE, idx_next = NULL_TREE, first = NULL_TREE;
5844 35 : int sub_one = 0;
5845 35 : if (kind == bitint_prec_large)
5846 : cnt = nelts;
5847 : else
5848 : {
5849 18 : rem = prec % limb_prec;
5850 18 : if (rem == 0 && (!TYPE_UNSIGNED (type) || ifn == IFN_CLRSB))
5851 : rem = limb_prec;
5852 18 : end = (prec - rem) / limb_prec;
5853 18 : cnt = 1 + (rem != 0);
5854 18 : if (ifn == IFN_CLRSB)
5855 5 : sub_one = 1;
5856 : }
5857 :
5858 35 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5859 35 : gsi_prev (&gsi);
5860 35 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5861 35 : edge_bb = e->src;
5862 35 : m_gsi = gsi_end_bb (edge_bb);
5863 :
5864 35 : if (ifn == IFN_CLZ)
5865 26 : bqp = XALLOCAVEC (struct bq_details, cnt);
5866 : else
5867 : {
5868 9 : gsi = gsi_for_stmt (stmt);
5869 9 : gsi_prev (&gsi);
5870 9 : e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5871 9 : edge_bb = e->src;
5872 9 : bqp = XALLOCAVEC (struct bq_details, 2 * cnt);
5873 : }
5874 :
5875 116 : for (unsigned i = 0; i < cnt; i++)
5876 : {
5877 81 : m_data_cnt = 0;
5878 81 : if (kind == bitint_prec_large)
5879 51 : idx = size_int (bitint_big_endian ? i : cnt - i - 1);
5880 30 : else if (i == cnt - 1)
5881 18 : idx = create_loop (size_int (bitint_big_endian ? i : end - 1),
5882 : &idx_next);
5883 : else
5884 12 : idx = bitint_big_endian ? size_zero_node : size_int (end);
5885 :
5886 81 : tree rhs1 = handle_operand (arg0, idx);
5887 81 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
5888 : {
5889 19 : if (ifn == IFN_CLZ && !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5890 0 : rhs1 = add_cast (unsigned_type_for (TREE_TYPE (rhs1)), rhs1);
5891 19 : else if (ifn == IFN_CLRSB && TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5892 0 : rhs1 = add_cast (signed_type_for (TREE_TYPE (rhs1)), rhs1);
5893 19 : rhs1 = add_cast (m_limb_type, rhs1);
5894 : }
5895 :
5896 81 : if (ifn == IFN_CLZ)
5897 : {
5898 59 : g = gimple_build_cond (NE_EXPR, rhs1,
5899 : build_zero_cst (m_limb_type),
5900 : NULL_TREE, NULL_TREE);
5901 59 : insert_before (g);
5902 59 : edge e1 = split_block (gsi_bb (m_gsi), g);
5903 59 : e1->flags = EDGE_FALSE_VALUE;
5904 59 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
5905 59 : e1->probability = profile_probability::unlikely ();
5906 59 : e2->probability = e1->probability.invert ();
5907 59 : if (i == 0)
5908 26 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5909 59 : m_gsi = gsi_after_labels (e1->dest);
5910 59 : bqp[i].e = e2;
5911 59 : bqp[i].val = rhs1;
5912 : }
5913 : else
5914 : {
5915 22 : if (i == 0)
5916 : {
5917 9 : first = rhs1;
5918 9 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5919 : PLUS_EXPR, rhs1,
5920 : build_int_cst (m_limb_type, 1));
5921 9 : insert_before (g);
5922 9 : g = gimple_build_cond (GT_EXPR, gimple_assign_lhs (g),
5923 : build_int_cst (m_limb_type, 1),
5924 : NULL_TREE, NULL_TREE);
5925 9 : insert_before (g);
5926 : }
5927 : else
5928 : {
5929 13 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5930 : BIT_XOR_EXPR, rhs1, first);
5931 13 : insert_before (g);
5932 13 : tree stype = signed_type_for (m_limb_type);
5933 13 : g = gimple_build_cond (LT_EXPR,
5934 : add_cast (stype,
5935 : gimple_assign_lhs (g)),
5936 : build_zero_cst (stype),
5937 : NULL_TREE, NULL_TREE);
5938 13 : insert_before (g);
5939 13 : edge e1 = split_block (gsi_bb (m_gsi), g);
5940 13 : e1->flags = EDGE_FALSE_VALUE;
5941 13 : edge e2 = make_edge (e1->src, gimple_bb (stmt),
5942 : EDGE_TRUE_VALUE);
5943 13 : e1->probability = profile_probability::unlikely ();
5944 13 : e2->probability = e1->probability.invert ();
5945 13 : if (i == 1)
5946 9 : set_immediate_dominator (CDI_DOMINATORS, e2->dest,
5947 : e2->src);
5948 13 : m_gsi = gsi_after_labels (e1->dest);
5949 13 : bqp[2 * i].e = e2;
5950 13 : g = gimple_build_cond (NE_EXPR, rhs1, first,
5951 : NULL_TREE, NULL_TREE);
5952 13 : insert_before (g);
5953 : }
5954 22 : edge e1 = split_block (gsi_bb (m_gsi), g);
5955 22 : e1->flags = EDGE_FALSE_VALUE;
5956 22 : edge e2 = make_edge (e1->src, edge_bb, EDGE_TRUE_VALUE);
5957 22 : e1->probability = profile_probability::unlikely ();
5958 22 : e2->probability = e1->probability.invert ();
5959 22 : if (i == 0)
5960 9 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5961 22 : m_gsi = gsi_after_labels (e1->dest);
5962 22 : bqp[2 * i + 1].e = e2;
5963 22 : bqp[i].val = rhs1;
5964 : }
5965 81 : if (tree_fits_uhwi_p (idx))
5966 126 : bqp[i].addend
5967 63 : = build_int_cst (integer_type_node,
5968 63 : (int) prec
5969 63 : - (((int) (bitint_big_endian
5970 0 : ? nelts - 1 - tree_to_uhwi (idx)
5971 63 : : tree_to_uhwi (idx)) + 1)
5972 63 : * limb_prec) - sub_one);
5973 : else
5974 : {
5975 18 : tree in, out;
5976 18 : in = build_int_cst (integer_type_node, rem - sub_one);
5977 18 : m_first = true;
5978 18 : in = prepare_data_in_out (in, idx, &out);
5979 18 : out = m_data[m_data_cnt + 1];
5980 18 : bqp[i].addend = in;
5981 18 : g = gimple_build_assign (out, PLUS_EXPR, in,
5982 : build_int_cst (integer_type_node,
5983 18 : limb_prec));
5984 18 : insert_before (g);
5985 18 : m_data[m_data_cnt] = out;
5986 : }
5987 :
5988 81 : m_first = false;
5989 81 : if (kind == bitint_prec_huge && i == cnt - 1)
5990 : {
5991 36 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
5992 : bitint_big_endian
5993 18 : ? size_one_node : size_int (-1));
5994 18 : insert_before (g);
5995 18 : g = gimple_build_cond (NE_EXPR, idx,
5996 : bitint_big_endian
5997 0 : ? size_int (nelts - 1) : size_zero_node,
5998 : NULL_TREE, NULL_TREE);
5999 18 : insert_before (g);
6000 18 : edge true_edge, false_edge;
6001 18 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
6002 : &true_edge, &false_edge);
6003 18 : m_gsi = gsi_after_labels (false_edge->dest);
6004 18 : m_bb = NULL;
6005 : }
6006 : }
6007 : }
6008 101 : switch (ifn)
6009 : {
6010 65 : case IFN_CLZ:
6011 65 : case IFN_CTZ:
6012 65 : case IFN_FFS:
6013 65 : gphi *phi1, *phi2, *phi3;
6014 65 : basic_block bb;
6015 65 : bb = gsi_bb (m_gsi);
6016 65 : remove_edge (find_edge (bb, gimple_bb (stmt)));
6017 65 : phi1 = create_phi_node (make_ssa_name (m_limb_type),
6018 : gimple_bb (stmt));
6019 65 : phi2 = create_phi_node (make_ssa_name (integer_type_node),
6020 : gimple_bb (stmt));
6021 293 : for (unsigned i = 0; i < cnt; i++)
6022 : {
6023 163 : add_phi_arg (phi1, bqp[i].val, bqp[i].e, UNKNOWN_LOCATION);
6024 163 : add_phi_arg (phi2, bqp[i].addend, bqp[i].e, UNKNOWN_LOCATION);
6025 : }
6026 65 : if (arg1 == NULL_TREE)
6027 : {
6028 35 : g = gimple_build_builtin_unreachable (m_loc);
6029 35 : insert_before (g);
6030 : }
6031 65 : m_gsi = gsi_for_stmt (stmt);
6032 65 : g = gimple_build_call (fndecl, 1, gimple_phi_result (phi1));
6033 65 : gimple_call_set_lhs (g, make_ssa_name (integer_type_node));
6034 65 : insert_before (g);
6035 65 : if (arg1 == NULL_TREE)
6036 35 : g = gimple_build_assign (lhs, PLUS_EXPR,
6037 : gimple_phi_result (phi2),
6038 : gimple_call_lhs (g));
6039 : else
6040 : {
6041 30 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6042 : PLUS_EXPR, gimple_phi_result (phi2),
6043 : gimple_call_lhs (g));
6044 30 : insert_before (g);
6045 30 : edge e1 = split_block (gimple_bb (stmt), g);
6046 30 : edge e2 = make_edge (bb, e1->dest, EDGE_FALLTHRU);
6047 30 : e2->probability = profile_probability::always ();
6048 30 : set_immediate_dominator (CDI_DOMINATORS, e1->dest,
6049 : get_immediate_dominator (CDI_DOMINATORS,
6050 : e1->src));
6051 30 : phi3 = create_phi_node (make_ssa_name (integer_type_node), e1->dest);
6052 30 : add_phi_arg (phi3, gimple_assign_lhs (g), e1, UNKNOWN_LOCATION);
6053 30 : add_phi_arg (phi3, arg1, e2, UNKNOWN_LOCATION);
6054 30 : m_gsi = gsi_for_stmt (stmt);
6055 30 : g = gimple_build_assign (lhs, gimple_phi_result (phi3));
6056 : }
6057 65 : gsi_replace (&m_gsi, g, true);
6058 65 : break;
6059 9 : case IFN_CLRSB:
6060 9 : bb = gsi_bb (m_gsi);
6061 9 : remove_edge (find_edge (bb, edge_bb));
6062 9 : edge e;
6063 9 : e = make_edge (bb, gimple_bb (stmt), EDGE_FALLTHRU);
6064 9 : e->probability = profile_probability::always ();
6065 9 : set_immediate_dominator (CDI_DOMINATORS, gimple_bb (stmt),
6066 : get_immediate_dominator (CDI_DOMINATORS,
6067 : edge_bb));
6068 9 : phi1 = create_phi_node (make_ssa_name (m_limb_type),
6069 : edge_bb);
6070 9 : phi2 = create_phi_node (make_ssa_name (integer_type_node),
6071 : edge_bb);
6072 9 : phi3 = create_phi_node (make_ssa_name (integer_type_node),
6073 : gimple_bb (stmt));
6074 40 : for (unsigned i = 0; i < cnt; i++)
6075 : {
6076 22 : add_phi_arg (phi1, bqp[i].val, bqp[2 * i + 1].e, UNKNOWN_LOCATION);
6077 22 : add_phi_arg (phi2, bqp[i].addend, bqp[2 * i + 1].e,
6078 : UNKNOWN_LOCATION);
6079 22 : tree a = bqp[i].addend;
6080 22 : if (i && kind == bitint_prec_large)
6081 8 : a = int_const_binop (PLUS_EXPR, a, integer_minus_one_node);
6082 22 : if (i)
6083 13 : add_phi_arg (phi3, a, bqp[2 * i].e, UNKNOWN_LOCATION);
6084 : }
6085 9 : add_phi_arg (phi3, build_int_cst (integer_type_node, prec - 1), e,
6086 : UNKNOWN_LOCATION);
6087 9 : m_gsi = gsi_after_labels (edge_bb);
6088 9 : g = gimple_build_call (fndecl, 1,
6089 : add_cast (signed_type_for (m_limb_type),
6090 : gimple_phi_result (phi1)));
6091 9 : gimple_call_set_lhs (g, make_ssa_name (integer_type_node));
6092 9 : insert_before (g);
6093 9 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6094 : PLUS_EXPR, gimple_call_lhs (g),
6095 : gimple_phi_result (phi2));
6096 9 : insert_before (g);
6097 9 : if (kind != bitint_prec_large)
6098 : {
6099 5 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6100 : PLUS_EXPR, gimple_assign_lhs (g),
6101 : integer_one_node);
6102 5 : insert_before (g);
6103 : }
6104 9 : add_phi_arg (phi3, gimple_assign_lhs (g),
6105 : find_edge (edge_bb, gimple_bb (stmt)), UNKNOWN_LOCATION);
6106 9 : m_gsi = gsi_for_stmt (stmt);
6107 9 : g = gimple_build_assign (lhs, gimple_phi_result (phi3));
6108 9 : gsi_replace (&m_gsi, g, true);
6109 9 : break;
6110 16 : case IFN_PARITY:
6111 16 : g = gimple_build_call (fndecl, 1, res);
6112 16 : gimple_call_set_lhs (g, lhs);
6113 16 : gsi_replace (&m_gsi, g, true);
6114 16 : break;
6115 11 : case IFN_POPCOUNT:
6116 11 : g = gimple_build_assign (lhs, res);
6117 11 : gsi_replace (&m_gsi, g, true);
6118 11 : break;
6119 0 : default:
6120 0 : gcc_unreachable ();
6121 : }
6122 : }
6123 :
6124 : /* Lower a .{BSWAP,BITREVERSE} call with one large/huge _BitInt argument. */
6125 :
6126 : void
6127 15 : bitint_large_huge::lower_bswap_bitreverse (tree obj, gimple *stmt)
6128 : {
6129 15 : tree arg = gimple_call_arg (stmt, 0);
6130 15 : tree lhs = gimple_call_lhs (stmt);
6131 15 : gimple *g;
6132 :
6133 15 : if (!lhs)
6134 : {
6135 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6136 0 : gsi_remove (&gsi, true);
6137 0 : return;
6138 : }
6139 15 : tree type = TREE_TYPE (arg);
6140 15 : gcc_assert (BITINT_TYPE_P (type));
6141 15 : bitint_prec_kind kind = bitint_precision_kind (type);
6142 15 : gcc_assert (kind >= bitint_prec_large);
6143 15 : if (!obj)
6144 : {
6145 2 : int part = var_to_partition (m_map, lhs);
6146 2 : gcc_assert (m_vars[part] != NULL_TREE);
6147 : obj = m_vars[part];
6148 : }
6149 15 : enum internal_fn ifn = gimple_call_internal_fn (stmt);
6150 15 : enum built_in_function bcode = END_BUILTINS;
6151 15 : switch (limb_prec)
6152 : {
6153 0 : case 16:
6154 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP16 : BUILT_IN_BITREVERSE16;
6155 : break;
6156 0 : case 32:
6157 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP32 : BUILT_IN_BITREVERSE32;
6158 : break;
6159 15 : case 64:
6160 15 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP64 : BUILT_IN_BITREVERSE64;
6161 : break;
6162 0 : case 128:
6163 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP128 : BUILT_IN_BITREVERSE128;
6164 : break;
6165 0 : default:
6166 0 : gcc_unreachable ();
6167 : }
6168 15 : tree fndecl = builtin_decl_explicit (bcode);
6169 15 : unsigned prec = TYPE_PRECISION (type);
6170 30 : tree p = build_int_cst (sizetype,
6171 15 : prec / limb_prec - (prec % limb_prec == 0));
6172 : /* For IFN .BSWAP or .BITREVERSE and
6173 : FN corresponding __builtin_bswapN or __builtin_bitreverseN where N
6174 : is limb_prec, lower
6175 : dst = IFN (src);
6176 : as
6177 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
6178 : if constexpr ((prec % limb_prec) == 0)
6179 : {
6180 : for (idx = 0; idx <= p; ++idx)
6181 : dst[p - idx] = FN (src[idx]);
6182 : }
6183 : else
6184 : {
6185 : unsigned n1 = prec % limb_prec;
6186 : unsigned n2 = limb_prec - n1;
6187 : dst[p] = FN (src[0]) >> n2;
6188 : for (idx = p - 1; (ssize_t) idx >= 0; --idx)
6189 : dst[idx] = FN ((src[p - idx] << n2) | (src[p - idx - 1] >> n1));
6190 : } */
6191 15 : if (prec % limb_prec == 0)
6192 : {
6193 3 : tree idx_next;
6194 3 : tree idx = create_loop (size_zero_node, &idx_next);
6195 3 : tree pmidx = make_ssa_name (sizetype);
6196 3 : g = gimple_build_assign (pmidx, MINUS_EXPR, p, idx);
6197 3 : insert_before (g);
6198 3 : m_data_cnt = 0;
6199 3 : tree t = handle_operand (arg, idx);
6200 3 : m_first = false;
6201 3 : g = gimple_build_call (fndecl, 1, t);
6202 3 : t = make_ssa_name (m_limb_type);
6203 3 : gimple_call_set_lhs (g, t);
6204 3 : insert_before (g);
6205 3 : tree l = limb_access (TREE_TYPE (lhs), obj, pmidx, true);
6206 3 : g = gimple_build_assign (l, t);
6207 3 : insert_before (g);
6208 3 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_one_node);
6209 3 : insert_before (g);
6210 3 : g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
6211 3 : insert_before (g);
6212 : }
6213 : else
6214 : {
6215 12 : tree n1 = build_int_cst (unsigned_type_node, prec % limb_prec);
6216 24 : tree n2 = build_int_cst (unsigned_type_node,
6217 12 : limb_prec - (prec % limb_prec));
6218 12 : m_data_cnt = 0;
6219 12 : tree t = handle_operand (arg, bitint_big_endian ? p : size_zero_node);
6220 12 : m_first = false;
6221 : /* Nothing is needed to bswap 8 bits or bitreverse 1 bit, so just
6222 : mask off higher bits in that case. */
6223 19 : if ((prec % limb_prec) != (ifn == IFN_BSWAP ? 8 : 1))
6224 : {
6225 8 : g = gimple_build_call (fndecl, 1, t);
6226 8 : t = make_ssa_name (m_limb_type);
6227 8 : gimple_call_set_lhs (g, t);
6228 8 : insert_before (g);
6229 8 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
6230 : t, n2);
6231 : }
6232 : else
6233 4 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_AND_EXPR,
6234 : t, build_int_cst (m_limb_type,
6235 : ifn == IFN_BSWAP
6236 6 : ? 0xff : 1));
6237 12 : insert_before (g);
6238 12 : t = gimple_assign_lhs (g);
6239 12 : tree l = limb_access (TREE_TYPE (lhs), obj,
6240 : bitint_big_endian ? size_zero_node : p, true);
6241 12 : g = gimple_build_assign (l, t);
6242 12 : insert_before (g);
6243 12 : tree pm1 = build_int_cst (sizetype, prec / limb_prec - 1);
6244 12 : tree idx_next;
6245 12 : tree idx = create_loop (pm1, &idx_next);
6246 12 : tree pmidx = make_ssa_name (sizetype);
6247 12 : g = gimple_build_assign (pmidx, MINUS_EXPR, p, idx);
6248 12 : insert_before (g);
6249 12 : m_data_cnt = 0;
6250 24 : t = handle_operand (arg, bitint_big_endian ? idx : pmidx);
6251 12 : m_data_cnt = 0;
6252 12 : g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
6253 : t, n2);
6254 12 : insert_before (g);
6255 12 : t = gimple_assign_lhs (g);
6256 12 : tree t2 = make_ssa_name (sizetype);
6257 12 : if (bitint_big_endian)
6258 0 : g = gimple_build_assign (t2, PLUS_EXPR, idx, size_one_node);
6259 : else
6260 12 : g = gimple_build_assign (t2, PLUS_EXPR, pmidx, size_int (-1));
6261 12 : insert_before (g);
6262 12 : t2 = handle_operand (arg, t2);
6263 12 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
6264 : t2, n1);
6265 12 : insert_before (g);
6266 12 : t2 = gimple_assign_lhs (g);
6267 12 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
6268 : t, t2);
6269 12 : insert_before (g);
6270 12 : t = gimple_assign_lhs (g);
6271 12 : g = gimple_build_call (fndecl, 1, t);
6272 12 : t = make_ssa_name (m_limb_type);
6273 12 : gimple_call_set_lhs (g, t);
6274 12 : insert_before (g);
6275 24 : l = limb_access (TREE_TYPE (lhs), obj,
6276 : bitint_big_endian ? pmidx : idx, true);
6277 12 : g = gimple_build_assign (l, t);
6278 12 : insert_before (g);
6279 12 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_int (-1));
6280 12 : insert_before (g);
6281 12 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node, NULL_TREE,
6282 : NULL_TREE);
6283 12 : insert_before (g);
6284 : }
6285 15 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6286 15 : if (bitint_extended == bitint_ext_full
6287 0 : && abi_limb_prec > limb_prec
6288 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
6289 0 : > CEIL (prec, limb_prec) * limb_prec))
6290 : {
6291 0 : m_gsi = gsi;
6292 0 : tree p2 = build_int_cst (sizetype,
6293 : CEIL (prec, abi_limb_prec)
6294 0 : * abi_limb_prec / limb_prec - 1);
6295 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
6296 0 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
6297 0 : insert_before (g);
6298 : }
6299 : }
6300 :
6301 : /* Lower a call statement with one or more large/huge _BitInt
6302 : arguments or large/huge _BitInt return value. */
6303 :
6304 : void
6305 8857 : bitint_large_huge::lower_call (tree obj, gimple *stmt)
6306 : {
6307 8857 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6308 8857 : unsigned int nargs = gimple_call_num_args (stmt);
6309 8857 : if (gimple_call_internal_p (stmt))
6310 4328 : switch (gimple_call_internal_fn (stmt))
6311 : {
6312 2733 : case IFN_ADD_OVERFLOW:
6313 2733 : case IFN_SUB_OVERFLOW:
6314 2733 : case IFN_UBSAN_CHECK_ADD:
6315 2733 : case IFN_UBSAN_CHECK_SUB:
6316 2733 : lower_addsub_overflow (obj, stmt);
6317 7061 : return;
6318 1479 : case IFN_MUL_OVERFLOW:
6319 1479 : case IFN_UBSAN_CHECK_MUL:
6320 1479 : lower_mul_overflow (obj, stmt);
6321 1479 : return;
6322 101 : case IFN_CLZ:
6323 101 : case IFN_CTZ:
6324 101 : case IFN_CLRSB:
6325 101 : case IFN_FFS:
6326 101 : case IFN_PARITY:
6327 101 : case IFN_POPCOUNT:
6328 101 : lower_bit_query (stmt);
6329 101 : return;
6330 15 : case IFN_BSWAP:
6331 15 : case IFN_BITREVERSE:
6332 15 : lower_bswap_bitreverse (obj, stmt);
6333 15 : return;
6334 : default:
6335 : break;
6336 : }
6337 4529 : bool returns_twice = (gimple_call_flags (stmt) & ECF_RETURNS_TWICE) != 0;
6338 10202 : for (unsigned int i = 0; i < nargs; ++i)
6339 : {
6340 5673 : tree arg = gimple_call_arg (stmt, i);
6341 9003 : if (TREE_CODE (arg) != SSA_NAME
6342 2440 : || !BITINT_TYPE_P (TREE_TYPE (arg))
6343 8051 : || bitint_precision_kind (TREE_TYPE (arg)) <= bitint_prec_middle)
6344 3330 : continue;
6345 2343 : if (SSA_NAME_IS_DEFAULT_DEF (arg)
6346 2343 : && (!SSA_NAME_VAR (arg) || VAR_P (SSA_NAME_VAR (arg))))
6347 : {
6348 1 : tree var = create_tmp_reg (TREE_TYPE (arg));
6349 1 : arg = get_or_create_ssa_default_def (cfun, var);
6350 : }
6351 : else
6352 : {
6353 2342 : int p = var_to_partition (m_map, arg);
6354 2342 : tree v = m_vars[p];
6355 2342 : gcc_assert (v != NULL_TREE);
6356 2342 : if (!types_compatible_p (TREE_TYPE (arg), TREE_TYPE (v)))
6357 2322 : v = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (arg), v);
6358 2342 : arg = make_ssa_name (TREE_TYPE (arg));
6359 2342 : gimple *g = gimple_build_assign (arg, v);
6360 2342 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
6361 2342 : if (returns_twice && bb_has_abnormal_pred (gimple_bb (stmt)))
6362 : {
6363 11 : m_returns_twice_calls.safe_push (stmt);
6364 11 : returns_twice = false;
6365 : }
6366 : }
6367 2343 : gimple_call_set_arg (stmt, i, arg);
6368 2343 : if (m_preserved == NULL)
6369 412 : m_preserved = BITMAP_ALLOC (NULL);
6370 2343 : bitmap_set_bit (m_preserved, SSA_NAME_VERSION (arg));
6371 : }
6372 4529 : tree lhs = gimple_call_lhs (stmt);
6373 4529 : if (lhs
6374 4398 : && TREE_CODE (lhs) == SSA_NAME
6375 4398 : && BITINT_TYPE_P (TREE_TYPE (lhs))
6376 8849 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large)
6377 : {
6378 4294 : int p = var_to_partition (m_map, lhs);
6379 4294 : tree v = m_vars[p];
6380 4294 : gcc_assert (v != NULL_TREE);
6381 4294 : if (!types_compatible_p (TREE_TYPE (lhs), TREE_TYPE (v)))
6382 4294 : v = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), v);
6383 4294 : gimple_call_set_lhs (stmt, v);
6384 4294 : SSA_NAME_DEF_STMT (lhs) = gimple_build_nop ();
6385 : }
6386 4529 : update_stmt (stmt);
6387 : }
6388 :
6389 : /* Lower __asm STMT which involves large/huge _BitInt values. */
6390 :
6391 : void
6392 3 : bitint_large_huge::lower_asm (gimple *stmt)
6393 : {
6394 3 : gasm *g = as_a <gasm *> (stmt);
6395 3 : unsigned noutputs = gimple_asm_noutputs (g);
6396 3 : unsigned ninputs = gimple_asm_ninputs (g);
6397 :
6398 5 : for (unsigned i = 0; i < noutputs; ++i)
6399 : {
6400 2 : tree t = gimple_asm_output_op (g, i);
6401 2 : tree s = TREE_VALUE (t);
6402 2 : if (TREE_CODE (s) == SSA_NAME
6403 1 : && BITINT_TYPE_P (TREE_TYPE (s))
6404 3 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
6405 : {
6406 1 : int part = var_to_partition (m_map, s);
6407 1 : gcc_assert (m_vars[part] != NULL_TREE);
6408 1 : TREE_VALUE (t) = m_vars[part];
6409 : }
6410 : }
6411 8 : for (unsigned i = 0; i < ninputs; ++i)
6412 : {
6413 5 : tree t = gimple_asm_input_op (g, i);
6414 5 : tree s = TREE_VALUE (t);
6415 5 : if (TREE_CODE (s) == SSA_NAME
6416 4 : && BITINT_TYPE_P (TREE_TYPE (s))
6417 9 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
6418 : {
6419 4 : if (SSA_NAME_IS_DEFAULT_DEF (s)
6420 4 : && (!SSA_NAME_VAR (s) || VAR_P (SSA_NAME_VAR (s))))
6421 : {
6422 1 : TREE_VALUE (t) = create_tmp_var (TREE_TYPE (s), "bitint");
6423 1 : mark_addressable (TREE_VALUE (t));
6424 : }
6425 : else
6426 : {
6427 3 : int part = var_to_partition (m_map, s);
6428 3 : gcc_assert (m_vars[part] != NULL_TREE);
6429 3 : TREE_VALUE (t) = m_vars[part];
6430 : }
6431 : }
6432 : }
6433 3 : update_stmt (stmt);
6434 3 : }
6435 :
6436 : /* Lower statement STMT which involves large/huge _BitInt values
6437 : into code accessing individual limbs. */
6438 :
6439 : void
6440 42895 : bitint_large_huge::lower_stmt (gimple *stmt)
6441 : {
6442 42895 : m_first = true;
6443 42895 : m_lhs = NULL_TREE;
6444 42895 : m_data.truncate (0);
6445 42895 : m_data_cnt = 0;
6446 42895 : m_gsi = gsi_for_stmt (stmt);
6447 42895 : m_after_stmt = NULL;
6448 42895 : m_bb = NULL;
6449 42895 : m_init_gsi = m_gsi;
6450 42895 : gsi_prev (&m_init_gsi);
6451 42895 : m_preheader_bb = NULL;
6452 42895 : m_upwards_2limb = 0;
6453 42895 : m_upwards = false;
6454 42895 : m_var_msb = false;
6455 42895 : m_cast_conditional = false;
6456 42895 : m_bitfld_load = 0;
6457 42895 : m_loc = gimple_location (stmt);
6458 42895 : if (is_gimple_call (stmt))
6459 : {
6460 7180 : lower_call (NULL_TREE, stmt);
6461 7180 : return;
6462 : }
6463 35715 : if (gimple_code (stmt) == GIMPLE_ASM)
6464 : {
6465 3 : lower_asm (stmt);
6466 3 : return;
6467 : }
6468 35712 : tree lhs = NULL_TREE, cmp_op1 = NULL_TREE, cmp_op2 = NULL_TREE;
6469 35712 : tree_code cmp_code = comparison_op (stmt, &cmp_op1, &cmp_op2);
6470 35712 : bool eq_p = (cmp_code == EQ_EXPR || cmp_code == NE_EXPR);
6471 35712 : bool mergeable_cast_p = false;
6472 35712 : bool final_cast_p = false;
6473 35712 : if (gimple_assign_cast_p (stmt))
6474 : {
6475 5449 : lhs = gimple_assign_lhs (stmt);
6476 5449 : tree rhs1 = gimple_assign_rhs1 (stmt);
6477 5449 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
6478 46 : rhs1 = TREE_OPERAND (rhs1, 0);
6479 9670 : if (BITINT_TYPE_P (TREE_TYPE (lhs))
6480 1228 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
6481 6641 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1)))
6482 : mergeable_cast_p = true;
6483 4547 : else if (BITINT_TYPE_P (TREE_TYPE (rhs1))
6484 4257 : && bitint_precision_kind (TREE_TYPE (rhs1)) >= bitint_prec_large
6485 8659 : && (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6486 36 : || POINTER_TYPE_P (TREE_TYPE (lhs))
6487 35 : || gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR))
6488 : {
6489 4257 : final_cast_p = true;
6490 4257 : if (((TREE_CODE (TREE_TYPE (lhs)) == INTEGER_TYPE
6491 572 : && TYPE_PRECISION (TREE_TYPE (lhs)) > MAX_FIXED_MODE_SIZE)
6492 4257 : || (!INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6493 36 : && !POINTER_TYPE_P (TREE_TYPE (lhs))))
6494 4292 : && gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)
6495 : {
6496 : /* Handle VIEW_CONVERT_EXPRs to not generally supported
6497 : huge INTEGER_TYPEs like uint256_t or uint512_t. These
6498 : are usually emitted from memcpy folding and backends
6499 : support moves with them but that is usually it.
6500 : Similarly handle VCEs to vector/complex types etc. */
6501 35 : gcc_assert (TREE_CODE (rhs1) == SSA_NAME);
6502 35 : if (SSA_NAME_IS_DEFAULT_DEF (rhs1)
6503 35 : && (!SSA_NAME_VAR (rhs1) || VAR_P (SSA_NAME_VAR (rhs1))))
6504 : {
6505 0 : tree var = create_tmp_reg (TREE_TYPE (lhs));
6506 0 : rhs1 = get_or_create_ssa_default_def (cfun, var);
6507 0 : gimple_assign_set_rhs1 (stmt, rhs1);
6508 0 : gimple_assign_set_rhs_code (stmt, SSA_NAME);
6509 : }
6510 35 : else if (m_names == NULL
6511 35 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))
6512 : {
6513 0 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6514 0 : gcc_assert (gimple_assign_load_p (g));
6515 0 : tree mem = gimple_assign_rhs1 (g);
6516 0 : tree ltype = TREE_TYPE (lhs);
6517 0 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (mem));
6518 0 : if (as != TYPE_ADDR_SPACE (ltype))
6519 0 : ltype
6520 0 : = build_qualified_type (ltype,
6521 0 : TYPE_QUALS (ltype)
6522 0 : | ENCODE_QUAL_ADDR_SPACE (as));
6523 0 : rhs1 = build1 (VIEW_CONVERT_EXPR, ltype, unshare_expr (mem));
6524 0 : gimple_assign_set_rhs1 (stmt, rhs1);
6525 : }
6526 : else
6527 : {
6528 35 : int part = var_to_partition (m_map, rhs1);
6529 35 : gcc_assert (m_vars[part] != NULL_TREE);
6530 35 : rhs1 = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs),
6531 : m_vars[part]);
6532 35 : gimple_assign_set_rhs1 (stmt, rhs1);
6533 : }
6534 35 : update_stmt (stmt);
6535 35 : return;
6536 : }
6537 4222 : if (TREE_CODE (rhs1) == SSA_NAME
6538 4222 : && (m_names == NULL
6539 4185 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
6540 : {
6541 1723 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6542 1723 : if (is_gimple_assign (g)
6543 1723 : && gimple_assign_rhs_code (g) == IMAGPART_EXPR)
6544 : {
6545 1640 : tree rhs2 = TREE_OPERAND (gimple_assign_rhs1 (g), 0);
6546 1640 : if (TREE_CODE (rhs2) == SSA_NAME
6547 1640 : && (m_names == NULL
6548 1603 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs2))))
6549 : {
6550 1640 : g = SSA_NAME_DEF_STMT (rhs2);
6551 1640 : int ovf = optimizable_arith_overflow (g);
6552 1640 : if (ovf == 2)
6553 : /* If .{ADD,SUB,MUL}_OVERFLOW has both REALPART_EXPR
6554 : and IMAGPART_EXPR uses, where the latter is cast to
6555 : non-_BitInt, it will be optimized when handling
6556 : the REALPART_EXPR. */
6557 : return;
6558 91 : if (ovf == 1)
6559 : {
6560 91 : lower_call (NULL_TREE, g);
6561 91 : return;
6562 : }
6563 : }
6564 : }
6565 : }
6566 : }
6567 145 : else if (BITINT_TYPE_P (TREE_TYPE (lhs))
6568 145 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
6569 145 : && !INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
6570 145 : && !POINTER_TYPE_P (TREE_TYPE (rhs1))
6571 290 : && gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)
6572 : {
6573 10 : int part = var_to_partition (m_map, lhs);
6574 10 : gcc_assert (m_vars[part] != NULL_TREE);
6575 10 : lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (rhs1), m_vars[part]);
6576 10 : insert_before (gimple_build_assign (lhs, rhs1));
6577 10 : return;
6578 : }
6579 : }
6580 34027 : if (gimple_store_p (stmt))
6581 : {
6582 9007 : tree rhs1 = gimple_assign_rhs1 (stmt);
6583 9007 : if (TREE_CODE (rhs1) == SSA_NAME
6584 9007 : && (m_names == NULL
6585 7947 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
6586 : {
6587 1639 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6588 1639 : m_loc = gimple_location (g);
6589 1639 : lhs = gimple_assign_lhs (stmt);
6590 1639 : if (is_gimple_assign (g) && !mergeable_op (g))
6591 643 : switch (gimple_assign_rhs_code (g))
6592 : {
6593 119 : case LSHIFT_EXPR:
6594 119 : case RSHIFT_EXPR:
6595 119 : lower_shift_stmt (lhs, g);
6596 478 : handled:
6597 478 : m_gsi = gsi_for_stmt (stmt);
6598 478 : unlink_stmt_vdef (stmt);
6599 956 : release_ssa_name (gimple_vdef (stmt));
6600 478 : gsi_remove (&m_gsi, true);
6601 478 : return;
6602 205 : case MULT_EXPR:
6603 205 : case TRUNC_DIV_EXPR:
6604 205 : case EXACT_DIV_EXPR:
6605 205 : case TRUNC_MOD_EXPR:
6606 205 : lower_muldiv_stmt (lhs, g);
6607 205 : goto handled;
6608 44 : case FIX_TRUNC_EXPR:
6609 44 : lower_float_conv_stmt (lhs, g);
6610 44 : goto handled;
6611 73 : case REALPART_EXPR:
6612 73 : case IMAGPART_EXPR:
6613 73 : lower_cplxpart_stmt (lhs, g);
6614 73 : goto handled;
6615 8 : case VIEW_CONVERT_EXPR:
6616 8 : {
6617 8 : tree rhs1 = gimple_assign_rhs1 (g);
6618 8 : rhs1 = TREE_OPERAND (rhs1, 0);
6619 8 : if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
6620 7 : && !POINTER_TYPE_P (TREE_TYPE (rhs1)))
6621 : {
6622 7 : tree ltype = TREE_TYPE (rhs1);
6623 7 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (lhs));
6624 7 : ltype
6625 14 : = build_qualified_type (ltype,
6626 7 : TYPE_QUALS (TREE_TYPE (lhs))
6627 7 : | ENCODE_QUAL_ADDR_SPACE (as));
6628 7 : lhs = build1 (VIEW_CONVERT_EXPR, ltype, lhs);
6629 7 : gimple_assign_set_lhs (stmt, lhs);
6630 7 : gimple_assign_set_rhs1 (stmt, rhs1);
6631 7 : gimple_assign_set_rhs_code (stmt, TREE_CODE (rhs1));
6632 7 : update_stmt (stmt);
6633 7 : return;
6634 : }
6635 : }
6636 : break;
6637 : default:
6638 : break;
6639 : }
6640 996 : else if (optimizable_arith_overflow (g) == 3)
6641 : {
6642 24 : lower_call (lhs, g);
6643 24 : goto handled;
6644 : }
6645 972 : else if (is_gimple_call (g) && gimple_call_internal_p (g))
6646 13 : switch (gimple_call_internal_fn (g))
6647 : {
6648 13 : case IFN_BSWAP:
6649 13 : case IFN_BITREVERSE:
6650 13 : lower_call (lhs, g);
6651 13 : goto handled;
6652 : default:
6653 : break;
6654 : }
6655 :
6656 1154 : m_loc = gimple_location (stmt);
6657 : }
6658 : }
6659 33542 : if (mergeable_op (stmt)
6660 22108 : || gimple_store_p (stmt)
6661 22108 : || gimple_assign_load_p (stmt)
6662 : || eq_p
6663 17298 : || mergeable_cast_p
6664 43709 : || (is_gimple_assign (stmt)
6665 9779 : && gimple_assign_rhs_code (stmt) == PAREN_EXPR))
6666 : {
6667 23377 : lhs = lower_mergeable_stmt (stmt, cmp_code, cmp_op1, cmp_op2);
6668 23377 : if (!eq_p)
6669 : return;
6670 : }
6671 10165 : else if (cmp_code != ERROR_MARK)
6672 722 : lhs = lower_comparison_stmt (stmt, cmp_code, cmp_op1, cmp_op2);
6673 16715 : if (cmp_code != ERROR_MARK)
6674 : {
6675 7272 : if (gimple_code (stmt) == GIMPLE_COND)
6676 : {
6677 6498 : gcond *cstmt = as_a <gcond *> (stmt);
6678 6498 : gimple_cond_set_lhs (cstmt, lhs);
6679 6498 : gimple_cond_set_rhs (cstmt, boolean_false_node);
6680 6498 : gimple_cond_set_code (cstmt, cmp_code);
6681 6498 : update_stmt (stmt);
6682 6498 : return;
6683 : }
6684 774 : if (gimple_assign_rhs_code (stmt) == COND_EXPR)
6685 : {
6686 0 : tree cond = build2 (cmp_code, boolean_type_node, lhs,
6687 : boolean_false_node);
6688 0 : gimple_assign_set_rhs1 (stmt, cond);
6689 0 : lhs = gimple_assign_lhs (stmt);
6690 0 : gcc_assert (!BITINT_TYPE_P (TREE_TYPE (lhs))
6691 : || (bitint_precision_kind (TREE_TYPE (lhs))
6692 : <= bitint_prec_middle));
6693 0 : update_stmt (stmt);
6694 0 : return;
6695 : }
6696 774 : gimple_assign_set_rhs1 (stmt, lhs);
6697 774 : gimple_assign_set_rhs2 (stmt, boolean_false_node);
6698 774 : gimple_assign_set_rhs_code (stmt, cmp_code);
6699 774 : update_stmt (stmt);
6700 774 : return;
6701 : }
6702 9443 : if (final_cast_p)
6703 : {
6704 2582 : tree lhs_type = TREE_TYPE (lhs);
6705 : /* Add support for 3 or more limbs filled in from normal integral
6706 : type if this assert fails. If no target chooses limb mode smaller
6707 : than half of largest supported normal integral type, this will not
6708 : be needed. */
6709 2582 : gcc_assert (TYPE_PRECISION (lhs_type) <= 2 * limb_prec);
6710 2582 : gimple *g;
6711 2546 : if ((BITINT_TYPE_P (lhs_type)
6712 36 : && bitint_precision_kind (lhs_type) == bitint_prec_middle)
6713 5149 : || POINTER_TYPE_P (lhs_type))
6714 16 : lhs_type = build_nonstandard_integer_type (TYPE_PRECISION (lhs_type),
6715 16 : TYPE_UNSIGNED (lhs_type));
6716 2582 : m_data_cnt = 0;
6717 2582 : tree rhs1 = gimple_assign_rhs1 (stmt);
6718 2582 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (rhs1));
6719 2582 : unsigned int cnt = CEIL (prec, limb_prec);
6720 2582 : tree r1 = handle_operand (rhs1, size_int (bitint_big_endian
6721 : ? cnt - 1 : 0));
6722 2582 : if (!useless_type_conversion_p (lhs_type, TREE_TYPE (r1)))
6723 2473 : r1 = add_cast (lhs_type, r1);
6724 2582 : if (TYPE_PRECISION (lhs_type) > limb_prec)
6725 : {
6726 70 : m_data_cnt = 0;
6727 70 : m_first = false;
6728 70 : tree r2 = handle_operand (rhs1, size_int (bitint_big_endian
6729 : ? cnt - 2 : 1));
6730 70 : r2 = add_cast (lhs_type, r2);
6731 70 : g = gimple_build_assign (make_ssa_name (lhs_type), LSHIFT_EXPR, r2,
6732 : build_int_cst (unsigned_type_node,
6733 70 : limb_prec));
6734 70 : insert_before (g);
6735 70 : g = gimple_build_assign (make_ssa_name (lhs_type), BIT_IOR_EXPR, r1,
6736 : gimple_assign_lhs (g));
6737 70 : insert_before (g);
6738 70 : r1 = gimple_assign_lhs (g);
6739 : }
6740 2582 : if (lhs_type != TREE_TYPE (lhs))
6741 16 : g = gimple_build_assign (lhs, NOP_EXPR, r1);
6742 : else
6743 2566 : g = gimple_build_assign (lhs, r1);
6744 2582 : gsi_replace (&m_gsi, g, true);
6745 2582 : return;
6746 : }
6747 6861 : if (is_gimple_assign (stmt))
6748 6861 : switch (gimple_assign_rhs_code (stmt))
6749 : {
6750 465 : case LSHIFT_EXPR:
6751 465 : case RSHIFT_EXPR:
6752 465 : lower_shift_stmt (NULL_TREE, stmt);
6753 465 : return;
6754 166 : case MULT_EXPR:
6755 166 : case TRUNC_DIV_EXPR:
6756 166 : case EXACT_DIV_EXPR:
6757 166 : case TRUNC_MOD_EXPR:
6758 166 : lower_muldiv_stmt (NULL_TREE, stmt);
6759 166 : return;
6760 283 : case FIX_TRUNC_EXPR:
6761 283 : case FLOAT_EXPR:
6762 283 : lower_float_conv_stmt (NULL_TREE, stmt);
6763 283 : return;
6764 5929 : case REALPART_EXPR:
6765 5929 : case IMAGPART_EXPR:
6766 5929 : lower_cplxpart_stmt (NULL_TREE, stmt);
6767 5929 : return;
6768 18 : case COMPLEX_EXPR:
6769 18 : lower_complexexpr_stmt (stmt);
6770 18 : return;
6771 : default:
6772 : break;
6773 : }
6774 0 : gcc_unreachable ();
6775 : }
6776 :
6777 : /* Helper for walk_non_aliased_vuses. Determine if we arrived at
6778 : the desired memory state. */
6779 :
6780 : void *
6781 2186 : vuse_eq (ao_ref *, tree vuse1, void *data)
6782 : {
6783 2186 : tree vuse2 = (tree) data;
6784 2186 : if (vuse1 == vuse2)
6785 820 : return data;
6786 :
6787 : return NULL;
6788 : }
6789 :
6790 : /* Return true if STMT uses a library function and needs to take
6791 : address of its inputs. We need to avoid bit-fields in those
6792 : cases. Similarly, we need to avoid overlap between destination
6793 : and source limb arrays. */
6794 :
6795 : bool
6796 15083 : stmt_needs_operand_addr (gimple *stmt)
6797 : {
6798 15083 : if (is_gimple_assign (stmt))
6799 10285 : switch (gimple_assign_rhs_code (stmt))
6800 : {
6801 585 : case MULT_EXPR:
6802 585 : case TRUNC_DIV_EXPR:
6803 585 : case EXACT_DIV_EXPR:
6804 585 : case TRUNC_MOD_EXPR:
6805 585 : case FLOAT_EXPR:
6806 585 : return true;
6807 : default:
6808 : break;
6809 : }
6810 4798 : else if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
6811 46 : switch (gimple_call_internal_fn (stmt))
6812 : {
6813 17 : case IFN_MUL_OVERFLOW:
6814 17 : case IFN_UBSAN_CHECK_MUL:
6815 17 : return true;
6816 : /* These two actually don't take address, but reshuffle
6817 : all bytes or bits, so need similar treatment. */
6818 4 : case IFN_BSWAP:
6819 4 : case IFN_BITREVERSE:
6820 4 : return true;
6821 : default:
6822 : break;
6823 : }
6824 : return false;
6825 : }
6826 :
6827 : /* Dominator walker used to discover which large/huge _BitInt
6828 : loads could be sunk into all their uses. */
6829 :
6830 624 : class bitint_dom_walker : public dom_walker
6831 : {
6832 : public:
6833 312 : bitint_dom_walker (bitmap names, bitmap loads)
6834 624 : : dom_walker (CDI_DOMINATORS), m_names (names), m_loads (loads) {}
6835 :
6836 : edge before_dom_children (basic_block) final override;
6837 :
6838 : private:
6839 : bitmap m_names, m_loads;
6840 : };
6841 :
6842 : edge
6843 4508 : bitint_dom_walker::before_dom_children (basic_block bb)
6844 : {
6845 4508 : gphi *phi = get_virtual_phi (bb);
6846 4508 : tree vop;
6847 4508 : if (phi)
6848 794 : vop = gimple_phi_result (phi);
6849 3714 : else if (bb == ENTRY_BLOCK_PTR_FOR_FN (cfun))
6850 : vop = NULL_TREE;
6851 : else
6852 3402 : vop = (tree) get_immediate_dominator (CDI_DOMINATORS, bb)->aux;
6853 :
6854 4508 : auto_vec<tree, 16> worklist;
6855 9016 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
6856 19980 : !gsi_end_p (gsi); gsi_next (&gsi))
6857 : {
6858 15472 : gimple *stmt = gsi_stmt (gsi);
6859 15472 : if (is_gimple_debug (stmt))
6860 2764 : continue;
6861 :
6862 15517 : if (!vop && gimple_vuse (stmt))
6863 : vop = gimple_vuse (stmt);
6864 :
6865 15083 : tree cvop = vop;
6866 28308 : if (gimple_vdef (stmt))
6867 : vop = gimple_vdef (stmt);
6868 :
6869 15083 : tree lhs = gimple_get_lhs (stmt);
6870 17458 : if (lhs
6871 10958 : && TREE_CODE (lhs) == SSA_NAME
6872 8737 : && BITINT_TYPE_P (TREE_TYPE (lhs))
6873 5860 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
6874 20795 : && !bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
6875 : /* If lhs of stmt is large/huge _BitInt SSA_NAME not in m_names,
6876 : it means it will be handled in a loop or straight line code
6877 : at the location of its (ultimate) immediate use, so for
6878 : vop checking purposes check these only at the ultimate
6879 : immediate use. */
6880 2375 : continue;
6881 :
6882 12708 : ssa_op_iter oi;
6883 12708 : use_operand_p use_p;
6884 21548 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, oi, SSA_OP_USE)
6885 : {
6886 8840 : tree s = USE_FROM_PTR (use_p);
6887 14260 : if (BITINT_TYPE_P (TREE_TYPE (s))
6888 8840 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
6889 3265 : worklist.safe_push (s);
6890 : }
6891 :
6892 12708 : bool needs_operand_addr = stmt_needs_operand_addr (stmt);
6893 31514 : while (worklist.length () > 0)
6894 : {
6895 6098 : tree s = worklist.pop ();
6896 :
6897 6098 : if (!bitmap_bit_p (m_names, SSA_NAME_VERSION (s)))
6898 : {
6899 2375 : gimple *g = SSA_NAME_DEF_STMT (s);
6900 2375 : needs_operand_addr |= stmt_needs_operand_addr (g);
6901 5447 : FOR_EACH_SSA_USE_OPERAND (use_p, g, oi, SSA_OP_USE)
6902 : {
6903 3072 : tree s2 = USE_FROM_PTR (use_p);
6904 3262 : if (BITINT_TYPE_P (TREE_TYPE (s2))
6905 3072 : && (bitint_precision_kind (TREE_TYPE (s2))
6906 : >= bitint_prec_large))
6907 2833 : worklist.safe_push (s2);
6908 : }
6909 3095 : continue;
6910 2375 : }
6911 3723 : if (!SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
6912 3723 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (s)))
6913 : {
6914 226 : tree rhs = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s));
6915 396 : if (TREE_CODE (rhs) == SSA_NAME
6916 226 : && bitmap_bit_p (m_loads, SSA_NAME_VERSION (rhs)))
6917 : s = rhs;
6918 : else
6919 170 : continue;
6920 : }
6921 3497 : else if (!bitmap_bit_p (m_loads, SSA_NAME_VERSION (s)))
6922 548 : continue;
6923 :
6924 3005 : gimple *g = SSA_NAME_DEF_STMT (s);
6925 3005 : tree rhs1 = gimple_assign_rhs1 (g);
6926 3005 : if (needs_operand_addr
6927 218 : && TREE_CODE (rhs1) == COMPONENT_REF
6928 3022 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (rhs1, 1)))
6929 : {
6930 4 : tree fld = TREE_OPERAND (rhs1, 1);
6931 : /* For little-endian, we can allow as inputs bit-fields
6932 : which start at a limb boundary. */
6933 6 : if (!bitint_big_endian
6934 4 : && DECL_OFFSET_ALIGN (fld) >= TYPE_ALIGN (TREE_TYPE (rhs1))
6935 4 : && tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld))
6936 4 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
6937 4 : % limb_prec) == 0)
6938 : ;
6939 : else
6940 : {
6941 2 : bitmap_clear_bit (m_loads, SSA_NAME_VERSION (s));
6942 2 : continue;
6943 : }
6944 : }
6945 :
6946 3003 : ao_ref ref;
6947 3003 : ao_ref_init (&ref, rhs1);
6948 3003 : tree lvop = gimple_vuse (g);
6949 3003 : unsigned limit = 64;
6950 3003 : tree vuse = cvop;
6951 3003 : if (vop != cvop
6952 1334 : && is_gimple_assign (stmt)
6953 1332 : && gimple_store_p (stmt)
6954 4335 : && (needs_operand_addr
6955 1139 : || !operand_equal_p (lhs, gimple_assign_rhs1 (g), 0)))
6956 : vuse = vop;
6957 3003 : if (vuse != lvop
6958 3003 : && walk_non_aliased_vuses (&ref, vuse, false, vuse_eq,
6959 : NULL, NULL, NULL, limit, lvop) == NULL)
6960 528 : bitmap_clear_bit (m_loads, SSA_NAME_VERSION (s));
6961 : }
6962 : }
6963 :
6964 4508 : bb->aux = (void *) vop;
6965 4508 : return NULL;
6966 4508 : }
6967 :
6968 : }
6969 :
6970 : /* Replacement for normal processing of STMT in tree-ssa-coalesce.cc
6971 : build_ssa_conflict_graph.
6972 : The differences are:
6973 : 1) don't process assignments with large/huge _BitInt lhs not in NAMES
6974 : 2) for large/huge _BitInt multiplication/division/modulo process def
6975 : only after processing uses rather than before to make uses conflict
6976 : with the definition
6977 : 3) for large/huge _BitInt uses not in NAMES mark the uses of their
6978 : SSA_NAME_DEF_STMT (recursively), because those uses will be sunk into
6979 : the final statement. */
6980 :
6981 : void
6982 84248 : build_bitint_stmt_ssa_conflicts (gimple *stmt, live_track *live,
6983 : ssa_conflicts *graph, bitmap names,
6984 : void (*def) (live_track *, tree,
6985 : ssa_conflicts *),
6986 : void (*use) (live_track *, tree),
6987 : void (*clear) (live_track *, tree))
6988 : {
6989 84248 : bool muldiv_p = false;
6990 84248 : tree lhs = NULL_TREE;
6991 84248 : if (is_gimple_assign (stmt))
6992 : {
6993 46254 : lhs = gimple_assign_lhs (stmt);
6994 46254 : if (TREE_CODE (lhs) == SSA_NAME)
6995 : {
6996 33434 : tree type = TREE_TYPE (lhs);
6997 33434 : if (TREE_CODE (type) == COMPLEX_TYPE)
6998 63 : type = TREE_TYPE (type);
6999 12866 : if (BITINT_TYPE_P (type)
7000 33439 : && bitint_precision_kind (type) >= bitint_prec_large)
7001 : {
7002 19955 : if (!bitmap_bit_p (names, SSA_NAME_VERSION (lhs)))
7003 4894 : return;
7004 :
7005 : /* A copy between 2 partitions does not introduce an interference
7006 : by itself. If they did, you would never be able to coalesce
7007 : two things which are copied. If the two variables really do
7008 : conflict, they will conflict elsewhere in the program.
7009 :
7010 : This is handled by simply removing the SRC of the copy from
7011 : the live list, and processing the stmt normally.
7012 :
7013 : Don't do this if lhs is not in names though, in such cases
7014 : it is actually used at some point later in the basic
7015 : block. */
7016 15061 : if (gimple_assign_copy_p (stmt))
7017 : {
7018 1686 : tree rhs1 = gimple_assign_rhs1 (stmt);
7019 1686 : if (TREE_CODE (rhs1) == SSA_NAME)
7020 45 : clear (live, rhs1);
7021 : }
7022 :
7023 15061 : switch (gimple_assign_rhs_code (stmt))
7024 : {
7025 166 : case MULT_EXPR:
7026 166 : case TRUNC_DIV_EXPR:
7027 166 : case EXACT_DIV_EXPR:
7028 166 : case TRUNC_MOD_EXPR:
7029 166 : muldiv_p = true;
7030 : default:
7031 : break;
7032 : }
7033 : }
7034 : }
7035 : }
7036 37994 : else if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
7037 4301 : switch (gimple_call_internal_fn (stmt))
7038 : {
7039 2699 : case IFN_ADD_OVERFLOW:
7040 2699 : case IFN_SUB_OVERFLOW:
7041 2699 : case IFN_UBSAN_CHECK_ADD:
7042 2699 : case IFN_UBSAN_CHECK_SUB:
7043 2699 : if (bitint_big_endian)
7044 : {
7045 0 : lhs = gimple_call_lhs (stmt);
7046 0 : if (lhs)
7047 79354 : muldiv_p = true;
7048 : }
7049 : break;
7050 1478 : case IFN_MUL_OVERFLOW:
7051 1478 : case IFN_UBSAN_CHECK_MUL:
7052 1478 : case IFN_BSWAP:
7053 1478 : case IFN_BITREVERSE:
7054 1478 : lhs = gimple_call_lhs (stmt);
7055 1478 : if (lhs)
7056 79354 : muldiv_p = true;
7057 : break;
7058 : default:
7059 : break;
7060 : }
7061 :
7062 158708 : auto_vec<tree, 16> worklist;
7063 79354 : ssa_op_iter iter;
7064 79354 : tree var;
7065 : /* On little-endian, mergeable ops process limbs from 0 up so except
7066 : for multiplication/division/modulo there is no risk in using the
7067 : same underlying variable for lhs and some operand, even when casts
7068 : are involved, the lhs limb is stored only after processing the source
7069 : limbs with the same index.
7070 : For multiplication/division/modulo, the libgcc library function requires
7071 : no aliasing between result and sources.
7072 : On big-endian, even mergeable ops limb processing can be problematic
7073 : though, because it can apply various index corrections e.g. when there
7074 : is a cast from operand with different number of limbs. So, make the
7075 : lhs conflict with all the operands which are (for now virtually) used on
7076 : the current stmt if there is any mismatch in the number of limbs between
7077 : operands and the lhs. */
7078 79354 : if (bitint_big_endian && lhs && !muldiv_p)
7079 : {
7080 0 : tree ltype = TREE_TYPE (lhs);
7081 0 : if (TREE_CODE (ltype) == COMPLEX_TYPE)
7082 : muldiv_p = true;
7083 0 : else if (TREE_CODE (lhs) == SSA_NAME
7084 0 : && BITINT_TYPE_P (ltype)
7085 0 : && bitint_precision_kind (ltype) >= bitint_prec_large)
7086 : {
7087 0 : unsigned lnelts = CEIL (TYPE_PRECISION (ltype), limb_prec);
7088 0 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_USE)
7089 : {
7090 0 : tree type = TREE_TYPE (var);
7091 0 : if (TREE_CODE (type) == COMPLEX_TYPE)
7092 0 : type = TREE_TYPE (type);
7093 0 : if (BITINT_TYPE_P (type)
7094 0 : && bitint_precision_kind (type) >= bitint_prec_large)
7095 : {
7096 0 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7097 : {
7098 0 : unsigned nelts = CEIL (TYPE_PRECISION (type), limb_prec);
7099 0 : if (TREE_CODE (TREE_TYPE (var)) == COMPLEX_TYPE
7100 0 : || lnelts != nelts)
7101 : {
7102 0 : muldiv_p = true;
7103 : break;
7104 : }
7105 : }
7106 : else
7107 0 : worklist.safe_push (var);
7108 : }
7109 : }
7110 :
7111 0 : while (!muldiv_p && worklist.length () > 0)
7112 : {
7113 0 : tree s = worklist.pop ();
7114 0 : FOR_EACH_SSA_TREE_OPERAND (var, SSA_NAME_DEF_STMT (s), iter,
7115 : SSA_OP_USE)
7116 : {
7117 0 : tree type = TREE_TYPE (var);
7118 0 : if (TREE_CODE (type) == COMPLEX_TYPE)
7119 0 : type = TREE_TYPE (type);
7120 0 : if (BITINT_TYPE_P (type)
7121 0 : && bitint_precision_kind (type) >= bitint_prec_large)
7122 : {
7123 0 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7124 : {
7125 0 : unsigned nelts = CEIL (TYPE_PRECISION (type),
7126 : limb_prec);
7127 0 : if (TREE_CODE (TREE_TYPE (var)) == COMPLEX_TYPE
7128 0 : || lnelts != nelts)
7129 : {
7130 : muldiv_p = true;
7131 : break;
7132 : }
7133 : }
7134 : else
7135 0 : worklist.safe_push (var);
7136 : }
7137 : }
7138 : }
7139 0 : worklist.truncate (0);
7140 : }
7141 : }
7142 :
7143 79354 : if (!muldiv_p)
7144 : {
7145 : /* For stmts with more than one SSA_NAME definition pretend all the
7146 : SSA_NAME outputs but the first one are live at this point, so
7147 : that conflicts are added in between all those even when they are
7148 : actually not really live after the asm, because expansion might
7149 : copy those into pseudos after the asm and if multiple outputs
7150 : share the same partition, it might overwrite those that should
7151 : be live. E.g.
7152 : asm volatile (".." : "=r" (a) : "=r" (b) : "0" (a), "1" (a));
7153 : return a;
7154 : See PR70593. */
7155 77710 : bool first = true;
7156 113735 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
7157 36025 : if (first)
7158 : first = false;
7159 : else
7160 0 : use (live, var);
7161 :
7162 113735 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
7163 36025 : def (live, var, graph);
7164 : }
7165 :
7166 135285 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_USE)
7167 : {
7168 55931 : tree type = TREE_TYPE (var);
7169 55931 : if (TREE_CODE (type) == COMPLEX_TYPE)
7170 6313 : type = TREE_TYPE (type);
7171 18877 : if (BITINT_TYPE_P (type)
7172 55948 : && bitint_precision_kind (type) >= bitint_prec_large)
7173 : {
7174 36072 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7175 30974 : use (live, var);
7176 : else
7177 5098 : worklist.safe_push (var);
7178 : }
7179 : }
7180 :
7181 87449 : while (worklist.length () > 0)
7182 : {
7183 8095 : tree s = worklist.pop ();
7184 16928 : FOR_EACH_SSA_TREE_OPERAND (var, SSA_NAME_DEF_STMT (s), iter, SSA_OP_USE)
7185 : {
7186 8833 : tree type = TREE_TYPE (var);
7187 8833 : if (TREE_CODE (type) == COMPLEX_TYPE)
7188 1675 : type = TREE_TYPE (type);
7189 536 : if (BITINT_TYPE_P (type)
7190 8838 : && bitint_precision_kind (type) >= bitint_prec_large)
7191 : {
7192 8167 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7193 5170 : use (live, var);
7194 : else
7195 2997 : worklist.safe_push (var);
7196 : }
7197 : }
7198 : }
7199 :
7200 79354 : if (muldiv_p)
7201 1644 : def (live, lhs, graph);
7202 : }
7203 :
7204 : /* If STMT is .{ADD,SUB,MUL}_OVERFLOW with INTEGER_CST arguments,
7205 : return the largest bitint_prec_kind of them, otherwise return
7206 : bitint_prec_small. */
7207 :
7208 : static bitint_prec_kind
7209 210929 : arith_overflow_arg_kind (gimple *stmt)
7210 : {
7211 210929 : bitint_prec_kind ret = bitint_prec_small;
7212 210929 : if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
7213 88560 : switch (gimple_call_internal_fn (stmt))
7214 : {
7215 : case IFN_ADD_OVERFLOW:
7216 : case IFN_SUB_OVERFLOW:
7217 : case IFN_MUL_OVERFLOW:
7218 224763 : for (int i = 0; i < 2; ++i)
7219 : {
7220 149842 : tree a = gimple_call_arg (stmt, i);
7221 149842 : if (TREE_CODE (a) == INTEGER_CST
7222 149842 : && BITINT_TYPE_P (TREE_TYPE (a)))
7223 : {
7224 5930 : bitint_prec_kind kind = bitint_precision_kind (TREE_TYPE (a));
7225 149842 : ret = MAX (ret, kind);
7226 : }
7227 : }
7228 : break;
7229 : default:
7230 : break;
7231 : }
7232 210929 : return ret;
7233 : }
7234 :
7235 : /* Entry point for _BitInt(N) operation lowering during optimization. */
7236 :
7237 : static unsigned int
7238 1516013 : gimple_lower_bitint (void)
7239 : {
7240 1516013 : small_max_prec = mid_min_prec = large_min_prec = huge_min_prec = 0;
7241 1516013 : limb_prec = abi_limb_prec = 0;
7242 1516013 : bitint_big_endian = false;
7243 :
7244 1516013 : unsigned int i;
7245 65026680 : for (i = 0; i < num_ssa_names; ++i)
7246 : {
7247 63518021 : tree s = ssa_name (i);
7248 63518021 : if (s == NULL)
7249 13588188 : continue;
7250 49929833 : tree type = TREE_TYPE (s);
7251 49929833 : if (TREE_CODE (type) == COMPLEX_TYPE)
7252 : {
7253 200253 : if (arith_overflow_arg_kind (SSA_NAME_DEF_STMT (s))
7254 : != bitint_prec_small)
7255 : break;
7256 200140 : type = TREE_TYPE (type);
7257 : }
7258 49912907 : if (BITINT_TYPE_P (type)
7259 49929727 : && bitint_precision_kind (type) != bitint_prec_small)
7260 : break;
7261 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
7262 : into memory. Such functions could have no large/huge SSA_NAMEs. */
7263 49922547 : if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
7264 : {
7265 21825038 : gimple *g = SSA_NAME_DEF_STMT (s);
7266 21825038 : if (is_gimple_assign (g) && gimple_store_p (g))
7267 : {
7268 11069958 : tree t = gimple_assign_rhs1 (g);
7269 22137464 : if (BITINT_TYPE_P (TREE_TYPE (t))
7270 11069958 : && (bitint_precision_kind (TREE_TYPE (t))
7271 : >= bitint_prec_large))
7272 : break;
7273 : }
7274 : }
7275 : /* Similarly, e.g. with -frounding-math casts from _BitInt INTEGER_CSTs
7276 : to floating point types need to be rewritten. */
7277 28097509 : else if (SCALAR_FLOAT_TYPE_P (type))
7278 : {
7279 2326470 : gimple *g = SSA_NAME_DEF_STMT (s);
7280 2326470 : if (is_gimple_assign (g) && gimple_assign_rhs_code (g) == FLOAT_EXPR)
7281 : {
7282 128345 : tree t = gimple_assign_rhs1 (g);
7283 128345 : if (TREE_CODE (t) == INTEGER_CST
7284 110 : && BITINT_TYPE_P (TREE_TYPE (t))
7285 128346 : && (bitint_precision_kind (TREE_TYPE (t))
7286 : != bitint_prec_small))
7287 : break;
7288 : }
7289 : }
7290 : }
7291 3032026 : if (i == num_ssa_names)
7292 : return 0;
7293 :
7294 7354 : basic_block bb;
7295 7354 : auto_vec<gimple *, 4> switch_statements;
7296 46170 : FOR_EACH_BB_FN (bb, cfun)
7297 : {
7298 115682 : if (gswitch *swtch = safe_dyn_cast <gswitch *> (*gsi_last_bb (bb)))
7299 : {
7300 23 : tree idx = gimple_switch_index (swtch);
7301 31 : if (!BITINT_TYPE_P (TREE_TYPE (idx))
7302 23 : || bitint_precision_kind (TREE_TYPE (idx)) < bitint_prec_large)
7303 12 : continue;
7304 :
7305 11 : if (optimize)
7306 6 : group_case_labels_stmt (swtch);
7307 11 : if (gimple_switch_num_labels (swtch) == 1)
7308 : {
7309 0 : single_succ_edge (bb)->flags |= EDGE_FALLTHRU;
7310 0 : gimple_stmt_iterator gsi = gsi_for_stmt (swtch);
7311 0 : gsi_remove (&gsi, true);
7312 : }
7313 : else
7314 11 : switch_statements.safe_push (swtch);
7315 : }
7316 : }
7317 :
7318 7354 : if (!switch_statements.is_empty ())
7319 : {
7320 11 : bool expanded = false;
7321 11 : gimple *stmt;
7322 11 : unsigned int j;
7323 11 : i = 0;
7324 22 : FOR_EACH_VEC_ELT (switch_statements, j, stmt)
7325 : {
7326 11 : gswitch *swtch = as_a<gswitch *> (stmt);
7327 11 : tree_switch_conversion::switch_decision_tree dt (swtch);
7328 11 : expanded |= dt.analyze_switch_statement ();
7329 11 : }
7330 :
7331 11 : if (expanded)
7332 : {
7333 11 : free_dominance_info (CDI_DOMINATORS);
7334 11 : free_dominance_info (CDI_POST_DOMINATORS);
7335 11 : mark_virtual_operands_for_renaming (cfun);
7336 11 : cleanup_tree_cfg (TODO_update_ssa);
7337 : }
7338 : }
7339 :
7340 7354 : if (flag_sanitize & (SANITIZE_ADDRESS | SANITIZE_HWADDRESS))
7341 : {
7342 14 : hash_map<tree, tree> shadow_vars_mapping;
7343 14 : bool need_commit_edge_insert = false;
7344 14 : bool any_asan_poison = false;
7345 129 : for (unsigned j = 0; j < num_ssa_names; ++j)
7346 : {
7347 115 : tree s = ssa_name (j);
7348 115 : if (s == NULL)
7349 31 : continue;
7350 84 : tree type = TREE_TYPE (s);
7351 37 : if (BITINT_TYPE_P (type)
7352 47 : && gimple_call_internal_p (SSA_NAME_DEF_STMT (s),
7353 : IFN_ASAN_POISON)
7354 112 : && bitint_precision_kind (type) >= bitint_prec_large)
7355 : {
7356 28 : any_asan_poison = true;
7357 28 : gimple_stmt_iterator gsi = gsi_for_stmt (SSA_NAME_DEF_STMT (s));
7358 28 : asan_expand_poison_ifn (&gsi, &need_commit_edge_insert,
7359 : shadow_vars_mapping);
7360 : }
7361 : }
7362 14 : if (any_asan_poison)
7363 : {
7364 14 : if (need_commit_edge_insert)
7365 0 : gsi_commit_edge_inserts ();
7366 14 : i = 0;
7367 14 : free_dominance_info (CDI_DOMINATORS);
7368 14 : free_dominance_info (CDI_POST_DOMINATORS);
7369 14 : mark_virtual_operands_for_renaming (cfun);
7370 14 : cleanup_tree_cfg (TODO_update_ssa);
7371 : }
7372 14 : }
7373 :
7374 7354 : struct bitint_large_huge large_huge;
7375 7354 : bool has_large_huge_parm_result = false;
7376 7354 : bool has_large_huge = false;
7377 7354 : unsigned int ret = 0, first_large_huge = ~0U;
7378 7354 : bool edge_insertions = false;
7379 126620 : for (; i < num_ssa_names; ++i)
7380 : {
7381 119266 : tree s = ssa_name (i);
7382 119266 : if (s == NULL)
7383 2793 : continue;
7384 116473 : tree type = TREE_TYPE (s);
7385 116473 : if (TREE_CODE (type) == COMPLEX_TYPE)
7386 : {
7387 5353 : if (arith_overflow_arg_kind (SSA_NAME_DEF_STMT (s))
7388 : >= bitint_prec_large)
7389 1958 : has_large_huge = true;
7390 5353 : type = TREE_TYPE (type);
7391 : }
7392 70309 : if (BITINT_TYPE_P (type)
7393 116499 : && bitint_precision_kind (type) >= bitint_prec_large)
7394 : {
7395 36158 : if (first_large_huge == ~0U)
7396 5811 : first_large_huge = i;
7397 36158 : gimple *stmt = SSA_NAME_DEF_STMT (s), *g;
7398 36158 : gimple_stmt_iterator gsi;
7399 36158 : tree_code rhs_code;
7400 : /* Unoptimize certain constructs to simpler alternatives to
7401 : avoid having to lower all of them. */
7402 36158 : if (is_gimple_assign (stmt) && gimple_bb (stmt))
7403 22268 : switch (rhs_code = gimple_assign_rhs_code (stmt))
7404 : {
7405 : default:
7406 : break;
7407 371 : case MULT_EXPR:
7408 371 : case TRUNC_DIV_EXPR:
7409 371 : case EXACT_DIV_EXPR:
7410 371 : case TRUNC_MOD_EXPR:
7411 371 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s))
7412 : {
7413 2 : location_t loc = gimple_location (stmt);
7414 2 : gsi = gsi_for_stmt (stmt);
7415 2 : tree rhs1 = gimple_assign_rhs1 (stmt);
7416 2 : tree rhs2 = gimple_assign_rhs2 (stmt);
7417 : /* For multiplication and division with (ab)
7418 : lhs and one or both operands force the operands
7419 : into new SSA_NAMEs to avoid coalescing failures. */
7420 2 : if (TREE_CODE (rhs1) == SSA_NAME
7421 2 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1))
7422 : {
7423 2 : first_large_huge = 0;
7424 2 : tree t = make_ssa_name (TREE_TYPE (rhs1));
7425 2 : g = gimple_build_assign (t, SSA_NAME, rhs1);
7426 2 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7427 2 : gimple_set_location (g, loc);
7428 2 : gimple_assign_set_rhs1 (stmt, t);
7429 2 : if (rhs1 == rhs2)
7430 : {
7431 0 : gimple_assign_set_rhs2 (stmt, t);
7432 0 : rhs2 = t;
7433 : }
7434 2 : update_stmt (stmt);
7435 : }
7436 2 : if (TREE_CODE (rhs2) == SSA_NAME
7437 2 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs2))
7438 : {
7439 0 : first_large_huge = 0;
7440 0 : tree t = make_ssa_name (TREE_TYPE (rhs2));
7441 0 : g = gimple_build_assign (t, SSA_NAME, rhs2);
7442 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7443 0 : gimple_set_location (g, loc);
7444 0 : gimple_assign_set_rhs2 (stmt, t);
7445 0 : update_stmt (stmt);
7446 : }
7447 : }
7448 : break;
7449 3 : case LROTATE_EXPR:
7450 3 : case RROTATE_EXPR:
7451 3 : {
7452 3 : first_large_huge = 0;
7453 3 : location_t loc = gimple_location (stmt);
7454 3 : gsi = gsi_for_stmt (stmt);
7455 3 : tree rhs1 = gimple_assign_rhs1 (stmt);
7456 3 : tree type = TREE_TYPE (rhs1);
7457 3 : tree n = gimple_assign_rhs2 (stmt), m;
7458 3 : tree p = build_int_cst (TREE_TYPE (n),
7459 3 : TYPE_PRECISION (type));
7460 3 : if (TREE_CODE (n) == INTEGER_CST)
7461 : {
7462 0 : if (integer_zerop (n))
7463 : m = n;
7464 : else
7465 0 : m = fold_build2 (MINUS_EXPR, TREE_TYPE (n), p, n);
7466 : }
7467 : else
7468 : {
7469 3 : tree tem = make_ssa_name (TREE_TYPE (n));
7470 3 : g = gimple_build_assign (tem, MINUS_EXPR, p, n);
7471 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7472 3 : gimple_set_location (g, loc);
7473 3 : m = make_ssa_name (TREE_TYPE (n));
7474 3 : g = gimple_build_assign (m, TRUNC_MOD_EXPR, tem, p);
7475 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7476 3 : gimple_set_location (g, loc);
7477 : }
7478 3 : if (!TYPE_UNSIGNED (type))
7479 : {
7480 0 : tree utype = build_bitint_type (TYPE_PRECISION (type),
7481 : 1);
7482 0 : if (TREE_CODE (rhs1) == INTEGER_CST)
7483 0 : rhs1 = fold_convert (utype, rhs1);
7484 : else
7485 : {
7486 0 : tree t = make_ssa_name (type);
7487 0 : g = gimple_build_assign (t, NOP_EXPR, rhs1);
7488 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7489 0 : gimple_set_location (g, loc);
7490 : }
7491 : }
7492 4 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7493 : rhs_code == LROTATE_EXPR
7494 : ? LSHIFT_EXPR : RSHIFT_EXPR,
7495 : rhs1, n);
7496 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7497 3 : gimple_set_location (g, loc);
7498 3 : tree op1 = gimple_assign_lhs (g);
7499 4 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7500 : rhs_code == LROTATE_EXPR
7501 : ? RSHIFT_EXPR : LSHIFT_EXPR,
7502 : rhs1, m);
7503 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7504 3 : gimple_set_location (g, loc);
7505 3 : tree op2 = gimple_assign_lhs (g);
7506 3 : tree lhs = gimple_assign_lhs (stmt);
7507 3 : if (!TYPE_UNSIGNED (type))
7508 : {
7509 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (op1)),
7510 : BIT_IOR_EXPR, op1, op2);
7511 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7512 0 : gimple_set_location (g, loc);
7513 0 : g = gimple_build_assign (lhs, NOP_EXPR,
7514 : gimple_assign_lhs (g));
7515 : }
7516 : else
7517 3 : g = gimple_build_assign (lhs, BIT_IOR_EXPR, op1, op2);
7518 3 : gsi_replace (&gsi, g, true);
7519 3 : gimple_set_location (g, loc);
7520 : }
7521 3 : break;
7522 17 : case ABS_EXPR:
7523 17 : case ABSU_EXPR:
7524 17 : case MIN_EXPR:
7525 17 : case MAX_EXPR:
7526 17 : case COND_EXPR:
7527 17 : first_large_huge = 0;
7528 17 : gsi = gsi_for_stmt (stmt);
7529 17 : tree lhs = gimple_assign_lhs (stmt);
7530 17 : tree rhs1 = gimple_assign_rhs1 (stmt), rhs2 = NULL_TREE;
7531 17 : location_t loc = gimple_location (stmt);
7532 17 : if (rhs_code == ABS_EXPR)
7533 4 : g = gimple_build_cond (LT_EXPR, rhs1,
7534 4 : build_zero_cst (TREE_TYPE (rhs1)),
7535 : NULL_TREE, NULL_TREE);
7536 13 : else if (rhs_code == ABSU_EXPR)
7537 : {
7538 4 : rhs2 = make_ssa_name (TREE_TYPE (lhs));
7539 4 : g = gimple_build_assign (rhs2, NOP_EXPR, rhs1);
7540 4 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7541 4 : gimple_set_location (g, loc);
7542 4 : g = gimple_build_cond (LT_EXPR, rhs1,
7543 4 : build_zero_cst (TREE_TYPE (rhs1)),
7544 : NULL_TREE, NULL_TREE);
7545 4 : rhs1 = rhs2;
7546 : }
7547 9 : else if (rhs_code == MIN_EXPR || rhs_code == MAX_EXPR)
7548 : {
7549 9 : rhs2 = gimple_assign_rhs2 (stmt);
7550 9 : if (TREE_CODE (rhs1) == INTEGER_CST)
7551 0 : std::swap (rhs1, rhs2);
7552 9 : g = gimple_build_cond (LT_EXPR, rhs1, rhs2,
7553 : NULL_TREE, NULL_TREE);
7554 9 : if (rhs_code == MAX_EXPR)
7555 5 : std::swap (rhs1, rhs2);
7556 : }
7557 : else
7558 : {
7559 0 : g = gimple_build_cond (NE_EXPR, rhs1,
7560 0 : build_zero_cst (TREE_TYPE (rhs1)),
7561 : NULL_TREE, NULL_TREE);
7562 0 : rhs1 = gimple_assign_rhs2 (stmt);
7563 0 : rhs2 = gimple_assign_rhs3 (stmt);
7564 : }
7565 17 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7566 17 : gimple_set_location (g, loc);
7567 17 : edge e1 = split_block (gsi_bb (gsi), g);
7568 17 : edge e2 = split_block (e1->dest, (gimple *) NULL);
7569 17 : edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
7570 17 : e3->probability = profile_probability::even ();
7571 17 : e1->flags = EDGE_TRUE_VALUE;
7572 17 : e1->probability = e3->probability.invert ();
7573 17 : if (dom_info_available_p (CDI_DOMINATORS))
7574 9 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
7575 17 : if (rhs_code == ABS_EXPR || rhs_code == ABSU_EXPR)
7576 : {
7577 8 : gsi = gsi_after_labels (e1->dest);
7578 8 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7579 : NEGATE_EXPR, rhs1);
7580 8 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7581 8 : gimple_set_location (g, loc);
7582 8 : rhs2 = gimple_assign_lhs (g);
7583 8 : std::swap (rhs1, rhs2);
7584 : }
7585 17 : gsi = gsi_for_stmt (stmt);
7586 17 : gsi_remove (&gsi, true);
7587 17 : gphi *phi = create_phi_node (lhs, e2->dest);
7588 17 : add_phi_arg (phi, rhs1, e2, UNKNOWN_LOCATION);
7589 17 : add_phi_arg (phi, rhs2, e3, UNKNOWN_LOCATION);
7590 17 : break;
7591 : }
7592 13890 : else if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
7593 76 : && is_gimple_call (stmt)
7594 13910 : && gimple_call_internal_p (stmt))
7595 20 : switch (gimple_call_internal_fn (stmt))
7596 : {
7597 12 : case IFN_UBSAN_CHECK_ADD:
7598 12 : case IFN_UBSAN_CHECK_SUB:
7599 12 : if (!bitint_big_endian)
7600 : break;
7601 : /* FALLTHRU */
7602 : case IFN_UBSAN_CHECK_MUL:
7603 : case IFN_BSWAP:
7604 : case IFN_BITREVERSE:
7605 22 : for (unsigned i = 0; i < gimple_call_num_args (stmt); ++i)
7606 : {
7607 14 : location_t loc = gimple_location (stmt);
7608 14 : gsi = gsi_for_stmt (stmt);
7609 : /* Similar case to multiplication/division with (ab)
7610 : above for internal functions which set muldiv_p. */
7611 14 : tree arg = gimple_call_arg (stmt, i);
7612 14 : if (TREE_CODE (arg) == SSA_NAME
7613 14 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg))
7614 : {
7615 8 : first_large_huge = 0;
7616 8 : tree t = make_ssa_name (TREE_TYPE (arg));
7617 8 : g = gimple_build_assign (t, SSA_NAME, arg);
7618 8 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7619 8 : gimple_set_location (g, loc);
7620 8 : gimple_call_set_arg (stmt, i, t);
7621 8 : if (i == 0
7622 8 : && gimple_call_num_args (stmt) >= 2
7623 14 : && gimple_call_arg (stmt, 1) == arg)
7624 0 : gimple_call_set_arg (stmt, 1, t);
7625 8 : update_stmt (stmt);
7626 : }
7627 : }
7628 : break;
7629 : default:
7630 : break;
7631 : }
7632 : }
7633 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
7634 : into memory. Such functions could have no large/huge SSA_NAMEs. */
7635 80315 : else if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
7636 : {
7637 51201 : gimple *g = SSA_NAME_DEF_STMT (s);
7638 51201 : if (is_gimple_assign (g) && gimple_store_p (g))
7639 : {
7640 16382 : tree t = gimple_assign_rhs1 (g);
7641 18565 : if (BITINT_TYPE_P (TREE_TYPE (t))
7642 16382 : && (bitint_precision_kind (TREE_TYPE (t))
7643 : >= bitint_prec_large))
7644 : has_large_huge = true;
7645 : }
7646 : }
7647 : /* Similarly, e.g. with -frounding-math casts from _BitInt INTEGER_CSTs
7648 : to floating point types need to be rewritten. */
7649 29114 : else if (SCALAR_FLOAT_TYPE_P (type))
7650 : {
7651 694 : gimple *g = SSA_NAME_DEF_STMT (s);
7652 694 : if (is_gimple_assign (g) && gimple_assign_rhs_code (g) == FLOAT_EXPR)
7653 : {
7654 187 : tree t = gimple_assign_rhs1 (g);
7655 187 : if (TREE_CODE (t) == INTEGER_CST
7656 1 : && BITINT_TYPE_P (TREE_TYPE (t))
7657 188 : && (bitint_precision_kind (TREE_TYPE (t))
7658 : >= bitint_prec_large))
7659 : has_large_huge = true;
7660 : }
7661 : }
7662 : }
7663 103195 : for (i = first_large_huge; i < num_ssa_names; ++i)
7664 : {
7665 95841 : tree s = ssa_name (i);
7666 95841 : if (s == NULL)
7667 2465 : continue;
7668 93376 : tree type = TREE_TYPE (s);
7669 93376 : if (TREE_CODE (type) == COMPLEX_TYPE)
7670 4124 : type = TREE_TYPE (type);
7671 56580 : if (BITINT_TYPE_P (type)
7672 93400 : && bitint_precision_kind (type) >= bitint_prec_large)
7673 : {
7674 36158 : use_operand_p use_p;
7675 36158 : gimple *use_stmt;
7676 36158 : has_large_huge = true;
7677 37822 : if (optimize
7678 53198 : && optimizable_arith_overflow (SSA_NAME_DEF_STMT (s)))
7679 6572 : continue;
7680 : /* Ignore large/huge _BitInt SSA_NAMEs which have single use in
7681 : the same bb and could be handled in the same loop with the
7682 : immediate use. */
7683 34494 : if (optimize
7684 15376 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
7685 15309 : && single_imm_use (s, &use_p, &use_stmt)
7686 49190 : && gimple_bb (SSA_NAME_DEF_STMT (s)) == gimple_bb (use_stmt))
7687 : {
7688 10334 : if (mergeable_op (SSA_NAME_DEF_STMT (s)))
7689 : {
7690 2219 : if (mergeable_op (use_stmt))
7691 1910 : continue;
7692 309 : tree_code cmp_code = comparison_op (use_stmt, NULL, NULL);
7693 309 : if (cmp_code == EQ_EXPR || cmp_code == NE_EXPR)
7694 26 : continue;
7695 283 : if (gimple_assign_cast_p (use_stmt))
7696 : {
7697 128 : tree lhs = gimple_assign_lhs (use_stmt);
7698 256 : if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
7699 : /* Don't merge with VIEW_CONVERT_EXPRs to
7700 : huge INTEGER_TYPEs used sometimes in memcpy
7701 : expansion. */
7702 244 : && (TREE_CODE (TREE_TYPE (lhs)) != INTEGER_TYPE
7703 6 : || (TYPE_PRECISION (TREE_TYPE (lhs))
7704 12 : <= MAX_FIXED_MODE_SIZE)))
7705 116 : continue;
7706 : }
7707 155 : else if (gimple_store_p (use_stmt)
7708 0 : && is_gimple_assign (use_stmt)
7709 0 : && !gimple_has_volatile_ops (use_stmt)
7710 155 : && !stmt_ends_bb_p (use_stmt))
7711 0 : continue;
7712 : }
7713 8282 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (s)))
7714 : {
7715 862 : tree rhs1 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s));
7716 862 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
7717 : {
7718 20 : rhs1 = TREE_OPERAND (rhs1, 0);
7719 20 : if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
7720 16 : && !POINTER_TYPE_P (TREE_TYPE (rhs1))
7721 16 : && gimple_store_p (use_stmt))
7722 7 : continue;
7723 : }
7724 1710 : if (INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
7725 764 : && ((is_gimple_assign (use_stmt)
7726 717 : && (gimple_assign_rhs_code (use_stmt)
7727 : != COMPLEX_EXPR))
7728 47 : || gimple_code (use_stmt) == GIMPLE_COND)
7729 754 : && (!gimple_store_p (use_stmt)
7730 130 : || (is_gimple_assign (use_stmt)
7731 130 : && !gimple_has_volatile_ops (use_stmt)
7732 130 : && !stmt_ends_bb_p (use_stmt)))
7733 1609 : && (TREE_CODE (rhs1) != SSA_NAME
7734 754 : || !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)))
7735 : {
7736 754 : if (is_gimple_assign (use_stmt))
7737 717 : switch (gimple_assign_rhs_code (use_stmt))
7738 : {
7739 54 : case TRUNC_DIV_EXPR:
7740 54 : case EXACT_DIV_EXPR:
7741 54 : case TRUNC_MOD_EXPR:
7742 54 : case FLOAT_EXPR:
7743 : /* For division, modulo and casts to floating
7744 : point, avoid representing unsigned operands
7745 : using negative prec if they were sign-extended
7746 : from narrower precision. */
7747 54 : if (TYPE_UNSIGNED (TREE_TYPE (s))
7748 28 : && !TYPE_UNSIGNED (TREE_TYPE (rhs1))
7749 63 : && (TYPE_PRECISION (TREE_TYPE (s))
7750 9 : > TYPE_PRECISION (TREE_TYPE (rhs1))))
7751 8 : goto force_name;
7752 : /* FALLTHRU */
7753 113 : case MULT_EXPR:
7754 128 : if (!BITINT_TYPE_P (TREE_TYPE (rhs1))
7755 113 : || (bitint_precision_kind (TREE_TYPE (rhs1))
7756 : < bitint_prec_large))
7757 48 : continue;
7758 : /* Uses which use handle_operand_addr can't
7759 : deal with nested casts. */
7760 65 : if (TREE_CODE (rhs1) == SSA_NAME
7761 65 : && gimple_assign_cast_p
7762 65 : (SSA_NAME_DEF_STMT (rhs1))
7763 43 : && has_single_use (rhs1)
7764 108 : && (gimple_bb (SSA_NAME_DEF_STMT (rhs1))
7765 43 : == gimple_bb (SSA_NAME_DEF_STMT (s))))
7766 43 : goto force_name;
7767 : break;
7768 0 : case VIEW_CONVERT_EXPR:
7769 0 : {
7770 0 : tree lhs = gimple_assign_lhs (use_stmt);
7771 : /* Don't merge with VIEW_CONVERT_EXPRs to
7772 : non-integral types. */
7773 0 : if (!INTEGRAL_TYPE_P (TREE_TYPE (lhs)))
7774 0 : goto force_name;
7775 : /* Don't merge with VIEW_CONVERT_EXPRs to
7776 : huge INTEGER_TYPEs used sometimes in memcpy
7777 : expansion. */
7778 0 : if (TREE_CODE (TREE_TYPE (lhs)) == INTEGER_TYPE
7779 0 : && (TYPE_PRECISION (TREE_TYPE (lhs))
7780 0 : > MAX_FIXED_MODE_SIZE))
7781 0 : goto force_name;
7782 : }
7783 : break;
7784 : default:
7785 : break;
7786 : }
7787 832 : if (!BITINT_TYPE_P (TREE_TYPE (rhs1))
7788 655 : || (bitint_precision_kind (TREE_TYPE (rhs1))
7789 : < bitint_prec_large))
7790 239 : continue;
7791 416 : if ((TYPE_PRECISION (TREE_TYPE (rhs1))
7792 416 : >= TYPE_PRECISION (TREE_TYPE (s)))
7793 416 : && mergeable_op (use_stmt))
7794 60 : continue;
7795 : /* Prevent merging a widening non-mergeable cast
7796 : on result of some narrower mergeable op
7797 : together with later mergeable operations. E.g.
7798 : result of _BitInt(223) addition shouldn't be
7799 : sign-extended to _BitInt(513) and have another
7800 : _BitInt(513) added to it, as handle_plus_minus
7801 : with its PHI node handling inside of handle_cast
7802 : will not work correctly. An exception is if
7803 : use_stmt is a store, this is handled directly
7804 : in lower_mergeable_stmt. */
7805 705 : if (TREE_CODE (rhs1) != SSA_NAME
7806 356 : || !has_single_use (rhs1)
7807 270 : || (gimple_bb (SSA_NAME_DEF_STMT (rhs1))
7808 270 : != gimple_bb (SSA_NAME_DEF_STMT (s)))
7809 214 : || !mergeable_op (SSA_NAME_DEF_STMT (rhs1))
7810 441 : || gimple_store_p (use_stmt))
7811 349 : continue;
7812 7 : if ((TYPE_PRECISION (TREE_TYPE (rhs1))
7813 7 : < TYPE_PRECISION (TREE_TYPE (s)))
7814 9 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (rhs1)))
7815 : {
7816 : /* Another exception is if the widening cast is
7817 : from mergeable same precision cast from something
7818 : not mergeable. */
7819 0 : tree rhs2
7820 0 : = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (rhs1));
7821 0 : if (BITINT_TYPE_P (TREE_TYPE (rhs2))
7822 0 : && (TYPE_PRECISION (TREE_TYPE (rhs1))
7823 0 : == TYPE_PRECISION (TREE_TYPE (rhs2))))
7824 : {
7825 0 : if (TREE_CODE (rhs2) != SSA_NAME
7826 0 : || !has_single_use (rhs2)
7827 0 : || (gimple_bb (SSA_NAME_DEF_STMT (rhs2))
7828 0 : != gimple_bb (SSA_NAME_DEF_STMT (s)))
7829 0 : || !mergeable_op (SSA_NAME_DEF_STMT (rhs2)))
7830 0 : continue;
7831 : }
7832 : }
7833 : }
7834 : }
7835 7528 : if (is_gimple_assign (SSA_NAME_DEF_STMT (s)))
7836 5418 : switch (gimple_assign_rhs_code (SSA_NAME_DEF_STMT (s)))
7837 : {
7838 1794 : case REALPART_EXPR:
7839 1794 : case IMAGPART_EXPR:
7840 1794 : {
7841 1794 : gimple *ds = SSA_NAME_DEF_STMT (s);
7842 1794 : tree rhs1 = gimple_assign_rhs1 (ds);
7843 1794 : rhs1 = TREE_OPERAND (rhs1, 0);
7844 1794 : if (TREE_CODE (rhs1) == SSA_NAME)
7845 : {
7846 1794 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
7847 1794 : if (optimizable_arith_overflow (g))
7848 : {
7849 1650 : if (gimple_assign_rhs_code (ds) == IMAGPART_EXPR)
7850 1640 : continue;
7851 10 : if (gimple_store_p (use_stmt))
7852 : {
7853 : /* Punt if the cast use of IMAGPART_EXPR stmt
7854 : appears before the store use_stmt, because
7855 : optimizable arith overflow can't be
7856 : lowered at the store location in that case.
7857 : See PR121828. */
7858 10 : gimple_stmt_iterator gsi
7859 10 : = gsi_for_stmt (use_stmt);
7860 10 : unsigned int cnt = 0;
7861 12 : do
7862 : {
7863 12 : gsi_prev_nondebug (&gsi);
7864 12 : if (gsi_end_p (gsi))
7865 : break;
7866 12 : gimple *g2 = gsi_stmt (gsi);
7867 12 : if (g2 == ds)
7868 : break;
7869 3 : if (++cnt == 64)
7870 : break;
7871 3 : if (!gimple_assign_cast_p (g2))
7872 2 : continue;
7873 1 : tree rhs2 = gimple_assign_rhs1 (g2);
7874 1 : if (TREE_CODE (rhs2) != SSA_NAME)
7875 0 : continue;
7876 1 : gimple *g3 = SSA_NAME_DEF_STMT (rhs2);
7877 1 : if (!is_gimple_assign (g3))
7878 0 : continue;
7879 1 : if (gimple_assign_rhs_code (g3)
7880 : != IMAGPART_EXPR)
7881 0 : continue;
7882 1 : rhs2 = gimple_assign_rhs1 (g3);
7883 1 : rhs2 = TREE_OPERAND (rhs2, 0);
7884 1 : if (rhs2 != rhs1)
7885 0 : continue;
7886 : cnt = 64;
7887 : break;
7888 : }
7889 : while (1);
7890 10 : if (cnt == 64)
7891 : break;
7892 : }
7893 : }
7894 : }
7895 : }
7896 : /* FALLTHRU */
7897 844 : case LSHIFT_EXPR:
7898 844 : case RSHIFT_EXPR:
7899 844 : case MULT_EXPR:
7900 844 : case TRUNC_DIV_EXPR:
7901 844 : case EXACT_DIV_EXPR:
7902 844 : case TRUNC_MOD_EXPR:
7903 844 : case FIX_TRUNC_EXPR:
7904 844 : if (gimple_store_p (use_stmt)
7905 444 : && is_gimple_assign (use_stmt)
7906 444 : && !gimple_has_volatile_ops (use_stmt)
7907 1288 : && !stmt_ends_bb_p (use_stmt))
7908 : {
7909 444 : tree lhs = gimple_assign_lhs (use_stmt);
7910 : /* As multiply/division passes address of the lhs
7911 : to library function and that assumes it can extend
7912 : it to whole number of limbs, avoid merging those
7913 : with bit-field stores. Don't allow it for
7914 : shifts etc. either, so that the bit-field store
7915 : handling doesn't have to be done everywhere. */
7916 444 : if (TREE_CODE (lhs) == COMPONENT_REF
7917 444 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
7918 : break;
7919 441 : continue;
7920 441 : }
7921 : break;
7922 : default:
7923 : break;
7924 : }
7925 2110 : else if (is_gimple_call (SSA_NAME_DEF_STMT (s))
7926 2110 : && gimple_call_internal_p (SSA_NAME_DEF_STMT (s)))
7927 15 : switch (gimple_call_internal_fn (SSA_NAME_DEF_STMT (s)))
7928 : {
7929 13 : case IFN_BSWAP:
7930 13 : case IFN_BITREVERSE:
7931 13 : if (gimple_store_p (use_stmt)
7932 13 : && is_gimple_assign (use_stmt)
7933 13 : && !gimple_has_volatile_ops (use_stmt)
7934 26 : && !stmt_ends_bb_p (use_stmt))
7935 : {
7936 13 : tree lhs = gimple_assign_lhs (use_stmt);
7937 13 : if (TREE_CODE (lhs) == COMPONENT_REF
7938 13 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
7939 : break;
7940 13 : continue;
7941 13 : }
7942 : break;
7943 : default:
7944 : break;
7945 : }
7946 : }
7947 :
7948 : /* Also ignore uninitialized uses. */
7949 29594 : if (SSA_NAME_IS_DEFAULT_DEF (s)
7950 29594 : && (!SSA_NAME_VAR (s) || VAR_P (SSA_NAME_VAR (s))))
7951 59 : continue;
7952 :
7953 29586 : force_name:
7954 29586 : if (!large_huge.m_names)
7955 5688 : large_huge.m_names = BITMAP_ALLOC (NULL);
7956 29586 : bitmap_set_bit (large_huge.m_names, SSA_NAME_VERSION (s));
7957 29586 : if (has_single_use (s))
7958 : {
7959 : tree s2 = s;
7960 : /* The coalescing hook special cases SSA_NAME copies.
7961 : Make sure not to mark in m_single_use_names single
7962 : use SSA_NAMEs copied from non-single use SSA_NAMEs. */
7963 25883 : while (gimple_assign_copy_p (SSA_NAME_DEF_STMT (s2)))
7964 : {
7965 938 : s2 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s2));
7966 938 : if (TREE_CODE (s2) != SSA_NAME)
7967 : break;
7968 41 : if (!has_single_use (s2))
7969 : {
7970 : s2 = NULL_TREE;
7971 : break;
7972 : }
7973 : }
7974 25870 : if (s2)
7975 : {
7976 25842 : if (!large_huge.m_single_use_names)
7977 5574 : large_huge.m_single_use_names = BITMAP_ALLOC (NULL);
7978 25842 : bitmap_set_bit (large_huge.m_single_use_names,
7979 25842 : SSA_NAME_VERSION (s));
7980 : }
7981 : }
7982 29586 : if (SSA_NAME_VAR (s)
7983 7370 : && ((TREE_CODE (SSA_NAME_VAR (s)) == PARM_DECL
7984 5420 : && SSA_NAME_IS_DEFAULT_DEF (s))
7985 1987 : || TREE_CODE (SSA_NAME_VAR (s)) == RESULT_DECL))
7986 : has_large_huge_parm_result = true;
7987 29586 : if (optimize
7988 10468 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
7989 10401 : && gimple_assign_load_p (SSA_NAME_DEF_STMT (s))
7990 6062 : && !gimple_has_volatile_ops (SSA_NAME_DEF_STMT (s))
7991 32568 : && !stmt_ends_bb_p (SSA_NAME_DEF_STMT (s)))
7992 : {
7993 2982 : use_operand_p use_p;
7994 2982 : imm_use_iterator iter;
7995 2982 : bool optimizable_load = true;
7996 6029 : FOR_EACH_IMM_USE_FAST (use_p, iter, s)
7997 : {
7998 3145 : gimple *use_stmt = USE_STMT (use_p);
7999 3145 : if (is_gimple_debug (use_stmt))
8000 0 : continue;
8001 3145 : if (is_gimple_call (use_stmt)
8002 3145 : && gimple_call_internal_p (use_stmt))
8003 66 : switch (gimple_call_internal_fn (use_stmt))
8004 : {
8005 10 : case IFN_CLZ:
8006 10 : case IFN_CTZ:
8007 10 : case IFN_CLRSB:
8008 10 : case IFN_FFS:
8009 10 : case IFN_PARITY:
8010 10 : case IFN_POPCOUNT:
8011 10 : case IFN_BSWAP:
8012 10 : case IFN_BITREVERSE:
8013 10 : continue;
8014 : default:
8015 : break;
8016 : }
8017 3135 : if (gimple_code (use_stmt) == GIMPLE_PHI
8018 : || is_gimple_call (use_stmt)
8019 : || gimple_code (use_stmt) == GIMPLE_ASM
8020 : || (is_gimple_assign (use_stmt)
8021 1903 : && (gimple_assign_rhs_code (use_stmt)
8022 : == COMPLEX_EXPR)))
8023 : {
8024 : optimizable_load = false;
8025 : break;
8026 : }
8027 2982 : }
8028 :
8029 2982 : ssa_op_iter oi;
8030 4027 : FOR_EACH_SSA_USE_OPERAND (use_p, SSA_NAME_DEF_STMT (s),
8031 : oi, SSA_OP_USE)
8032 : {
8033 1045 : tree s2 = USE_FROM_PTR (use_p);
8034 1045 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s2))
8035 : {
8036 : optimizable_load = false;
8037 : break;
8038 : }
8039 : }
8040 :
8041 2982 : if (optimizable_load && !stmt_ends_bb_p (SSA_NAME_DEF_STMT (s)))
8042 : {
8043 2884 : if (!large_huge.m_loads)
8044 312 : large_huge.m_loads = BITMAP_ALLOC (NULL);
8045 2884 : bitmap_set_bit (large_huge.m_loads, SSA_NAME_VERSION (s));
8046 : }
8047 : }
8048 : }
8049 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
8050 : into memory. Such functions could have no large/huge SSA_NAMEs. */
8051 57218 : else if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
8052 : {
8053 40410 : gimple *g = SSA_NAME_DEF_STMT (s);
8054 40410 : if (is_gimple_assign (g) && gimple_store_p (g))
8055 : {
8056 13739 : tree t = gimple_assign_rhs1 (g);
8057 15618 : if (BITINT_TYPE_P (TREE_TYPE (t))
8058 13739 : && bitint_precision_kind (TREE_TYPE (t)) >= bitint_prec_large)
8059 : has_large_huge = true;
8060 : }
8061 : }
8062 : }
8063 :
8064 7354 : if (large_huge.m_names || has_large_huge)
8065 : {
8066 5886 : ret = TODO_update_ssa_only_virtuals | TODO_cleanup_cfg;
8067 5886 : calculate_dominance_info (CDI_DOMINATORS);
8068 5886 : if (optimize)
8069 3102 : enable_ranger (cfun);
8070 5886 : if (large_huge.m_loads)
8071 : {
8072 312 : basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
8073 312 : entry->aux = NULL;
8074 624 : bitint_dom_walker (large_huge.m_names,
8075 312 : large_huge.m_loads).walk (entry);
8076 312 : bitmap_and_compl_into (large_huge.m_names, large_huge.m_loads);
8077 312 : clear_aux_for_blocks ();
8078 312 : BITMAP_FREE (large_huge.m_loads);
8079 : }
8080 5886 : large_huge.m_limb_type = build_nonstandard_integer_type (limb_prec, 1);
8081 5886 : large_huge.m_limb_size
8082 5886 : = tree_to_uhwi (TYPE_SIZE_UNIT (large_huge.m_limb_type));
8083 : }
8084 7354 : if (large_huge.m_names)
8085 : {
8086 5688 : large_huge.m_map
8087 11376 : = init_var_map (num_ssa_names, NULL, large_huge.m_names);
8088 5688 : coalesce_ssa_name (large_huge.m_map);
8089 5688 : partition_view_normal (large_huge.m_map);
8090 5688 : if (dump_file && (dump_flags & TDF_DETAILS))
8091 : {
8092 0 : fprintf (dump_file, "After Coalescing:\n");
8093 0 : dump_var_map (dump_file, large_huge.m_map);
8094 : }
8095 5688 : large_huge.m_vars
8096 5688 : = XCNEWVEC (tree, num_var_partitions (large_huge.m_map));
8097 5688 : bitmap_iterator bi;
8098 5688 : if (has_large_huge_parm_result)
8099 19742 : EXECUTE_IF_SET_IN_BITMAP (large_huge.m_names, 0, i, bi)
8100 : {
8101 15530 : tree s = ssa_name (i);
8102 15530 : if (SSA_NAME_VAR (s)
8103 6224 : && ((TREE_CODE (SSA_NAME_VAR (s)) == PARM_DECL
8104 5420 : && SSA_NAME_IS_DEFAULT_DEF (s))
8105 841 : || TREE_CODE (SSA_NAME_VAR (s)) == RESULT_DECL))
8106 : {
8107 5383 : int p = var_to_partition (large_huge.m_map, s);
8108 5383 : if (large_huge.m_vars[p] == NULL_TREE)
8109 : {
8110 5383 : large_huge.m_vars[p] = SSA_NAME_VAR (s);
8111 5383 : mark_addressable (SSA_NAME_VAR (s));
8112 : }
8113 : }
8114 : }
8115 5688 : tree atype = NULL_TREE;
8116 5688 : if (dump_file && (dump_flags & TDF_DETAILS))
8117 0 : fprintf (dump_file, "Mapping SSA_NAMEs to decls:\n");
8118 32920 : EXECUTE_IF_SET_IN_BITMAP (large_huge.m_names, 0, i, bi)
8119 : {
8120 27232 : tree s = ssa_name (i);
8121 27232 : int p = var_to_partition (large_huge.m_map, s);
8122 27232 : if (large_huge.m_vars[p] == NULL_TREE)
8123 : {
8124 18923 : if (atype == NULL_TREE
8125 33054 : || !tree_int_cst_equal (TYPE_SIZE (atype),
8126 14131 : TYPE_SIZE (TREE_TYPE (s))))
8127 : {
8128 7769 : unsigned HOST_WIDE_INT nelts
8129 7769 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (s))) / limb_prec;
8130 7769 : atype = build_array_type_nelts (large_huge.m_limb_type,
8131 7769 : nelts);
8132 : }
8133 18923 : large_huge.m_vars[p] = create_tmp_var (atype, "bitint");
8134 18923 : mark_addressable (large_huge.m_vars[p]);
8135 : }
8136 27232 : if (dump_file && (dump_flags & TDF_DETAILS))
8137 : {
8138 0 : print_generic_expr (dump_file, s, TDF_SLIM);
8139 0 : fprintf (dump_file, " -> ");
8140 0 : print_generic_expr (dump_file, large_huge.m_vars[p], TDF_SLIM);
8141 0 : fprintf (dump_file, "\n");
8142 : }
8143 : }
8144 : }
8145 :
8146 46457 : FOR_EACH_BB_REVERSE_FN (bb, cfun)
8147 : {
8148 39103 : gimple_stmt_iterator prev;
8149 197804 : for (gimple_stmt_iterator gsi = gsi_last_bb (bb); !gsi_end_p (gsi);
8150 119598 : gsi = prev)
8151 : {
8152 119598 : prev = gsi;
8153 119598 : gsi_prev (&prev);
8154 119598 : ssa_op_iter iter;
8155 119598 : gimple *stmt = gsi_stmt (gsi);
8156 119598 : if (is_gimple_debug (stmt))
8157 76703 : continue;
8158 114624 : bitint_prec_kind kind = bitint_prec_small;
8159 114624 : tree t;
8160 344904 : FOR_EACH_SSA_TREE_OPERAND (t, stmt, iter, SSA_OP_ALL_OPERANDS)
8161 230280 : if (BITINT_TYPE_P (TREE_TYPE (t)))
8162 : {
8163 79215 : bitint_prec_kind this_kind
8164 79215 : = bitint_precision_kind (TREE_TYPE (t));
8165 230488 : kind = MAX (kind, this_kind);
8166 : }
8167 114624 : if (is_gimple_assign (stmt) && gimple_store_p (stmt))
8168 : {
8169 16404 : t = gimple_assign_rhs1 (stmt);
8170 16404 : if (BITINT_TYPE_P (TREE_TYPE (t)))
8171 : {
8172 14203 : bitint_prec_kind this_kind
8173 14203 : = bitint_precision_kind (TREE_TYPE (t));
8174 14203 : kind = MAX (kind, this_kind);
8175 : }
8176 : }
8177 114624 : if (is_gimple_assign (stmt)
8178 114624 : && gimple_assign_rhs_code (stmt) == FLOAT_EXPR)
8179 : {
8180 189 : t = gimple_assign_rhs1 (stmt);
8181 218 : if (BITINT_TYPE_P (TREE_TYPE (t))
8182 191 : && TREE_CODE (t) == INTEGER_CST)
8183 : {
8184 1 : bitint_prec_kind this_kind
8185 1 : = bitint_precision_kind (TREE_TYPE (t));
8186 1 : kind = MAX (kind, this_kind);
8187 : }
8188 : }
8189 114624 : if (is_gimple_call (stmt))
8190 : {
8191 25860 : t = gimple_call_lhs (stmt);
8192 25860 : if (t && TREE_CODE (TREE_TYPE (t)) == COMPLEX_TYPE)
8193 : {
8194 5323 : bitint_prec_kind this_kind = arith_overflow_arg_kind (stmt);
8195 5323 : kind = MAX (kind, this_kind);
8196 5323 : if (BITINT_TYPE_P (TREE_TYPE (TREE_TYPE (t))))
8197 : {
8198 5198 : this_kind
8199 5198 : = bitint_precision_kind (TREE_TYPE (TREE_TYPE (t)));
8200 5198 : kind = MAX (kind, this_kind);
8201 : }
8202 : }
8203 : }
8204 114302 : if (kind == bitint_prec_small)
8205 45089 : continue;
8206 69535 : switch (gimple_code (stmt))
8207 : {
8208 11137 : case GIMPLE_CALL:
8209 : /* For now. We'll need to handle some internal functions and
8210 : perhaps some builtins. */
8211 11137 : if (kind == bitint_prec_middle)
8212 2280 : continue;
8213 : break;
8214 4 : case GIMPLE_ASM:
8215 4 : if (kind == bitint_prec_middle)
8216 1 : continue;
8217 : break;
8218 1124 : case GIMPLE_RETURN:
8219 1124 : continue;
8220 48647 : case GIMPLE_ASSIGN:
8221 48647 : if (gimple_clobber_p (stmt))
8222 3511 : continue;
8223 45136 : if (kind >= bitint_prec_large)
8224 : break;
8225 8732 : if (gimple_assign_single_p (stmt))
8226 : /* No need to lower copies, loads or stores. */
8227 5788 : continue;
8228 2944 : if (gimple_assign_cast_p (stmt))
8229 : {
8230 2379 : tree lhs = gimple_assign_lhs (stmt);
8231 2379 : tree rhs = gimple_assign_rhs1 (stmt);
8232 4758 : if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
8233 2379 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs))
8234 4752 : && (TYPE_PRECISION (TREE_TYPE (lhs))
8235 2373 : == TYPE_PRECISION (TREE_TYPE (rhs))))
8236 : /* No need to lower casts to same precision. */
8237 28 : continue;
8238 : }
8239 : break;
8240 : default:
8241 : break;
8242 1124 : }
8243 :
8244 11539 : if (kind == bitint_prec_middle)
8245 : {
8246 5041 : tree type = NULL_TREE;
8247 : /* Middle _BitInt(N) is rewritten to casts to INTEGER_TYPEs
8248 : with the same precision and back. */
8249 5041 : unsigned int nops = gimple_num_ops (stmt);
8250 16928 : for (unsigned int i = is_gimple_assign (stmt) ? 1 : 0;
8251 16928 : i < nops; ++i)
8252 11887 : if (tree op = gimple_op (stmt, i))
8253 : {
8254 7645 : tree nop = maybe_cast_middle_bitint (&gsi, op, type);
8255 7645 : if (nop != op)
8256 6765 : gimple_set_op (stmt, i, nop);
8257 880 : else if (COMPARISON_CLASS_P (op))
8258 : {
8259 0 : TREE_OPERAND (op, 0)
8260 0 : = maybe_cast_middle_bitint (&gsi,
8261 0 : TREE_OPERAND (op, 0),
8262 : type);
8263 0 : TREE_OPERAND (op, 1)
8264 0 : = maybe_cast_middle_bitint (&gsi,
8265 0 : TREE_OPERAND (op, 1),
8266 : type);
8267 : }
8268 880 : else if (TREE_CODE (op) == CASE_LABEL_EXPR)
8269 : {
8270 24 : CASE_LOW (op)
8271 24 : = maybe_cast_middle_bitint (&gsi, CASE_LOW (op),
8272 : type);
8273 48 : CASE_HIGH (op)
8274 48 : = maybe_cast_middle_bitint (&gsi, CASE_HIGH (op),
8275 : type);
8276 : }
8277 : }
8278 5041 : if (tree lhs = gimple_get_lhs (stmt))
8279 4464 : if (BITINT_TYPE_P (TREE_TYPE (lhs))
8280 2916 : && (bitint_precision_kind (TREE_TYPE (lhs))
8281 : == bitint_prec_middle))
8282 : {
8283 1367 : int prec = TYPE_PRECISION (TREE_TYPE (lhs));
8284 1367 : int uns = TYPE_UNSIGNED (TREE_TYPE (lhs));
8285 1367 : type = build_nonstandard_integer_type (prec, uns);
8286 1367 : tree lhs2 = make_ssa_name (type);
8287 1367 : gimple_set_lhs (stmt, lhs2);
8288 1367 : gimple *g = gimple_build_assign (lhs, NOP_EXPR, lhs2);
8289 1367 : if (stmt_ends_bb_p (stmt))
8290 : {
8291 4 : edge e = find_fallthru_edge (gsi_bb (gsi)->succs);
8292 4 : gsi_insert_on_edge (e, g);
8293 4 : edge_insertions = true;
8294 : }
8295 : else
8296 1363 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
8297 : }
8298 5041 : update_stmt (stmt);
8299 5041 : continue;
8300 5041 : }
8301 :
8302 51762 : if (tree lhs = gimple_get_lhs (stmt))
8303 45130 : if (TREE_CODE (lhs) == SSA_NAME)
8304 : {
8305 36123 : tree type = TREE_TYPE (lhs);
8306 36123 : if (TREE_CODE (type) == COMPLEX_TYPE)
8307 4197 : type = TREE_TYPE (type);
8308 5444 : if (BITINT_TYPE_P (type)
8309 30691 : && bitint_precision_kind (type) >= bitint_prec_large
8310 66612 : && (large_huge.m_names == NULL
8311 30284 : || !bitmap_bit_p (large_huge.m_names,
8312 30284 : SSA_NAME_VERSION (lhs))))
8313 8867 : continue;
8314 : }
8315 :
8316 42895 : large_huge.lower_stmt (stmt);
8317 : }
8318 :
8319 39103 : tree atype = NULL_TREE;
8320 47744 : for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
8321 8641 : gsi_next (&gsi))
8322 : {
8323 8641 : gphi *phi = gsi.phi ();
8324 8641 : tree lhs = gimple_phi_result (phi);
8325 17048 : if (!BITINT_TYPE_P (TREE_TYPE (lhs))
8326 8641 : || bitint_precision_kind (TREE_TYPE (lhs)) < bitint_prec_large)
8327 8448 : continue;
8328 193 : int p1 = var_to_partition (large_huge.m_map, lhs);
8329 193 : gcc_assert (large_huge.m_vars[p1] != NULL_TREE);
8330 : tree v1 = large_huge.m_vars[p1];
8331 694 : for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
8332 : {
8333 501 : tree arg = gimple_phi_arg_def (phi, i);
8334 501 : edge e = gimple_phi_arg_edge (phi, i);
8335 501 : gimple *g;
8336 501 : switch (TREE_CODE (arg))
8337 : {
8338 99 : case INTEGER_CST:
8339 99 : if (integer_zerop (arg) && VAR_P (v1))
8340 : {
8341 45 : tree zero = build_zero_cst (TREE_TYPE (v1));
8342 45 : g = gimple_build_assign (v1, zero);
8343 45 : gsi_insert_on_edge (e, g);
8344 45 : edge_insertions = true;
8345 162 : break;
8346 : }
8347 54 : int ext;
8348 54 : unsigned int min_prec, prec, rem;
8349 54 : tree c;
8350 54 : prec = TYPE_PRECISION (TREE_TYPE (arg));
8351 54 : rem = prec % (2 * limb_prec);
8352 54 : min_prec = bitint_min_cst_precision (arg, ext);
8353 54 : if (min_prec > prec - rem - 2 * limb_prec
8354 12 : && min_prec > (unsigned) limb_prec)
8355 : /* Constant which has enough significant bits that it
8356 : isn't worth trying to save .rodata space by extending
8357 : from smaller number. */
8358 : min_prec = prec;
8359 : else
8360 : {
8361 45 : min_prec = CEIL (min_prec, limb_prec) * limb_prec;
8362 45 : if (min_prec > (unsigned) limb_prec
8363 3 : && abi_limb_prec > limb_prec)
8364 : {
8365 : /* For targets with ABI limb precision higher than
8366 : limb precision round to ABI limb precision,
8367 : otherwise c can contain padding bits. */
8368 0 : min_prec
8369 0 : = CEIL (min_prec, abi_limb_prec) * abi_limb_prec;
8370 0 : if (min_prec > prec - rem - 2 * limb_prec)
8371 9 : min_prec = prec;
8372 : }
8373 : }
8374 54 : if (min_prec == 0)
8375 : c = NULL_TREE;
8376 51 : else if (min_prec == prec)
8377 9 : c = tree_output_constant_def (arg);
8378 42 : else if (min_prec == (unsigned) limb_prec)
8379 39 : c = fold_convert (large_huge.m_limb_type, arg);
8380 : else
8381 : {
8382 3 : tree ctype = build_bitint_type (min_prec, 1);
8383 3 : c = tree_output_constant_def (fold_convert (ctype, arg));
8384 : }
8385 51 : if (c)
8386 : {
8387 51 : if (VAR_P (v1) && min_prec == prec)
8388 : {
8389 8 : tree v2 = build1 (VIEW_CONVERT_EXPR,
8390 8 : TREE_TYPE (v1), c);
8391 8 : g = gimple_build_assign (v1, v2);
8392 8 : gsi_insert_on_edge (e, g);
8393 8 : edge_insertions = true;
8394 8 : break;
8395 : }
8396 43 : if (TREE_CODE (TREE_TYPE (c)) == INTEGER_TYPE)
8397 : {
8398 39 : if (bitint_big_endian)
8399 : {
8400 0 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8401 0 : tree sz1 = TYPE_SIZE_UNIT (TREE_TYPE (v1));
8402 0 : tree sz2 = TYPE_SIZE_UNIT (TREE_TYPE (c));
8403 0 : tree off = build_int_cst (ptype,
8404 0 : tree_to_uhwi (sz1)
8405 0 : - tree_to_uhwi (sz2));
8406 0 : tree vd = build2 (MEM_REF, TREE_TYPE (c),
8407 : build_fold_addr_expr (v1),
8408 : off);
8409 0 : g = gimple_build_assign (vd, c);
8410 : }
8411 : else
8412 39 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8413 39 : TREE_TYPE (c),
8414 : v1), c);
8415 : }
8416 : else
8417 : {
8418 4 : unsigned HOST_WIDE_INT nelts
8419 4 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (c)))
8420 4 : / limb_prec;
8421 4 : tree vtype
8422 8 : = build_array_type_nelts (large_huge.m_limb_type,
8423 4 : nelts);
8424 4 : tree vd;
8425 4 : if (bitint_big_endian)
8426 : {
8427 0 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8428 0 : tree sz1 = TYPE_SIZE_UNIT (TREE_TYPE (v1));
8429 0 : tree sz2 = TYPE_SIZE_UNIT (vtype);
8430 0 : tree off = build_int_cst (ptype,
8431 0 : tree_to_uhwi (sz1)
8432 0 : - tree_to_uhwi (sz2));
8433 0 : vd = build2 (MEM_REF, vtype,
8434 : build_fold_addr_expr (v1), off);
8435 : }
8436 : else
8437 4 : vd = build1 (VIEW_CONVERT_EXPR, vtype, v1);
8438 4 : g = gimple_build_assign (vd,
8439 : build1 (VIEW_CONVERT_EXPR,
8440 : vtype, c));
8441 : }
8442 43 : gsi_insert_on_edge (e, g);
8443 43 : if (min_prec == prec)
8444 : {
8445 : edge_insertions = true;
8446 : break;
8447 : }
8448 : }
8449 45 : if (ext == 0)
8450 : {
8451 39 : unsigned HOST_WIDE_INT nelts
8452 39 : = (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (v1)))
8453 39 : - min_prec) / limb_prec;
8454 39 : tree vtype
8455 78 : = build_array_type_nelts (large_huge.m_limb_type,
8456 39 : nelts);
8457 39 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8458 39 : tree off;
8459 39 : if (c && !bitint_big_endian)
8460 38 : off = fold_convert (ptype,
8461 : TYPE_SIZE_UNIT (TREE_TYPE (c)));
8462 : else
8463 1 : off = build_zero_cst (ptype);
8464 39 : tree vd = build2 (MEM_REF, vtype,
8465 : build_fold_addr_expr (v1), off);
8466 39 : g = gimple_build_assign (vd, build_zero_cst (vtype));
8467 : }
8468 : else
8469 : {
8470 6 : tree vd = v1;
8471 6 : if (c && !bitint_big_endian)
8472 : {
8473 4 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8474 4 : tree off
8475 4 : = fold_convert (ptype,
8476 : TYPE_SIZE_UNIT (TREE_TYPE (c)));
8477 4 : vd = build2 (MEM_REF, large_huge.m_limb_type,
8478 : build_fold_addr_expr (v1), off);
8479 : }
8480 6 : vd = build_fold_addr_expr (vd);
8481 6 : unsigned HOST_WIDE_INT nbytes
8482 6 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (v1)));
8483 6 : if (c)
8484 4 : nbytes
8485 4 : -= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (c)));
8486 6 : tree fn = builtin_decl_implicit (BUILT_IN_MEMSET);
8487 6 : g = gimple_build_call (fn, 3, vd,
8488 : integer_minus_one_node,
8489 : build_int_cst (sizetype,
8490 6 : nbytes));
8491 : }
8492 45 : gsi_insert_on_edge (e, g);
8493 45 : edge_insertions = true;
8494 45 : break;
8495 0 : default:
8496 0 : gcc_unreachable ();
8497 402 : case SSA_NAME:
8498 402 : if (gimple_code (SSA_NAME_DEF_STMT (arg)) == GIMPLE_NOP)
8499 : {
8500 9 : if (large_huge.m_names == NULL
8501 18 : || !bitmap_bit_p (large_huge.m_names,
8502 9 : SSA_NAME_VERSION (arg)))
8503 384 : continue;
8504 : }
8505 402 : int p2 = var_to_partition (large_huge.m_map, arg);
8506 402 : if (p1 == p2)
8507 384 : continue;
8508 18 : gcc_assert (large_huge.m_vars[p2] != NULL_TREE);
8509 18 : tree v2 = large_huge.m_vars[p2];
8510 18 : if (VAR_P (v1) && VAR_P (v2))
8511 18 : g = gimple_build_assign (v1, v2);
8512 0 : else if (VAR_P (v1))
8513 0 : g = gimple_build_assign (v1, build1 (VIEW_CONVERT_EXPR,
8514 0 : TREE_TYPE (v1), v2));
8515 0 : else if (VAR_P (v2))
8516 0 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8517 0 : TREE_TYPE (v2), v1), v2);
8518 : else
8519 : {
8520 0 : if (atype == NULL_TREE
8521 0 : || !tree_int_cst_equal (TYPE_SIZE (atype),
8522 0 : TYPE_SIZE (TREE_TYPE (lhs))))
8523 : {
8524 0 : unsigned HOST_WIDE_INT nelts
8525 0 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (lhs)))
8526 0 : / limb_prec;
8527 0 : atype
8528 0 : = build_array_type_nelts (large_huge.m_limb_type,
8529 0 : nelts);
8530 : }
8531 0 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8532 : atype, v1),
8533 : build1 (VIEW_CONVERT_EXPR,
8534 : atype, v2));
8535 : }
8536 18 : gsi_insert_on_edge (e, g);
8537 18 : edge_insertions = true;
8538 18 : break;
8539 : }
8540 : }
8541 : }
8542 : }
8543 :
8544 7354 : if (large_huge.m_names || has_large_huge)
8545 : {
8546 5886 : gimple *nop = NULL;
8547 386915 : for (i = 0; i < num_ssa_names; ++i)
8548 : {
8549 381029 : tree s = ssa_name (i);
8550 381029 : if (s == NULL_TREE)
8551 15534 : continue;
8552 365495 : tree type = TREE_TYPE (s);
8553 365495 : if (TREE_CODE (type) == COMPLEX_TYPE)
8554 17605 : type = TREE_TYPE (type);
8555 325596 : if (BITINT_TYPE_P (type)
8556 365522 : && bitint_precision_kind (type) >= bitint_prec_large)
8557 : {
8558 40844 : if (large_huge.m_preserved
8559 45342 : && bitmap_bit_p (large_huge.m_preserved,
8560 6841 : SSA_NAME_VERSION (s)))
8561 2343 : continue;
8562 36158 : gimple *g = SSA_NAME_DEF_STMT (s);
8563 36158 : if (gimple_code (g) == GIMPLE_NOP)
8564 : {
8565 9736 : if (SSA_NAME_VAR (s))
8566 5452 : set_ssa_default_def (cfun, SSA_NAME_VAR (s), NULL_TREE);
8567 9736 : release_ssa_name (s);
8568 9736 : continue;
8569 : }
8570 26422 : if (gimple_bb (g) == NULL)
8571 : {
8572 0 : release_ssa_name (s);
8573 0 : continue;
8574 : }
8575 26422 : if (gimple_code (g) != GIMPLE_ASM)
8576 : {
8577 26421 : gimple_stmt_iterator gsi = gsi_for_stmt (g);
8578 26421 : bool save_vta = flag_var_tracking_assignments;
8579 26421 : flag_var_tracking_assignments = false;
8580 26421 : gsi_remove (&gsi, true);
8581 26421 : flag_var_tracking_assignments = save_vta;
8582 : }
8583 26422 : if (nop == NULL)
8584 4917 : nop = gimple_build_nop ();
8585 26422 : SSA_NAME_DEF_STMT (s) = nop;
8586 26422 : release_ssa_name (s);
8587 : }
8588 : }
8589 5886 : if (optimize)
8590 3102 : disable_ranger (cfun);
8591 : }
8592 :
8593 7354 : if (edge_insertions)
8594 56 : gsi_commit_edge_inserts ();
8595 :
8596 : /* Fix up arguments of ECF_RETURNS_TWICE calls. Those were temporarily
8597 : inserted before the call, but that is invalid IL, so move them to the
8598 : right place and add corresponding PHIs. */
8599 7354 : if (!large_huge.m_returns_twice_calls.is_empty ())
8600 : {
8601 9 : auto_vec<gimple *, 16> arg_stmts;
8602 29 : while (!large_huge.m_returns_twice_calls.is_empty ())
8603 : {
8604 11 : gimple *stmt = large_huge.m_returns_twice_calls.pop ();
8605 11 : gimple_stmt_iterator gsi = gsi_after_labels (gimple_bb (stmt));
8606 36 : while (gsi_stmt (gsi) != stmt)
8607 : {
8608 25 : if (is_gimple_debug (gsi_stmt (gsi)))
8609 2 : gsi_next (&gsi);
8610 : else
8611 : {
8612 23 : arg_stmts.safe_push (gsi_stmt (gsi));
8613 23 : gsi_remove (&gsi, false);
8614 : }
8615 : }
8616 11 : gimple *g;
8617 11 : basic_block bb = NULL;
8618 11 : edge e = NULL, ead = NULL;
8619 34 : FOR_EACH_VEC_ELT (arg_stmts, i, g)
8620 : {
8621 23 : gsi_safe_insert_before (&gsi, g);
8622 23 : if (i == 0)
8623 : {
8624 11 : bb = gimple_bb (stmt);
8625 11 : gcc_checking_assert (EDGE_COUNT (bb->preds) == 2);
8626 11 : e = EDGE_PRED (bb, 0);
8627 11 : ead = EDGE_PRED (bb, 1);
8628 11 : if ((ead->flags & EDGE_ABNORMAL) == 0)
8629 0 : std::swap (e, ead);
8630 11 : gcc_checking_assert ((e->flags & EDGE_ABNORMAL) == 0
8631 : && (ead->flags & EDGE_ABNORMAL));
8632 : }
8633 23 : tree lhs = gimple_assign_lhs (g);
8634 23 : tree arg = lhs;
8635 23 : gphi *phi = create_phi_node (copy_ssa_name (arg), bb);
8636 23 : add_phi_arg (phi, arg, e, UNKNOWN_LOCATION);
8637 23 : tree var = create_tmp_reg (TREE_TYPE (arg));
8638 23 : suppress_warning (var, OPT_Wuninitialized);
8639 23 : arg = get_or_create_ssa_default_def (cfun, var);
8640 23 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg) = 1;
8641 23 : add_phi_arg (phi, arg, ead, UNKNOWN_LOCATION);
8642 23 : arg = gimple_phi_result (phi);
8643 23 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg) = 1;
8644 23 : imm_use_iterator iter;
8645 23 : gimple *use_stmt;
8646 69 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
8647 : {
8648 46 : if (use_stmt == phi)
8649 23 : continue;
8650 23 : gcc_checking_assert (use_stmt == stmt);
8651 23 : use_operand_p use_p;
8652 46 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
8653 23 : SET_USE (use_p, arg);
8654 23 : }
8655 : }
8656 11 : update_stmt (stmt);
8657 11 : arg_stmts.truncate (0);
8658 : }
8659 9 : }
8660 :
8661 7354 : return ret;
8662 7354 : }
8663 :
8664 : namespace {
8665 :
8666 : const pass_data pass_data_lower_bitint =
8667 : {
8668 : GIMPLE_PASS, /* type */
8669 : "bitintlower", /* name */
8670 : OPTGROUP_NONE, /* optinfo_flags */
8671 : TV_NONE, /* tv_id */
8672 : PROP_ssa, /* properties_required */
8673 : PROP_gimple_lbitint, /* properties_provided */
8674 : 0, /* properties_destroyed */
8675 : 0, /* todo_flags_start */
8676 : 0, /* todo_flags_finish */
8677 : };
8678 :
8679 : class pass_lower_bitint : public gimple_opt_pass
8680 : {
8681 : public:
8682 588392 : pass_lower_bitint (gcc::context *ctxt)
8683 1176784 : : gimple_opt_pass (pass_data_lower_bitint, ctxt)
8684 : {}
8685 :
8686 : /* opt_pass methods: */
8687 294196 : opt_pass * clone () final override { return new pass_lower_bitint (m_ctxt); }
8688 1062786 : unsigned int execute (function *) final override
8689 : {
8690 1062786 : return gimple_lower_bitint ();
8691 : }
8692 :
8693 : }; // class pass_lower_bitint
8694 :
8695 : } // anon namespace
8696 :
8697 : gimple_opt_pass *
8698 294196 : make_pass_lower_bitint (gcc::context *ctxt)
8699 : {
8700 294196 : return new pass_lower_bitint (ctxt);
8701 : }
8702 :
8703 :
8704 : namespace {
8705 :
8706 : const pass_data pass_data_lower_bitint_O0 =
8707 : {
8708 : GIMPLE_PASS, /* type */
8709 : "bitintlower0", /* name */
8710 : OPTGROUP_NONE, /* optinfo_flags */
8711 : TV_NONE, /* tv_id */
8712 : PROP_cfg, /* properties_required */
8713 : PROP_gimple_lbitint, /* properties_provided */
8714 : 0, /* properties_destroyed */
8715 : 0, /* todo_flags_start */
8716 : 0, /* todo_flags_finish */
8717 : };
8718 :
8719 : class pass_lower_bitint_O0 : public gimple_opt_pass
8720 : {
8721 : public:
8722 294196 : pass_lower_bitint_O0 (gcc::context *ctxt)
8723 588392 : : gimple_opt_pass (pass_data_lower_bitint_O0, ctxt)
8724 : {}
8725 :
8726 : /* opt_pass methods: */
8727 1515908 : bool gate (function *fun) final override
8728 : {
8729 : /* With errors, normal optimization passes are not run. If we don't
8730 : lower bitint operations at all, rtl expansion will abort. */
8731 1515908 : return !(fun->curr_properties & PROP_gimple_lbitint);
8732 : }
8733 :
8734 453227 : unsigned int execute (function *) final override
8735 : {
8736 453227 : return gimple_lower_bitint ();
8737 : }
8738 :
8739 : }; // class pass_lower_bitint_O0
8740 :
8741 : } // anon namespace
8742 :
8743 : gimple_opt_pass *
8744 294196 : make_pass_lower_bitint_O0 (gcc::context *ctxt)
8745 : {
8746 294196 : return new pass_lower_bitint_O0 (ctxt);
8747 : }
|
