Line data Source code
1 : /* Lower _BitInt(N) operations to scalar operations.
2 : Copyright (C) 2023-2026 Free Software Foundation, Inc.
3 : Contributed by Jakub Jelinek <jakub@redhat.com>.
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it
8 : under the terms of the GNU General Public License as published by the
9 : Free Software Foundation; either version 3, or (at your option) any
10 : later version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT
13 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "cfghooks.h"
29 : #include "tree-pass.h"
30 : #include "ssa.h"
31 : #include "fold-const.h"
32 : #include "gimplify.h"
33 : #include "gimple-iterator.h"
34 : #include "tree-cfg.h"
35 : #include "tree-dfa.h"
36 : #include "cfgloop.h"
37 : #include "cfganal.h"
38 : #include "target.h"
39 : #include "tree-ssa-live.h"
40 : #include "tree-ssa-coalesce.h"
41 : #include "domwalk.h"
42 : #include "memmodel.h"
43 : #include "optabs.h"
44 : #include "varasm.h"
45 : #include "gimple-range.h"
46 : #include "value-range.h"
47 : #include "langhooks.h"
48 : #include "gimplify-me.h"
49 : #include "diagnostic-core.h"
50 : #include "tree-eh.h"
51 : #include "tree-pretty-print.h"
52 : #include "alloc-pool.h"
53 : #include "tree-into-ssa.h"
54 : #include "tree-cfgcleanup.h"
55 : #include "tree-switch-conversion.h"
56 : #include "ubsan.h"
57 : #include "stor-layout.h"
58 : #include "gimple-lower-bitint.h"
59 :
60 : /* Split BITINT_TYPE precisions in 4 categories. Small _BitInt, where
61 : target hook says it is a single limb, middle _BitInt which per ABI
62 : does not, but there is some INTEGER_TYPE in which arithmetics can be
63 : performed (operations on such _BitInt are lowered to casts to that
64 : arithmetic type and cast back; e.g. on x86_64 limb is DImode, but
65 : target supports TImode, so _BitInt(65) to _BitInt(128) are middle
66 : ones), large _BitInt which should by straight line code and
67 : finally huge _BitInt which should be handled by loops over the limbs. */
68 :
69 : enum bitint_prec_kind {
70 : bitint_prec_small,
71 : bitint_prec_middle,
72 : bitint_prec_large,
73 : bitint_prec_huge
74 : };
75 :
76 : /* Caches to speed up bitint_precision_kind. */
77 :
78 : static int small_max_prec, mid_min_prec, large_min_prec, huge_min_prec;
79 : static int limb_prec, abi_limb_prec;
80 : static bool bitint_big_endian;
81 : static enum bitint_ext bitint_extended;
82 :
83 : /* Categorize _BitInt(PREC) as small, middle, large or huge. */
84 :
85 : static bitint_prec_kind
86 484447 : bitint_precision_kind (int prec)
87 : {
88 484447 : if (prec <= small_max_prec)
89 : return bitint_prec_small;
90 466732 : if (huge_min_prec && prec >= huge_min_prec)
91 : return bitint_prec_huge;
92 264147 : if (large_min_prec && prec >= large_min_prec)
93 : return bitint_prec_large;
94 52134 : if (mid_min_prec && prec >= mid_min_prec)
95 : return bitint_prec_middle;
96 :
97 9682 : struct bitint_info info;
98 9682 : bool ok = targetm.c.bitint_type_info (prec, &info);
99 9682 : gcc_assert (ok);
100 9682 : scalar_int_mode limb_mode = as_a <scalar_int_mode> (info.limb_mode);
101 9682 : if (prec <= GET_MODE_PRECISION (limb_mode))
102 : {
103 2334 : small_max_prec = prec;
104 2334 : return bitint_prec_small;
105 : }
106 7348 : bitint_big_endian = info.big_endian;
107 7348 : bitint_extended = info.extended;
108 7348 : if (info.limb_mode == info.abi_limb_mode && bitint_extended == bitint_ext_full)
109 0 : bitint_extended = bitint_ext_partial;
110 7348 : if (!large_min_prec
111 14606 : && GET_MODE_PRECISION (limb_mode) <= MAX_FIXED_MODE_SIZE)
112 14516 : large_min_prec = MAX_FIXED_MODE_SIZE + 1;
113 7348 : if (!limb_prec)
114 7258 : limb_prec = GET_MODE_PRECISION (limb_mode);
115 7348 : if (!abi_limb_prec)
116 7258 : abi_limb_prec
117 7258 : = GET_MODE_PRECISION (as_a <scalar_int_mode> (info.abi_limb_mode));
118 : /* For bitint_ext_full with different limb_mode from abi_limb_mode we
119 : currently only support only abi_limb_mode twice the precision of
120 : limb_mode, and don't support big endian in that case either. */
121 7348 : gcc_assert (bitint_extended != bitint_ext_full
122 : || (abi_limb_prec == 2 * limb_prec
123 : && !bitint_big_endian));
124 7348 : if (!huge_min_prec)
125 : {
126 14516 : if (4 * limb_prec >= MAX_FIXED_MODE_SIZE)
127 7258 : huge_min_prec = 4 * limb_prec;
128 : else
129 0 : huge_min_prec = MAX_FIXED_MODE_SIZE + 1;
130 : }
131 14696 : if (prec <= MAX_FIXED_MODE_SIZE)
132 : {
133 1609 : if (!mid_min_prec || prec < mid_min_prec)
134 1609 : mid_min_prec = prec;
135 1609 : return bitint_prec_middle;
136 : }
137 5739 : if (huge_min_prec && prec >= huge_min_prec)
138 : return bitint_prec_huge;
139 : return bitint_prec_large;
140 : }
141 :
142 : /* Same for a TYPE. */
143 :
144 : static bitint_prec_kind
145 479005 : bitint_precision_kind (tree type)
146 : {
147 479005 : return bitint_precision_kind (TYPE_PRECISION (type));
148 : }
149 :
150 : /* Return minimum precision needed to describe INTEGER_CST
151 : CST. All bits above that precision up to precision of
152 : TREE_TYPE (CST) are cleared if EXT is set to 0, or set
153 : if EXT is set to -1. */
154 :
155 : static unsigned
156 5334 : bitint_min_cst_precision (tree cst, int &ext)
157 : {
158 5334 : ext = tree_int_cst_sgn (cst) < 0 ? -1 : 0;
159 5334 : wide_int w = wi::to_wide (cst);
160 5334 : unsigned min_prec = wi::min_precision (w, TYPE_SIGN (TREE_TYPE (cst)));
161 : /* For signed values, we don't need to count the sign bit,
162 : we'll use constant 0 or -1 for the upper bits. */
163 5334 : if (!TYPE_UNSIGNED (TREE_TYPE (cst)))
164 3244 : --min_prec;
165 : else
166 : {
167 : /* For unsigned values, also try signed min_precision
168 : in case the constant has lots of most significant bits set. */
169 2090 : unsigned min_prec2 = wi::min_precision (w, SIGNED) - 1;
170 2090 : if (min_prec2 < min_prec)
171 : {
172 993 : ext = -1;
173 993 : return min_prec2;
174 : }
175 : }
176 : return min_prec;
177 5334 : }
178 :
179 : namespace {
180 :
181 : /* If OP is middle _BitInt, cast it to corresponding INTEGER_TYPE
182 : cached in TYPE and return it. */
183 :
184 : tree
185 7768 : maybe_cast_middle_bitint (gimple_stmt_iterator *gsi, tree op, tree &type)
186 : {
187 7768 : if (op == NULL_TREE
188 7740 : || !BITINT_TYPE_P (TREE_TYPE (op))
189 14631 : || bitint_precision_kind (TREE_TYPE (op)) != bitint_prec_middle)
190 908 : return op;
191 :
192 6860 : int prec = TYPE_PRECISION (TREE_TYPE (op));
193 6860 : int uns = TYPE_UNSIGNED (TREE_TYPE (op));
194 6860 : if (type == NULL_TREE
195 2538 : || TYPE_PRECISION (type) != prec
196 9398 : || TYPE_UNSIGNED (type) != uns)
197 4322 : type = build_nonstandard_integer_type (prec, uns);
198 :
199 6860 : if (TREE_CODE (op) != SSA_NAME)
200 : {
201 2351 : tree nop = fold_convert (type, op);
202 2351 : if (is_gimple_val (nop))
203 : return nop;
204 : }
205 :
206 4509 : tree nop = make_ssa_name (type);
207 4509 : gimple *g = gimple_build_assign (nop, NOP_EXPR, op);
208 4509 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
209 4509 : return nop;
210 : }
211 :
212 : /* Return true if STMT can be handled in a loop from least to most
213 : significant limb together with its dependencies. */
214 :
215 : bool
216 47614 : mergeable_op (gimple *stmt)
217 : {
218 47614 : if (!is_gimple_assign (stmt))
219 : return false;
220 38944 : switch (gimple_assign_rhs_code (stmt))
221 : {
222 : case PLUS_EXPR:
223 : case MINUS_EXPR:
224 : case NEGATE_EXPR:
225 : case BIT_AND_EXPR:
226 : case BIT_IOR_EXPR:
227 : case BIT_XOR_EXPR:
228 : case BIT_NOT_EXPR:
229 : case SSA_NAME:
230 : case INTEGER_CST:
231 : case BIT_FIELD_REF:
232 : return true;
233 410 : case LSHIFT_EXPR:
234 410 : {
235 410 : tree cnt = gimple_assign_rhs2 (stmt);
236 410 : if (tree_fits_uhwi_p (cnt)
237 198 : && tree_to_uhwi (cnt) < (unsigned HOST_WIDE_INT) limb_prec)
238 : return true;
239 : }
240 : break;
241 6219 : CASE_CONVERT:
242 6219 : case VIEW_CONVERT_EXPR:
243 6219 : {
244 6219 : tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt));
245 6219 : tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (stmt));
246 6219 : if (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
247 6177 : && BITINT_TYPE_P (lhs_type)
248 3640 : && BITINT_TYPE_P (rhs_type)
249 3293 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
250 3257 : && bitint_precision_kind (rhs_type) >= bitint_prec_large
251 9331 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
252 3112 : == CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
253 : {
254 2174 : if (TYPE_PRECISION (rhs_type) >= TYPE_PRECISION (lhs_type))
255 : return true;
256 168 : if ((unsigned) TYPE_PRECISION (lhs_type) % (2 * limb_prec) != 0)
257 : return true;
258 17 : if (bitint_precision_kind (lhs_type) == bitint_prec_large)
259 : return true;
260 : }
261 : break;
262 : }
263 : default:
264 : break;
265 : }
266 : return false;
267 : }
268 :
269 : /* Return non-zero if stmt is .{ADD,SUB,MUL}_OVERFLOW call with
270 : _Complex large/huge _BitInt lhs which has at most two immediate uses,
271 : at most one use in REALPART_EXPR stmt in the same bb and exactly one
272 : IMAGPART_EXPR use in the same bb with a single use which casts it to
273 : non-BITINT_TYPE integral type. If there is a REALPART_EXPR use,
274 : return 2. Such cases (most common uses of those builtins) can be
275 : optimized by marking their lhs and lhs of IMAGPART_EXPR and maybe lhs
276 : of REALPART_EXPR as not needed to be backed up by a stack variable.
277 : For .UBSAN_CHECK_{ADD,SUB,MUL} return 3. */
278 :
279 : int
280 21105 : optimizable_arith_overflow (gimple *stmt)
281 : {
282 21105 : bool is_ubsan = false;
283 21105 : if (!is_gimple_call (stmt) || !gimple_call_internal_p (stmt))
284 : return false;
285 5197 : switch (gimple_call_internal_fn (stmt))
286 : {
287 : case IFN_ADD_OVERFLOW:
288 : case IFN_SUB_OVERFLOW:
289 : case IFN_MUL_OVERFLOW:
290 : break;
291 48 : case IFN_UBSAN_CHECK_ADD:
292 48 : case IFN_UBSAN_CHECK_SUB:
293 48 : case IFN_UBSAN_CHECK_MUL:
294 48 : is_ubsan = true;
295 48 : break;
296 : default:
297 : return 0;
298 : }
299 5171 : tree lhs = gimple_call_lhs (stmt);
300 5171 : if (!lhs)
301 : return 0;
302 5171 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
303 : return 0;
304 5171 : tree type = is_ubsan ? TREE_TYPE (lhs) : TREE_TYPE (TREE_TYPE (lhs));
305 0 : if (!BITINT_TYPE_P (type)
306 5171 : || bitint_precision_kind (type) < bitint_prec_large)
307 0 : return 0;
308 :
309 5171 : if (is_ubsan)
310 : {
311 48 : use_operand_p use_p;
312 48 : gimple *use_stmt;
313 48 : if (!single_imm_use (lhs, &use_p, &use_stmt)
314 48 : || gimple_bb (use_stmt) != gimple_bb (stmt)
315 48 : || !gimple_store_p (use_stmt)
316 48 : || !is_gimple_assign (use_stmt)
317 48 : || gimple_has_volatile_ops (use_stmt)
318 96 : || stmt_ends_bb_p (use_stmt))
319 0 : return 0;
320 : return 3;
321 : }
322 :
323 5123 : imm_use_iterator ui;
324 5123 : use_operand_p use_p;
325 5123 : int seen = 0;
326 5123 : gimple *realpart = NULL, *cast = NULL;
327 19827 : FOR_EACH_IMM_USE_FAST (use_p, ui, lhs)
328 : {
329 9777 : gimple *g = USE_STMT (use_p);
330 9777 : if (is_gimple_debug (g))
331 0 : continue;
332 9777 : if (!is_gimple_assign (g) || gimple_bb (g) != gimple_bb (stmt))
333 : return 0;
334 9777 : if (gimple_assign_rhs_code (g) == REALPART_EXPR)
335 : {
336 4654 : if ((seen & 1) != 0)
337 : return 0;
338 4654 : seen |= 1;
339 4654 : realpart = g;
340 : }
341 5123 : else if (gimple_assign_rhs_code (g) == IMAGPART_EXPR)
342 : {
343 5123 : if ((seen & 2) != 0)
344 196 : return 0;
345 5123 : seen |= 2;
346 :
347 5123 : use_operand_p use2_p;
348 5123 : gimple *use_stmt;
349 5123 : tree lhs2 = gimple_assign_lhs (g);
350 5123 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs2))
351 : return 0;
352 5123 : if (!single_imm_use (lhs2, &use2_p, &use_stmt)
353 5123 : || gimple_bb (use_stmt) != gimple_bb (stmt)
354 10246 : || !gimple_assign_cast_p (use_stmt))
355 : return 0;
356 :
357 4931 : lhs2 = gimple_assign_lhs (use_stmt);
358 9862 : if (!INTEGRAL_TYPE_P (TREE_TYPE (lhs2))
359 9862 : || BITINT_TYPE_P (TREE_TYPE (lhs2)))
360 : return 0;
361 4927 : cast = use_stmt;
362 : }
363 : else
364 : return 0;
365 196 : }
366 4927 : if ((seen & 2) == 0)
367 : return 0;
368 4927 : if (seen == 3)
369 : {
370 : /* Punt if the cast stmt appears before realpart stmt, because
371 : if both appear, the lowering wants to emit all the code
372 : at the location of realpart stmt. */
373 4654 : gimple_stmt_iterator gsi = gsi_for_stmt (realpart);
374 4654 : unsigned int cnt = 0;
375 4657 : do
376 : {
377 4657 : gsi_prev_nondebug (&gsi);
378 4657 : if (gsi_end_p (gsi) || gsi_stmt (gsi) == cast)
379 : return 0;
380 4654 : if (gsi_stmt (gsi) == stmt)
381 : return 2;
382 : /* If realpart is too far from stmt, punt as well.
383 : Usually it will appear right after it. */
384 3 : if (++cnt == 32)
385 : return 0;
386 : }
387 : while (1);
388 : }
389 : return 1;
390 : }
391 :
392 : /* If STMT is some kind of comparison (GIMPLE_COND, comparison assignment)
393 : comparing large/huge _BitInt types, return the comparison code and if
394 : non-NULL fill in the comparison operands to *POP1 and *POP2. */
395 :
396 : tree_code
397 35797 : comparison_op (gimple *stmt, tree *pop1, tree *pop2)
398 : {
399 35797 : tree op1 = NULL_TREE, op2 = NULL_TREE;
400 35797 : tree_code code = ERROR_MARK;
401 35797 : if (gimple_code (stmt) == GIMPLE_COND)
402 : {
403 6536 : code = gimple_cond_code (stmt);
404 6536 : op1 = gimple_cond_lhs (stmt);
405 6536 : op2 = gimple_cond_rhs (stmt);
406 : }
407 29261 : else if (is_gimple_assign (stmt))
408 : {
409 29246 : code = gimple_assign_rhs_code (stmt);
410 29246 : op1 = gimple_assign_rhs1 (stmt);
411 29246 : if (TREE_CODE_CLASS (code) == tcc_comparison
412 29246 : || TREE_CODE_CLASS (code) == tcc_binary)
413 2187 : op2 = gimple_assign_rhs2 (stmt);
414 : }
415 35797 : if (TREE_CODE_CLASS (code) != tcc_comparison)
416 : return ERROR_MARK;
417 7310 : tree type = TREE_TYPE (op1);
418 10 : if (!BITINT_TYPE_P (type)
419 7320 : || bitint_precision_kind (type) < bitint_prec_large)
420 0 : return ERROR_MARK;
421 7310 : if (pop1)
422 : {
423 7248 : *pop1 = op1;
424 7248 : *pop2 = op2;
425 : }
426 : return code;
427 : }
428 :
429 : /* Class used during large/huge _BitInt lowering containing all the
430 : state for the methods. */
431 :
432 : struct bitint_large_huge
433 : {
434 7256 : bitint_large_huge ()
435 7256 : : m_names (NULL), m_loads (NULL), m_preserved (NULL),
436 7256 : m_single_use_names (NULL), m_map (NULL), m_vars (NULL),
437 7256 : m_limb_type (NULL_TREE), m_data (vNULL),
438 7256 : m_returns_twice_calls (vNULL) {}
439 :
440 : ~bitint_large_huge ();
441 :
442 : void insert_before (gimple *);
443 : tree limb_access_type (tree, tree);
444 : tree limb_access (tree, tree, tree, bool, bool = false);
445 : tree build_bit_field_ref (tree, tree, unsigned HOST_WIDE_INT,
446 : unsigned HOST_WIDE_INT);
447 : void if_then (gimple *, profile_probability, edge &, edge &);
448 : void if_then_else (gimple *, profile_probability, edge &, edge &);
449 : void if_then_if_then_else (gimple *g, gimple *,
450 : profile_probability, profile_probability,
451 : edge &, edge &, edge &);
452 : tree handle_operand (tree, tree);
453 : tree prepare_data_in_out (tree, tree, tree *, tree = NULL_TREE);
454 : tree add_cast (tree, tree);
455 : tree handle_plus_minus (tree_code, tree, tree, tree);
456 : tree handle_lshift (tree, tree, tree);
457 : tree handle_cast (tree, tree, tree);
458 : tree handle_bit_field_ref (tree, tree);
459 : tree handle_load (gimple *, tree);
460 : tree handle_stmt (gimple *, tree);
461 : tree handle_operand_addr (tree, gimple *, int *, int *);
462 : tree create_loop (tree, tree *);
463 : tree lower_mergeable_stmt (gimple *, tree_code &, tree, tree);
464 : tree lower_comparison_stmt (gimple *, tree_code &, tree, tree);
465 : void lower_shift_stmt (tree, gimple *);
466 : void lower_muldiv_stmt (tree, gimple *);
467 : void lower_float_conv_stmt (tree, gimple *);
468 : tree arith_overflow_extract_bits (unsigned int, unsigned int, tree,
469 : unsigned int, bool);
470 : void finish_arith_overflow (tree, tree, tree, tree, tree, tree, gimple *,
471 : unsigned, tree_code);
472 : void lower_addsub_overflow (tree, gimple *);
473 : void lower_mul_overflow (tree, gimple *);
474 : void lower_cplxpart_stmt (tree, gimple *);
475 : void lower_complexexpr_stmt (gimple *);
476 : void lower_bit_query (gimple *);
477 : void lower_bswap_bitreverse (tree, gimple *);
478 : void lower_call (tree, gimple *);
479 : void lower_asm (gimple *);
480 : void lower_stmt (gimple *);
481 :
482 : /* Bitmap of large/huge _BitInt SSA_NAMEs except those can be
483 : merged with their uses. */
484 : bitmap m_names;
485 : /* Subset of those for lhs of load statements. These will be
486 : cleared in m_names if the loads will be mergeable with all
487 : their uses. */
488 : bitmap m_loads;
489 : /* Bitmap of large/huge _BitInt SSA_NAMEs that should survive
490 : to later passes (arguments or return values of calls). */
491 : bitmap m_preserved;
492 : /* Subset of m_names which have a single use. As the lowering
493 : can replace various original statements with their lowered
494 : form even before it is done iterating over all basic blocks,
495 : testing has_single_use for the purpose of emitting clobbers
496 : doesn't work properly. */
497 : bitmap m_single_use_names;
498 : /* Used for coalescing/partitioning of large/huge _BitInt SSA_NAMEs
499 : set in m_names. */
500 : var_map m_map;
501 : /* Mapping of the partitions to corresponding decls. */
502 : tree *m_vars;
503 : /* Unsigned integer type with limb precision. */
504 : tree m_limb_type;
505 : /* Its TYPE_SIZE_UNIT. */
506 : unsigned HOST_WIDE_INT m_limb_size;
507 : /* Location of a gimple stmt which is being currently lowered. */
508 : location_t m_loc;
509 : /* Current stmt iterator where code is being lowered currently. */
510 : gimple_stmt_iterator m_gsi;
511 : /* Statement after which any clobbers should be added if non-NULL. */
512 : gimple *m_after_stmt;
513 : /* Set when creating loops to the loop header bb and its preheader. */
514 : basic_block m_bb, m_preheader_bb;
515 : /* Stmt iterator after which initialization statements should be emitted. */
516 : gimple_stmt_iterator m_init_gsi;
517 : /* Decl into which a mergeable statement stores result. */
518 : tree m_lhs;
519 : /* handle_operand/handle_stmt can be invoked in various ways.
520 :
521 : lower_mergeable_stmt for large _BitInt calls those with constant
522 : idx only, expanding to straight line code, for huge _BitInt
523 : emits a loop from least significant limb upwards, where each loop
524 : iteration handles 2 limbs, plus there can be up to one full limb
525 : and one partial limb processed after the loop, where handle_operand
526 : and/or handle_stmt are called with constant idx. m_upwards_2limb
527 : is set for this case, false otherwise. m_upwards is true if it
528 : is either large or huge _BitInt handled by lower_mergeable_stmt,
529 : i.e. indexes always increase.
530 :
531 : Another way is used by lower_comparison_stmt, which walks limbs
532 : from most significant to least significant, partial limb if any
533 : processed first with constant idx and then loop processing a single
534 : limb per iteration with non-constant idx.
535 :
536 : Another way is used in lower_shift_stmt, where for LSHIFT_EXPR
537 : destination limbs are processed from most significant to least
538 : significant or for RSHIFT_EXPR the other way around, in loops or
539 : straight line code, but idx usually is non-constant (so from
540 : handle_operand/handle_stmt POV random access). The LSHIFT_EXPR
541 : handling there can access even partial limbs using non-constant
542 : idx (then m_var_msb should be true, for all the other cases
543 : including lower_mergeable_stmt/lower_comparison_stmt that is
544 : not the case and so m_var_msb should be false.
545 :
546 : m_first should be set the first time handle_operand/handle_stmt
547 : is called and clear when it is called for some other limb with
548 : the same argument. If the lowering of an operand (e.g. INTEGER_CST)
549 : or statement (e.g. +/-/<< with < limb_prec constant) needs some
550 : state between the different calls, when m_first is true it should
551 : push some trees to m_data vector and also make sure m_data_cnt is
552 : incremented by how many trees were pushed, and when m_first is
553 : false, it can use the m_data[m_data_cnt] etc. data or update them,
554 : just needs to bump m_data_cnt by the same amount as when it was
555 : called with m_first set. The toplevel calls to
556 : handle_operand/handle_stmt should set m_data_cnt to 0 and truncate
557 : m_data vector when setting m_first to true.
558 :
559 : m_cast_conditional and m_bitfld_load are used when handling a
560 : bit-field load inside of a widening cast. handle_cast sometimes
561 : needs to do runtime comparisons and handle_operand only conditionally
562 : or even in two separate conditional blocks for one idx (once with
563 : constant index after comparing the runtime one for equality with the
564 : constant). In these cases, m_cast_conditional is set to true and
565 : the bit-field load then communicates its m_data_cnt to handle_cast
566 : using m_bitfld_load. */
567 : bool m_first;
568 : bool m_var_msb;
569 : unsigned m_upwards_2limb;
570 : bool m_upwards;
571 : bool m_cast_conditional;
572 : unsigned m_bitfld_load;
573 : vec<tree> m_data;
574 : unsigned int m_data_cnt;
575 : vec<gimple *> m_returns_twice_calls;
576 : };
577 :
578 7256 : bitint_large_huge::~bitint_large_huge ()
579 : {
580 7256 : BITMAP_FREE (m_names);
581 7256 : BITMAP_FREE (m_loads);
582 7256 : BITMAP_FREE (m_preserved);
583 7256 : BITMAP_FREE (m_single_use_names);
584 7256 : if (m_map)
585 5612 : delete_var_map (m_map);
586 7256 : XDELETEVEC (m_vars);
587 7256 : m_data.release ();
588 7256 : m_returns_twice_calls.release ();
589 7256 : }
590 :
591 : /* Insert gimple statement G before current location
592 : and set its gimple_location. */
593 :
594 : void
595 361607 : bitint_large_huge::insert_before (gimple *g)
596 : {
597 361607 : gimple_set_location (g, m_loc);
598 361607 : gsi_insert_before (&m_gsi, g, GSI_SAME_STMT);
599 361607 : }
600 :
601 : /* Return type for accessing limb IDX of BITINT_TYPE TYPE.
602 : This is normally m_limb_type, except for a partial most
603 : significant limb if any. */
604 :
605 : tree
606 130331 : bitint_large_huge::limb_access_type (tree type, tree idx)
607 : {
608 130331 : if (type == NULL_TREE)
609 5878 : return m_limb_type;
610 124453 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
611 124453 : unsigned int prec = TYPE_PRECISION (type);
612 124453 : gcc_assert (i * limb_prec < prec
613 : || (bitint_extended == bitint_ext_full
614 : && abi_limb_prec > limb_prec
615 : && i * limb_prec
616 : < CEIL (prec, abi_limb_prec) * abi_limb_prec));
617 248906 : if (bitint_big_endian
618 124453 : ? (i != 0 || (prec % limb_prec) == 0)
619 124453 : : (i + 1) * limb_prec <= prec)
620 81330 : return m_limb_type;
621 : else
622 86246 : return build_nonstandard_integer_type (prec % limb_prec,
623 43123 : TYPE_UNSIGNED (type));
624 : }
625 :
626 : /* Return a tree how to access limb IDX of VAR corresponding to BITINT_TYPE
627 : TYPE. If WRITE_P is true, it will be a store, otherwise a read. */
628 :
629 : tree
630 155371 : bitint_large_huge::limb_access (tree type, tree var, tree idx, bool write_p,
631 : bool abi_load_p)
632 : {
633 155371 : tree atype = (tree_fits_uhwi_p (idx)
634 155371 : ? limb_access_type (type, idx) : m_limb_type);
635 :
636 155371 : tree ltype = (bitint_extended && abi_load_p) ? atype : m_limb_type;
637 :
638 155371 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (var));
639 155371 : tree ret;
640 155371 : if (DECL_P (var) && tree_fits_uhwi_p (idx))
641 : {
642 95638 : if (as != TYPE_ADDR_SPACE (ltype))
643 0 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
644 0 : | ENCODE_QUAL_ADDR_SPACE (as));
645 95638 : tree ptype = build_pointer_type (strip_array_types (TREE_TYPE (var)));
646 95638 : unsigned HOST_WIDE_INT off = tree_to_uhwi (idx) * m_limb_size;
647 95638 : if (bitint_big_endian)
648 0 : off += m_limb_size - tree_to_uhwi (TYPE_SIZE_UNIT (ltype));
649 95638 : ret = build2 (MEM_REF, ltype,
650 : build_fold_addr_expr (var),
651 95638 : build_int_cst (ptype, off));
652 95638 : TREE_THIS_VOLATILE (ret) = TREE_THIS_VOLATILE (var);
653 95638 : TREE_SIDE_EFFECTS (ret) = TREE_SIDE_EFFECTS (var);
654 95638 : }
655 59733 : else if (TREE_CODE (var) == MEM_REF && tree_fits_uhwi_p (idx))
656 : {
657 4302 : if (as != TYPE_ADDR_SPACE (ltype))
658 0 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
659 0 : | ENCODE_QUAL_ADDR_SPACE (as));
660 4302 : unsigned HOST_WIDE_INT off = tree_to_uhwi (idx) * m_limb_size;
661 4302 : if (bitint_big_endian)
662 0 : off += m_limb_size - tree_to_uhwi (TYPE_SIZE_UNIT (ltype));
663 4302 : ret
664 8604 : = build2 (MEM_REF, ltype, unshare_expr (TREE_OPERAND (var, 0)),
665 8604 : size_binop (PLUS_EXPR, TREE_OPERAND (var, 1),
666 : build_int_cst (TREE_TYPE (TREE_OPERAND (var, 1)),
667 : off)));
668 4302 : TREE_THIS_VOLATILE (ret) = TREE_THIS_VOLATILE (var);
669 4302 : TREE_SIDE_EFFECTS (ret) = TREE_SIDE_EFFECTS (var);
670 4302 : TREE_THIS_NOTRAP (ret) = TREE_THIS_NOTRAP (var);
671 4302 : }
672 : else
673 : {
674 55431 : ltype = m_limb_type;
675 55431 : if (as != TYPE_ADDR_SPACE (ltype))
676 17 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
677 17 : | ENCODE_QUAL_ADDR_SPACE (as));
678 55431 : var = unshare_expr (var);
679 55431 : if (TREE_CODE (TREE_TYPE (var)) != ARRAY_TYPE
680 83198 : || !useless_type_conversion_p (m_limb_type,
681 27767 : TREE_TYPE (TREE_TYPE (var))))
682 : {
683 28606 : unsigned HOST_WIDE_INT nelts
684 28606 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var))), limb_prec);
685 28606 : tree atype = build_array_type_nelts (ltype, nelts);
686 28606 : var = build1 (VIEW_CONVERT_EXPR, atype, var);
687 : }
688 55431 : ret = build4 (ARRAY_REF, ltype, var, idx, NULL_TREE, NULL_TREE);
689 : }
690 155371 : if (!write_p && !useless_type_conversion_p (atype, ltype))
691 : {
692 18928 : gimple *g = gimple_build_assign (make_ssa_name (m_limb_type), ret);
693 18928 : insert_before (g);
694 18928 : ret = gimple_assign_lhs (g);
695 18928 : ret = build1 (NOP_EXPR, atype, ret);
696 : }
697 155371 : return ret;
698 : }
699 :
700 : /* Build a BIT_FIELD_REF to access BITSIZE bits with FTYPE type at
701 : offset BITPOS inside of OBJ. */
702 :
703 : tree
704 273 : bitint_large_huge::build_bit_field_ref (tree ftype, tree obj,
705 : unsigned HOST_WIDE_INT bitsize,
706 : unsigned HOST_WIDE_INT bitpos)
707 : {
708 546 : if (INTEGRAL_TYPE_P (TREE_TYPE (obj))
709 282 : && !type_has_mode_precision_p (TREE_TYPE (obj)))
710 : {
711 9 : unsigned HOST_WIDE_INT nelts
712 9 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))), limb_prec);
713 9 : tree ltype = m_limb_type;
714 9 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (obj));
715 9 : if (as != TYPE_ADDR_SPACE (ltype))
716 0 : ltype = build_qualified_type (ltype, TYPE_QUALS (ltype)
717 0 : | ENCODE_QUAL_ADDR_SPACE (as));
718 9 : tree atype = build_array_type_nelts (ltype, nelts);
719 9 : obj = build1 (VIEW_CONVERT_EXPR, atype, obj);
720 : }
721 273 : return build3 (BIT_FIELD_REF, ftype, obj, bitsize_int (bitsize),
722 273 : bitsize_int (bitpos));
723 : }
724 :
725 : /* Emit a half diamond,
726 : if (COND)
727 : |\
728 : | \
729 : | \
730 : | new_bb1
731 : | /
732 : | /
733 : |/
734 : or if (COND) new_bb1;
735 : PROB is the probability that the condition is true.
736 : Updates m_gsi to start of new_bb1.
737 : Sets EDGE_TRUE to edge from new_bb1 to successor and
738 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND) bb. */
739 :
740 : void
741 4282 : bitint_large_huge::if_then (gimple *cond, profile_probability prob,
742 : edge &edge_true, edge &edge_false)
743 : {
744 4282 : insert_before (cond);
745 4282 : edge e1 = split_block (gsi_bb (m_gsi), cond);
746 4282 : edge e2 = split_block (e1->dest, (gimple *) NULL);
747 4282 : edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
748 4282 : e1->flags = EDGE_TRUE_VALUE;
749 4282 : e1->probability = prob;
750 4282 : e3->probability = prob.invert ();
751 4282 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
752 4282 : edge_true = e2;
753 4282 : edge_false = e3;
754 4282 : m_gsi = gsi_after_labels (e1->dest);
755 4282 : }
756 :
757 : /* Emit a full diamond,
758 : if (COND)
759 : /\
760 : / \
761 : / \
762 : new_bb1 new_bb2
763 : \ /
764 : \ /
765 : \/
766 : or if (COND) new_bb2; else new_bb1;
767 : PROB is the probability that the condition is true.
768 : Updates m_gsi to start of new_bb2.
769 : Sets EDGE_TRUE to edge from new_bb1 to successor and
770 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND) bb. */
771 :
772 : void
773 112 : bitint_large_huge::if_then_else (gimple *cond, profile_probability prob,
774 : edge &edge_true, edge &edge_false)
775 : {
776 112 : insert_before (cond);
777 112 : edge e1 = split_block (gsi_bb (m_gsi), cond);
778 112 : edge e2 = split_block (e1->dest, (gimple *) NULL);
779 112 : basic_block bb = create_empty_bb (e1->dest);
780 112 : add_bb_to_loop (bb, e1->dest->loop_father);
781 112 : edge e3 = make_edge (e1->src, bb, EDGE_TRUE_VALUE);
782 112 : e1->flags = EDGE_FALSE_VALUE;
783 112 : e3->probability = prob;
784 112 : e1->probability = prob.invert ();
785 112 : bb->count = e1->src->count.apply_probability (prob);
786 112 : set_immediate_dominator (CDI_DOMINATORS, bb, e1->src);
787 112 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
788 112 : edge_true = make_single_succ_edge (bb, e2->dest, EDGE_FALLTHRU);
789 112 : edge_false = e2;
790 112 : m_gsi = gsi_after_labels (bb);
791 112 : }
792 :
793 : /* Emit a half diamond with full diamond in it
794 : if (COND1)
795 : |\
796 : | \
797 : | \
798 : | if (COND2)
799 : | / \
800 : | / \
801 : |new_bb1 new_bb2
802 : | | /
803 : \ | /
804 : \ | /
805 : \ | /
806 : \|/
807 : or if (COND1) { if (COND2) new_bb2; else new_bb1; }
808 : PROB1 is the probability that the condition 1 is true.
809 : PROB2 is the probability that the condition 2 is true.
810 : Updates m_gsi to start of new_bb1.
811 : Sets EDGE_TRUE_TRUE to edge from new_bb2 to successor,
812 : EDGE_TRUE_FALSE to edge from new_bb1 to successor and
813 : EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND1) bb.
814 : If COND2 is NULL, this is equivalent to
815 : if_then (COND1, PROB1, EDGE_TRUE_FALSE, EDGE_FALSE);
816 : EDGE_TRUE_TRUE = NULL; */
817 :
818 : void
819 2080 : bitint_large_huge::if_then_if_then_else (gimple *cond1, gimple *cond2,
820 : profile_probability prob1,
821 : profile_probability prob2,
822 : edge &edge_true_true,
823 : edge &edge_true_false,
824 : edge &edge_false)
825 : {
826 2080 : edge e2, e3, e4 = NULL;
827 2080 : if_then (cond1, prob1, e2, e3);
828 2080 : if (cond2 == NULL)
829 : {
830 1240 : edge_true_true = NULL;
831 1240 : edge_true_false = e2;
832 1240 : edge_false = e3;
833 1240 : return;
834 : }
835 840 : insert_before (cond2);
836 840 : e2 = split_block (gsi_bb (m_gsi), cond2);
837 840 : basic_block bb = create_empty_bb (e2->dest);
838 840 : add_bb_to_loop (bb, e2->dest->loop_father);
839 840 : e4 = make_edge (e2->src, bb, EDGE_TRUE_VALUE);
840 840 : set_immediate_dominator (CDI_DOMINATORS, bb, e2->src);
841 840 : e4->probability = prob2;
842 840 : e2->flags = EDGE_FALSE_VALUE;
843 840 : e2->probability = prob2.invert ();
844 840 : bb->count = e2->src->count.apply_probability (prob2);
845 840 : e4 = make_single_succ_edge (bb, e3->dest, EDGE_FALLTHRU);
846 840 : e2 = find_edge (e2->dest, e3->dest);
847 840 : edge_true_true = e4;
848 840 : edge_true_false = e2;
849 840 : edge_false = e3;
850 840 : m_gsi = gsi_after_labels (e2->src);
851 : }
852 :
853 : /* Emit code to access limb IDX from OP. */
854 :
855 : tree
856 110695 : bitint_large_huge::handle_operand (tree op, tree idx)
857 : {
858 110695 : switch (TREE_CODE (op))
859 : {
860 76395 : case SSA_NAME:
861 76395 : if (m_names == NULL
862 76395 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (op)))
863 : {
864 14246 : if (SSA_NAME_IS_DEFAULT_DEF (op))
865 : {
866 5 : if (m_first)
867 : {
868 2 : tree v = create_tmp_reg (m_limb_type);
869 2 : if (SSA_NAME_VAR (op) && VAR_P (SSA_NAME_VAR (op)))
870 : {
871 2 : DECL_NAME (v) = DECL_NAME (SSA_NAME_VAR (op));
872 2 : DECL_SOURCE_LOCATION (v)
873 2 : = DECL_SOURCE_LOCATION (SSA_NAME_VAR (op));
874 : }
875 2 : v = get_or_create_ssa_default_def (cfun, v);
876 2 : m_data.safe_push (v);
877 : }
878 5 : tree ret = m_data[m_data_cnt];
879 5 : m_data_cnt++;
880 5 : if (tree_fits_uhwi_p (idx))
881 : {
882 3 : tree type = limb_access_type (TREE_TYPE (op), idx);
883 3 : ret = add_cast (type, ret);
884 : }
885 5 : return ret;
886 : }
887 14241 : location_t loc_save = m_loc;
888 14241 : m_loc = gimple_location (SSA_NAME_DEF_STMT (op));
889 14241 : tree ret = handle_stmt (SSA_NAME_DEF_STMT (op), idx);
890 14241 : m_loc = loc_save;
891 14241 : return ret;
892 : }
893 62149 : int p;
894 62149 : gimple *g;
895 62149 : tree t;
896 62149 : p = var_to_partition (m_map, op);
897 62149 : gcc_assert (m_vars[p] != NULL_TREE);
898 62149 : t = limb_access (TREE_TYPE (op), m_vars[p], idx, false);
899 62149 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t);
900 62149 : insert_before (g);
901 62149 : t = gimple_assign_lhs (g);
902 62149 : if (m_first
903 22486 : && m_single_use_names
904 21614 : && m_vars[p] != m_lhs
905 21517 : && m_after_stmt
906 70837 : && bitmap_bit_p (m_single_use_names, SSA_NAME_VERSION (op)))
907 : {
908 8404 : tree clobber = build_clobber (TREE_TYPE (m_vars[p]),
909 : CLOBBER_STORAGE_END);
910 8404 : g = gimple_build_assign (m_vars[p], clobber);
911 8404 : gimple_stmt_iterator gsi = gsi_for_stmt (m_after_stmt);
912 8404 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
913 : }
914 : return t;
915 34300 : case INTEGER_CST:
916 34300 : if (tree_fits_uhwi_p (idx))
917 : {
918 23781 : tree c, type = limb_access_type (TREE_TYPE (op), idx);
919 23781 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
920 23781 : if (m_first)
921 : {
922 6206 : m_data.safe_push (NULL_TREE);
923 6206 : m_data.safe_push (NULL_TREE);
924 : }
925 23781 : if (bitint_big_endian)
926 0 : i = CEIL (TYPE_PRECISION (TREE_TYPE (op)), limb_prec) - 1 - i;
927 23781 : if (limb_prec != HOST_BITS_PER_WIDE_INT)
928 : {
929 0 : wide_int w = wi::rshift (wi::to_wide (op), i * limb_prec,
930 0 : TYPE_SIGN (TREE_TYPE (op)));
931 0 : c = wide_int_to_tree (type,
932 0 : wide_int::from (w, TYPE_PRECISION (type),
933 : UNSIGNED));
934 0 : }
935 23781 : else if (i >= TREE_INT_CST_EXT_NUNITS (op))
936 7469 : c = build_int_cst (type,
937 13274 : tree_int_cst_sgn (op) < 0 ? -1 : 0);
938 : else
939 16312 : c = build_int_cst (type, TREE_INT_CST_ELT (op, i));
940 23781 : m_data_cnt += 2;
941 23781 : return c;
942 : }
943 10519 : if (m_first
944 10519 : || (m_data[m_data_cnt] == NULL_TREE
945 159 : && m_data[m_data_cnt + 1] == NULL_TREE))
946 : {
947 5305 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
948 5305 : unsigned int rem = prec % ((m_upwards_2limb ? 2 : 1) * limb_prec);
949 5305 : int ext;
950 5305 : unsigned min_prec = bitint_min_cst_precision (op, ext);
951 5305 : if (m_first)
952 : {
953 5146 : m_data.safe_push (NULL_TREE);
954 5146 : m_data.safe_push (NULL_TREE);
955 : }
956 5305 : if (integer_zerop (op))
957 : {
958 837 : tree c = build_zero_cst (m_limb_type);
959 837 : m_data[m_data_cnt] = c;
960 837 : m_data[m_data_cnt + 1] = c;
961 : }
962 4468 : else if (integer_all_onesp (op))
963 : {
964 670 : tree c = build_all_ones_cst (m_limb_type);
965 670 : m_data[m_data_cnt] = c;
966 670 : m_data[m_data_cnt + 1] = c;
967 : }
968 3798 : else if (m_upwards_2limb && min_prec <= (unsigned) limb_prec)
969 : {
970 : /* Single limb constant. Use a phi with that limb from
971 : the preheader edge and 0 or -1 constant from the other edge
972 : and for the second limb in the loop. */
973 864 : tree out;
974 864 : gcc_assert (m_first);
975 864 : m_data.pop ();
976 864 : m_data.pop ();
977 864 : prepare_data_in_out (fold_convert (m_limb_type, op), idx, &out,
978 864 : build_int_cst (m_limb_type, ext));
979 864 : }
980 2934 : else if (min_prec > prec - rem - 2 * limb_prec)
981 : {
982 : /* Constant which has enough significant bits that it isn't
983 : worth trying to save .rodata space by extending from smaller
984 : number. */
985 2415 : tree type;
986 2415 : if (m_var_msb)
987 25 : type = TREE_TYPE (op);
988 : else
989 : /* If we have a guarantee the most significant partial limb
990 : (if any) will be only accessed through handle_operand
991 : with INTEGER_CST idx, we don't need to include the partial
992 : limb in .rodata. */
993 2390 : type = build_bitint_type (prec - rem, 1);
994 2415 : tree c = tree_output_constant_def (fold_convert (type, op));
995 2415 : m_data[m_data_cnt] = c;
996 2415 : m_data[m_data_cnt + 1] = NULL_TREE;
997 : }
998 519 : else if (m_upwards_2limb)
999 : {
1000 : /* Constant with smaller number of bits. Trade conditional
1001 : code for .rodata space by extending from smaller number. */
1002 444 : min_prec = CEIL (min_prec, 2 * limb_prec) * (2 * limb_prec);
1003 444 : tree type = build_bitint_type (min_prec, 1);
1004 444 : tree c = tree_output_constant_def (fold_convert (type, op));
1005 444 : tree ridx = idx;
1006 444 : if (bitint_big_endian)
1007 : {
1008 0 : ridx = make_ssa_name (sizetype);
1009 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1010 0 : size_int (min_prec / limb_prec
1011 : - ((HOST_WIDE_INT)
1012 : CEIL (prec,
1013 : limb_prec))));
1014 0 : insert_before (g);
1015 : }
1016 444 : tree ridx2 = make_ssa_name (sizetype);
1017 444 : g = gimple_build_assign (ridx2, PLUS_EXPR, ridx,
1018 : bitint_big_endian
1019 0 : ? size_int (-1) : size_one_node);
1020 444 : insert_before (g);
1021 444 : if (bitint_big_endian)
1022 0 : g = gimple_build_cond (GE_EXPR, idx,
1023 0 : size_int (CEIL (prec, limb_prec)
1024 : - min_prec / limb_prec),
1025 : NULL_TREE, NULL_TREE);
1026 : else
1027 444 : g = gimple_build_cond (LT_EXPR, idx,
1028 444 : size_int (min_prec / limb_prec),
1029 : NULL_TREE, NULL_TREE);
1030 444 : edge edge_true, edge_false;
1031 888 : if_then (g, (min_prec >= (prec - rem) / 2
1032 312 : ? profile_probability::likely ()
1033 132 : : profile_probability::unlikely ()),
1034 : edge_true, edge_false);
1035 444 : tree c1 = limb_access (TREE_TYPE (op), c, ridx, false);
1036 444 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c1)), c1);
1037 444 : insert_before (g);
1038 444 : c1 = gimple_assign_lhs (g);
1039 444 : tree c2 = limb_access (TREE_TYPE (op), c, ridx2, false);
1040 444 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c2)), c2);
1041 444 : insert_before (g);
1042 444 : c2 = gimple_assign_lhs (g);
1043 444 : tree c3 = build_int_cst (m_limb_type, ext);
1044 444 : m_gsi = gsi_after_labels (edge_true->dest);
1045 444 : m_data[m_data_cnt] = make_ssa_name (m_limb_type);
1046 444 : m_data[m_data_cnt + 1] = make_ssa_name (m_limb_type);
1047 444 : gphi *phi = create_phi_node (m_data[m_data_cnt],
1048 : edge_true->dest);
1049 444 : add_phi_arg (phi, c1, edge_true, UNKNOWN_LOCATION);
1050 444 : add_phi_arg (phi, c3, edge_false, UNKNOWN_LOCATION);
1051 444 : phi = create_phi_node (m_data[m_data_cnt + 1], edge_true->dest);
1052 444 : add_phi_arg (phi, c2, edge_true, UNKNOWN_LOCATION);
1053 444 : add_phi_arg (phi, c3, edge_false, UNKNOWN_LOCATION);
1054 : }
1055 : else
1056 : {
1057 : /* Constant with smaller number of bits. Trade conditional
1058 : code for .rodata space by extending from smaller number.
1059 : Version for loops with random access to the limbs or
1060 : downwards loops. */
1061 75 : min_prec = CEIL (min_prec, limb_prec) * limb_prec;
1062 75 : tree c;
1063 75 : if (min_prec <= (unsigned) limb_prec)
1064 21 : c = fold_convert (m_limb_type, op);
1065 : else
1066 : {
1067 54 : tree type = build_bitint_type (min_prec, 1);
1068 54 : c = tree_output_constant_def (fold_convert (type, op));
1069 : }
1070 75 : m_data[m_data_cnt] = c;
1071 75 : m_data[m_data_cnt + 1] = integer_type_node;
1072 : }
1073 5305 : t = m_data[m_data_cnt];
1074 : }
1075 : else
1076 5214 : t = m_data[m_data_cnt + 1];
1077 10519 : if (m_data[m_data_cnt + 1] == NULL_TREE)
1078 : {
1079 4774 : tree ridx = idx;
1080 4774 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
1081 4774 : tree c = m_data[m_data_cnt];
1082 4774 : unsigned int min_prec = TYPE_PRECISION (TREE_TYPE (c));
1083 4774 : if (bitint_big_endian
1084 0 : && CEIL (min_prec, limb_prec) != CEIL (prec, limb_prec))
1085 : {
1086 0 : ridx = make_ssa_name (sizetype);
1087 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1088 0 : size_int (CEIL (min_prec, limb_prec)
1089 : - ((HOST_WIDE_INT)
1090 : CEIL (prec, limb_prec))));
1091 0 : insert_before (g);
1092 : }
1093 4774 : t = limb_access (TREE_TYPE (op), c, ridx, false);
1094 4774 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t);
1095 4774 : insert_before (g);
1096 4774 : t = gimple_assign_lhs (g);
1097 : }
1098 5745 : else if (m_data[m_data_cnt + 1] == integer_type_node)
1099 : {
1100 115 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
1101 115 : unsigned rem = prec % ((m_upwards_2limb ? 2 : 1) * limb_prec);
1102 115 : int ext = wi::neg_p (wi::to_wide (op)) ? -1 : 0;
1103 115 : tree c = m_data[m_data_cnt];
1104 115 : unsigned min_prec = TYPE_PRECISION (TREE_TYPE (c));
1105 115 : if (bitint_big_endian)
1106 0 : g = gimple_build_cond (GE_EXPR, idx,
1107 0 : size_int (CEIL (prec, limb_prec)
1108 : - min_prec / limb_prec),
1109 : NULL_TREE, NULL_TREE);
1110 : else
1111 115 : g = gimple_build_cond (LT_EXPR, idx,
1112 115 : size_int (min_prec / limb_prec),
1113 : NULL_TREE, NULL_TREE);
1114 115 : edge edge_true, edge_false;
1115 230 : if_then (g, (min_prec >= (prec - rem) / 2
1116 29 : ? profile_probability::likely ()
1117 86 : : profile_probability::unlikely ()),
1118 : edge_true, edge_false);
1119 115 : if (min_prec > (unsigned) limb_prec)
1120 : {
1121 70 : tree ridx = idx;
1122 70 : if (bitint_big_endian)
1123 : {
1124 0 : ridx = make_ssa_name (sizetype);
1125 0 : g = gimple_build_assign (ridx, PLUS_EXPR, idx,
1126 0 : size_int (min_prec / limb_prec
1127 : - ((HOST_WIDE_INT)
1128 : CEIL (prec,
1129 : limb_prec))));
1130 0 : insert_before (g);
1131 : }
1132 70 : c = limb_access (TREE_TYPE (op), c, ridx, false);
1133 70 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (c)), c);
1134 70 : insert_before (g);
1135 70 : c = gimple_assign_lhs (g);
1136 : }
1137 115 : tree c2 = build_int_cst (m_limb_type, ext);
1138 115 : m_gsi = gsi_after_labels (edge_true->dest);
1139 115 : t = make_ssa_name (m_limb_type);
1140 115 : gphi *phi = create_phi_node (t, edge_true->dest);
1141 115 : add_phi_arg (phi, c, edge_true, UNKNOWN_LOCATION);
1142 115 : add_phi_arg (phi, c2, edge_false, UNKNOWN_LOCATION);
1143 : }
1144 10519 : m_data_cnt += 2;
1145 10519 : return t;
1146 0 : default:
1147 0 : gcc_unreachable ();
1148 : }
1149 : }
1150 :
1151 : /* Helper method, add a PHI node with VAL from preheader edge if
1152 : inside of a loop and m_first. Keep state in a pair of m_data
1153 : elements. If VAL_OUT is non-NULL, use that as PHI argument from
1154 : the latch edge, otherwise create a new SSA_NAME for it and let
1155 : caller initialize it. */
1156 :
1157 : tree
1158 15648 : bitint_large_huge::prepare_data_in_out (tree val, tree idx, tree *data_out,
1159 : tree val_out)
1160 : {
1161 15648 : if (!m_first)
1162 : {
1163 9325 : *data_out = tree_fits_uhwi_p (idx) ? NULL_TREE : m_data[m_data_cnt + 1];
1164 9325 : return m_data[m_data_cnt];
1165 : }
1166 :
1167 6323 : *data_out = NULL_TREE;
1168 6323 : if (tree_fits_uhwi_p (idx))
1169 : {
1170 1988 : m_data.safe_push (val);
1171 1988 : m_data.safe_push (NULL_TREE);
1172 1988 : return val;
1173 : }
1174 :
1175 4335 : tree in = make_ssa_name (TREE_TYPE (val));
1176 4335 : gphi *phi = create_phi_node (in, m_bb);
1177 4335 : edge e1 = find_edge (m_preheader_bb, m_bb);
1178 4335 : edge e2 = EDGE_PRED (m_bb, 0);
1179 4335 : if (e1 == e2)
1180 4335 : e2 = EDGE_PRED (m_bb, 1);
1181 4335 : add_phi_arg (phi, val, e1, UNKNOWN_LOCATION);
1182 4335 : tree out = val_out ? val_out : make_ssa_name (TREE_TYPE (val));
1183 4335 : add_phi_arg (phi, out, e2, UNKNOWN_LOCATION);
1184 4335 : m_data.safe_push (in);
1185 4335 : m_data.safe_push (out);
1186 4335 : return in;
1187 : }
1188 :
1189 : /* Return VAL cast to TYPE. If VAL is INTEGER_CST, just
1190 : convert it without emitting any code, otherwise emit
1191 : the conversion statement before the current location. */
1192 :
1193 : tree
1194 38267 : bitint_large_huge::add_cast (tree type, tree val)
1195 : {
1196 38267 : if (TREE_CODE (val) == INTEGER_CST)
1197 4509 : return fold_convert (type, val);
1198 :
1199 33758 : tree lhs = make_ssa_name (type);
1200 33758 : gimple *g = gimple_build_assign (lhs, NOP_EXPR, val);
1201 33758 : insert_before (g);
1202 33758 : return lhs;
1203 : }
1204 :
1205 : /* Helper of handle_stmt method, handle PLUS_EXPR or MINUS_EXPR. */
1206 :
1207 : tree
1208 13208 : bitint_large_huge::handle_plus_minus (tree_code code, tree rhs1, tree rhs2,
1209 : tree idx)
1210 : {
1211 13208 : tree lhs, data_out, ctype;
1212 13208 : tree rhs1_type = TREE_TYPE (rhs1);
1213 13208 : gimple *g;
1214 13208 : tree data_in = prepare_data_in_out (build_zero_cst (m_limb_type), idx,
1215 : &data_out);
1216 :
1217 18936 : if (optab_handler (code == PLUS_EXPR ? uaddc5_optab : usubc5_optab,
1218 13208 : TYPE_MODE (m_limb_type)) != CODE_FOR_nothing)
1219 : {
1220 13208 : ctype = build_complex_type (m_limb_type);
1221 13208 : if (!types_compatible_p (rhs1_type, m_limb_type))
1222 : {
1223 1040 : if (!TYPE_UNSIGNED (rhs1_type))
1224 : {
1225 329 : tree type = unsigned_type_for (rhs1_type);
1226 329 : rhs1 = add_cast (type, rhs1);
1227 329 : rhs2 = add_cast (type, rhs2);
1228 : }
1229 1040 : rhs1 = add_cast (m_limb_type, rhs1);
1230 1040 : rhs2 = add_cast (m_limb_type, rhs2);
1231 : }
1232 13208 : lhs = make_ssa_name (ctype);
1233 18936 : g = gimple_build_call_internal (code == PLUS_EXPR
1234 : ? IFN_UADDC : IFN_USUBC,
1235 : 3, rhs1, rhs2, data_in);
1236 13208 : gimple_call_set_lhs (g, lhs);
1237 13208 : insert_before (g);
1238 13208 : if (data_out == NULL_TREE)
1239 11025 : data_out = make_ssa_name (m_limb_type);
1240 13208 : g = gimple_build_assign (data_out, IMAGPART_EXPR,
1241 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1242 13208 : insert_before (g);
1243 : }
1244 0 : else if (types_compatible_p (rhs1_type, m_limb_type))
1245 : {
1246 0 : ctype = build_complex_type (m_limb_type);
1247 0 : lhs = make_ssa_name (ctype);
1248 0 : g = gimple_build_call_internal (code == PLUS_EXPR
1249 : ? IFN_ADD_OVERFLOW : IFN_SUB_OVERFLOW,
1250 : 2, rhs1, rhs2);
1251 0 : gimple_call_set_lhs (g, lhs);
1252 0 : insert_before (g);
1253 0 : if (data_out == NULL_TREE)
1254 0 : data_out = make_ssa_name (m_limb_type);
1255 0 : if (!integer_zerop (data_in))
1256 : {
1257 0 : rhs1 = make_ssa_name (m_limb_type);
1258 0 : g = gimple_build_assign (rhs1, REALPART_EXPR,
1259 : build1 (REALPART_EXPR, m_limb_type, lhs));
1260 0 : insert_before (g);
1261 0 : rhs2 = make_ssa_name (m_limb_type);
1262 0 : g = gimple_build_assign (rhs2, IMAGPART_EXPR,
1263 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1264 0 : insert_before (g);
1265 0 : lhs = make_ssa_name (ctype);
1266 0 : g = gimple_build_call_internal (code == PLUS_EXPR
1267 : ? IFN_ADD_OVERFLOW
1268 : : IFN_SUB_OVERFLOW,
1269 : 2, rhs1, data_in);
1270 0 : gimple_call_set_lhs (g, lhs);
1271 0 : insert_before (g);
1272 0 : data_in = make_ssa_name (m_limb_type);
1273 0 : g = gimple_build_assign (data_in, IMAGPART_EXPR,
1274 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1275 0 : insert_before (g);
1276 0 : g = gimple_build_assign (data_out, PLUS_EXPR, rhs2, data_in);
1277 0 : insert_before (g);
1278 : }
1279 : else
1280 : {
1281 0 : g = gimple_build_assign (data_out, IMAGPART_EXPR,
1282 : build1 (IMAGPART_EXPR, m_limb_type, lhs));
1283 0 : insert_before (g);
1284 : }
1285 : }
1286 : else
1287 : {
1288 0 : tree in = add_cast (rhs1_type, data_in);
1289 0 : lhs = make_ssa_name (rhs1_type);
1290 0 : g = gimple_build_assign (lhs, code, rhs1, rhs2);
1291 0 : insert_before (g);
1292 0 : rhs1 = make_ssa_name (rhs1_type);
1293 0 : g = gimple_build_assign (rhs1, code, lhs, in);
1294 0 : insert_before (g);
1295 0 : m_data[m_data_cnt] = NULL_TREE;
1296 0 : m_data_cnt += 2;
1297 0 : return rhs1;
1298 : }
1299 13208 : rhs1 = make_ssa_name (m_limb_type);
1300 13208 : g = gimple_build_assign (rhs1, REALPART_EXPR,
1301 : build1 (REALPART_EXPR, m_limb_type, lhs));
1302 13208 : insert_before (g);
1303 13208 : if (!types_compatible_p (rhs1_type, m_limb_type))
1304 1040 : rhs1 = add_cast (rhs1_type, rhs1);
1305 13208 : m_data[m_data_cnt] = data_out;
1306 13208 : m_data_cnt += 2;
1307 13208 : return rhs1;
1308 : }
1309 :
1310 : /* Helper function for handle_stmt method, handle LSHIFT_EXPR by
1311 : count in [0, limb_prec - 1] range. */
1312 :
1313 : tree
1314 152 : bitint_large_huge::handle_lshift (tree rhs1, tree rhs2, tree idx)
1315 : {
1316 152 : unsigned HOST_WIDE_INT cnt = tree_to_uhwi (rhs2);
1317 152 : gcc_checking_assert (cnt < (unsigned) limb_prec);
1318 152 : if (cnt == 0)
1319 : return rhs1;
1320 :
1321 152 : tree lhs, data_out, rhs1_type = TREE_TYPE (rhs1);
1322 152 : gimple *g;
1323 152 : tree data_in = prepare_data_in_out (build_zero_cst (m_limb_type), idx,
1324 : &data_out);
1325 :
1326 152 : if (!integer_zerop (data_in))
1327 : {
1328 136 : lhs = make_ssa_name (m_limb_type);
1329 136 : g = gimple_build_assign (lhs, RSHIFT_EXPR, data_in,
1330 : build_int_cst (unsigned_type_node,
1331 136 : limb_prec - cnt));
1332 136 : insert_before (g);
1333 136 : if (!types_compatible_p (rhs1_type, m_limb_type))
1334 35 : lhs = add_cast (rhs1_type, lhs);
1335 : data_in = lhs;
1336 : }
1337 152 : if (types_compatible_p (rhs1_type, m_limb_type))
1338 : {
1339 117 : if (data_out == NULL_TREE)
1340 82 : data_out = make_ssa_name (m_limb_type);
1341 117 : g = gimple_build_assign (data_out, rhs1);
1342 117 : insert_before (g);
1343 : }
1344 152 : if (cnt < (unsigned) TYPE_PRECISION (rhs1_type))
1345 : {
1346 137 : lhs = make_ssa_name (rhs1_type);
1347 137 : g = gimple_build_assign (lhs, LSHIFT_EXPR, rhs1, rhs2);
1348 137 : insert_before (g);
1349 137 : if (!integer_zerop (data_in))
1350 : {
1351 121 : rhs1 = lhs;
1352 121 : lhs = make_ssa_name (rhs1_type);
1353 121 : g = gimple_build_assign (lhs, BIT_IOR_EXPR, rhs1, data_in);
1354 121 : insert_before (g);
1355 : }
1356 : }
1357 : else
1358 : lhs = data_in;
1359 152 : m_data[m_data_cnt] = data_out;
1360 152 : m_data_cnt += 2;
1361 152 : return lhs;
1362 : }
1363 :
1364 : /* Helper function for handle_stmt method, handle an integral
1365 : to integral conversion. */
1366 :
1367 : tree
1368 7457 : bitint_large_huge::handle_cast (tree lhs_type, tree rhs1, tree idx)
1369 : {
1370 7457 : tree rhs_type = TREE_TYPE (rhs1);
1371 7457 : gimple *g;
1372 7457 : if ((TREE_CODE (rhs1) == SSA_NAME || TREE_CODE (rhs1) == INTEGER_CST)
1373 7457 : && BITINT_TYPE_P (lhs_type)
1374 7457 : && BITINT_TYPE_P (rhs_type)
1375 6468 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
1376 13925 : && bitint_precision_kind (rhs_type) >= bitint_prec_large)
1377 : {
1378 5877 : if (TYPE_PRECISION (rhs_type) >= TYPE_PRECISION (lhs_type)
1379 : /* If lhs has bigger precision than rhs, we can use
1380 : the simple case only if there is a guarantee that
1381 : the most significant limb is handled in straight
1382 : line code. If m_var_msb (on left shifts) or
1383 : if m_upwards_2limb * limb_prec is equal to
1384 : lhs precision or if not m_upwards_2limb and lhs_type
1385 : has precision which is multiple of limb_prec that is
1386 : not the case. */
1387 5877 : || (!m_var_msb
1388 1481 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1389 1481 : == CEIL (TYPE_PRECISION (rhs_type), limb_prec))
1390 346 : && ((!m_upwards_2limb
1391 182 : && (TYPE_PRECISION (lhs_type) % limb_prec != 0))
1392 243 : || (m_upwards_2limb
1393 328 : && (m_upwards_2limb * limb_prec
1394 164 : < TYPE_PRECISION (lhs_type))))))
1395 : {
1396 4637 : tree ridx = idx;
1397 4637 : if (bitint_big_endian
1398 4637 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1399 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1400 : {
1401 0 : HOST_WIDE_INT diff = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1402 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1403 0 : if (tree_fits_uhwi_p (idx))
1404 0 : ridx = size_int (tree_to_uhwi (idx) + diff);
1405 : else
1406 : {
1407 0 : tree t = make_ssa_name (sizetype);
1408 0 : g = gimple_build_assign (t, PLUS_EXPR, idx, size_int (diff));
1409 0 : insert_before (g);
1410 0 : ridx = t;
1411 : }
1412 : }
1413 4637 : rhs1 = handle_operand (rhs1, ridx);
1414 4637 : if (tree_fits_uhwi_p (idx))
1415 : {
1416 2372 : tree type = limb_access_type (lhs_type, idx);
1417 2372 : if (!types_compatible_p (type, TREE_TYPE (rhs1)))
1418 1241 : rhs1 = add_cast (type, rhs1);
1419 : }
1420 4637 : return rhs1;
1421 : }
1422 1240 : tree t;
1423 : /* Indexes lower than this don't need any special processing. */
1424 1240 : unsigned low = ((unsigned) TYPE_PRECISION (rhs_type)
1425 1240 : - !TYPE_UNSIGNED (rhs_type)) / limb_prec;
1426 : /* Indexes >= than this always contain an extension. */
1427 1240 : unsigned high = CEIL ((unsigned) TYPE_PRECISION (rhs_type), limb_prec);
1428 1240 : unsigned lcnt = CEIL ((unsigned) TYPE_PRECISION (lhs_type), limb_prec);
1429 1240 : unsigned lowe = bitint_big_endian ? lcnt - 1 - low : low;
1430 1240 : bool save_first = m_first;
1431 1240 : if (m_first)
1432 : {
1433 405 : m_data.safe_push (NULL_TREE);
1434 405 : m_data.safe_push (NULL_TREE);
1435 405 : m_data.safe_push (NULL_TREE);
1436 405 : if (TYPE_UNSIGNED (rhs_type))
1437 : /* No need to keep state between iterations. */
1438 : ;
1439 192 : else if (m_upwards && !m_upwards_2limb)
1440 : /* We need to keep state between iterations, but
1441 : not within any loop, everything is straight line
1442 : code with only increasing indexes. */
1443 : ;
1444 152 : else if (!m_upwards_2limb)
1445 : {
1446 3 : unsigned save_data_cnt = m_data_cnt;
1447 3 : gimple_stmt_iterator save_gsi = m_gsi;
1448 3 : m_gsi = m_init_gsi;
1449 3 : if (gsi_end_p (m_gsi))
1450 0 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
1451 : else
1452 3 : gsi_next (&m_gsi);
1453 3 : m_data_cnt = save_data_cnt + 3;
1454 3 : t = handle_operand (rhs1, size_int (bitint_big_endian
1455 : ? high - 1 - low : low));
1456 3 : m_first = false;
1457 3 : m_data[save_data_cnt + 2]
1458 3 : = build_int_cst (NULL_TREE, m_data_cnt);
1459 3 : m_data_cnt = save_data_cnt;
1460 3 : t = add_cast (signed_type_for (m_limb_type), t);
1461 3 : tree lpm1 = build_int_cst (unsigned_type_node, limb_prec - 1);
1462 3 : tree n = make_ssa_name (TREE_TYPE (t));
1463 3 : g = gimple_build_assign (n, RSHIFT_EXPR, t, lpm1);
1464 3 : insert_before (g);
1465 3 : m_data[save_data_cnt + 1] = add_cast (m_limb_type, n);
1466 3 : m_init_gsi = m_gsi;
1467 3 : if (gsi_end_p (m_init_gsi))
1468 0 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
1469 : else
1470 3 : gsi_prev (&m_init_gsi);
1471 3 : m_gsi = save_gsi;
1472 : }
1473 149 : else if (m_upwards_2limb * limb_prec < TYPE_PRECISION (rhs_type))
1474 : /* We need to keep state between iterations, but
1475 : fortunately not within the loop, only afterwards. */
1476 : ;
1477 : else
1478 : {
1479 145 : tree out;
1480 145 : m_data.truncate (m_data_cnt);
1481 145 : prepare_data_in_out (build_zero_cst (m_limb_type), idx, &out);
1482 145 : m_data.safe_push (NULL_TREE);
1483 : }
1484 : }
1485 :
1486 1240 : unsigned save_data_cnt = m_data_cnt;
1487 1240 : m_data_cnt += 3;
1488 1240 : if (!tree_fits_uhwi_p (idx))
1489 : {
1490 670 : if (m_upwards_2limb
1491 650 : && low >= m_upwards_2limb - m_first)
1492 : {
1493 158 : if (bitint_big_endian
1494 158 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1495 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1496 : {
1497 0 : HOST_WIDE_INT diff
1498 0 : = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1499 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1500 0 : tree t = make_ssa_name (sizetype);
1501 0 : g = gimple_build_assign (t, PLUS_EXPR, idx, size_int (diff));
1502 0 : insert_before (g);
1503 0 : idx = t;
1504 : }
1505 158 : rhs1 = handle_operand (rhs1, idx);
1506 158 : if (m_first)
1507 131 : m_data[save_data_cnt + 2]
1508 262 : = build_int_cst (NULL_TREE, m_data_cnt);
1509 158 : m_first = save_first;
1510 158 : return rhs1;
1511 : }
1512 1289 : bool single_comparison
1513 512 : = low == high || (m_upwards_2limb && (low & 1) == m_first);
1514 265 : tree idxc = idx;
1515 265 : if (!single_comparison
1516 265 : && m_upwards_2limb
1517 245 : && !m_first
1518 112 : && low + 1 == m_upwards_2limb)
1519 : /* In this case we know that idx <= low always,
1520 : so effectively we just needs a single comparison,
1521 : idx < low or idx == low, but we'd need to emit different
1522 : code for the 2 branches than single_comparison normally
1523 : emits. So, instead of special-casing that, emit a
1524 : low <= low comparison which cfg cleanup will clean up
1525 : at the end of the pass. */
1526 89 : idxc = size_int (lowe);
1527 512 : if (bitint_big_endian)
1528 0 : g = gimple_build_cond (single_comparison ? GT_EXPR : GE_EXPR,
1529 0 : idxc, size_int (lowe),
1530 : NULL_TREE, NULL_TREE);
1531 : else
1532 777 : g = gimple_build_cond (single_comparison ? LT_EXPR : LE_EXPR,
1533 512 : idxc, size_int (low), NULL_TREE, NULL_TREE);
1534 512 : edge edge_true_true, edge_true_false, edge_false;
1535 777 : if_then_if_then_else (g, (single_comparison ? NULL
1536 265 : : gimple_build_cond (EQ_EXPR, idx,
1537 265 : size_int (lowe),
1538 : NULL_TREE,
1539 : NULL_TREE)),
1540 : profile_probability::likely (),
1541 : profile_probability::unlikely (),
1542 : edge_true_true, edge_true_false, edge_false);
1543 512 : bool save_cast_conditional = m_cast_conditional;
1544 512 : m_cast_conditional = true;
1545 512 : m_bitfld_load = 0;
1546 512 : tree t1 = idx, t2 = NULL_TREE;
1547 512 : if (bitint_big_endian
1548 512 : && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
1549 0 : != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
1550 : {
1551 0 : HOST_WIDE_INT diff = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
1552 0 : diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
1553 0 : t1 = make_ssa_name (sizetype);
1554 0 : g = gimple_build_assign (t1, PLUS_EXPR, idx, size_int (diff));
1555 0 : insert_before (g);
1556 : }
1557 512 : t1 = handle_operand (rhs1, t1);
1558 512 : if (m_first)
1559 199 : m_data[save_data_cnt + 2]
1560 398 : = build_int_cst (NULL_TREE, m_data_cnt);
1561 512 : tree ext = NULL_TREE;
1562 512 : tree bitfld = NULL_TREE;
1563 512 : if (!single_comparison)
1564 : {
1565 265 : m_gsi = gsi_after_labels (edge_true_true->src);
1566 265 : m_first = false;
1567 265 : m_data_cnt = save_data_cnt + 3;
1568 265 : if (m_bitfld_load)
1569 : {
1570 4 : bitfld = m_data[m_bitfld_load];
1571 4 : m_data[m_bitfld_load] = m_data[m_bitfld_load + 2];
1572 4 : m_bitfld_load = 0;
1573 : }
1574 265 : t2 = handle_operand (rhs1, size_int (bitint_big_endian
1575 : ? high - 1 - low : low));
1576 265 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (t2)))
1577 220 : t2 = add_cast (m_limb_type, t2);
1578 265 : if (!TYPE_UNSIGNED (rhs_type) && m_upwards_2limb)
1579 : {
1580 145 : ext = add_cast (signed_type_for (m_limb_type), t2);
1581 290 : tree lpm1 = build_int_cst (unsigned_type_node,
1582 145 : limb_prec - 1);
1583 145 : tree n = make_ssa_name (TREE_TYPE (ext));
1584 145 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
1585 145 : insert_before (g);
1586 145 : ext = add_cast (m_limb_type, n);
1587 : }
1588 : }
1589 512 : tree t3;
1590 512 : if (TYPE_UNSIGNED (rhs_type))
1591 262 : t3 = build_zero_cst (m_limb_type);
1592 250 : else if (m_upwards_2limb && (save_first || ext != NULL_TREE))
1593 167 : t3 = m_data[save_data_cnt];
1594 : else
1595 83 : t3 = m_data[save_data_cnt + 1];
1596 512 : m_gsi = gsi_after_labels (edge_true_false->dest);
1597 512 : t = make_ssa_name (m_limb_type);
1598 512 : gphi *phi = create_phi_node (t, edge_true_false->dest);
1599 512 : add_phi_arg (phi, t1, edge_true_false, UNKNOWN_LOCATION);
1600 512 : add_phi_arg (phi, t3, edge_false, UNKNOWN_LOCATION);
1601 512 : if (edge_true_true)
1602 265 : add_phi_arg (phi, t2, edge_true_true, UNKNOWN_LOCATION);
1603 512 : if (ext)
1604 : {
1605 145 : tree t4 = make_ssa_name (m_limb_type);
1606 145 : phi = create_phi_node (t4, edge_true_false->dest);
1607 145 : add_phi_arg (phi, build_zero_cst (m_limb_type), edge_true_false,
1608 : UNKNOWN_LOCATION);
1609 145 : add_phi_arg (phi, m_data[save_data_cnt], edge_false,
1610 : UNKNOWN_LOCATION);
1611 145 : add_phi_arg (phi, ext, edge_true_true, UNKNOWN_LOCATION);
1612 145 : if (!save_cast_conditional)
1613 : {
1614 135 : g = gimple_build_assign (m_data[save_data_cnt + 1], t4);
1615 135 : insert_before (g);
1616 : }
1617 : else
1618 10 : for (basic_block bb = gsi_bb (m_gsi);;)
1619 : {
1620 10 : edge e1 = single_succ_edge (bb);
1621 10 : edge e2 = find_edge (e1->dest, m_bb), e3;
1622 10 : tree t5 = (e2 ? m_data[save_data_cnt + 1]
1623 10 : : make_ssa_name (m_limb_type));
1624 10 : phi = create_phi_node (t5, e1->dest);
1625 10 : edge_iterator ei;
1626 30 : FOR_EACH_EDGE (e3, ei, e1->dest->preds)
1627 30 : add_phi_arg (phi, (e3 == e1 ? t4
1628 10 : : build_zero_cst (m_limb_type)),
1629 : e3, UNKNOWN_LOCATION);
1630 10 : if (e2)
1631 : break;
1632 0 : t4 = t5;
1633 0 : bb = e1->dest;
1634 0 : }
1635 : }
1636 512 : if (m_bitfld_load)
1637 : {
1638 8 : tree t4;
1639 8 : if (!save_first && !save_cast_conditional)
1640 2 : t4 = m_data[m_bitfld_load + 1];
1641 : else
1642 6 : t4 = make_ssa_name (m_limb_type);
1643 8 : phi = create_phi_node (t4, edge_true_false->dest);
1644 12 : add_phi_arg (phi,
1645 4 : edge_true_true ? bitfld : m_data[m_bitfld_load],
1646 : edge_true_false, UNKNOWN_LOCATION);
1647 8 : add_phi_arg (phi, m_data[m_bitfld_load + 2],
1648 : edge_false, UNKNOWN_LOCATION);
1649 8 : if (edge_true_true)
1650 4 : add_phi_arg (phi, m_data[m_bitfld_load], edge_true_true,
1651 : UNKNOWN_LOCATION);
1652 8 : if (save_cast_conditional)
1653 4 : for (basic_block bb = gsi_bb (m_gsi);;)
1654 : {
1655 4 : edge e1 = single_succ_edge (bb);
1656 4 : edge e2 = find_edge (e1->dest, m_bb), e3;
1657 4 : tree t5 = ((e2 && !save_first) ? m_data[m_bitfld_load + 1]
1658 4 : : make_ssa_name (m_limb_type));
1659 4 : phi = create_phi_node (t5, e1->dest);
1660 4 : edge_iterator ei;
1661 14 : FOR_EACH_EDGE (e3, ei, e1->dest->preds)
1662 16 : add_phi_arg (phi, (e3 == e1 ? t4
1663 6 : : build_zero_cst (m_limb_type)),
1664 : e3, UNKNOWN_LOCATION);
1665 4 : t4 = t5;
1666 4 : if (e2)
1667 : break;
1668 0 : bb = e1->dest;
1669 0 : }
1670 8 : m_data[m_bitfld_load] = t4;
1671 8 : m_data[m_bitfld_load + 2] = t4;
1672 8 : m_bitfld_load = 0;
1673 : }
1674 512 : m_cast_conditional = save_cast_conditional;
1675 512 : m_first = save_first;
1676 512 : return t;
1677 : }
1678 : else
1679 : {
1680 570 : unsigned tidx = tree_to_uhwi (idx);
1681 570 : if (bitint_big_endian)
1682 0 : tidx = lcnt - 1 - tidx;
1683 570 : if (tidx < low)
1684 : {
1685 152 : t = handle_operand (rhs1, (bitint_big_endian
1686 0 : ? size_int (high - 1 - tidx) : idx));
1687 152 : if (m_first)
1688 71 : m_data[save_data_cnt + 2]
1689 142 : = build_int_cst (NULL_TREE, m_data_cnt);
1690 : }
1691 418 : else if (tidx < high)
1692 : {
1693 68 : t = handle_operand (rhs1, size_int (bitint_big_endian
1694 : ? high - 1 - low : low));
1695 68 : if (m_first)
1696 1 : m_data[save_data_cnt + 2]
1697 2 : = build_int_cst (NULL_TREE, m_data_cnt);
1698 68 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (t)))
1699 60 : t = add_cast (m_limb_type, t);
1700 68 : tree ext = NULL_TREE;
1701 68 : if (!TYPE_UNSIGNED (rhs_type) && m_upwards)
1702 : {
1703 44 : ext = add_cast (signed_type_for (m_limb_type), t);
1704 88 : tree lpm1 = build_int_cst (unsigned_type_node,
1705 44 : limb_prec - 1);
1706 44 : tree n = make_ssa_name (TREE_TYPE (ext));
1707 44 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
1708 44 : insert_before (g);
1709 44 : ext = add_cast (m_limb_type, n);
1710 44 : m_data[save_data_cnt + 1] = ext;
1711 : }
1712 : }
1713 : else
1714 : {
1715 350 : if (TYPE_UNSIGNED (rhs_type) && m_first)
1716 : {
1717 0 : handle_operand (rhs1, (bitint_big_endian
1718 0 : ? size_int (high - 1)
1719 : : size_zero_node));
1720 0 : m_data[save_data_cnt + 2]
1721 0 : = build_int_cst (NULL_TREE, m_data_cnt);
1722 : }
1723 : else
1724 350 : m_data_cnt = tree_to_uhwi (m_data[save_data_cnt + 2]);
1725 350 : if (TYPE_UNSIGNED (rhs_type))
1726 184 : t = build_zero_cst (m_limb_type);
1727 166 : else if (m_bb
1728 16 : && m_data[save_data_cnt]
1729 179 : && ((tidx & 1) == 0 || tidx != low + 1))
1730 : t = m_data[save_data_cnt];
1731 : else
1732 160 : t = m_data[save_data_cnt + 1];
1733 : }
1734 570 : tree type = limb_access_type (lhs_type, idx);
1735 570 : if (!useless_type_conversion_p (type, m_limb_type))
1736 291 : t = add_cast (type, t);
1737 570 : m_first = save_first;
1738 570 : return t;
1739 : }
1740 : }
1741 0 : else if (BITINT_TYPE_P (lhs_type)
1742 1580 : && bitint_precision_kind (lhs_type) >= bitint_prec_large
1743 3160 : && INTEGRAL_TYPE_P (rhs_type))
1744 : {
1745 : /* Add support for 3 or more limbs filled in from normal integral
1746 : type if this assert fails. If no target chooses limb mode smaller
1747 : than half of largest supported normal integral type, this will not
1748 : be needed. */
1749 1580 : gcc_assert (TYPE_PRECISION (rhs_type) <= 2 * limb_prec);
1750 1580 : tree r1 = NULL_TREE, r2 = NULL_TREE, rext = NULL_TREE;
1751 1580 : if (m_first)
1752 : {
1753 577 : gimple_stmt_iterator save_gsi = m_gsi;
1754 577 : m_gsi = m_init_gsi;
1755 577 : if (gsi_end_p (m_gsi))
1756 57 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
1757 : else
1758 520 : gsi_next (&m_gsi);
1759 374 : if (BITINT_TYPE_P (rhs_type)
1760 577 : && bitint_precision_kind (rhs_type) == bitint_prec_middle)
1761 : {
1762 63 : tree type = NULL_TREE;
1763 63 : rhs1 = maybe_cast_middle_bitint (&m_gsi, rhs1, type);
1764 63 : rhs_type = TREE_TYPE (rhs1);
1765 : }
1766 577 : r1 = rhs1;
1767 577 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
1768 509 : r1 = add_cast (m_limb_type, rhs1);
1769 577 : if (TYPE_PRECISION (rhs_type) > limb_prec)
1770 : {
1771 109 : g = gimple_build_assign (make_ssa_name (rhs_type),
1772 : RSHIFT_EXPR, rhs1,
1773 : build_int_cst (unsigned_type_node,
1774 109 : limb_prec));
1775 109 : insert_before (g);
1776 109 : r2 = add_cast (m_limb_type, gimple_assign_lhs (g));
1777 : }
1778 577 : if (TYPE_UNSIGNED (rhs_type))
1779 279 : rext = build_zero_cst (m_limb_type);
1780 : else
1781 : {
1782 298 : rext = add_cast (signed_type_for (m_limb_type), r2 ? r2 : r1);
1783 298 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rext)),
1784 : RSHIFT_EXPR, rext,
1785 : build_int_cst (unsigned_type_node,
1786 298 : limb_prec - 1));
1787 298 : insert_before (g);
1788 298 : rext = add_cast (m_limb_type, gimple_assign_lhs (g));
1789 : }
1790 577 : m_init_gsi = m_gsi;
1791 577 : if (gsi_end_p (m_init_gsi))
1792 564 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
1793 : else
1794 295 : gsi_prev (&m_init_gsi);
1795 577 : m_gsi = save_gsi;
1796 : }
1797 1580 : tree t;
1798 1580 : if (m_upwards_2limb)
1799 : {
1800 724 : if (m_first)
1801 : {
1802 280 : tree out1, out2;
1803 280 : prepare_data_in_out (r1, idx, &out1, rext);
1804 280 : if (TYPE_PRECISION (rhs_type) > limb_prec)
1805 : {
1806 70 : prepare_data_in_out (r2, idx, &out2, rext);
1807 70 : m_data.pop ();
1808 70 : t = m_data.pop ();
1809 70 : m_data[m_data_cnt + 1] = t;
1810 : }
1811 : else
1812 210 : m_data[m_data_cnt + 1] = rext;
1813 280 : m_data.safe_push (rext);
1814 280 : t = m_data[m_data_cnt];
1815 : }
1816 444 : else if (!tree_fits_uhwi_p (idx))
1817 280 : t = m_data[m_data_cnt + 1];
1818 : else
1819 : {
1820 164 : tree type = limb_access_type (lhs_type, idx);
1821 164 : t = m_data[m_data_cnt + 2];
1822 164 : if (!useless_type_conversion_p (type, m_limb_type))
1823 136 : t = add_cast (type, t);
1824 : }
1825 724 : m_data_cnt += 3;
1826 724 : return t;
1827 : }
1828 856 : else if (m_first)
1829 : {
1830 297 : m_data.safe_push (r1);
1831 297 : m_data.safe_push (r2);
1832 297 : m_data.safe_push (rext);
1833 : }
1834 856 : unsigned lcnt = CEIL ((unsigned) TYPE_PRECISION (lhs_type), limb_prec);
1835 856 : if (tree_fits_uhwi_p (idx))
1836 : {
1837 812 : tree type = limb_access_type (lhs_type, idx);
1838 812 : if (bitint_big_endian
1839 812 : ? tree_to_uhwi (idx) == lcnt - 1 : integer_zerop (idx))
1840 269 : t = m_data[m_data_cnt];
1841 543 : else if (TYPE_PRECISION (rhs_type) > limb_prec
1842 543 : && (bitint_big_endian
1843 72 : ? tree_to_uhwi (idx) == lcnt - 2
1844 72 : : integer_onep (idx)))
1845 33 : t = m_data[m_data_cnt + 1];
1846 : else
1847 510 : t = m_data[m_data_cnt + 2];
1848 812 : if (!useless_type_conversion_p (type, m_limb_type))
1849 250 : t = add_cast (type, t);
1850 812 : m_data_cnt += 3;
1851 812 : return t;
1852 : }
1853 44 : g = gimple_build_cond (NE_EXPR, idx,
1854 : bitint_big_endian
1855 0 : ? size_int (lcnt - 1) : size_zero_node,
1856 : NULL_TREE, NULL_TREE);
1857 44 : edge e2, e3, e4 = NULL;
1858 44 : if_then (g, profile_probability::likely (), e2, e3);
1859 44 : if (m_data[m_data_cnt + 1])
1860 : {
1861 14 : g = gimple_build_cond (EQ_EXPR, idx,
1862 : bitint_big_endian
1863 0 : ? size_int (lcnt - 2) : size_one_node,
1864 : NULL_TREE, NULL_TREE);
1865 14 : insert_before (g);
1866 14 : edge e5 = split_block (gsi_bb (m_gsi), g);
1867 14 : e4 = make_edge (e5->src, e2->dest, EDGE_TRUE_VALUE);
1868 14 : e2 = find_edge (e5->dest, e2->dest);
1869 14 : e4->probability = profile_probability::unlikely ();
1870 14 : e5->flags = EDGE_FALSE_VALUE;
1871 14 : e5->probability = e4->probability.invert ();
1872 : }
1873 44 : m_gsi = gsi_after_labels (e2->dest);
1874 44 : t = make_ssa_name (m_limb_type);
1875 44 : gphi *phi = create_phi_node (t, e2->dest);
1876 44 : add_phi_arg (phi, m_data[m_data_cnt + 2], e2, UNKNOWN_LOCATION);
1877 44 : add_phi_arg (phi, m_data[m_data_cnt], e3, UNKNOWN_LOCATION);
1878 44 : if (e4)
1879 14 : add_phi_arg (phi, m_data[m_data_cnt + 1], e4, UNKNOWN_LOCATION);
1880 44 : m_data_cnt += 3;
1881 44 : return t;
1882 : }
1883 : return NULL_TREE;
1884 : }
1885 :
1886 : /* Helper function for handle_stmt method, handle a BIT_FIELD_REF. */
1887 :
1888 : tree
1889 23 : bitint_large_huge::handle_bit_field_ref (tree op, tree idx)
1890 : {
1891 23 : if (tree_fits_uhwi_p (idx))
1892 : {
1893 13 : if (m_first)
1894 4 : m_data.safe_push (NULL);
1895 13 : ++m_data_cnt;
1896 13 : unsigned HOST_WIDE_INT sz = tree_to_uhwi (TYPE_SIZE (m_limb_type));
1897 13 : unsigned i = tree_to_uhwi (idx);
1898 13 : if (bitint_big_endian)
1899 0 : i = CEIL (TYPE_PRECISION (TREE_TYPE (op)), limb_prec) - 1 - i;
1900 26 : tree bfr = build3 (BIT_FIELD_REF, m_limb_type,
1901 13 : TREE_OPERAND (op, 0),
1902 13 : TYPE_SIZE (m_limb_type),
1903 13 : size_binop (PLUS_EXPR, TREE_OPERAND (op, 2),
1904 : bitsize_int (i * sz)));
1905 13 : tree r = make_ssa_name (m_limb_type);
1906 13 : gimple *g = gimple_build_assign (r, bfr);
1907 13 : insert_before (g);
1908 13 : tree type = limb_access_type (TREE_TYPE (op), idx);
1909 13 : if (!useless_type_conversion_p (type, m_limb_type))
1910 0 : r = add_cast (type, r);
1911 13 : return r;
1912 : }
1913 10 : tree var;
1914 10 : if (m_first)
1915 : {
1916 5 : unsigned HOST_WIDE_INT sz = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (op)));
1917 5 : machine_mode mode;
1918 5 : tree type, bfr;
1919 5 : if (bitwise_mode_for_size (sz).exists (&mode)
1920 0 : && known_eq (GET_MODE_BITSIZE (mode), sz))
1921 0 : type = bitwise_type_for_mode (mode);
1922 : else
1923 : {
1924 5 : mode = VOIDmode;
1925 5 : type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_OPERAND (op, 0)));
1926 : }
1927 5 : if (TYPE_ALIGN (type) < TYPE_ALIGN (TREE_TYPE (op)))
1928 0 : type = build_aligned_type (type, TYPE_ALIGN (TREE_TYPE (op)));
1929 5 : var = create_tmp_var (type);
1930 5 : TREE_ADDRESSABLE (var) = 1;
1931 5 : gimple *g;
1932 5 : if (mode != VOIDmode)
1933 : {
1934 0 : bfr = build3 (BIT_FIELD_REF, type, TREE_OPERAND (op, 0),
1935 0 : TYPE_SIZE (type), TREE_OPERAND (op, 2));
1936 0 : g = gimple_build_assign (make_ssa_name (type),
1937 : BIT_FIELD_REF, bfr);
1938 0 : gimple_set_location (g, m_loc);
1939 0 : gsi_insert_after (&m_init_gsi, g, GSI_NEW_STMT);
1940 0 : bfr = gimple_assign_lhs (g);
1941 : }
1942 : else
1943 5 : bfr = TREE_OPERAND (op, 0);
1944 5 : g = gimple_build_assign (var, bfr);
1945 5 : gimple_set_location (g, m_loc);
1946 5 : gsi_insert_after (&m_init_gsi, g, GSI_NEW_STMT);
1947 5 : if (mode == VOIDmode)
1948 : {
1949 5 : unsigned HOST_WIDE_INT nelts
1950 5 : = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (op))), limb_prec);
1951 5 : tree atype = build_array_type_nelts (m_limb_type, nelts);
1952 5 : var = build2 (MEM_REF, atype, build_fold_addr_expr (var),
1953 : build_int_cst (build_pointer_type (type),
1954 5 : tree_to_uhwi (TREE_OPERAND (op, 2))
1955 5 : / BITS_PER_UNIT));
1956 : }
1957 5 : m_data.safe_push (var);
1958 : }
1959 : else
1960 5 : var = unshare_expr (m_data[m_data_cnt]);
1961 10 : ++m_data_cnt;
1962 10 : var = limb_access (TREE_TYPE (op), var, idx, false);
1963 10 : tree r = make_ssa_name (m_limb_type);
1964 10 : gimple *g = gimple_build_assign (r, var);
1965 10 : insert_before (g);
1966 10 : return r;
1967 : }
1968 :
1969 : /* Add a new EH edge from SRC to EH_EDGE->dest, where EH_EDGE
1970 : is an older EH edge, and except for virtual PHIs duplicate the
1971 : PHI argument from the EH_EDGE to the new EH edge. */
1972 :
1973 : static void
1974 20 : add_eh_edge (basic_block src, edge eh_edge)
1975 : {
1976 20 : edge e = make_edge (src, eh_edge->dest, EDGE_EH);
1977 20 : e->probability = profile_probability::very_unlikely ();
1978 20 : for (gphi_iterator gsi = gsi_start_phis (eh_edge->dest);
1979 27 : !gsi_end_p (gsi); gsi_next (&gsi))
1980 : {
1981 7 : gphi *phi = gsi.phi ();
1982 7 : tree lhs = gimple_phi_result (phi);
1983 14 : if (virtual_operand_p (lhs))
1984 4 : continue;
1985 3 : const phi_arg_d *arg = gimple_phi_arg (phi, eh_edge->dest_idx);
1986 3 : add_phi_arg (phi, arg->def, e, arg->locus);
1987 : }
1988 20 : }
1989 :
1990 : /* Helper function for handle_stmt method, handle a load from memory. */
1991 :
1992 : tree
1993 21336 : bitint_large_huge::handle_load (gimple *stmt, tree idx)
1994 : {
1995 21336 : tree rhs1 = gimple_assign_rhs1 (stmt);
1996 21336 : tree rhs_type = TREE_TYPE (rhs1);
1997 21336 : bool eh = stmt_ends_bb_p (stmt);
1998 21336 : bool load_bitfield_p = false;
1999 21336 : edge eh_edge = NULL;
2000 21336 : gimple *g;
2001 :
2002 21336 : if (TREE_CODE (rhs1) == BIT_FIELD_REF
2003 21336 : && integer_zerop (TREE_OPERAND (rhs1, 2)))
2004 2 : rhs1 = TREE_OPERAND (rhs1, 0);
2005 :
2006 21336 : if (eh)
2007 : {
2008 10 : edge_iterator ei;
2009 10 : basic_block bb = gimple_bb (stmt);
2010 :
2011 10 : FOR_EACH_EDGE (eh_edge, ei, bb->succs)
2012 10 : if (eh_edge->flags & EDGE_EH)
2013 : break;
2014 : }
2015 :
2016 21336 : if (TREE_CODE (rhs1) == COMPONENT_REF
2017 21336 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (rhs1, 1)))
2018 : {
2019 1225 : tree fld = TREE_OPERAND (rhs1, 1);
2020 : /* For little-endian, we can allow as inputs bit-fields
2021 : which start at a limb boundary. */
2022 1225 : gcc_assert (tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld)));
2023 1225 : if (!bitint_big_endian
2024 1225 : && DECL_OFFSET_ALIGN (fld) >= TYPE_ALIGN (TREE_TYPE (rhs1))
2025 2450 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % limb_prec) == 0)
2026 : {
2027 739 : load_bitfield_p = true;
2028 751 : goto normal_load;
2029 : }
2030 : /* Even if DECL_FIELD_BIT_OFFSET (fld) is a multiple of BITS_PER_UNIT,
2031 : handle it normally for now. */
2032 486 : if (!bitint_big_endian
2033 486 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % BITS_PER_UNIT) == 0)
2034 : {
2035 12 : load_bitfield_p = true;
2036 12 : goto normal_load;
2037 : }
2038 474 : tree repr = DECL_BIT_FIELD_REPRESENTATIVE (fld);
2039 474 : poly_int64 bitoffset;
2040 474 : poly_uint64 field_offset, repr_offset;
2041 474 : bool var_field_off = false;
2042 474 : if (poly_int_tree_p (DECL_FIELD_OFFSET (fld), &field_offset)
2043 948 : && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
2044 474 : bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
2045 : else
2046 : {
2047 : bitoffset = 0;
2048 : var_field_off = true;
2049 : }
2050 474 : bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2051 474 : - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
2052 948 : tree nrhs1 = build3 (COMPONENT_REF, TREE_TYPE (repr),
2053 474 : TREE_OPERAND (rhs1, 0), repr,
2054 0 : var_field_off ? TREE_OPERAND (rhs1, 2) : NULL_TREE);
2055 474 : HOST_WIDE_INT bo = bitoffset.to_constant ();
2056 474 : unsigned bo_idx = (unsigned HOST_WIDE_INT) bo / limb_prec;
2057 474 : unsigned bo_bit = (unsigned HOST_WIDE_INT) bo % limb_prec;
2058 474 : unsigned bo_last = 0;
2059 474 : unsigned bo_shift = bo_bit;
2060 474 : unsigned nelts = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
2061 474 : if (bitint_big_endian)
2062 : {
2063 0 : bo_last = CEIL (TYPE_PRECISION (rhs_type) + bo_bit, limb_prec) - 1;
2064 0 : bo_shift = (TYPE_PRECISION (rhs_type) + bo_bit) % limb_prec;
2065 0 : if (bo_shift)
2066 0 : bo_shift = limb_prec - bo_shift;
2067 : }
2068 474 : if (m_first)
2069 : {
2070 141 : if (m_upwards && bo_shift)
2071 : {
2072 138 : gimple_stmt_iterator save_gsi = m_gsi;
2073 138 : m_gsi = m_init_gsi;
2074 138 : if (gsi_end_p (m_gsi))
2075 57 : m_gsi = gsi_after_labels (gsi_bb (m_gsi));
2076 : else
2077 81 : gsi_next (&m_gsi);
2078 138 : tree t = limb_access (NULL_TREE, nrhs1,
2079 138 : size_int (bo_idx + bo_last), true);
2080 138 : tree iv = make_ssa_name (m_limb_type);
2081 138 : g = gimple_build_assign (iv, t);
2082 138 : insert_before (g);
2083 138 : if (eh)
2084 : {
2085 2 : maybe_duplicate_eh_stmt (g, stmt);
2086 2 : if (eh_edge)
2087 : {
2088 2 : edge e = split_block (gsi_bb (m_gsi), g);
2089 2 : add_eh_edge (e->src, eh_edge);
2090 2 : m_gsi = gsi_after_labels (e->dest);
2091 2 : if (gsi_bb (save_gsi) == e->src)
2092 : {
2093 1 : if (gsi_end_p (save_gsi))
2094 0 : save_gsi = gsi_end_bb (e->dest);
2095 : else
2096 1 : save_gsi = gsi_for_stmt (gsi_stmt (save_gsi));
2097 : }
2098 2 : if (m_preheader_bb == e->src)
2099 1 : m_preheader_bb = e->dest;
2100 : }
2101 : }
2102 138 : m_init_gsi = m_gsi;
2103 138 : if (gsi_end_p (m_init_gsi))
2104 222 : m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
2105 : else
2106 27 : gsi_prev (&m_init_gsi);
2107 138 : m_gsi = save_gsi;
2108 138 : tree out;
2109 138 : prepare_data_in_out (iv, idx, &out);
2110 138 : out = m_data[m_data_cnt];
2111 138 : m_data.safe_push (out);
2112 138 : }
2113 : else
2114 : {
2115 3 : m_data.safe_push (NULL_TREE);
2116 3 : m_data.safe_push (NULL_TREE);
2117 3 : m_data.safe_push (NULL_TREE);
2118 : }
2119 : }
2120 :
2121 474 : tree nidx0 = NULL_TREE, nidx1 = NULL_TREE;
2122 474 : tree iv = m_data[m_data_cnt];
2123 474 : if (m_cast_conditional && iv)
2124 : {
2125 12 : gcc_assert (!m_bitfld_load);
2126 12 : m_bitfld_load = m_data_cnt;
2127 : }
2128 474 : if (tree_fits_uhwi_p (idx))
2129 : {
2130 272 : unsigned prec = TYPE_PRECISION (rhs_type);
2131 272 : unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
2132 272 : if (bitint_big_endian)
2133 0 : i = nelts - 1 - i;
2134 272 : gcc_assert (i * limb_prec < prec);
2135 272 : if (bo_shift)
2136 272 : nidx1 = size_int (bo_idx + (bitint_big_endian
2137 : ? bo_last - i - 1 : i + 1));
2138 272 : if ((i + 1) * limb_prec > prec)
2139 : {
2140 100 : prec %= limb_prec;
2141 100 : if (prec + bo_bit <= (unsigned) limb_prec)
2142 272 : nidx1 = NULL_TREE;
2143 : }
2144 272 : if (!iv)
2145 4 : nidx0 = size_int (bo_idx + (bitint_big_endian ? bo_last - i : i));
2146 : }
2147 : else
2148 : {
2149 202 : HOST_WIDE_INT adj = bo_idx;
2150 202 : if (bitint_big_endian)
2151 0 : adj += (HOST_WIDE_INT) bo_last + 1 - nelts;
2152 202 : if (!iv)
2153 : {
2154 4 : if (adj == 0)
2155 : nidx0 = idx;
2156 : else
2157 : {
2158 0 : nidx0 = make_ssa_name (sizetype);
2159 0 : g = gimple_build_assign (nidx0, PLUS_EXPR, idx,
2160 0 : size_int (adj));
2161 0 : insert_before (g);
2162 : }
2163 : }
2164 202 : if (bo_shift)
2165 : {
2166 202 : if (bitint_big_endian && adj == 1)
2167 : nidx1 = idx;
2168 : else
2169 : {
2170 202 : nidx1 = make_ssa_name (sizetype);
2171 202 : g = gimple_build_assign (nidx1, PLUS_EXPR, idx,
2172 404 : size_int (adj + (bitint_big_endian
2173 : ? -1 : 1)));
2174 202 : insert_before (g);
2175 : }
2176 : }
2177 : }
2178 :
2179 680 : tree iv2 = NULL_TREE;
2180 206 : if (nidx0)
2181 : {
2182 8 : tree t = limb_access (NULL_TREE, nrhs1, nidx0, true);
2183 8 : iv = make_ssa_name (m_limb_type);
2184 8 : g = gimple_build_assign (iv, t);
2185 8 : insert_before (g);
2186 8 : if (eh)
2187 : {
2188 0 : maybe_duplicate_eh_stmt (g, stmt);
2189 0 : if (eh_edge)
2190 : {
2191 0 : edge e = split_block (gsi_bb (m_gsi), g);
2192 0 : m_gsi = gsi_after_labels (e->dest);
2193 0 : add_eh_edge (e->src, eh_edge);
2194 : }
2195 : }
2196 : }
2197 474 : if (nidx1)
2198 : {
2199 385 : bool conditional = m_var_msb && !tree_fits_uhwi_p (idx);
2200 385 : unsigned prec = TYPE_PRECISION (rhs_type);
2201 385 : if (conditional)
2202 : {
2203 3 : if ((prec % limb_prec) == 0
2204 3 : || ((prec % limb_prec) + bo_bit > (unsigned) limb_prec))
2205 382 : conditional = false;
2206 : }
2207 385 : edge edge_true = NULL, edge_false = NULL;
2208 385 : if (conditional)
2209 : {
2210 6 : g = gimple_build_cond (NE_EXPR, idx,
2211 : bitint_big_endian
2212 : ? size_zero_node
2213 3 : : size_int (prec / limb_prec),
2214 : NULL_TREE, NULL_TREE);
2215 3 : if_then (g, profile_probability::likely (),
2216 : edge_true, edge_false);
2217 : }
2218 385 : tree t = limb_access (NULL_TREE, nrhs1, nidx1, true);
2219 385 : if (m_upwards_2limb
2220 283 : && !m_first
2221 184 : && !m_bitfld_load
2222 178 : && !tree_fits_uhwi_p (idx))
2223 95 : iv2 = m_data[m_data_cnt + 1];
2224 : else
2225 290 : iv2 = make_ssa_name (m_limb_type);
2226 385 : g = gimple_build_assign (iv2, t);
2227 385 : insert_before (g);
2228 385 : if (eh)
2229 : {
2230 5 : maybe_duplicate_eh_stmt (g, stmt);
2231 5 : if (eh_edge)
2232 : {
2233 5 : edge e = split_block (gsi_bb (m_gsi), g);
2234 5 : m_gsi = gsi_after_labels (e->dest);
2235 5 : add_eh_edge (e->src, eh_edge);
2236 : }
2237 : }
2238 385 : if (conditional)
2239 : {
2240 3 : tree iv3 = make_ssa_name (m_limb_type);
2241 3 : if (eh)
2242 0 : edge_true = find_edge (gsi_bb (m_gsi), edge_false->dest);
2243 3 : gphi *phi = create_phi_node (iv3, edge_true->dest);
2244 3 : add_phi_arg (phi, iv2, edge_true, UNKNOWN_LOCATION);
2245 3 : add_phi_arg (phi, build_zero_cst (m_limb_type),
2246 : edge_false, UNKNOWN_LOCATION);
2247 3 : m_gsi = gsi_after_labels (edge_true->dest);
2248 3 : iv2 = iv3;
2249 : }
2250 : }
2251 474 : if (bo_shift)
2252 : {
2253 474 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
2254 : iv, build_int_cst (unsigned_type_node,
2255 474 : bo_shift));
2256 474 : insert_before (g);
2257 474 : iv = gimple_assign_lhs (g);
2258 : }
2259 474 : if (iv2)
2260 : {
2261 385 : g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
2262 : iv2, build_int_cst (unsigned_type_node,
2263 385 : limb_prec - bo_shift));
2264 385 : insert_before (g);
2265 385 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
2266 : gimple_assign_lhs (g), iv);
2267 385 : insert_before (g);
2268 385 : iv = gimple_assign_lhs (g);
2269 385 : if (m_data[m_data_cnt])
2270 379 : m_data[m_data_cnt] = iv2;
2271 : }
2272 474 : if (tree_fits_uhwi_p (idx))
2273 : {
2274 272 : tree atype = limb_access_type (rhs_type, idx);
2275 272 : if (!useless_type_conversion_p (atype, TREE_TYPE (iv)))
2276 100 : iv = add_cast (atype, iv);
2277 : }
2278 474 : m_data_cnt += 3;
2279 474 : return iv;
2280 : }
2281 :
2282 20862 : normal_load:
2283 : /* Use write_p = true for loads with EH edges to make
2284 : sure limb_access doesn't add a cast as separate
2285 : statement after it. */
2286 20862 : rhs1 = limb_access (rhs_type, rhs1, idx, eh, !load_bitfield_p);
2287 20862 : tree ret = make_ssa_name (TREE_TYPE (rhs1));
2288 20862 : g = gimple_build_assign (ret, rhs1);
2289 20862 : insert_before (g);
2290 20862 : if (eh)
2291 : {
2292 3 : maybe_duplicate_eh_stmt (g, stmt);
2293 3 : if (eh_edge)
2294 : {
2295 3 : edge e = split_block (gsi_bb (m_gsi), g);
2296 3 : m_gsi = gsi_after_labels (e->dest);
2297 3 : add_eh_edge (e->src, eh_edge);
2298 : }
2299 3 : if (tree_fits_uhwi_p (idx))
2300 : {
2301 1 : tree atype = limb_access_type (rhs_type, idx);
2302 1 : if (!useless_type_conversion_p (atype, TREE_TYPE (rhs1)))
2303 1 : ret = add_cast (atype, ret);
2304 : }
2305 : }
2306 : return ret;
2307 : }
2308 :
2309 : /* Return a limb IDX from a mergeable statement STMT. */
2310 :
2311 : tree
2312 38849 : bitint_large_huge::handle_stmt (gimple *stmt, tree idx)
2313 : {
2314 38849 : tree lhs, rhs1, rhs2 = NULL_TREE;
2315 38849 : gimple *g;
2316 38849 : switch (gimple_code (stmt))
2317 : {
2318 38849 : case GIMPLE_ASSIGN:
2319 38849 : if (gimple_assign_load_p (stmt))
2320 21336 : return handle_load (stmt, idx);
2321 17513 : switch (gimple_assign_rhs_code (stmt))
2322 : {
2323 964 : case BIT_AND_EXPR:
2324 964 : case BIT_IOR_EXPR:
2325 964 : case BIT_XOR_EXPR:
2326 964 : rhs2 = handle_operand (gimple_assign_rhs2 (stmt), idx);
2327 : /* FALLTHRU */
2328 1360 : case BIT_NOT_EXPR:
2329 1360 : rhs1 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2330 1360 : lhs = make_ssa_name (TREE_TYPE (rhs1));
2331 1360 : g = gimple_build_assign (lhs, gimple_assign_rhs_code (stmt),
2332 : rhs1, rhs2);
2333 1360 : insert_before (g);
2334 1360 : return lhs;
2335 3950 : case PLUS_EXPR:
2336 3950 : case MINUS_EXPR:
2337 3950 : rhs1 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2338 3950 : rhs2 = handle_operand (gimple_assign_rhs2 (stmt), idx);
2339 3950 : return handle_plus_minus (gimple_assign_rhs_code (stmt),
2340 3950 : rhs1, rhs2, idx);
2341 123 : case NEGATE_EXPR:
2342 123 : rhs2 = handle_operand (gimple_assign_rhs1 (stmt), idx);
2343 123 : rhs1 = build_zero_cst (TREE_TYPE (rhs2));
2344 123 : return handle_plus_minus (MINUS_EXPR, rhs1, rhs2, idx);
2345 152 : case LSHIFT_EXPR:
2346 152 : return handle_lshift (handle_operand (gimple_assign_rhs1 (stmt),
2347 : idx),
2348 152 : gimple_assign_rhs2 (stmt), idx);
2349 5032 : case SSA_NAME:
2350 5032 : case PAREN_EXPR:
2351 5032 : case INTEGER_CST:
2352 5032 : return handle_operand (gimple_assign_rhs1 (stmt), idx);
2353 6865 : CASE_CONVERT:
2354 6865 : return handle_cast (TREE_TYPE (gimple_assign_lhs (stmt)),
2355 6865 : gimple_assign_rhs1 (stmt), idx);
2356 8 : case VIEW_CONVERT_EXPR:
2357 8 : return handle_cast (TREE_TYPE (gimple_assign_lhs (stmt)),
2358 8 : TREE_OPERAND (gimple_assign_rhs1 (stmt), 0),
2359 8 : idx);
2360 23 : case BIT_FIELD_REF:
2361 23 : return handle_bit_field_ref (gimple_assign_rhs1 (stmt), idx);
2362 : default:
2363 : break;
2364 : }
2365 : break;
2366 : default:
2367 : break;
2368 : }
2369 0 : gcc_unreachable ();
2370 : }
2371 :
2372 : /* Return minimum precision of OP at STMT.
2373 : Positive value is minimum precision above which all bits
2374 : are zero, negative means all bits above negation of the
2375 : value are copies of the sign bit. */
2376 :
2377 : static int
2378 8380 : range_to_prec (tree op, gimple *stmt)
2379 : {
2380 8380 : int_range_max r;
2381 8380 : wide_int w;
2382 8380 : tree type = TREE_TYPE (op);
2383 8380 : unsigned int prec = TYPE_PRECISION (type);
2384 :
2385 8380 : if (!optimize
2386 7398 : || !get_range_query (cfun)->range_of_expr (r, op, stmt)
2387 12079 : || r.undefined_p ())
2388 : {
2389 4682 : if (TYPE_UNSIGNED (type))
2390 1900 : return prec;
2391 : else
2392 2782 : return MIN ((int) -prec, -2);
2393 : }
2394 :
2395 3698 : if (!TYPE_UNSIGNED (TREE_TYPE (op)))
2396 : {
2397 2297 : w = r.lower_bound ();
2398 2297 : if (wi::neg_p (w))
2399 : {
2400 1897 : int min_prec1 = wi::min_precision (w, SIGNED);
2401 1897 : w = r.upper_bound ();
2402 1897 : int min_prec2 = wi::min_precision (w, SIGNED);
2403 1897 : int min_prec = MAX (min_prec1, min_prec2);
2404 1897 : return MIN (-min_prec, -2);
2405 : }
2406 : }
2407 :
2408 1801 : w = r.upper_bound ();
2409 1801 : int min_prec = wi::min_precision (w, UNSIGNED);
2410 1801 : return MAX (min_prec, 1);
2411 8380 : }
2412 :
2413 : /* Return address of the first limb of OP and write into *PREC
2414 : its precision. If positive, the operand is zero extended
2415 : from that precision, if it is negative, the operand is sign-extended
2416 : from -*PREC. If PREC_STORED is NULL, it is the toplevel call,
2417 : otherwise *PREC_STORED is prec from the innermost call without
2418 : range optimizations (0 for uninitialized SSA_NAME). */
2419 :
2420 : tree
2421 3788 : bitint_large_huge::handle_operand_addr (tree op, gimple *stmt,
2422 : int *prec_stored, int *prec)
2423 : {
2424 3788 : wide_int w;
2425 3788 : location_t loc_save = m_loc;
2426 3788 : tree ret = NULL_TREE;
2427 3788 : int precs = 0;
2428 3808 : if ((!BITINT_TYPE_P (TREE_TYPE (op))
2429 3774 : || bitint_precision_kind (TREE_TYPE (op)) < bitint_prec_large)
2430 3897 : && TREE_CODE (op) != INTEGER_CST)
2431 : {
2432 115 : do_int:
2433 115 : *prec = range_to_prec (op, stmt);
2434 115 : bitint_prec_kind kind = bitint_prec_small;
2435 115 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (op)));
2436 115 : if (BITINT_TYPE_P (TREE_TYPE (op)))
2437 104 : kind = bitint_precision_kind (TREE_TYPE (op));
2438 104 : if (kind == bitint_prec_middle)
2439 : {
2440 12 : tree type = NULL_TREE;
2441 12 : op = maybe_cast_middle_bitint (&m_gsi, op, type);
2442 : }
2443 115 : tree op_type = TREE_TYPE (op);
2444 115 : unsigned HOST_WIDE_INT nelts
2445 115 : = CEIL (TYPE_PRECISION (op_type), limb_prec);
2446 : /* Add support for 3 or more limbs filled in from normal
2447 : integral type if this assert fails. If no target chooses
2448 : limb mode smaller than half of largest supported normal
2449 : integral type, this will not be needed. */
2450 115 : gcc_assert (nelts <= 2);
2451 115 : precs = (TYPE_UNSIGNED (op_type)
2452 115 : ? TYPE_PRECISION (op_type) : -TYPE_PRECISION (op_type));
2453 115 : if (*prec <= limb_prec && *prec >= -limb_prec)
2454 : {
2455 102 : nelts = 1;
2456 102 : if (TYPE_UNSIGNED (op_type))
2457 : {
2458 26 : if (precs > limb_prec)
2459 115 : precs = limb_prec;
2460 : }
2461 76 : else if (precs < -limb_prec)
2462 115 : precs = -limb_prec;
2463 : }
2464 115 : if (prec_stored)
2465 0 : *prec_stored = precs;
2466 115 : tree atype = build_array_type_nelts (m_limb_type, nelts);
2467 115 : tree var = create_tmp_var (atype);
2468 115 : tree t1 = op;
2469 115 : if (!useless_type_conversion_p (m_limb_type, op_type))
2470 115 : t1 = add_cast (m_limb_type, t1);
2471 115 : tree v = build4 (ARRAY_REF, m_limb_type, var,
2472 0 : bitint_big_endian && nelts > 1
2473 : ? size_one_node : size_zero_node,
2474 : NULL_TREE, NULL_TREE);
2475 115 : gimple *g = gimple_build_assign (v, t1);
2476 115 : insert_before (g);
2477 115 : if (nelts > 1)
2478 : {
2479 13 : tree lp = build_int_cst (unsigned_type_node, limb_prec);
2480 13 : g = gimple_build_assign (make_ssa_name (op_type),
2481 : RSHIFT_EXPR, op, lp);
2482 13 : insert_before (g);
2483 13 : tree t2 = gimple_assign_lhs (g);
2484 13 : t2 = add_cast (m_limb_type, t2);
2485 13 : v = build4 (ARRAY_REF, m_limb_type, var,
2486 : bitint_big_endian ? size_zero_node : size_one_node,
2487 : NULL_TREE, NULL_TREE);
2488 13 : g = gimple_build_assign (v, t2);
2489 13 : insert_before (g);
2490 : }
2491 115 : ret = build_fold_addr_expr (var);
2492 115 : if (!stmt_ends_bb_p (gsi_stmt (m_gsi)))
2493 : {
2494 114 : tree clobber = build_clobber (atype, CLOBBER_STORAGE_END);
2495 114 : g = gimple_build_assign (var, clobber);
2496 114 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
2497 : }
2498 115 : m_loc = loc_save;
2499 115 : goto do_ret;
2500 : }
2501 3715 : switch (TREE_CODE (op))
2502 : {
2503 2883 : case SSA_NAME:
2504 2883 : if (m_names == NULL
2505 2883 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (op)))
2506 : {
2507 96 : gimple *g = SSA_NAME_DEF_STMT (op);
2508 96 : m_loc = gimple_location (g);
2509 96 : if (gimple_assign_load_p (g))
2510 : {
2511 36 : *prec = range_to_prec (op, NULL);
2512 36 : precs = (TYPE_UNSIGNED (TREE_TYPE (op))
2513 36 : ? TYPE_PRECISION (TREE_TYPE (op))
2514 25 : : -TYPE_PRECISION (TREE_TYPE (op)));
2515 36 : if (prec_stored)
2516 6 : *prec_stored = precs;
2517 36 : ret = build_fold_addr_expr (gimple_assign_rhs1 (g));
2518 36 : ret = force_gimple_operand_gsi (&m_gsi, ret, true,
2519 : NULL_TREE, true, GSI_SAME_STMT);
2520 : }
2521 60 : else if (gimple_code (g) == GIMPLE_NOP)
2522 : {
2523 2 : *prec = TYPE_UNSIGNED (TREE_TYPE (op)) ? limb_prec : -limb_prec;
2524 2 : precs = *prec;
2525 2 : if (prec_stored)
2526 1 : *prec_stored = 0;
2527 2 : tree var = create_tmp_var (m_limb_type);
2528 2 : TREE_ADDRESSABLE (var) = 1;
2529 2 : ret = build_fold_addr_expr (var);
2530 2 : if (!stmt_ends_bb_p (gsi_stmt (m_gsi)))
2531 : {
2532 2 : tree clobber = build_clobber (m_limb_type,
2533 : CLOBBER_STORAGE_END);
2534 2 : g = gimple_build_assign (var, clobber);
2535 2 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
2536 : }
2537 : }
2538 : else
2539 : {
2540 58 : gcc_assert (gimple_assign_cast_p (g));
2541 58 : tree rhs1 = gimple_assign_rhs1 (g);
2542 58 : bitint_prec_kind kind = bitint_prec_small;
2543 58 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
2544 1 : rhs1 = TREE_OPERAND (rhs1, 0);
2545 58 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (rhs1)));
2546 58 : if (BITINT_TYPE_P (TREE_TYPE (rhs1)))
2547 49 : kind = bitint_precision_kind (TREE_TYPE (rhs1));
2548 49 : if (kind >= bitint_prec_large)
2549 : {
2550 16 : tree lhs_type = TREE_TYPE (op);
2551 16 : tree rhs_type = TREE_TYPE (rhs1);
2552 16 : int prec_stored_val = 0;
2553 16 : ret = handle_operand_addr (rhs1, g, &prec_stored_val, prec);
2554 16 : precs = prec_stored_val;
2555 16 : if (prec_stored)
2556 0 : *prec_stored = prec_stored_val;
2557 16 : if (precs == 0)
2558 : {
2559 1 : gcc_assert (*prec == limb_prec || *prec == -limb_prec);
2560 : precs = *prec;
2561 : }
2562 16 : if (TYPE_PRECISION (lhs_type) > TYPE_PRECISION (rhs_type))
2563 : {
2564 4 : if (TYPE_UNSIGNED (lhs_type)
2565 4 : && !TYPE_UNSIGNED (rhs_type))
2566 1 : gcc_assert (*prec >= 0 || prec_stored == NULL);
2567 : }
2568 : else
2569 : {
2570 12 : if (prec_stored_val == 0)
2571 : /* Non-widening cast of uninitialized value. */;
2572 11 : else if (*prec > 0 && *prec < TYPE_PRECISION (lhs_type))
2573 : ;
2574 11 : else if (TYPE_UNSIGNED (lhs_type))
2575 : {
2576 8 : gcc_assert (*prec > 0
2577 : || prec_stored_val > 0
2578 : || (-prec_stored_val
2579 : >= TYPE_PRECISION (lhs_type)));
2580 8 : *prec = TYPE_PRECISION (lhs_type);
2581 : }
2582 3 : else if (*prec < 0 && -*prec < TYPE_PRECISION (lhs_type))
2583 : ;
2584 : else
2585 3 : *prec = -TYPE_PRECISION (lhs_type);
2586 : }
2587 : }
2588 : else
2589 : {
2590 42 : op = rhs1;
2591 42 : stmt = g;
2592 42 : goto do_int;
2593 : }
2594 : }
2595 54 : m_loc = loc_save;
2596 54 : goto do_ret;
2597 : }
2598 : else
2599 : {
2600 2787 : int p = var_to_partition (m_map, op);
2601 2787 : gcc_assert (m_vars[p] != NULL_TREE);
2602 2787 : *prec = range_to_prec (op, stmt);
2603 2787 : precs = (TYPE_UNSIGNED (TREE_TYPE (op))
2604 2787 : ? TYPE_PRECISION (TREE_TYPE (op))
2605 1652 : : -TYPE_PRECISION (TREE_TYPE (op)));
2606 2787 : if (prec_stored)
2607 9 : *prec_stored = precs;
2608 2787 : ret = build_fold_addr_expr (m_vars[p]);
2609 2787 : goto do_ret;
2610 : }
2611 832 : case INTEGER_CST:
2612 832 : unsigned int min_prec, mp;
2613 832 : tree type;
2614 832 : w = wi::to_wide (op);
2615 832 : if (tree_int_cst_sgn (op) >= 0)
2616 : {
2617 617 : min_prec = wi::min_precision (w, UNSIGNED);
2618 617 : *prec = MAX (min_prec, 1);
2619 : }
2620 : else
2621 : {
2622 215 : min_prec = wi::min_precision (w, SIGNED);
2623 215 : *prec = MIN ((int) -min_prec, -2);
2624 : }
2625 832 : mp = CEIL (min_prec, limb_prec) * limb_prec;
2626 832 : if (mp == 0)
2627 : mp = 1;
2628 832 : if (mp >= (unsigned) TYPE_PRECISION (TREE_TYPE (op))
2629 832 : && (BITINT_TYPE_P (TREE_TYPE (op))
2630 12 : || TYPE_PRECISION (TREE_TYPE (op)) <= limb_prec))
2631 295 : type = TREE_TYPE (op);
2632 : else
2633 537 : type = build_bitint_type (mp, 1);
2634 8 : if (!BITINT_TYPE_P (type)
2635 832 : || bitint_precision_kind (type) == bitint_prec_small)
2636 : {
2637 562 : if (TYPE_PRECISION (type) <= limb_prec)
2638 562 : type = m_limb_type;
2639 : else
2640 : {
2641 0 : while (bitint_precision_kind (mp) == bitint_prec_small)
2642 0 : mp += limb_prec;
2643 : /* This case is for targets which e.g. have 64-bit
2644 : limb but categorize up to 128-bits _BitInts as
2645 : small. We could use type of m_limb_type[2] and
2646 : similar instead to save space. */
2647 0 : type = build_bitint_type (mp, 1);
2648 : }
2649 : }
2650 832 : if (tree_int_cst_sgn (op) >= 0)
2651 617 : precs = MAX (TYPE_PRECISION (type), 1);
2652 : else
2653 215 : precs = MIN ((int) -TYPE_PRECISION (type), -2);
2654 832 : if (prec_stored)
2655 0 : *prec_stored = precs;
2656 832 : op = tree_output_constant_def (fold_convert (type, op));
2657 832 : ret = build_fold_addr_expr (op);
2658 832 : goto do_ret;
2659 0 : default:
2660 0 : gcc_unreachable ();
2661 : }
2662 3788 : do_ret:
2663 3788 : if (bitint_big_endian && prec_stored == NULL)
2664 : {
2665 0 : int p1 = *prec < 0 ? -*prec : *prec;
2666 0 : int p2 = precs < 0 ? -precs : precs;
2667 0 : int c1 = CEIL (p1, limb_prec);
2668 0 : int c2 = CEIL (p2, limb_prec);
2669 0 : gcc_assert (c1 <= c2);
2670 0 : if (c1 != c2)
2671 : {
2672 0 : gimple *g
2673 0 : = gimple_build_assign (make_ssa_name (TREE_TYPE (ret)),
2674 : POINTER_PLUS_EXPR, ret,
2675 0 : size_int ((c2 - c1) * m_limb_size));
2676 0 : insert_before (g);
2677 0 : ret = gimple_assign_lhs (g);
2678 : }
2679 : }
2680 3788 : return ret;
2681 3788 : }
2682 :
2683 : /* Helper function, create a loop before the current location,
2684 : start with sizetype INIT value from the preheader edge. Return
2685 : a PHI result and set *IDX_NEXT to SSA_NAME it creates and uses
2686 : from the latch edge. */
2687 :
2688 : tree
2689 14549 : bitint_large_huge::create_loop (tree init, tree *idx_next)
2690 : {
2691 14549 : if (!gsi_end_p (m_gsi))
2692 12385 : gsi_prev (&m_gsi);
2693 : else
2694 4328 : m_gsi = gsi_last_bb (gsi_bb (m_gsi));
2695 14549 : edge e1 = split_block (gsi_bb (m_gsi), gsi_stmt (m_gsi));
2696 14549 : edge e2 = split_block (e1->dest, (gimple *) NULL);
2697 14549 : edge e3 = make_edge (e1->dest, e1->dest, EDGE_TRUE_VALUE);
2698 14549 : e3->probability = profile_probability::very_unlikely ();
2699 14549 : e2->flags = EDGE_FALSE_VALUE;
2700 14549 : e2->probability = e3->probability.invert ();
2701 14549 : tree idx = make_ssa_name (sizetype);
2702 14549 : gphi *phi = create_phi_node (idx, e1->dest);
2703 14549 : add_phi_arg (phi, init, e1, UNKNOWN_LOCATION);
2704 14549 : *idx_next = make_ssa_name (sizetype);
2705 14549 : add_phi_arg (phi, *idx_next, e3, UNKNOWN_LOCATION);
2706 14549 : m_gsi = gsi_after_labels (e1->dest);
2707 14549 : m_bb = e1->dest;
2708 14549 : m_preheader_bb = e1->src;
2709 14549 : class loop *loop = alloc_loop ();
2710 14549 : loop->header = e1->dest;
2711 14549 : add_loop (loop, e1->src->loop_father);
2712 14549 : return idx;
2713 : }
2714 :
2715 : /* Lower large/huge _BitInt statement mergeable or similar STMT which can be
2716 : lowered using iteration from the least significant limb up to the most
2717 : significant limb. For large _BitInt it is emitted as straight line code
2718 : before current location, for huge _BitInt as a loop handling two limbs
2719 : at once, followed by handling up to limbs in straight line code (at most
2720 : one full and one partial limb). It can also handle EQ_EXPR/NE_EXPR
2721 : comparisons, in that case CMP_CODE should be the comparison code and
2722 : CMP_OP1/CMP_OP2 the comparison operands. */
2723 :
2724 : tree
2725 23253 : bitint_large_huge::lower_mergeable_stmt (gimple *stmt, tree_code &cmp_code,
2726 : tree cmp_op1, tree cmp_op2)
2727 : {
2728 23253 : bool eq_p = cmp_code != ERROR_MARK;
2729 23253 : tree type;
2730 23253 : if (eq_p)
2731 6526 : type = TREE_TYPE (cmp_op1);
2732 : else
2733 16727 : type = TREE_TYPE (gimple_assign_lhs (stmt));
2734 23253 : gcc_assert (BITINT_TYPE_P (type));
2735 23253 : bitint_prec_kind kind = bitint_precision_kind (type);
2736 23253 : gcc_assert (kind >= bitint_prec_large);
2737 23253 : gimple *g;
2738 23253 : tree lhs = gimple_get_lhs (stmt);
2739 23253 : tree rhs1, lhs_type = lhs ? TREE_TYPE (lhs) : NULL_TREE;
2740 17159 : if (lhs
2741 17159 : && TREE_CODE (lhs) == SSA_NAME
2742 8695 : && BITINT_TYPE_P (TREE_TYPE (lhs))
2743 8263 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large)
2744 : {
2745 8263 : int p = var_to_partition (m_map, lhs);
2746 8263 : gcc_assert (m_vars[p] != NULL_TREE);
2747 8263 : m_lhs = lhs = m_vars[p];
2748 : }
2749 23253 : unsigned cnt, rem = 0, end = 0, prec = TYPE_PRECISION (type);
2750 23253 : bool sext = false;
2751 23253 : tree ext = NULL_TREE, store_operand = NULL_TREE;
2752 23253 : bool eh = false;
2753 23253 : basic_block eh_pad = NULL;
2754 23253 : tree nlhs = NULL_TREE;
2755 23253 : unsigned HOST_WIDE_INT bo_idx = 0;
2756 23253 : unsigned HOST_WIDE_INT bo_bit = 0;
2757 23253 : unsigned bo_shift = 0;
2758 23253 : unsigned bo_last = 0;
2759 23253 : bool bo_be_p = false;
2760 23253 : tree bf_cur = NULL_TREE, bf_next = NULL_TREE;
2761 23253 : if (gimple_store_p (stmt))
2762 : {
2763 8464 : store_operand = gimple_assign_rhs1 (stmt);
2764 8464 : eh = stmt_ends_bb_p (stmt);
2765 8464 : if (eh)
2766 : {
2767 2 : edge e;
2768 2 : edge_iterator ei;
2769 2 : basic_block bb = gimple_bb (stmt);
2770 :
2771 2 : FOR_EACH_EDGE (e, ei, bb->succs)
2772 2 : if (e->flags & EDGE_EH)
2773 : {
2774 2 : eh_pad = e->dest;
2775 2 : break;
2776 : }
2777 : }
2778 8464 : if (TREE_CODE (lhs) == COMPONENT_REF
2779 8464 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
2780 : {
2781 162 : tree fld = TREE_OPERAND (lhs, 1);
2782 162 : gcc_assert (tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld)));
2783 162 : tree repr = DECL_BIT_FIELD_REPRESENTATIVE (fld);
2784 162 : poly_int64 bitoffset;
2785 162 : poly_uint64 field_offset, repr_offset;
2786 162 : if (!bitint_big_endian
2787 162 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2788 162 : % BITS_PER_UNIT) == 0)
2789 : nlhs = lhs;
2790 : else
2791 : {
2792 143 : bool var_field_off = false;
2793 143 : if (poly_int_tree_p (DECL_FIELD_OFFSET (fld), &field_offset)
2794 286 : && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
2795 143 : bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
2796 : else
2797 : {
2798 : bitoffset = 0;
2799 : var_field_off = true;
2800 : }
2801 143 : bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
2802 143 : - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
2803 286 : nlhs = build3 (COMPONENT_REF, TREE_TYPE (repr),
2804 143 : TREE_OPERAND (lhs, 0), repr,
2805 : var_field_off
2806 0 : ? TREE_OPERAND (lhs, 2) : NULL_TREE);
2807 143 : HOST_WIDE_INT bo = bitoffset.to_constant ();
2808 143 : bo_idx = (unsigned HOST_WIDE_INT) bo / limb_prec;
2809 143 : bo_bit = (unsigned HOST_WIDE_INT) bo % limb_prec;
2810 143 : bo_shift = bo_bit;
2811 143 : if (bitint_big_endian)
2812 : {
2813 0 : bo_last = CEIL (prec + bo_bit, limb_prec) - 1;
2814 0 : bo_shift = (prec + bo_bit) % limb_prec;
2815 0 : bo_be_p = true;
2816 0 : if (bo_shift)
2817 0 : bo_shift = limb_prec - bo_shift;
2818 : }
2819 : }
2820 : }
2821 : }
2822 8464 : if ((store_operand
2823 8464 : && TREE_CODE (store_operand) == SSA_NAME
2824 7488 : && (m_names == NULL
2825 7473 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (store_operand)))
2826 1125 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (store_operand)))
2827 31171 : || gimple_assign_cast_p (stmt))
2828 : {
2829 2134 : rhs1 = gimple_assign_rhs1 (store_operand
2830 546 : ? SSA_NAME_DEF_STMT (store_operand)
2831 : : stmt);
2832 1588 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
2833 2 : rhs1 = TREE_OPERAND (rhs1, 0);
2834 : /* Optimize mergeable ops ending with widening cast to _BitInt
2835 : (or followed by store). We can lower just the limbs of the
2836 : cast operand and widen afterwards. */
2837 1588 : if (TREE_CODE (rhs1) == SSA_NAME
2838 1588 : && (m_names == NULL
2839 1583 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))
2840 607 : && BITINT_TYPE_P (TREE_TYPE (rhs1))
2841 469 : && bitint_precision_kind (TREE_TYPE (rhs1)) >= bitint_prec_large
2842 2010 : && (CEIL ((unsigned) TYPE_PRECISION (TREE_TYPE (rhs1)),
2843 422 : limb_prec) < CEIL (prec, limb_prec)
2844 306 : || (kind == bitint_prec_huge
2845 262 : && TYPE_PRECISION (TREE_TYPE (rhs1)) < prec)))
2846 : {
2847 122 : store_operand = rhs1;
2848 122 : prec = TYPE_PRECISION (TREE_TYPE (rhs1));
2849 122 : kind = bitint_precision_kind (TREE_TYPE (rhs1));
2850 122 : if (!TYPE_UNSIGNED (TREE_TYPE (rhs1)))
2851 46 : sext = true;
2852 : }
2853 : }
2854 23253 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
2855 23253 : if (kind == bitint_prec_large)
2856 12177 : cnt = CEIL (prec, limb_prec);
2857 : else
2858 : {
2859 11076 : rem = (prec % (2 * limb_prec));
2860 11076 : end = (prec - rem) / limb_prec;
2861 11076 : cnt = 2 + CEIL (rem, limb_prec);
2862 11076 : idx = idx_first = create_loop (bitint_big_endian
2863 11076 : ? size_int (cnt - 2 + end - 1)
2864 : : size_zero_node, &idx_next);
2865 : }
2866 :
2867 23253 : basic_block edge_bb = NULL;
2868 23253 : if (eq_p)
2869 : {
2870 6526 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
2871 6526 : gsi_prev (&gsi);
2872 6526 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
2873 6526 : edge_bb = e->src;
2874 6526 : if (kind == bitint_prec_large)
2875 7170 : m_gsi = gsi_end_bb (edge_bb);
2876 : }
2877 : else
2878 16727 : m_after_stmt = stmt;
2879 23253 : if (kind != bitint_prec_large)
2880 11076 : m_upwards_2limb = end;
2881 23253 : m_upwards = true;
2882 :
2883 23253 : bool separate_ext
2884 23253 : = (prec != (unsigned) TYPE_PRECISION (type)
2885 23253 : && (CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)
2886 122 : > CEIL (prec, limb_prec)));
2887 23253 : bool zero_ms_limb = false;
2888 23253 : if (bitint_extended == bitint_ext_full
2889 0 : && !eq_p
2890 0 : && !nlhs
2891 0 : && abi_limb_prec > limb_prec
2892 23253 : && ((CEIL ((unsigned) TYPE_PRECISION (type), abi_limb_prec)
2893 0 : * abi_limb_prec / limb_prec)
2894 0 : > CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)))
2895 : {
2896 0 : if (prec == (unsigned) TYPE_PRECISION (type))
2897 : {
2898 0 : sext = !TYPE_UNSIGNED (type);
2899 0 : separate_ext = true;
2900 : }
2901 0 : else if (TYPE_UNSIGNED (type) && sext)
2902 : zero_ms_limb = true;
2903 : else
2904 : separate_ext = true;
2905 : }
2906 23369 : unsigned dst_idx_off = 0;
2907 23253 : if (separate_ext && bitint_big_endian)
2908 0 : dst_idx_off = (CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)
2909 0 : - CEIL (prec, limb_prec));
2910 :
2911 93256 : for (unsigned i = 0; i < cnt; i++)
2912 : {
2913 70003 : m_data_cnt = 0;
2914 70003 : if (kind == bitint_prec_large)
2915 37142 : idx = size_int (bitint_big_endian ? cnt - 1 - i : i);
2916 32861 : else if (i >= 2)
2917 10709 : idx = size_int (bitint_big_endian ? cnt - 1 - i : end + (i > 2));
2918 70003 : if (eq_p)
2919 : {
2920 19769 : rhs1 = handle_operand (cmp_op1, idx);
2921 19769 : tree rhs2 = handle_operand (cmp_op2, idx);
2922 19769 : g = gimple_build_cond (NE_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
2923 19769 : insert_before (g);
2924 19769 : edge e1 = split_block (gsi_bb (m_gsi), g);
2925 19769 : e1->flags = EDGE_FALSE_VALUE;
2926 19769 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
2927 19769 : e1->probability = profile_probability::unlikely ();
2928 19769 : e2->probability = e1->probability.invert ();
2929 19769 : if (i == 0)
2930 6526 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
2931 19769 : m_gsi = gsi_after_labels (e1->dest);
2932 : }
2933 : else
2934 : {
2935 50234 : if (store_operand)
2936 25626 : rhs1 = handle_operand (store_operand, idx);
2937 : else
2938 24608 : rhs1 = handle_stmt (stmt, idx);
2939 50234 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
2940 11978 : rhs1 = add_cast (m_limb_type, rhs1);
2941 50234 : if (sext && i == cnt - 1)
2942 50234 : ext = rhs1;
2943 50234 : tree nidx = idx;
2944 50234 : HOST_WIDE_INT adj = bo_idx;
2945 50234 : if (bo_be_p)
2946 0 : adj += bo_last - (CEIL (prec, limb_prec) - 1);
2947 : else
2948 50234 : adj += dst_idx_off;
2949 50234 : if (adj)
2950 : {
2951 220 : if (tree_fits_uhwi_p (idx))
2952 116 : nidx = size_int (tree_to_uhwi (idx) + adj);
2953 : else
2954 : {
2955 104 : nidx = make_ssa_name (sizetype);
2956 104 : g = gimple_build_assign (nidx, PLUS_EXPR, idx,
2957 104 : size_int (adj));
2958 104 : insert_before (g);
2959 : }
2960 : }
2961 50234 : bool done = false;
2962 50234 : basic_block new_bb = NULL;
2963 : /* Handle stores into bit-fields. */
2964 50234 : if (bo_shift)
2965 : {
2966 455 : if (i == 0)
2967 : {
2968 143 : edge e2 = NULL;
2969 143 : if (kind != bitint_prec_large)
2970 : {
2971 112 : prepare_data_in_out (build_zero_cst (m_limb_type),
2972 : idx, &bf_next);
2973 112 : bf_next = m_data.pop ();
2974 112 : bf_cur = m_data.pop ();
2975 112 : g = gimple_build_cond (EQ_EXPR, idx,
2976 : bitint_big_endian
2977 0 : ? size_int (CEIL (prec,
2978 : limb_prec) - 1)
2979 : : size_zero_node,
2980 : NULL_TREE, NULL_TREE);
2981 112 : edge edge_true;
2982 112 : if_then_else (g, profile_probability::unlikely (),
2983 : edge_true, e2);
2984 112 : new_bb = e2->dest;
2985 : }
2986 143 : tree ftype
2987 143 : = build_nonstandard_integer_type (limb_prec - bo_shift, 1);
2988 286 : tree bfr = build_bit_field_ref (ftype, unshare_expr (nlhs),
2989 143 : limb_prec - bo_shift,
2990 : bitint_big_endian
2991 0 : ? (bo_idx + bo_last)
2992 0 : * limb_prec
2993 143 : : bo_idx * limb_prec
2994 : + bo_bit);
2995 143 : tree t = add_cast (ftype, rhs1);
2996 143 : g = gimple_build_assign (bfr, t);
2997 143 : insert_before (g);
2998 143 : if (eh)
2999 : {
3000 0 : maybe_duplicate_eh_stmt (g, stmt);
3001 0 : if (eh_pad)
3002 : {
3003 0 : edge e = split_block (gsi_bb (m_gsi), g);
3004 0 : m_gsi = gsi_after_labels (e->dest);
3005 0 : add_eh_edge (e->src,
3006 : find_edge (gimple_bb (stmt), eh_pad));
3007 : }
3008 : }
3009 143 : if (kind == bitint_prec_large)
3010 : {
3011 : bf_cur = rhs1;
3012 : done = true;
3013 : }
3014 112 : else if (e2)
3015 112 : m_gsi = gsi_after_labels (e2->src);
3016 : }
3017 112 : if (!done)
3018 : {
3019 424 : tree t1 = make_ssa_name (m_limb_type);
3020 424 : tree t2 = make_ssa_name (m_limb_type);
3021 424 : tree t3 = make_ssa_name (m_limb_type);
3022 424 : g = gimple_build_assign (t1, RSHIFT_EXPR, bf_cur,
3023 : build_int_cst (unsigned_type_node,
3024 424 : limb_prec
3025 424 : - bo_shift));
3026 424 : insert_before (g);
3027 424 : g = gimple_build_assign (t2, LSHIFT_EXPR, rhs1,
3028 : build_int_cst (unsigned_type_node,
3029 424 : bo_shift));
3030 424 : insert_before (g);
3031 424 : bf_cur = rhs1;
3032 424 : g = gimple_build_assign (t3, BIT_IOR_EXPR, t1, t2);
3033 424 : insert_before (g);
3034 424 : rhs1 = t3;
3035 424 : if (bf_next && i == 1)
3036 : {
3037 112 : g = gimple_build_assign (bf_next, bf_cur);
3038 112 : insert_before (g);
3039 : }
3040 : }
3041 : }
3042 31 : if (!done)
3043 : {
3044 : /* Handle bit-field access to partial last limb if needed. */
3045 50203 : if (nlhs
3046 480 : && i == cnt - 1
3047 162 : && !separate_ext
3048 138 : && tree_fits_uhwi_p (idx))
3049 : {
3050 123 : unsigned int tprec = TYPE_PRECISION (type);
3051 123 : unsigned int rprec = (tprec - 1) % limb_prec + 1;
3052 123 : if (rprec + bo_shift < (unsigned) limb_prec)
3053 : {
3054 46 : tree ftype
3055 46 : = build_nonstandard_integer_type (rprec + bo_shift, 1);
3056 46 : tree bfr
3057 92 : = build_bit_field_ref (ftype, unshare_expr (nlhs),
3058 : rprec + bo_shift,
3059 : bitint_big_endian
3060 0 : ? bo_idx * limb_prec + bo_bit
3061 46 : : (bo_idx + tprec / limb_prec)
3062 46 : * limb_prec);
3063 46 : tree t = add_cast (ftype, rhs1);
3064 46 : g = gimple_build_assign (bfr, t);
3065 46 : done = true;
3066 46 : bf_cur = NULL_TREE;
3067 : }
3068 77 : else if (rprec + bo_shift == (unsigned) limb_prec)
3069 50157 : bf_cur = NULL_TREE;
3070 : }
3071 : /* Otherwise, stores to any other lhs. */
3072 46 : if (!done)
3073 : {
3074 99880 : tree l = limb_access (nlhs ? NULL_TREE : lhs_type,
3075 : nlhs ? nlhs : lhs, nidx, true);
3076 50157 : g = gimple_build_assign (l, rhs1);
3077 : }
3078 50203 : insert_before (g);
3079 50203 : if (eh)
3080 : {
3081 6 : maybe_duplicate_eh_stmt (g, stmt);
3082 6 : if (eh_pad)
3083 : {
3084 6 : edge e = split_block (gsi_bb (m_gsi), g);
3085 6 : m_gsi = gsi_after_labels (e->dest);
3086 6 : add_eh_edge (e->src,
3087 : find_edge (gimple_bb (stmt), eh_pad));
3088 : }
3089 : }
3090 50203 : if (new_bb)
3091 112 : m_gsi = gsi_after_labels (new_bb);
3092 : }
3093 : }
3094 70003 : m_first = false;
3095 70003 : if (kind == bitint_prec_huge && i <= 1)
3096 : {
3097 22152 : if (i == 0)
3098 : {
3099 11076 : idx = make_ssa_name (sizetype);
3100 11076 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
3101 : bitint_big_endian
3102 0 : ? size_int (-1) : size_one_node);
3103 11076 : insert_before (g);
3104 : }
3105 : else
3106 : {
3107 11076 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
3108 22152 : size_int (bitint_big_endian ? -2 : 2));
3109 11076 : insert_before (g);
3110 11076 : if (bitint_big_endian)
3111 0 : g = gimple_build_cond (NE_EXPR, idx_first, size_int (cnt - 1),
3112 : NULL_TREE, NULL_TREE);
3113 : else
3114 11076 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (end),
3115 : NULL_TREE, NULL_TREE);
3116 11076 : insert_before (g);
3117 11076 : if (eq_p)
3118 2941 : m_gsi = gsi_after_labels (edge_bb);
3119 : else
3120 8135 : m_gsi = gsi_for_stmt (stmt);
3121 11076 : m_bb = NULL;
3122 : }
3123 : }
3124 : }
3125 :
3126 23253 : if (separate_ext)
3127 : {
3128 116 : if (sext)
3129 : {
3130 43 : ext = add_cast (signed_type_for (m_limb_type), ext);
3131 86 : tree lpm1 = build_int_cst (unsigned_type_node,
3132 43 : limb_prec - 1);
3133 43 : tree n = make_ssa_name (TREE_TYPE (ext));
3134 43 : g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
3135 43 : insert_before (g);
3136 43 : ext = add_cast (m_limb_type, n);
3137 : }
3138 : else
3139 73 : ext = build_zero_cst (m_limb_type);
3140 116 : kind = bitint_precision_kind (type);
3141 116 : unsigned start = CEIL (prec, limb_prec);
3142 116 : prec = TYPE_PRECISION (type);
3143 116 : if (bitint_extended == bitint_ext_full
3144 0 : && !nlhs
3145 0 : && !zero_ms_limb
3146 0 : && abi_limb_prec > limb_prec)
3147 : {
3148 0 : prec = CEIL (prec, abi_limb_prec) * abi_limb_prec;
3149 0 : kind = bitint_precision_kind (prec);
3150 : }
3151 116 : unsigned total = CEIL (prec, limb_prec);
3152 116 : idx = idx_first = idx_next = NULL_TREE;
3153 116 : if (prec <= (start + 2 + (bo_shift != 0)) * limb_prec)
3154 : kind = bitint_prec_large;
3155 78 : if (kind == bitint_prec_large)
3156 38 : cnt = total - start;
3157 : else
3158 : {
3159 78 : rem = prec % limb_prec;
3160 78 : end = (prec - rem) / limb_prec;
3161 148 : cnt = (bo_shift != 0) + 1 + (rem != 0);
3162 : }
3163 116 : if (bitint_big_endian && bo_shift != 0 && (prec % limb_prec) == 0)
3164 0 : ++total;
3165 296 : for (unsigned i = 0; i < cnt; i++)
3166 : {
3167 180 : if (kind == bitint_prec_large || (i == 0 && bo_shift != 0))
3168 48 : idx = size_int (bo_idx
3169 : + (bitint_big_endian
3170 : ? total - 1 - start - i : start + i));
3171 132 : else if (i == cnt - 1 && rem != 0)
3172 108 : idx = size_int (bo_idx + (bitint_big_endian ? 0 : end));
3173 78 : else if (i == (bo_shift != 0))
3174 78 : idx = create_loop (size_int (bo_idx
3175 : + (bitint_big_endian
3176 : ? total - 1 - start - i
3177 : : start + i)), &idx_next);
3178 180 : rhs1 = ext;
3179 180 : if (bf_cur != NULL_TREE && bf_cur != ext)
3180 : {
3181 20 : tree t1 = make_ssa_name (m_limb_type);
3182 20 : g = gimple_build_assign (t1, RSHIFT_EXPR, bf_cur,
3183 : build_int_cst (unsigned_type_node,
3184 20 : limb_prec - bo_shift));
3185 20 : insert_before (g);
3186 20 : if (integer_zerop (ext))
3187 : rhs1 = t1;
3188 : else
3189 : {
3190 10 : tree t2 = make_ssa_name (m_limb_type);
3191 10 : rhs1 = make_ssa_name (m_limb_type);
3192 10 : g = gimple_build_assign (t2, LSHIFT_EXPR, ext,
3193 : build_int_cst (unsigned_type_node,
3194 10 : bo_shift));
3195 10 : insert_before (g);
3196 10 : g = gimple_build_assign (rhs1, BIT_IOR_EXPR, t1, t2);
3197 10 : insert_before (g);
3198 : }
3199 : bf_cur = ext;
3200 : }
3201 180 : bool done = false;
3202 : /* Handle bit-field access to partial last limb if needed. */
3203 180 : if (nlhs && i == cnt - 1)
3204 : {
3205 24 : unsigned int tprec = TYPE_PRECISION (type);
3206 24 : unsigned int rprec = (tprec - 1) % limb_prec + 1;
3207 24 : if (rprec + bo_shift < (unsigned) limb_prec)
3208 : {
3209 12 : tree ftype
3210 12 : = build_nonstandard_integer_type (rprec + bo_shift, 1);
3211 12 : tree bfr
3212 24 : = build_bit_field_ref (ftype, unshare_expr (nlhs),
3213 : rprec + bo_shift,
3214 : bitint_big_endian
3215 0 : ? bo_idx * limb_prec + bo_bit
3216 12 : : (bo_idx + tprec / limb_prec)
3217 12 : * limb_prec);
3218 12 : tree t = add_cast (ftype, rhs1);
3219 12 : g = gimple_build_assign (bfr, t);
3220 12 : done = true;
3221 12 : bf_cur = NULL_TREE;
3222 : }
3223 12 : else if (rprec + bo_shift == (unsigned) limb_prec)
3224 : bf_cur = NULL_TREE;
3225 : }
3226 : /* Otherwise, stores to any other lhs. */
3227 12 : if (!done)
3228 : {
3229 306 : tree l = limb_access (nlhs ? NULL_TREE : lhs_type,
3230 : nlhs ? nlhs : lhs, idx, true);
3231 :
3232 168 : if (bitint_extended
3233 0 : && sext
3234 0 : && TYPE_UNSIGNED (lhs_type)
3235 0 : && tree_fits_uhwi_p (idx)
3236 168 : && !nlhs)
3237 : {
3238 0 : rhs1 = add_cast (limb_access_type (lhs_type, idx), rhs1);
3239 0 : rhs1 = add_cast (TREE_TYPE (l), rhs1);
3240 : }
3241 :
3242 168 : g = gimple_build_assign (l, rhs1);
3243 : }
3244 180 : insert_before (g);
3245 180 : if (eh)
3246 : {
3247 0 : maybe_duplicate_eh_stmt (g, stmt);
3248 0 : if (eh_pad)
3249 : {
3250 0 : edge e = split_block (gsi_bb (m_gsi), g);
3251 0 : m_gsi = gsi_after_labels (e->dest);
3252 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3253 : }
3254 : }
3255 180 : if (kind == bitint_prec_huge && i == (bo_shift != 0))
3256 : {
3257 78 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3258 : bitint_big_endian
3259 0 : ? size_int (-1) : size_one_node);
3260 78 : insert_before (g);
3261 78 : if (bitint_big_endian && rem != 0)
3262 0 : g = gimple_build_cond (NE_EXPR, idx,
3263 0 : size_int (bo_idx + 1),
3264 : NULL_TREE, NULL_TREE);
3265 : else
3266 78 : g = gimple_build_cond (NE_EXPR, idx_next,
3267 156 : size_int (bo_idx
3268 : + (bitint_big_endian
3269 : ? 0 : end)),
3270 : NULL_TREE, NULL_TREE);
3271 78 : insert_before (g);
3272 78 : m_gsi = gsi_for_stmt (stmt);
3273 78 : m_bb = NULL;
3274 : }
3275 : }
3276 : }
3277 23253 : if (bf_cur != NULL_TREE)
3278 : {
3279 72 : unsigned int tprec = TYPE_PRECISION (type);
3280 72 : unsigned int rprec = (tprec + bo_shift) % limb_prec;
3281 72 : tree ftype = build_nonstandard_integer_type (rprec, 1);
3282 144 : tree bfr = build_bit_field_ref (ftype, unshare_expr (nlhs),
3283 : rprec,
3284 : bitint_big_endian
3285 0 : ? bo_idx * limb_prec + bo_bit
3286 72 : : (bo_idx + (tprec + bo_bit) / limb_prec)
3287 : * limb_prec);
3288 72 : rhs1 = bf_cur;
3289 72 : if (bf_cur != ext)
3290 : {
3291 64 : rhs1 = make_ssa_name (TREE_TYPE (rhs1));
3292 64 : g = gimple_build_assign (rhs1, RSHIFT_EXPR, bf_cur,
3293 : build_int_cst (unsigned_type_node,
3294 64 : limb_prec - bo_shift));
3295 64 : insert_before (g);
3296 : }
3297 72 : rhs1 = add_cast (ftype, rhs1);
3298 72 : g = gimple_build_assign (bfr, rhs1);
3299 72 : insert_before (g);
3300 72 : if (eh)
3301 : {
3302 0 : maybe_duplicate_eh_stmt (g, stmt);
3303 0 : if (eh_pad)
3304 : {
3305 0 : edge e = split_block (gsi_bb (m_gsi), g);
3306 0 : m_gsi = gsi_after_labels (e->dest);
3307 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3308 : }
3309 : }
3310 : }
3311 23253 : if (zero_ms_limb)
3312 : {
3313 0 : tree p2 = build_int_cst (sizetype,
3314 0 : CEIL ((unsigned) TYPE_PRECISION (type),
3315 : abi_limb_prec)
3316 0 : * abi_limb_prec / limb_prec - 1);
3317 0 : tree l = limb_access (lhs_type, lhs, p2, true);
3318 0 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
3319 0 : insert_before (g);
3320 0 : if (eh)
3321 : {
3322 0 : maybe_duplicate_eh_stmt (g, stmt);
3323 0 : if (eh_pad)
3324 : {
3325 0 : edge e = split_block (gsi_bb (m_gsi), g);
3326 0 : m_gsi = gsi_after_labels (e->dest);
3327 0 : add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
3328 : }
3329 : }
3330 : }
3331 :
3332 23253 : if (gimple_store_p (stmt))
3333 : {
3334 8464 : unlink_stmt_vdef (stmt);
3335 16928 : release_ssa_name (gimple_vdef (stmt));
3336 8464 : gsi_remove (&m_gsi, true);
3337 : }
3338 23253 : if (eq_p)
3339 : {
3340 6526 : lhs = make_ssa_name (boolean_type_node);
3341 6526 : basic_block bb = gimple_bb (stmt);
3342 6526 : gphi *phi = create_phi_node (lhs, bb);
3343 6526 : edge e = find_edge (gsi_bb (m_gsi), bb);
3344 6526 : unsigned int n = EDGE_COUNT (bb->preds);
3345 32821 : for (unsigned int i = 0; i < n; i++)
3346 : {
3347 26295 : edge e2 = EDGE_PRED (bb, i);
3348 26295 : add_phi_arg (phi, e == e2 ? boolean_true_node : boolean_false_node,
3349 : e2, UNKNOWN_LOCATION);
3350 : }
3351 6526 : cmp_code = cmp_code == EQ_EXPR ? NE_EXPR : EQ_EXPR;
3352 6526 : return lhs;
3353 : }
3354 : else
3355 : return NULL_TREE;
3356 : }
3357 :
3358 : /* Handle a large/huge _BitInt comparison statement STMT other than
3359 : EQ_EXPR/NE_EXPR. CMP_CODE, CMP_OP1 and CMP_OP2 meaning is like in
3360 : lower_mergeable_stmt. The {GT,GE,LT,LE}_EXPR comparisons are
3361 : lowered by iteration from the most significant limb downwards to
3362 : the least significant one, for large _BitInt in straight line code,
3363 : otherwise with most significant limb handled in
3364 : straight line code followed by a loop handling one limb at a time.
3365 : Comparisons with unsigned huge _BitInt with precisions which are
3366 : multiples of limb precision can use just the loop and don't need to
3367 : handle most significant limb before the loop. The loop or straight
3368 : line code jumps to final basic block if a particular pair of limbs
3369 : is not equal. */
3370 :
3371 : tree
3372 722 : bitint_large_huge::lower_comparison_stmt (gimple *stmt, tree_code &cmp_code,
3373 : tree cmp_op1, tree cmp_op2)
3374 : {
3375 722 : tree type = TREE_TYPE (cmp_op1);
3376 722 : gcc_assert (BITINT_TYPE_P (type));
3377 722 : bitint_prec_kind kind = bitint_precision_kind (type);
3378 722 : gcc_assert (kind >= bitint_prec_large);
3379 722 : gimple *g;
3380 722 : if (!TYPE_UNSIGNED (type)
3381 441 : && integer_zerop (cmp_op2)
3382 750 : && (cmp_code == GE_EXPR || cmp_code == LT_EXPR))
3383 : {
3384 28 : unsigned end = CEIL ((unsigned) TYPE_PRECISION (type), limb_prec) - 1;
3385 56 : tree idx = size_int (bitint_big_endian ? 0 : end);
3386 28 : m_data_cnt = 0;
3387 28 : tree rhs1 = handle_operand (cmp_op1, idx);
3388 28 : if (TYPE_UNSIGNED (TREE_TYPE (rhs1)))
3389 : {
3390 24 : tree stype = signed_type_for (TREE_TYPE (rhs1));
3391 24 : rhs1 = add_cast (stype, rhs1);
3392 : }
3393 28 : tree lhs = make_ssa_name (boolean_type_node);
3394 28 : g = gimple_build_assign (lhs, cmp_code, rhs1,
3395 28 : build_zero_cst (TREE_TYPE (rhs1)));
3396 28 : insert_before (g);
3397 28 : cmp_code = NE_EXPR;
3398 28 : return lhs;
3399 : }
3400 :
3401 694 : unsigned cnt, rem = 0, end = 0;
3402 694 : tree idx = NULL_TREE, idx_next = NULL_TREE;
3403 694 : if (kind == bitint_prec_large)
3404 377 : cnt = CEIL ((unsigned) TYPE_PRECISION (type), limb_prec);
3405 : else
3406 : {
3407 317 : rem = ((unsigned) TYPE_PRECISION (type) % limb_prec);
3408 317 : if (rem == 0 && !TYPE_UNSIGNED (type))
3409 : rem = limb_prec;
3410 317 : end = ((unsigned) TYPE_PRECISION (type) - rem) / limb_prec;
3411 317 : cnt = 1 + (rem != 0);
3412 : }
3413 :
3414 694 : basic_block edge_bb = NULL;
3415 694 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
3416 694 : gsi_prev (&gsi);
3417 694 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
3418 694 : edge_bb = e->src;
3419 694 : m_gsi = gsi_end_bb (edge_bb);
3420 :
3421 694 : edge *edges = XALLOCAVEC (edge, cnt * 2);
3422 2519 : for (unsigned i = 0; i < cnt; i++)
3423 : {
3424 1825 : m_data_cnt = 0;
3425 1825 : if (kind == bitint_prec_large)
3426 1246 : idx = size_int (bitint_big_endian ? i : cnt - i - 1);
3427 579 : else if (i == cnt - 1)
3428 317 : idx = create_loop (size_int (bitint_big_endian ? cnt - 1 : end - 1),
3429 : &idx_next);
3430 : else
3431 524 : idx = size_int (bitint_big_endian ? 0 : end);
3432 1825 : tree rhs1 = handle_operand (cmp_op1, idx);
3433 1825 : tree rhs2 = handle_operand (cmp_op2, idx);
3434 1825 : if (i == 0
3435 694 : && !TYPE_UNSIGNED (type)
3436 2238 : && TYPE_UNSIGNED (TREE_TYPE (rhs1)))
3437 : {
3438 112 : tree stype = signed_type_for (TREE_TYPE (rhs1));
3439 112 : rhs1 = add_cast (stype, rhs1);
3440 112 : rhs2 = add_cast (stype, rhs2);
3441 : }
3442 1825 : g = gimple_build_cond (GT_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
3443 1825 : insert_before (g);
3444 1825 : edge e1 = split_block (gsi_bb (m_gsi), g);
3445 1825 : e1->flags = EDGE_FALSE_VALUE;
3446 1825 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
3447 1825 : e1->probability = profile_probability::likely ();
3448 1825 : e2->probability = e1->probability.invert ();
3449 1825 : if (i == 0)
3450 694 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
3451 1825 : m_gsi = gsi_after_labels (e1->dest);
3452 1825 : edges[2 * i] = e2;
3453 1825 : g = gimple_build_cond (LT_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
3454 1825 : insert_before (g);
3455 1825 : e1 = split_block (gsi_bb (m_gsi), g);
3456 1825 : e1->flags = EDGE_FALSE_VALUE;
3457 1825 : e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
3458 1825 : e1->probability = profile_probability::unlikely ();
3459 1825 : e2->probability = e1->probability.invert ();
3460 1825 : m_gsi = gsi_after_labels (e1->dest);
3461 1825 : edges[2 * i + 1] = e2;
3462 1825 : m_first = false;
3463 1825 : if (kind == bitint_prec_huge && i == cnt - 1)
3464 : {
3465 634 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3466 : bitint_big_endian ? size_one_node
3467 317 : : size_int (-1));
3468 317 : insert_before (g);
3469 317 : g = gimple_build_cond (NE_EXPR, idx,
3470 : bitint_big_endian
3471 0 : ? size_int (end - 1 + (cnt != 1))
3472 : : size_zero_node,
3473 : NULL_TREE, NULL_TREE);
3474 317 : insert_before (g);
3475 317 : edge true_edge, false_edge;
3476 317 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
3477 : &true_edge, &false_edge);
3478 317 : m_gsi = gsi_after_labels (false_edge->dest);
3479 317 : m_bb = NULL;
3480 : }
3481 : }
3482 :
3483 694 : tree lhs = make_ssa_name (boolean_type_node);
3484 694 : basic_block bb = gimple_bb (stmt);
3485 694 : gphi *phi = create_phi_node (lhs, bb);
3486 4344 : for (unsigned int i = 0; i < cnt * 2; i++)
3487 : {
3488 3650 : tree val = ((cmp_code == GT_EXPR || cmp_code == GE_EXPR)
3489 3650 : ^ (i & 1)) ? boolean_true_node : boolean_false_node;
3490 3650 : add_phi_arg (phi, val, edges[i], UNKNOWN_LOCATION);
3491 : }
3492 694 : add_phi_arg (phi, (cmp_code == GE_EXPR || cmp_code == LE_EXPR)
3493 : ? boolean_true_node : boolean_false_node,
3494 : find_edge (gsi_bb (m_gsi), bb), UNKNOWN_LOCATION);
3495 694 : cmp_code = NE_EXPR;
3496 694 : return lhs;
3497 : }
3498 :
3499 : /* Lower large/huge _BitInt left and right shift except for left
3500 : shift by < limb_prec constant. */
3501 :
3502 : void
3503 584 : bitint_large_huge::lower_shift_stmt (tree obj, gimple *stmt)
3504 : {
3505 584 : tree rhs1 = gimple_assign_rhs1 (stmt);
3506 584 : tree lhs = gimple_assign_lhs (stmt);
3507 584 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
3508 584 : tree type = TREE_TYPE (rhs1);
3509 584 : gimple *final_stmt = gsi_stmt (m_gsi);
3510 584 : gcc_assert (BITINT_TYPE_P (type)
3511 : && bitint_precision_kind (type) >= bitint_prec_large);
3512 584 : int prec = TYPE_PRECISION (type);
3513 584 : tree n = gimple_assign_rhs2 (stmt), n1, n2, n3, n4;
3514 584 : gimple *g;
3515 584 : if (obj == NULL_TREE)
3516 : {
3517 465 : int part = var_to_partition (m_map, lhs);
3518 465 : gcc_assert (m_vars[part] != NULL_TREE);
3519 : obj = m_vars[part];
3520 : }
3521 : /* Preparation code common for both left and right shifts.
3522 : unsigned n1 = n % limb_prec;
3523 : size_t n2 = n / limb_prec;
3524 : size_t n3 = n1 != 0;
3525 : unsigned n4 = (limb_prec - n1) % limb_prec;
3526 : (for power of 2 limb_prec n4 can be -n1 & limb_prec). */
3527 584 : if (TREE_CODE (n) == INTEGER_CST)
3528 : {
3529 249 : tree lp = build_int_cst (TREE_TYPE (n), limb_prec);
3530 249 : n1 = int_const_binop (TRUNC_MOD_EXPR, n, lp);
3531 249 : n2 = fold_convert (sizetype, int_const_binop (TRUNC_DIV_EXPR, n, lp));
3532 249 : n3 = size_int (!integer_zerop (n1));
3533 249 : n4 = int_const_binop (TRUNC_MOD_EXPR,
3534 249 : int_const_binop (MINUS_EXPR, lp, n1), lp);
3535 : }
3536 : else
3537 : {
3538 335 : n1 = make_ssa_name (TREE_TYPE (n));
3539 335 : n2 = make_ssa_name (sizetype);
3540 335 : n3 = make_ssa_name (sizetype);
3541 335 : n4 = make_ssa_name (TREE_TYPE (n));
3542 335 : if (pow2p_hwi (limb_prec))
3543 : {
3544 335 : tree lpm1 = build_int_cst (TREE_TYPE (n), limb_prec - 1);
3545 335 : g = gimple_build_assign (n1, BIT_AND_EXPR, n, lpm1);
3546 335 : insert_before (g);
3547 989 : g = gimple_build_assign (useless_type_conversion_p (sizetype,
3548 335 : TREE_TYPE (n))
3549 319 : ? n2 : make_ssa_name (TREE_TYPE (n)),
3550 : RSHIFT_EXPR, n,
3551 335 : build_int_cst (TREE_TYPE (n),
3552 670 : exact_log2 (limb_prec)));
3553 335 : insert_before (g);
3554 335 : if (gimple_assign_lhs (g) != n2)
3555 : {
3556 319 : g = gimple_build_assign (n2, NOP_EXPR, gimple_assign_lhs (g));
3557 319 : insert_before (g);
3558 : }
3559 335 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (n)),
3560 : NEGATE_EXPR, n1);
3561 335 : insert_before (g);
3562 335 : g = gimple_build_assign (n4, BIT_AND_EXPR, gimple_assign_lhs (g),
3563 : lpm1);
3564 335 : insert_before (g);
3565 : }
3566 : else
3567 : {
3568 0 : tree lp = build_int_cst (TREE_TYPE (n), limb_prec);
3569 0 : g = gimple_build_assign (n1, TRUNC_MOD_EXPR, n, lp);
3570 0 : insert_before (g);
3571 0 : g = gimple_build_assign (useless_type_conversion_p (sizetype,
3572 0 : TREE_TYPE (n))
3573 0 : ? n2 : make_ssa_name (TREE_TYPE (n)),
3574 : TRUNC_DIV_EXPR, n, lp);
3575 0 : insert_before (g);
3576 0 : if (gimple_assign_lhs (g) != n2)
3577 : {
3578 0 : g = gimple_build_assign (n2, NOP_EXPR, gimple_assign_lhs (g));
3579 0 : insert_before (g);
3580 : }
3581 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (n)),
3582 : MINUS_EXPR, lp, n1);
3583 0 : insert_before (g);
3584 0 : g = gimple_build_assign (n4, TRUNC_MOD_EXPR, gimple_assign_lhs (g),
3585 : lp);
3586 0 : insert_before (g);
3587 : }
3588 335 : g = gimple_build_assign (make_ssa_name (boolean_type_node), NE_EXPR, n1,
3589 335 : build_zero_cst (TREE_TYPE (n)));
3590 335 : insert_before (g);
3591 335 : g = gimple_build_assign (n3, NOP_EXPR, gimple_assign_lhs (g));
3592 335 : insert_before (g);
3593 : }
3594 1168 : tree p = build_int_cst (sizetype,
3595 584 : prec / limb_prec - (prec % limb_prec == 0));
3596 584 : if (rhs_code == RSHIFT_EXPR)
3597 : {
3598 : /* Lower
3599 : dst = src >> n;
3600 : as
3601 : unsigned n1 = n % limb_prec;
3602 : size_t n2 = n / limb_prec;
3603 : size_t n3 = n1 != 0;
3604 : unsigned n4 = (limb_prec - n1) % limb_prec;
3605 : size_t idx;
3606 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
3607 : int signed_p = (typeof (src) -1) < 0;
3608 : for (idx = n2; idx < ((!signed_p && (prec % limb_prec == 0))
3609 : ? p : p - n3); ++idx)
3610 : dst[idx - n2] = (src[idx] >> n1) | (src[idx + n3] << n4);
3611 : limb_type ext;
3612 : if (prec % limb_prec == 0)
3613 : ext = src[p];
3614 : else if (signed_p)
3615 : ext = ((signed limb_type) (src[p] << (limb_prec
3616 : - (prec % limb_prec))))
3617 : >> (limb_prec - (prec % limb_prec));
3618 : else
3619 : ext = src[p] & (((limb_type) 1 << (prec % limb_prec)) - 1);
3620 : if (!signed_p && (prec % limb_prec == 0))
3621 : ;
3622 : else if (idx < prec / 64)
3623 : {
3624 : dst[idx - n2] = (src[idx] >> n1) | (ext << n4);
3625 : ++idx;
3626 : }
3627 : idx -= n2;
3628 : if (signed_p)
3629 : {
3630 : dst[idx] = ((signed limb_type) ext) >> n1;
3631 : ext = ((signed limb_type) ext) >> (limb_prec - 1);
3632 : }
3633 : else
3634 : {
3635 : dst[idx] = ext >> n1;
3636 : ext = 0;
3637 : }
3638 : for (++idx; idx <= p; ++idx)
3639 : dst[idx] = ext; */
3640 387 : tree pmn3;
3641 387 : if (TYPE_UNSIGNED (type) && prec % limb_prec == 0)
3642 100 : pmn3 = bitint_big_endian ? size_zero_node : p;
3643 287 : else if (bitint_big_endian)
3644 : pmn3 = n3;
3645 287 : else if (TREE_CODE (n3) == INTEGER_CST)
3646 114 : pmn3 = int_const_binop (MINUS_EXPR, p, n3);
3647 : else
3648 : {
3649 173 : pmn3 = make_ssa_name (sizetype);
3650 173 : g = gimple_build_assign (pmn3, MINUS_EXPR, p, n3);
3651 173 : insert_before (g);
3652 : }
3653 387 : tree pmn2 = NULL_TREE;
3654 387 : if (bitint_big_endian)
3655 : {
3656 0 : if (TREE_CODE (n2) == INTEGER_CST)
3657 0 : pmn2 = int_const_binop (MINUS_EXPR, p, n2);
3658 : else
3659 : {
3660 0 : pmn2 = make_ssa_name (sizetype);
3661 0 : g = gimple_build_assign (pmn2, MINUS_EXPR, p, n2);
3662 0 : insert_before (g);
3663 : }
3664 0 : g = gimple_build_cond (GT_EXPR, pmn2, pmn3, NULL_TREE, NULL_TREE);
3665 : }
3666 : else
3667 387 : g = gimple_build_cond (LT_EXPR, n2, pmn3, NULL_TREE, NULL_TREE);
3668 387 : edge edge_true, edge_false;
3669 387 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3670 387 : tree idx_next;
3671 774 : tree idx = create_loop (bitint_big_endian ? pmn2 : n2, &idx_next);
3672 387 : tree idxmn2 = make_ssa_name (sizetype);
3673 387 : tree idxpn3 = make_ssa_name (sizetype);
3674 774 : g = gimple_build_assign (idxmn2,
3675 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3676 : idx, n2);
3677 387 : insert_before (g);
3678 774 : g = gimple_build_assign (idxpn3,
3679 : bitint_big_endian ? MINUS_EXPR : PLUS_EXPR,
3680 : idx, n3);
3681 387 : insert_before (g);
3682 387 : m_data_cnt = 0;
3683 387 : tree t1 = handle_operand (rhs1, idx);
3684 387 : m_first = false;
3685 387 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3686 : RSHIFT_EXPR, t1, n1);
3687 387 : insert_before (g);
3688 387 : t1 = gimple_assign_lhs (g);
3689 387 : if (!integer_zerop (n3))
3690 : {
3691 299 : m_data_cnt = 0;
3692 299 : tree t2 = handle_operand (rhs1, idxpn3);
3693 299 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3694 : LSHIFT_EXPR, t2, n4);
3695 299 : insert_before (g);
3696 299 : t2 = gimple_assign_lhs (g);
3697 299 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3698 : BIT_IOR_EXPR, t1, t2);
3699 299 : insert_before (g);
3700 299 : t1 = gimple_assign_lhs (g);
3701 : }
3702 387 : tree l = limb_access (TREE_TYPE (lhs), obj, idxmn2, true);
3703 387 : g = gimple_build_assign (l, t1);
3704 387 : insert_before (g);
3705 387 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3706 0 : bitint_big_endian ? size_int (-1)
3707 : : size_one_node);
3708 387 : insert_before (g);
3709 774 : g = gimple_build_cond (bitint_big_endian ? GT_EXPR : LT_EXPR,
3710 : idx_next, pmn3, NULL_TREE, NULL_TREE);
3711 387 : insert_before (g);
3712 387 : idx = make_ssa_name (sizetype);
3713 387 : m_gsi = gsi_for_stmt (final_stmt);
3714 387 : gphi *phi = create_phi_node (idx, gsi_bb (m_gsi));
3715 387 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3716 387 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3717 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3718 774 : add_phi_arg (phi, bitint_big_endian ? pmn2 : n2, edge_false,
3719 : UNKNOWN_LOCATION);
3720 387 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3721 387 : m_data_cnt = 0;
3722 387 : tree ms = handle_operand (rhs1, bitint_big_endian ? size_zero_node : p);
3723 387 : tree ext = ms;
3724 387 : if (!types_compatible_p (TREE_TYPE (ms), m_limb_type))
3725 230 : ext = add_cast (m_limb_type, ms);
3726 587 : if (!(TYPE_UNSIGNED (type) && prec % limb_prec == 0)
3727 487 : && !integer_zerop (n3))
3728 : {
3729 249 : if (bitint_big_endian)
3730 0 : g = gimple_build_cond (GT_EXPR, idx, size_zero_node,
3731 : NULL_TREE, NULL_TREE);
3732 : else
3733 249 : g = gimple_build_cond (LT_EXPR, idx, p, NULL_TREE, NULL_TREE);
3734 249 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3735 249 : m_data_cnt = 0;
3736 249 : t1 = handle_operand (rhs1, idx);
3737 249 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3738 : RSHIFT_EXPR, t1, n1);
3739 249 : insert_before (g);
3740 249 : t1 = gimple_assign_lhs (g);
3741 249 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3742 : LSHIFT_EXPR, ext, n4);
3743 249 : insert_before (g);
3744 249 : tree t2 = gimple_assign_lhs (g);
3745 249 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3746 : BIT_IOR_EXPR, t1, t2);
3747 249 : insert_before (g);
3748 249 : t1 = gimple_assign_lhs (g);
3749 249 : idxmn2 = make_ssa_name (sizetype);
3750 498 : g = gimple_build_assign (idxmn2, bitint_big_endian
3751 : ? PLUS_EXPR : MINUS_EXPR, idx, n2);
3752 249 : insert_before (g);
3753 249 : l = limb_access (TREE_TYPE (lhs), obj, idxmn2, true);
3754 249 : g = gimple_build_assign (l, t1);
3755 249 : insert_before (g);
3756 249 : idx_next = make_ssa_name (sizetype);
3757 249 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3758 : bitint_big_endian
3759 0 : ? size_int (-1) : size_one_node);
3760 249 : insert_before (g);
3761 249 : m_gsi = gsi_for_stmt (final_stmt);
3762 249 : tree nidx = make_ssa_name (sizetype);
3763 249 : phi = create_phi_node (nidx, gsi_bb (m_gsi));
3764 249 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3765 249 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3766 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3767 249 : add_phi_arg (phi, idx, edge_false, UNKNOWN_LOCATION);
3768 249 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3769 249 : idx = nidx;
3770 : }
3771 774 : g = gimple_build_assign (make_ssa_name (sizetype),
3772 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3773 : idx, n2);
3774 387 : insert_before (g);
3775 387 : idx = gimple_assign_lhs (g);
3776 387 : tree sext = ext;
3777 387 : if (!TYPE_UNSIGNED (type))
3778 187 : sext = add_cast (signed_type_for (m_limb_type), ext);
3779 387 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (sext)),
3780 : RSHIFT_EXPR, sext, n1);
3781 387 : insert_before (g);
3782 387 : t1 = gimple_assign_lhs (g);
3783 387 : if (!TYPE_UNSIGNED (type))
3784 : {
3785 187 : t1 = add_cast (m_limb_type, t1);
3786 187 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (sext)),
3787 : RSHIFT_EXPR, sext,
3788 187 : build_int_cst (TREE_TYPE (n),
3789 187 : limb_prec - 1));
3790 187 : insert_before (g);
3791 187 : ext = add_cast (m_limb_type, gimple_assign_lhs (g));
3792 : }
3793 : else
3794 200 : ext = build_zero_cst (m_limb_type);
3795 387 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3796 387 : g = gimple_build_assign (l, t1);
3797 387 : insert_before (g);
3798 387 : g = gimple_build_assign (make_ssa_name (sizetype), PLUS_EXPR, idx,
3799 : bitint_big_endian
3800 0 : ? size_int (-1) : size_one_node);
3801 387 : insert_before (g);
3802 387 : tree p2 = p;
3803 387 : if (bitint_big_endian)
3804 : {
3805 0 : tree new_idx = gimple_assign_lhs (g);
3806 0 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
3807 : NULL_TREE, NULL_TREE);
3808 0 : idx = new_idx;
3809 : }
3810 : else
3811 : {
3812 387 : if (bitint_extended == bitint_ext_full
3813 0 : && abi_limb_prec > limb_prec)
3814 0 : p2 = build_int_cst (sizetype,
3815 0 : CEIL (prec, abi_limb_prec)
3816 0 : * abi_limb_prec / limb_prec - 1);
3817 387 : idx = gimple_assign_lhs (g);
3818 387 : g = gimple_build_cond (LE_EXPR, idx, p2, NULL_TREE, NULL_TREE);
3819 : }
3820 387 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3821 387 : idx = create_loop (idx, &idx_next);
3822 387 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3823 387 : g = gimple_build_assign (l, ext);
3824 387 : insert_before (g);
3825 387 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3826 : bitint_big_endian
3827 0 : ? size_int (-1) : size_one_node);
3828 387 : insert_before (g);
3829 387 : if (bitint_big_endian)
3830 0 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
3831 : NULL_TREE, NULL_TREE);
3832 : else
3833 387 : g = gimple_build_cond (LE_EXPR, idx_next, p2, NULL_TREE, NULL_TREE);
3834 387 : insert_before (g);
3835 : }
3836 : else
3837 : {
3838 : /* Lower
3839 : dst = src << n;
3840 : as
3841 : unsigned n1 = n % limb_prec;
3842 : size_t n2 = n / limb_prec;
3843 : size_t n3 = n1 != 0;
3844 : unsigned n4 = (limb_prec - n1) % limb_prec;
3845 : size_t idx;
3846 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
3847 : for (idx = p; (ssize_t) idx >= (ssize_t) (n2 + n3); --idx)
3848 : dst[idx] = (src[idx - n2] << n1) | (src[idx - n2 - n3] >> n4);
3849 : if (n1)
3850 : {
3851 : dst[idx] = src[idx - n2] << n1;
3852 : --idx;
3853 : }
3854 : for (; (ssize_t) idx >= 0; --idx)
3855 : dst[idx] = 0; */
3856 197 : tree n2pn3;
3857 197 : if (TREE_CODE (n2) == INTEGER_CST && TREE_CODE (n3) == INTEGER_CST)
3858 71 : n2pn3 = int_const_binop (PLUS_EXPR, n2, n3);
3859 : else
3860 : {
3861 126 : n2pn3 = make_ssa_name (sizetype);
3862 126 : g = gimple_build_assign (n2pn3, PLUS_EXPR, n2, n3);
3863 126 : insert_before (g);
3864 : }
3865 197 : if (bitint_big_endian)
3866 : {
3867 0 : if (TREE_CODE (n2pn3) == INTEGER_CST)
3868 0 : n2pn3 = int_const_binop (MINUS_EXPR, p, n2pn3);
3869 : else
3870 : {
3871 0 : g = gimple_build_assign (make_ssa_name (sizetype),
3872 : MINUS_EXPR, p, n2pn3);
3873 0 : insert_before (g);
3874 0 : n2pn3 = gimple_assign_lhs (g);
3875 : }
3876 : }
3877 : /* For LSHIFT_EXPR, we can use handle_operand with non-INTEGER_CST
3878 : idx even to access the most significant partial limb. */
3879 197 : m_var_msb = true;
3880 197 : if (integer_zerop (n3))
3881 : /* For n3 == 0 p >= n2 + n3 is always true for all valid shift
3882 : counts. Emit if (true) condition that can be optimized later. */
3883 45 : g = gimple_build_cond (NE_EXPR, boolean_true_node, boolean_false_node,
3884 : NULL_TREE, NULL_TREE);
3885 152 : else if (bitint_big_endian)
3886 0 : g = gimple_build_cond (NE_EXPR, n2pn3, size_int (-1), NULL_TREE,
3887 : NULL_TREE);
3888 : else
3889 152 : g = gimple_build_cond (LE_EXPR, n2pn3, p, NULL_TREE, NULL_TREE);
3890 197 : edge edge_true, edge_false;
3891 197 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3892 197 : tree idx_next;
3893 197 : tree idx = create_loop (bitint_big_endian ? size_zero_node : p,
3894 : &idx_next);
3895 197 : tree idxmn2 = make_ssa_name (sizetype);
3896 197 : tree idxmn2mn3 = make_ssa_name (sizetype);
3897 394 : g = gimple_build_assign (idxmn2,
3898 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3899 : idx, n2);
3900 197 : insert_before (g);
3901 394 : g = gimple_build_assign (idxmn2mn3,
3902 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3903 : idxmn2, n3);
3904 197 : insert_before (g);
3905 197 : m_data_cnt = 0;
3906 197 : tree t1 = handle_operand (rhs1, idxmn2);
3907 197 : m_first = false;
3908 197 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3909 : LSHIFT_EXPR, t1, n1);
3910 197 : insert_before (g);
3911 197 : t1 = gimple_assign_lhs (g);
3912 197 : if (!integer_zerop (n3))
3913 : {
3914 152 : m_data_cnt = 0;
3915 152 : tree t2 = handle_operand (rhs1, idxmn2mn3);
3916 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3917 : RSHIFT_EXPR, t2, n4);
3918 152 : insert_before (g);
3919 152 : t2 = gimple_assign_lhs (g);
3920 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3921 : BIT_IOR_EXPR, t1, t2);
3922 152 : insert_before (g);
3923 152 : t1 = gimple_assign_lhs (g);
3924 : }
3925 197 : tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3926 197 : g = gimple_build_assign (l, t1);
3927 197 : insert_before (g);
3928 394 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3929 : bitint_big_endian
3930 197 : ? size_one_node : size_int (-1));
3931 197 : insert_before (g);
3932 197 : tree sn2pn3 = add_cast (ssizetype, n2pn3);
3933 394 : g = gimple_build_cond (bitint_big_endian ? LE_EXPR : GE_EXPR,
3934 : add_cast (ssizetype, idx_next), sn2pn3,
3935 : NULL_TREE, NULL_TREE);
3936 197 : insert_before (g);
3937 197 : idx = make_ssa_name (sizetype);
3938 197 : m_gsi = gsi_for_stmt (final_stmt);
3939 197 : gphi *phi = create_phi_node (idx, gsi_bb (m_gsi));
3940 197 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3941 197 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3942 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3943 197 : add_phi_arg (phi, bitint_big_endian ? size_zero_node : p,
3944 : edge_false, UNKNOWN_LOCATION);
3945 197 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3946 197 : m_data_cnt = 0;
3947 197 : if (!integer_zerop (n3))
3948 : {
3949 152 : g = gimple_build_cond (NE_EXPR, n3, size_zero_node,
3950 : NULL_TREE, NULL_TREE);
3951 152 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3952 152 : idxmn2 = make_ssa_name (sizetype);
3953 304 : g = gimple_build_assign (idxmn2,
3954 : bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
3955 : idx, n2);
3956 152 : insert_before (g);
3957 152 : m_data_cnt = 0;
3958 152 : t1 = handle_operand (rhs1, idxmn2);
3959 152 : g = gimple_build_assign (make_ssa_name (m_limb_type),
3960 : LSHIFT_EXPR, t1, n1);
3961 152 : insert_before (g);
3962 152 : t1 = gimple_assign_lhs (g);
3963 152 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3964 152 : g = gimple_build_assign (l, t1);
3965 152 : insert_before (g);
3966 152 : idx_next = make_ssa_name (sizetype);
3967 304 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3968 : bitint_big_endian
3969 152 : ? size_one_node : size_int (-1));
3970 152 : insert_before (g);
3971 152 : m_gsi = gsi_for_stmt (final_stmt);
3972 152 : tree nidx = make_ssa_name (sizetype);
3973 152 : phi = create_phi_node (nidx, gsi_bb (m_gsi));
3974 152 : edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
3975 152 : edge_true = EDGE_PRED (gsi_bb (m_gsi),
3976 : EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
3977 152 : add_phi_arg (phi, idx, edge_false, UNKNOWN_LOCATION);
3978 152 : add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
3979 152 : idx = nidx;
3980 : }
3981 197 : if (bitint_big_endian)
3982 0 : g = gimple_build_cond (LE_EXPR, idx, p, NULL_TREE, NULL_TREE);
3983 : else
3984 197 : g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx),
3985 : ssize_int (0), NULL_TREE, NULL_TREE);
3986 197 : if_then (g, profile_probability::likely (), edge_true, edge_false);
3987 197 : idx = create_loop (idx, &idx_next);
3988 197 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
3989 197 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
3990 197 : insert_before (g);
3991 394 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
3992 : bitint_big_endian
3993 197 : ? size_one_node : size_int (-1));
3994 197 : insert_before (g);
3995 197 : if (bitint_big_endian)
3996 0 : g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
3997 : else
3998 197 : g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx_next),
3999 : ssize_int (0), NULL_TREE, NULL_TREE);
4000 197 : insert_before (g);
4001 197 : if (bitint_extended && prec % limb_prec != 0)
4002 : {
4003 : /* The most significant limb has been updated either in the
4004 : loop or in the if after it. To simplify the code, just
4005 : read it back from memory and extend. */
4006 0 : m_gsi = gsi_after_labels (edge_false->dest);
4007 0 : idx = bitint_big_endian ? size_zero_node : p;
4008 0 : tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
4009 0 : tree type = limb_access_type (TREE_TYPE (lhs), idx);
4010 0 : tree v = make_ssa_name (m_limb_type);
4011 0 : g = gimple_build_assign (v, l);
4012 0 : insert_before (g);
4013 0 : v = add_cast (type, v);
4014 0 : l = limb_access (TREE_TYPE (lhs), obj, idx, true);
4015 0 : v = add_cast (m_limb_type, v);
4016 0 : g = gimple_build_assign (l, v);
4017 0 : insert_before (g);
4018 0 : if (bitint_extended == bitint_ext_full
4019 0 : && abi_limb_prec > limb_prec
4020 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4021 0 : > CEIL (prec, limb_prec) * limb_prec))
4022 : {
4023 0 : tree p2 = build_int_cst (sizetype,
4024 : CEIL (prec, abi_limb_prec)
4025 0 : * abi_limb_prec / limb_prec - 1);
4026 0 : if (TYPE_UNSIGNED (TREE_TYPE (lhs)))
4027 0 : v = build_zero_cst (m_limb_type);
4028 : else
4029 : {
4030 0 : v = add_cast (signed_type_for (m_limb_type), v);
4031 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (v)),
4032 : RSHIFT_EXPR, v,
4033 : build_int_cst (unsigned_type_node,
4034 0 : limb_prec - 1));
4035 0 : insert_before (g);
4036 0 : v = add_cast (m_limb_type, gimple_assign_lhs (g));
4037 : }
4038 0 : l = limb_access (TREE_TYPE (lhs), obj, p2, true);
4039 0 : g = gimple_build_assign (l, v);
4040 0 : insert_before (g);
4041 : }
4042 : }
4043 197 : else if (bitint_extended == bitint_ext_full
4044 0 : && abi_limb_prec > limb_prec
4045 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4046 0 : > CEIL (prec, limb_prec) * limb_prec))
4047 : {
4048 0 : m_gsi = gsi_after_labels (edge_false->dest);
4049 0 : tree p2 = build_int_cst (sizetype,
4050 : CEIL (prec, abi_limb_prec)
4051 0 : * abi_limb_prec / limb_prec - 1);
4052 0 : tree v;
4053 0 : if (TYPE_UNSIGNED (TREE_TYPE (lhs)))
4054 0 : v = build_zero_cst (m_limb_type);
4055 : else
4056 : {
4057 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p, true);
4058 0 : v = make_ssa_name (m_limb_type);
4059 0 : g = gimple_build_assign (v, l);
4060 0 : insert_before (g);
4061 0 : v = add_cast (signed_type_for (m_limb_type), v);
4062 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (v)),
4063 : RSHIFT_EXPR, v,
4064 : build_int_cst (unsigned_type_node,
4065 0 : limb_prec - 1));
4066 0 : insert_before (g);
4067 0 : v = add_cast (m_limb_type, gimple_assign_lhs (g));
4068 : }
4069 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
4070 0 : g = gimple_build_assign (l, v);
4071 0 : insert_before (g);
4072 : }
4073 : }
4074 584 : }
4075 :
4076 : /* Lower large/huge _BitInt multiplication or division. */
4077 :
4078 : void
4079 356 : bitint_large_huge::lower_muldiv_stmt (tree obj, gimple *stmt)
4080 : {
4081 356 : tree rhs1 = gimple_assign_rhs1 (stmt);
4082 356 : tree rhs2 = gimple_assign_rhs2 (stmt);
4083 356 : tree lhs = gimple_assign_lhs (stmt);
4084 356 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
4085 356 : tree type = TREE_TYPE (rhs1);
4086 356 : gcc_assert (BITINT_TYPE_P (type)
4087 : && bitint_precision_kind (type) >= bitint_prec_large);
4088 356 : int prec = TYPE_PRECISION (type), prec1, prec2;
4089 356 : bool ext_ms_limb = false;
4090 356 : bool do_ext = false;
4091 356 : rhs1 = handle_operand_addr (rhs1, stmt, NULL, &prec1);
4092 356 : rhs2 = handle_operand_addr (rhs2, stmt, NULL, &prec2);
4093 356 : if (obj == NULL_TREE)
4094 : {
4095 151 : int part = var_to_partition (m_map, lhs);
4096 151 : gcc_assert (m_vars[part] != NULL_TREE);
4097 151 : obj = m_vars[part];
4098 151 : lhs = build_fold_addr_expr (obj);
4099 : }
4100 : else
4101 : {
4102 205 : lhs = build_fold_addr_expr (obj);
4103 205 : lhs = force_gimple_operand_gsi (&m_gsi, lhs, true,
4104 : NULL_TREE, true, GSI_SAME_STMT);
4105 : }
4106 356 : if (bitint_extended && TYPE_OVERFLOW_WRAPS (type))
4107 : {
4108 0 : if (rhs_code == MULT_EXPR)
4109 : do_ext = true;
4110 : /* For signed division with -fwrapv, minimum negative / -1 needs
4111 : is minimum negative and the padding bits above it should be all
4112 : set. */
4113 0 : else if (!TYPE_UNSIGNED (type)
4114 0 : && (rhs_code == TRUNC_DIV_EXPR || rhs_code == EXACT_DIV_EXPR))
4115 356 : do_ext = true;
4116 : }
4117 356 : if (bitint_extended == bitint_ext_full
4118 0 : && abi_limb_prec > limb_prec
4119 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4120 0 : > CEIL (prec, limb_prec) * limb_prec))
4121 : {
4122 : /* unsigned multiplication needs to wrap around, so we can't
4123 : increase prec. Similarly for -fwrapv. */
4124 0 : if (do_ext)
4125 : ext_ms_limb = true;
4126 : else
4127 0 : prec = CEIL (prec, abi_limb_prec) * abi_limb_prec;
4128 : }
4129 356 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
4130 356 : gimple *g;
4131 356 : switch (rhs_code)
4132 : {
4133 201 : case MULT_EXPR:
4134 201 : g = gimple_build_call_internal (IFN_MULBITINT, 6,
4135 201 : lhs, build_int_cst (sitype, prec),
4136 201 : rhs1, build_int_cst (sitype, prec1),
4137 201 : rhs2, build_int_cst (sitype, prec2));
4138 201 : insert_before (g);
4139 201 : break;
4140 103 : case TRUNC_DIV_EXPR:
4141 103 : case EXACT_DIV_EXPR:
4142 103 : g = gimple_build_call_internal (IFN_DIVMODBITINT, 8,
4143 103 : lhs, build_int_cst (sitype, prec),
4144 : null_pointer_node,
4145 : build_int_cst (sitype, 0),
4146 103 : rhs1, build_int_cst (sitype, prec1),
4147 103 : rhs2, build_int_cst (sitype, prec2));
4148 103 : if (!stmt_ends_bb_p (stmt))
4149 102 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4150 103 : insert_before (g);
4151 103 : break;
4152 52 : case TRUNC_MOD_EXPR:
4153 52 : g = gimple_build_call_internal (IFN_DIVMODBITINT, 8, null_pointer_node,
4154 : build_int_cst (sitype, 0),
4155 52 : lhs, build_int_cst (sitype, prec),
4156 52 : rhs1, build_int_cst (sitype, prec1),
4157 52 : rhs2, build_int_cst (sitype, prec2));
4158 52 : if (!stmt_ends_bb_p (stmt))
4159 49 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4160 52 : insert_before (g);
4161 52 : break;
4162 0 : default:
4163 0 : gcc_unreachable ();
4164 : }
4165 356 : if (stmt_ends_bb_p (stmt))
4166 : {
4167 4 : maybe_duplicate_eh_stmt (g, stmt);
4168 4 : edge e1;
4169 4 : edge_iterator ei;
4170 4 : basic_block bb = gimple_bb (stmt);
4171 :
4172 4 : FOR_EACH_EDGE (e1, ei, bb->succs)
4173 4 : if (e1->flags & EDGE_EH)
4174 : break;
4175 4 : if (e1)
4176 : {
4177 4 : edge e2 = split_block (gsi_bb (m_gsi), g);
4178 4 : m_gsi = gsi_after_labels (e2->dest);
4179 4 : add_eh_edge (e2->src, e1);
4180 : }
4181 : }
4182 356 : if (do_ext
4183 356 : && ((prec % limb_prec) != 0 || (ext_ms_limb && !TYPE_UNSIGNED (type))))
4184 : {
4185 : /* Unsigned multiplication wraps, but libgcc function will return the
4186 : bits beyond prec within the top limb as another limb of the full
4187 : multiplication. So, clear the padding bits here. */
4188 0 : tree idx = size_int (bitint_big_endian ? 0 : prec / limb_prec);
4189 0 : tree l = limb_access (type, obj, idx, true);
4190 0 : tree ctype = limb_access_type (type, idx);
4191 0 : tree v = make_ssa_name (m_limb_type);
4192 0 : g = gimple_build_assign (v, l);
4193 0 : insert_before (g);
4194 0 : tree v2 = v;
4195 0 : if ((prec % limb_prec) != 0)
4196 : {
4197 0 : v = add_cast (ctype, v);
4198 0 : l = limb_access (type, obj, idx, true);
4199 0 : v = add_cast (m_limb_type, v);
4200 0 : v2 = v;
4201 0 : g = gimple_build_assign (l, v);
4202 0 : insert_before (g);
4203 : }
4204 0 : if (ext_ms_limb && !TYPE_UNSIGNED (type))
4205 : {
4206 0 : v2 = add_cast (signed_type_for (m_limb_type), v2);
4207 0 : tree lpm1 = build_int_cst (unsigned_type_node, limb_prec - 1);
4208 0 : v = make_ssa_name (TREE_TYPE (v2));
4209 0 : g = gimple_build_assign (v, RSHIFT_EXPR, v2, lpm1);
4210 0 : insert_before (g);
4211 0 : unsigned int i
4212 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4213 0 : v = add_cast (m_limb_type, v);
4214 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1),
4215 : true), v);
4216 0 : insert_before (g);
4217 0 : ext_ms_limb = false;
4218 : }
4219 : }
4220 356 : if (ext_ms_limb)
4221 : {
4222 0 : unsigned int i = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4223 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1), true),
4224 : build_zero_cst (m_limb_type));
4225 0 : insert_before (g);
4226 : }
4227 356 : }
4228 :
4229 : /* Lower large/huge _BitInt conversion to/from floating point. */
4230 :
4231 : void
4232 319 : bitint_large_huge::lower_float_conv_stmt (tree obj, gimple *stmt)
4233 : {
4234 319 : tree rhs1 = gimple_assign_rhs1 (stmt);
4235 319 : tree lhs = gimple_assign_lhs (stmt);
4236 319 : tree_code rhs_code = gimple_assign_rhs_code (stmt);
4237 319 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
4238 319 : gimple *g;
4239 319 : if (rhs_code == FIX_TRUNC_EXPR)
4240 : {
4241 179 : tree type = TREE_TYPE (lhs);
4242 179 : int prec = TYPE_PRECISION (type);
4243 179 : bool extend_ms_limb = false;
4244 179 : if (bitint_extended == bitint_ext_full
4245 0 : && abi_limb_prec > limb_prec
4246 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4247 0 : > CEIL (prec, limb_prec) * limb_prec))
4248 179 : extend_ms_limb = true;
4249 179 : if (!TYPE_UNSIGNED (type))
4250 93 : prec = -prec;
4251 179 : if (obj == NULL_TREE)
4252 : {
4253 135 : int part = var_to_partition (m_map, lhs);
4254 135 : gcc_assert (m_vars[part] != NULL_TREE);
4255 135 : obj = m_vars[part];
4256 135 : lhs = build_fold_addr_expr (obj);
4257 : }
4258 : else
4259 : {
4260 44 : lhs = build_fold_addr_expr (obj);
4261 44 : lhs = force_gimple_operand_gsi (&m_gsi, lhs, true,
4262 : NULL_TREE, true, GSI_SAME_STMT);
4263 : }
4264 179 : scalar_mode from_mode
4265 179 : = as_a <scalar_mode> (TYPE_MODE (TREE_TYPE (rhs1)));
4266 : #ifdef HAVE_SFmode
4267 : /* IEEE single is a full superset of both IEEE half and
4268 : bfloat formats, convert to float first and then to _BitInt
4269 : to avoid the need of another 2 library routines. */
4270 179 : if ((REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format
4271 179 : || REAL_MODE_FORMAT (from_mode) == &ieee_half_format)
4272 191 : && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
4273 : {
4274 12 : tree type = lang_hooks.types.type_for_mode (SFmode, 0);
4275 12 : if (type)
4276 12 : rhs1 = add_cast (type, rhs1);
4277 : }
4278 : #endif
4279 179 : g = gimple_build_call_internal (IFN_FLOATTOBITINT, 3,
4280 179 : lhs, build_int_cst (sitype, prec),
4281 : rhs1);
4282 179 : insert_before (g);
4283 179 : if (extend_ms_limb)
4284 : {
4285 0 : unsigned int i
4286 0 : = (CEIL (prec < 0 ? -prec : prec, abi_limb_prec)
4287 0 : * abi_limb_prec / limb_prec);
4288 0 : tree val;
4289 0 : if (prec < 0)
4290 : {
4291 0 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4292 : limb_access (type, obj,
4293 0 : size_int (i - 2),
4294 : true));
4295 0 : insert_before (g);
4296 0 : val = add_cast (signed_type_for (m_limb_type),
4297 : gimple_assign_lhs (g));
4298 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (val)),
4299 : RSHIFT_EXPR, val,
4300 : build_int_cst (unsigned_type_node,
4301 0 : limb_prec - 1));
4302 0 : insert_before (g);
4303 0 : val = add_cast (m_limb_type, gimple_assign_lhs (g));
4304 : }
4305 : else
4306 0 : val = build_zero_cst (m_limb_type);
4307 0 : g = gimple_build_assign (limb_access (type, obj, size_int (i - 1),
4308 : true), val);
4309 0 : insert_before (g);
4310 : }
4311 : }
4312 : else
4313 : {
4314 140 : int prec;
4315 140 : rhs1 = handle_operand_addr (rhs1, stmt, NULL, &prec);
4316 140 : g = gimple_build_call_internal (IFN_BITINTTOFLOAT, 2,
4317 140 : rhs1, build_int_cst (sitype, prec));
4318 140 : gimple_call_set_lhs (g, lhs);
4319 140 : if (!stmt_ends_bb_p (stmt))
4320 139 : gimple_call_set_nothrow (as_a <gcall *> (g), true);
4321 140 : gsi_replace (&m_gsi, g, true);
4322 : }
4323 319 : }
4324 :
4325 : /* Helper method for lower_addsub_overflow and lower_mul_overflow.
4326 : If check_zero is true, caller wants to check if all bits in [start, end)
4327 : are zero, otherwise if bits in [start, end) are either all zero or
4328 : all ones. L is the limb with index LIMB, START and END are measured
4329 : in bits. */
4330 :
4331 : tree
4332 6423 : bitint_large_huge::arith_overflow_extract_bits (unsigned int start,
4333 : unsigned int end, tree l,
4334 : unsigned int limb,
4335 : bool check_zero)
4336 : {
4337 6423 : unsigned startlimb = start / limb_prec;
4338 6423 : unsigned endlimb = (end - 1) / limb_prec;
4339 6423 : gimple *g;
4340 :
4341 6423 : if ((start % limb_prec) == 0 && (end % limb_prec) == 0)
4342 : return l;
4343 6155 : if (startlimb == endlimb && limb == startlimb)
4344 : {
4345 2020 : if (check_zero)
4346 : {
4347 1479 : wide_int w = wi::shifted_mask (start % limb_prec,
4348 1479 : end - start, false, limb_prec);
4349 2958 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4350 : BIT_AND_EXPR, l,
4351 1479 : wide_int_to_tree (m_limb_type, w));
4352 1479 : insert_before (g);
4353 1479 : return gimple_assign_lhs (g);
4354 1479 : }
4355 541 : unsigned int shift = start % limb_prec;
4356 541 : if ((end % limb_prec) != 0)
4357 : {
4358 336 : unsigned int lshift = (-end) % limb_prec;
4359 336 : shift += lshift;
4360 336 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4361 : LSHIFT_EXPR, l,
4362 : build_int_cst (unsigned_type_node,
4363 336 : lshift));
4364 336 : insert_before (g);
4365 336 : l = gimple_assign_lhs (g);
4366 : }
4367 541 : l = add_cast (signed_type_for (m_limb_type), l);
4368 541 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4369 : RSHIFT_EXPR, l,
4370 541 : build_int_cst (unsigned_type_node, shift));
4371 541 : insert_before (g);
4372 541 : return add_cast (m_limb_type, gimple_assign_lhs (g));
4373 : }
4374 4135 : else if (limb == startlimb)
4375 : {
4376 1981 : if ((start % limb_prec) == 0)
4377 : return l;
4378 1867 : if (!check_zero)
4379 941 : l = add_cast (signed_type_for (m_limb_type), l);
4380 1867 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4381 : RSHIFT_EXPR, l,
4382 : build_int_cst (unsigned_type_node,
4383 1867 : start % limb_prec));
4384 1867 : insert_before (g);
4385 1867 : l = gimple_assign_lhs (g);
4386 1867 : if (!check_zero)
4387 941 : l = add_cast (m_limb_type, l);
4388 1867 : return l;
4389 : }
4390 2154 : else if (limb == endlimb)
4391 : {
4392 1703 : if ((end % limb_prec) == 0)
4393 : return l;
4394 1686 : if (check_zero)
4395 : {
4396 880 : wide_int w = wi::mask (end % limb_prec, false, limb_prec);
4397 1760 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4398 : BIT_AND_EXPR, l,
4399 880 : wide_int_to_tree (m_limb_type, w));
4400 880 : insert_before (g);
4401 880 : return gimple_assign_lhs (g);
4402 880 : }
4403 806 : unsigned int shift = (-end) % limb_prec;
4404 806 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4405 : LSHIFT_EXPR, l,
4406 806 : build_int_cst (unsigned_type_node, shift));
4407 806 : insert_before (g);
4408 806 : l = add_cast (signed_type_for (m_limb_type), gimple_assign_lhs (g));
4409 806 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
4410 : RSHIFT_EXPR, l,
4411 806 : build_int_cst (unsigned_type_node, shift));
4412 806 : insert_before (g);
4413 806 : return add_cast (m_limb_type, gimple_assign_lhs (g));
4414 : }
4415 : return l;
4416 : }
4417 :
4418 : /* Helper method for lower_addsub_overflow and lower_mul_overflow. Store
4419 : result including overflow flag into the right locations. */
4420 :
4421 : void
4422 4181 : bitint_large_huge::finish_arith_overflow (tree var, tree obj, tree type,
4423 : tree ovf, tree lhs, tree orig_obj,
4424 : gimple *stmt, unsigned nelts,
4425 : tree_code code)
4426 : {
4427 4181 : gimple *g;
4428 :
4429 4181 : if (obj == NULL_TREE
4430 4181 : && (!BITINT_TYPE_P (type)
4431 231 : || bitint_precision_kind (type) < bitint_prec_large))
4432 : {
4433 : /* Add support for 3 or more limbs filled in from normal integral
4434 : type if this assert fails. If no target chooses limb mode smaller
4435 : than half of largest supported normal integral type, this will not
4436 : be needed. */
4437 247 : gcc_assert (TYPE_PRECISION (type) <= 2 * limb_prec);
4438 247 : tree lhs_type = type;
4439 107 : if (BITINT_TYPE_P (type)
4440 247 : && bitint_precision_kind (type) == bitint_prec_middle)
4441 46 : lhs_type = build_nonstandard_integer_type (TYPE_PRECISION (type),
4442 46 : TYPE_UNSIGNED (type));
4443 247 : tree r1 = limb_access (NULL_TREE, var,
4444 : bitint_big_endian
4445 0 : ? size_int (nelts - 1) : size_zero_node, true);
4446 247 : g = gimple_build_assign (make_ssa_name (m_limb_type), r1);
4447 247 : insert_before (g);
4448 247 : r1 = gimple_assign_lhs (g);
4449 247 : if (!useless_type_conversion_p (lhs_type, TREE_TYPE (r1)))
4450 247 : r1 = add_cast (lhs_type, r1);
4451 247 : if (TYPE_PRECISION (lhs_type) > limb_prec)
4452 : {
4453 90 : tree r2 = limb_access (NULL_TREE, var,
4454 : bitint_big_endian
4455 0 : ? size_int (nelts - 2) : size_one_node, true);
4456 90 : g = gimple_build_assign (make_ssa_name (m_limb_type), r2);
4457 90 : insert_before (g);
4458 90 : r2 = gimple_assign_lhs (g);
4459 90 : r2 = add_cast (lhs_type, r2);
4460 90 : g = gimple_build_assign (make_ssa_name (lhs_type), LSHIFT_EXPR, r2,
4461 : build_int_cst (unsigned_type_node,
4462 90 : limb_prec));
4463 90 : insert_before (g);
4464 90 : g = gimple_build_assign (make_ssa_name (lhs_type), BIT_IOR_EXPR, r1,
4465 : gimple_assign_lhs (g));
4466 90 : insert_before (g);
4467 90 : r1 = gimple_assign_lhs (g);
4468 : }
4469 247 : if (lhs_type != type)
4470 46 : r1 = add_cast (type, r1);
4471 247 : ovf = add_cast (lhs_type, ovf);
4472 247 : if (lhs_type != type)
4473 46 : ovf = add_cast (type, ovf);
4474 247 : g = gimple_build_assign (lhs, COMPLEX_EXPR, r1, ovf);
4475 247 : m_gsi = gsi_for_stmt (stmt);
4476 247 : gsi_replace (&m_gsi, g, true);
4477 : }
4478 : else
4479 : {
4480 3934 : unsigned HOST_WIDE_INT obj_nelts = 0;
4481 3934 : tree atype = NULL_TREE;
4482 3934 : if (obj)
4483 : {
4484 3843 : obj_nelts = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))) / limb_prec;
4485 3843 : if (orig_obj == NULL_TREE)
4486 2269 : obj_nelts >>= 1;
4487 3843 : atype = build_array_type_nelts (m_limb_type, obj_nelts);
4488 : }
4489 3934 : if (bitint_extended && (var || obj))
4490 : {
4491 0 : unsigned prec = TYPE_PRECISION (type);
4492 0 : unsigned prec_limbs = CEIL (prec, limb_prec);
4493 0 : bool ext_ms_limb
4494 : = (bitint_extended == bitint_ext_full
4495 0 : && abi_limb_prec > limb_prec
4496 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
4497 0 : > CEIL (prec, limb_prec) * limb_prec));
4498 : /* For .{ADD,SUB}_OVERFLOW the partial limb if any is
4499 : already extended in lower_addsub_overflow. */
4500 0 : if ((code == MULT_EXPR && (prec % limb_prec) != 0)
4501 0 : || (ext_ms_limb && !TYPE_UNSIGNED (type)))
4502 : {
4503 0 : tree plm1idx = size_int (bitint_big_endian
4504 : ? nelts - obj_nelts : prec_limbs - 1);
4505 0 : tree plm1type
4506 0 : = limb_access_type (type, bitint_big_endian
4507 : ? size_zero_node : plm1idx);
4508 0 : tree l = limb_access (bitint_big_endian ? NULL_TREE : type,
4509 : var ? var : obj, plm1idx, true);
4510 0 : tree rhs = make_ssa_name (TREE_TYPE (l));
4511 0 : g = gimple_build_assign (rhs, l);
4512 0 : insert_before (g);
4513 0 : if (code == MULT_EXPR && (prec % limb_prec) != 0)
4514 : {
4515 0 : if (!useless_type_conversion_p (plm1type, TREE_TYPE (rhs)))
4516 0 : rhs = add_cast (plm1type, rhs);
4517 0 : if (!useless_type_conversion_p (TREE_TYPE (l),
4518 0 : TREE_TYPE (rhs)))
4519 0 : rhs = add_cast (TREE_TYPE (l), rhs);
4520 0 : l = limb_access (bitint_big_endian ? NULL_TREE : type,
4521 : var ? var : obj, plm1idx, true);
4522 0 : g = gimple_build_assign (l, rhs);
4523 0 : insert_before (g);
4524 : }
4525 0 : if (ext_ms_limb && !TYPE_UNSIGNED (type))
4526 : {
4527 0 : rhs = add_cast (signed_type_for (m_limb_type), rhs);
4528 0 : tree lpm1 = build_int_cst (unsigned_type_node,
4529 0 : limb_prec - 1);
4530 0 : tree v = make_ssa_name (TREE_TYPE (rhs));
4531 0 : g = gimple_build_assign (v, RSHIFT_EXPR, rhs, lpm1);
4532 0 : insert_before (g);
4533 0 : unsigned int i
4534 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4535 0 : v = add_cast (m_limb_type, v);
4536 0 : g = gimple_build_assign (limb_access (type, var ? var : obj,
4537 0 : size_int (i - 1),
4538 : true), v);
4539 0 : insert_before (g);
4540 0 : ext_ms_limb = false;
4541 : }
4542 : }
4543 0 : if (ext_ms_limb)
4544 : {
4545 0 : unsigned int i
4546 0 : = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
4547 0 : g = gimple_build_assign (limb_access (type, var ? var : obj,
4548 0 : size_int (i - 1), true),
4549 : build_zero_cst (m_limb_type));
4550 0 : insert_before (g);
4551 : }
4552 : }
4553 3934 : if (var && obj)
4554 : {
4555 512 : tree v1, v2;
4556 512 : tree off;
4557 512 : if (orig_obj == NULL_TREE)
4558 : {
4559 32 : off = build_zero_cst (build_pointer_type (TREE_TYPE (obj)));
4560 32 : v1 = build2 (MEM_REF, atype,
4561 : build_fold_addr_expr (unshare_expr (obj)), off);
4562 : }
4563 480 : else if (!useless_type_conversion_p (atype, TREE_TYPE (obj)))
4564 8 : v1 = build1 (VIEW_CONVERT_EXPR, atype, unshare_expr (obj));
4565 : else
4566 472 : v1 = unshare_expr (obj);
4567 512 : off = build_int_cst (build_pointer_type (TREE_TYPE (var)),
4568 : bitint_big_endian
4569 512 : ? (nelts - obj_nelts) * m_limb_size : 0);
4570 512 : v2 = build2 (MEM_REF, atype, build_fold_addr_expr (var), off);
4571 512 : g = gimple_build_assign (v1, v2);
4572 512 : insert_before (g);
4573 : }
4574 3422 : else if (obj && bitint_big_endian && nelts != obj_nelts)
4575 : {
4576 0 : gcc_assert (nelts > obj_nelts);
4577 0 : tree fn = builtin_decl_implicit (BUILT_IN_MEMMOVE);
4578 0 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (obj)),
4579 0 : (nelts - obj_nelts) * m_limb_size);
4580 0 : tree src = build2 (MEM_REF, atype,
4581 : build_fold_addr_expr (unshare_expr (obj)), off);
4582 0 : g = gimple_build_call (fn, 3,
4583 : build_fold_addr_expr (unshare_expr (obj)),
4584 : src, build_int_cst (size_type_node,
4585 0 : obj_nelts * m_limb_size));
4586 0 : insert_before (g);
4587 : }
4588 3934 : if (orig_obj == NULL_TREE && obj)
4589 : {
4590 2269 : ovf = add_cast (m_limb_type, ovf);
4591 2269 : tree l = limb_access (NULL_TREE, obj,
4592 2269 : size_int (bitint_big_endian
4593 : ? obj_nelts * 2 - 1 : obj_nelts),
4594 : true);
4595 2269 : g = gimple_build_assign (l, ovf);
4596 2269 : insert_before (g);
4597 2269 : if (obj_nelts > 1)
4598 : {
4599 2269 : atype = build_array_type_nelts (m_limb_type, obj_nelts - 1);
4600 2269 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (obj)),
4601 2269 : (obj_nelts + !bitint_big_endian)
4602 2269 : * m_limb_size);
4603 2269 : tree v1 = build2 (MEM_REF, atype,
4604 : build_fold_addr_expr (unshare_expr (obj)),
4605 : off);
4606 2269 : g = gimple_build_assign (v1, build_zero_cst (atype));
4607 2269 : insert_before (g);
4608 : }
4609 : }
4610 1665 : else if (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE)
4611 : {
4612 1638 : imm_use_iterator ui;
4613 1638 : use_operand_p use_p;
4614 1638 : FOR_EACH_IMM_USE_FAST (use_p, ui, lhs)
4615 : {
4616 1638 : g = USE_STMT (use_p);
4617 1638 : if (!is_gimple_assign (g)
4618 1638 : || gimple_assign_rhs_code (g) != IMAGPART_EXPR)
4619 0 : continue;
4620 1638 : tree lhs2 = gimple_assign_lhs (g);
4621 1638 : gimple *use_stmt;
4622 1638 : single_imm_use (lhs2, &use_p, &use_stmt);
4623 1638 : lhs2 = gimple_assign_lhs (use_stmt);
4624 1638 : gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
4625 1638 : if (useless_type_conversion_p (TREE_TYPE (lhs2), TREE_TYPE (ovf)))
4626 1617 : g = gimple_build_assign (lhs2, ovf);
4627 : else
4628 21 : g = gimple_build_assign (lhs2, NOP_EXPR, ovf);
4629 1638 : gsi_replace (&gsi, g, true);
4630 1638 : if (gsi_stmt (m_gsi) == use_stmt)
4631 91 : m_gsi = gsi_for_stmt (g);
4632 1638 : break;
4633 1638 : }
4634 : }
4635 27 : else if (ovf != boolean_false_node)
4636 : {
4637 27 : g = gimple_build_cond (NE_EXPR, ovf, boolean_false_node,
4638 : NULL_TREE, NULL_TREE);
4639 27 : edge edge_true, edge_false;
4640 27 : if_then (g, profile_probability::very_unlikely (),
4641 : edge_true, edge_false);
4642 27 : tree zero = build_zero_cst (TREE_TYPE (lhs));
4643 27 : tree fn = ubsan_build_overflow_builtin (code, m_loc,
4644 27 : TREE_TYPE (lhs),
4645 : zero, zero, NULL);
4646 27 : force_gimple_operand_gsi (&m_gsi, fn, true, NULL_TREE,
4647 : true, GSI_SAME_STMT);
4648 27 : m_gsi = gsi_after_labels (edge_true->dest);
4649 : }
4650 : }
4651 4181 : if (var)
4652 : {
4653 771 : tree clobber = build_clobber (TREE_TYPE (var), CLOBBER_STORAGE_END);
4654 771 : g = gimple_build_assign (var, clobber);
4655 771 : gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
4656 : }
4657 4181 : }
4658 :
4659 : /* Helper function for lower_addsub_overflow and lower_mul_overflow.
4660 : Given precisions of result TYPE (PREC), argument 0 precision PREC0,
4661 : argument 1 precision PREC1 and minimum precision for the result
4662 : PREC2, compute *START, *END, *CHECK_ZERO and return OVF. */
4663 :
4664 : static tree
4665 4181 : arith_overflow (tree_code code, tree type, int prec, int prec0, int prec1,
4666 : int prec2, unsigned *start, unsigned *end, bool *check_zero)
4667 : {
4668 4181 : *start = 0;
4669 4181 : *end = 0;
4670 4181 : *check_zero = true;
4671 : /* Ignore this special rule for subtraction, even if both
4672 : prec0 >= 0 and prec1 >= 0, their subtraction can be negative
4673 : in infinite precision. */
4674 4181 : if (code != MINUS_EXPR && prec0 >= 0 && prec1 >= 0)
4675 : {
4676 : /* Result in [0, prec2) is unsigned, if prec > prec2,
4677 : all bits above it will be zero. */
4678 681 : if ((prec - !TYPE_UNSIGNED (type)) >= prec2)
4679 0 : return boolean_false_node;
4680 : else
4681 : {
4682 : /* ovf if any of bits in [start, end) is non-zero. */
4683 681 : *start = prec - !TYPE_UNSIGNED (type);
4684 681 : *end = prec2;
4685 : }
4686 : }
4687 3500 : else if (TYPE_UNSIGNED (type))
4688 : {
4689 : /* If result in [0, prec2) is signed and if prec > prec2,
4690 : all bits above it will be sign bit copies. */
4691 1942 : if (prec >= prec2)
4692 : {
4693 : /* ovf if bit prec - 1 is non-zero. */
4694 184 : *start = prec - 1;
4695 184 : *end = prec;
4696 : }
4697 : else
4698 : {
4699 : /* ovf if any of bits in [start, end) is non-zero. */
4700 1758 : *start = prec;
4701 1758 : *end = prec2;
4702 : }
4703 : }
4704 1558 : else if (prec >= prec2)
4705 0 : return boolean_false_node;
4706 : else
4707 : {
4708 : /* ovf if [start, end) bits aren't all zeros or all ones. */
4709 1558 : *start = prec - 1;
4710 1558 : *end = prec2;
4711 1558 : *check_zero = false;
4712 : }
4713 : return NULL_TREE;
4714 : }
4715 :
4716 : /* Lower a .{ADD,SUB}_OVERFLOW call with at least one large/huge _BitInt
4717 : argument or return type _Complex large/huge _BitInt. */
4718 :
4719 : void
4720 2721 : bitint_large_huge::lower_addsub_overflow (tree obj, gimple *stmt)
4721 : {
4722 2721 : tree arg0 = gimple_call_arg (stmt, 0);
4723 2721 : tree arg1 = gimple_call_arg (stmt, 1);
4724 2721 : tree lhs = gimple_call_lhs (stmt);
4725 2721 : gimple *g;
4726 :
4727 2721 : if (!lhs)
4728 : {
4729 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
4730 0 : gsi_remove (&gsi, true);
4731 0 : return;
4732 : }
4733 2721 : gimple *final_stmt = gsi_stmt (m_gsi);
4734 2721 : tree type = TREE_TYPE (lhs);
4735 2721 : if (TREE_CODE (type) == COMPLEX_TYPE)
4736 2703 : type = TREE_TYPE (type);
4737 2721 : int prec = TYPE_PRECISION (type);
4738 2721 : int prec0 = range_to_prec (arg0, stmt);
4739 2721 : int prec1 = range_to_prec (arg1, stmt);
4740 : /* If PREC0 >= 0 && PREC1 >= 0 and CODE is not MINUS_EXPR, PREC2 is
4741 : the minimum unsigned precision of any possible operation's
4742 : result, otherwise it is minimum signed precision.
4743 : Some examples:
4744 : If PREC0 or PREC1 is 8, it means that argument is [0, 0xff],
4745 : if PREC0 or PREC1 is 10, it means that argument is [0, 0x3ff],
4746 : if PREC0 or PREC1 is -8, it means that argument is [-0x80, 0x7f],
4747 : if PREC0 or PREC1 is -10, it means that argument is [-0x200, 0x1ff].
4748 : PREC0 CODE PREC1 RESULT PREC2 SIGNED vs. UNSIGNED
4749 : 8 + 8 [0, 0x1fe] 9 UNSIGNED
4750 : 8 + 10 [0, 0x4fe] 11 UNSIGNED
4751 : -8 + -8 [-0x100, 0xfe] 9 SIGNED
4752 : -8 + -10 [-0x280, 0x27e] 11 SIGNED
4753 : 8 + -8 [-0x80, 0x17e] 10 SIGNED
4754 : 8 + -10 [-0x200, 0x2fe] 11 SIGNED
4755 : 10 + -8 [-0x80, 0x47e] 12 SIGNED
4756 : 8 - 8 [-0xff, 0xff] 9 SIGNED
4757 : 8 - 10 [-0x3ff, 0xff] 11 SIGNED
4758 : 10 - 8 [-0xff, 0x3ff] 11 SIGNED
4759 : -8 - -8 [-0xff, 0xff] 9 SIGNED
4760 : -8 - -10 [-0x27f, 0x27f] 11 SIGNED
4761 : -10 - -8 [-0x27f, 0x27f] 11 SIGNED
4762 : 8 - -8 [-0x7f, 0x17f] 10 SIGNED
4763 : 8 - -10 [-0x1ff, 0x2ff] 11 SIGNED
4764 : 10 - -8 [-0x7f, 0x47f] 12 SIGNED
4765 : -8 - 8 [-0x17f, 0x7f] 10 SIGNED
4766 : -8 - 10 [-0x47f, 0x7f] 12 SIGNED
4767 : -10 - 8 [-0x2ff, 0x1ff] 11 SIGNED */
4768 2721 : int prec2 = MAX (prec0 < 0 ? -prec0 : prec0,
4769 : prec1 < 0 ? -prec1 : prec1);
4770 : /* If operands are either both signed or both unsigned,
4771 : we need just one additional bit. */
4772 3760 : prec2 = (((prec0 < 0) == (prec1 < 0)
4773 : /* If one operand is signed and one unsigned and
4774 : the signed one has larger precision, we need
4775 : just one extra bit, otherwise two. */
4776 702 : || (prec0 < 0 ? (prec2 == -prec0 && prec2 != prec1)
4777 337 : : (prec2 == -prec1 && prec2 != prec0)))
4778 2721 : ? prec2 + 1 : prec2 + 2);
4779 2721 : int prec3 = MAX (prec0 < 0 ? -prec0 : prec0,
4780 : prec1 < 0 ? -prec1 : prec1);
4781 2721 : prec3 = MAX (prec3, prec);
4782 2721 : tree var = NULL_TREE;
4783 2721 : tree orig_obj = obj;
4784 2721 : if (obj == NULL_TREE
4785 1741 : && BITINT_TYPE_P (type)
4786 1640 : && bitint_precision_kind (type) >= bitint_prec_large
4787 1524 : && m_names
4788 4221 : && bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
4789 : {
4790 1445 : int part = var_to_partition (m_map, lhs);
4791 1445 : gcc_assert (m_vars[part] != NULL_TREE);
4792 1445 : obj = m_vars[part];
4793 1445 : if (TREE_TYPE (lhs) == type)
4794 2 : orig_obj = obj;
4795 : }
4796 101 : if (!BITINT_TYPE_P (type)
4797 2721 : || bitint_precision_kind (type) < bitint_prec_large)
4798 : {
4799 217 : unsigned HOST_WIDE_INT nelts = CEIL (prec, limb_prec);
4800 217 : tree atype = build_array_type_nelts (m_limb_type, nelts);
4801 217 : var = create_tmp_var (atype);
4802 : }
4803 :
4804 2721 : enum tree_code code;
4805 2721 : switch (gimple_call_internal_fn (stmt))
4806 : {
4807 : case IFN_ADD_OVERFLOW:
4808 : case IFN_UBSAN_CHECK_ADD:
4809 : code = PLUS_EXPR;
4810 : break;
4811 1393 : case IFN_SUB_OVERFLOW:
4812 1393 : case IFN_UBSAN_CHECK_SUB:
4813 1393 : code = MINUS_EXPR;
4814 1393 : break;
4815 0 : default:
4816 0 : gcc_unreachable ();
4817 : }
4818 2721 : unsigned start, end;
4819 2721 : bool check_zero;
4820 2721 : tree ovf = arith_overflow (code, type, prec, prec0, prec1, prec2,
4821 : &start, &end, &check_zero);
4822 :
4823 2721 : unsigned startlimb, endlimb;
4824 2721 : if (ovf)
4825 : {
4826 : startlimb = ~0U;
4827 : endlimb = ~0U;
4828 : }
4829 : else
4830 : {
4831 2721 : startlimb = start / limb_prec;
4832 2721 : endlimb = (end - 1) / limb_prec;
4833 : }
4834 :
4835 2721 : int prec4 = ovf != NULL_TREE ? prec : prec3;
4836 2721 : bitint_prec_kind kind = bitint_precision_kind (prec4);
4837 2721 : unsigned cnt, rem = 0, fin = 0, nelts;
4838 2721 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
4839 5442 : bool last_ovf = (ovf == NULL_TREE
4840 2721 : && CEIL (prec2, limb_prec) > CEIL (prec3, limb_prec));
4841 2721 : if (kind != bitint_prec_huge)
4842 1539 : nelts = cnt = CEIL (prec4, limb_prec) + last_ovf;
4843 : else
4844 : {
4845 1182 : rem = prec4 % (2 * limb_prec);
4846 1182 : fin = (prec4 - rem) / limb_prec;
4847 1182 : cnt = 2 + CEIL (rem, limb_prec) + last_ovf;
4848 1182 : nelts = fin + cnt - 2;
4849 1182 : idx = idx_first = create_loop (bitint_big_endian
4850 1182 : ? size_int (nelts - 1) : size_zero_node,
4851 : &idx_next);
4852 : }
4853 :
4854 2721 : if (kind == bitint_prec_huge)
4855 1182 : m_upwards_2limb = fin;
4856 2721 : m_upwards = true;
4857 :
4858 2721 : tree type0 = TREE_TYPE (arg0);
4859 2721 : tree type1 = TREE_TYPE (arg1);
4860 2721 : int prec5 = prec3;
4861 2721 : if (bitint_precision_kind (prec5) < bitint_prec_large)
4862 10 : prec5 = MAX (TYPE_PRECISION (type0), TYPE_PRECISION (type1));
4863 2721 : if (TYPE_PRECISION (type0) < prec5)
4864 : {
4865 146 : type0 = build_bitint_type (prec5, TYPE_UNSIGNED (type0));
4866 146 : if (TREE_CODE (arg0) == INTEGER_CST)
4867 27 : arg0 = fold_convert (type0, arg0);
4868 : }
4869 2721 : if (TYPE_PRECISION (type1) < prec5)
4870 : {
4871 156 : type1 = build_bitint_type (prec5, TYPE_UNSIGNED (type1));
4872 156 : if (TREE_CODE (arg1) == INTEGER_CST)
4873 76 : arg1 = fold_convert (type1, arg1);
4874 : }
4875 2721 : unsigned int data_cnt = 0;
4876 2721 : tree last_rhs1 = NULL_TREE, last_rhs2 = NULL_TREE;
4877 2721 : tree cmp = build_zero_cst (m_limb_type);
4878 2721 : unsigned prec_limbs = CEIL ((unsigned) prec, limb_prec);
4879 2721 : tree ovf_out = NULL_TREE, cmp_out = NULL_TREE;
4880 11856 : for (unsigned i = 0; i < cnt; i++)
4881 : {
4882 9135 : m_data_cnt = 0;
4883 9135 : tree rhs1, rhs2;
4884 9135 : if (kind != bitint_prec_huge)
4885 5303 : idx = size_int (bitint_big_endian ? nelts - 1 - i : i);
4886 3832 : else if (i >= 2)
4887 1468 : idx = size_int (bitint_big_endian ? nelts + 1 - fin - i : fin + i - 2);
4888 9135 : if (!last_ovf || i < cnt - 1)
4889 : {
4890 8189 : tree idx0 = idx, idx1 = idx;
4891 8189 : if (bitint_big_endian
4892 8189 : && CEIL ((unsigned) TYPE_PRECISION (type0), limb_prec) != nelts)
4893 : {
4894 0 : HOST_WIDE_INT diff
4895 0 : = ((HOST_WIDE_INT) CEIL (TYPE_PRECISION (type0), limb_prec)
4896 0 : - (HOST_WIDE_INT) nelts);
4897 0 : if (tree_fits_uhwi_p (idx))
4898 0 : idx0 = size_int (tree_to_uhwi (idx) + diff);
4899 : else
4900 : {
4901 0 : idx0 = make_ssa_name (sizetype);
4902 0 : g = gimple_build_assign (idx0, PLUS_EXPR, idx,
4903 0 : size_int (diff));
4904 0 : insert_before (g);
4905 : }
4906 : }
4907 8189 : if (type0 != TREE_TYPE (arg0))
4908 334 : rhs1 = handle_cast (type0, arg0, idx0);
4909 : else
4910 7855 : rhs1 = handle_operand (arg0, idx0);
4911 8189 : if (bitint_big_endian
4912 8189 : && CEIL ((unsigned) TYPE_PRECISION (type1), limb_prec) != nelts)
4913 : {
4914 0 : HOST_WIDE_INT diff
4915 0 : = ((HOST_WIDE_INT) CEIL (TYPE_PRECISION (type1), limb_prec)
4916 0 : - (HOST_WIDE_INT) nelts);
4917 0 : if (tree_fits_uhwi_p (idx))
4918 0 : idx1 = size_int (tree_to_uhwi (idx) + diff);
4919 : else
4920 : {
4921 0 : idx1 = make_ssa_name (sizetype);
4922 0 : g = gimple_build_assign (idx1, PLUS_EXPR, idx,
4923 0 : size_int (diff));
4924 0 : insert_before (g);
4925 : }
4926 : }
4927 8189 : if (type1 != TREE_TYPE (arg1))
4928 250 : rhs2 = handle_cast (type1, arg1, idx1);
4929 : else
4930 7939 : rhs2 = handle_operand (arg1, idx1);
4931 8189 : if (i == 0)
4932 2721 : data_cnt = m_data_cnt;
4933 8189 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
4934 1971 : rhs1 = add_cast (m_limb_type, rhs1);
4935 8189 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs2)))
4936 1971 : rhs2 = add_cast (m_limb_type, rhs2);
4937 : last_rhs1 = rhs1;
4938 : last_rhs2 = rhs2;
4939 : }
4940 : else
4941 : {
4942 946 : m_data_cnt = data_cnt;
4943 946 : if (TYPE_UNSIGNED (type0) || prec0 >= 0)
4944 421 : rhs1 = build_zero_cst (m_limb_type);
4945 : else
4946 : {
4947 525 : rhs1 = add_cast (signed_type_for (m_limb_type), last_rhs1);
4948 525 : if (TREE_CODE (rhs1) == INTEGER_CST)
4949 52 : rhs1 = build_int_cst (m_limb_type,
4950 74 : tree_int_cst_sgn (rhs1) < 0 ? -1 : 0);
4951 : else
4952 : {
4953 946 : tree lpm1 = build_int_cst (unsigned_type_node,
4954 473 : limb_prec - 1);
4955 473 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
4956 : RSHIFT_EXPR, rhs1, lpm1);
4957 473 : insert_before (g);
4958 473 : rhs1 = add_cast (m_limb_type, gimple_assign_lhs (g));
4959 : }
4960 : }
4961 946 : if (TYPE_UNSIGNED (type1) || prec1 >= 0)
4962 543 : rhs2 = build_zero_cst (m_limb_type);
4963 : else
4964 : {
4965 403 : rhs2 = add_cast (signed_type_for (m_limb_type), last_rhs2);
4966 403 : if (TREE_CODE (rhs2) == INTEGER_CST)
4967 114 : rhs2 = build_int_cst (m_limb_type,
4968 153 : tree_int_cst_sgn (rhs2) < 0 ? -1 : 0);
4969 : else
4970 : {
4971 578 : tree lpm1 = build_int_cst (unsigned_type_node,
4972 289 : limb_prec - 1);
4973 289 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs2)),
4974 : RSHIFT_EXPR, rhs2, lpm1);
4975 289 : insert_before (g);
4976 289 : rhs2 = add_cast (m_limb_type, gimple_assign_lhs (g));
4977 : }
4978 : }
4979 : }
4980 9135 : tree rhs = handle_plus_minus (code, rhs1, rhs2, idx);
4981 9135 : if (ovf != boolean_false_node)
4982 : {
4983 9135 : if (tree_fits_uhwi_p (idx))
4984 : {
4985 6771 : unsigned limb = tree_to_uhwi (idx);
4986 6771 : if (bitint_big_endian)
4987 0 : limb = nelts - 1 - limb;
4988 6771 : if (limb >= startlimb && limb <= endlimb)
4989 : {
4990 3358 : tree l = arith_overflow_extract_bits (start, end, rhs,
4991 : limb, check_zero);
4992 3358 : tree this_ovf = make_ssa_name (boolean_type_node);
4993 3358 : if (ovf == NULL_TREE && !check_zero)
4994 : {
4995 895 : cmp = l;
4996 895 : g = gimple_build_assign (make_ssa_name (m_limb_type),
4997 : PLUS_EXPR, l,
4998 : build_int_cst (m_limb_type, 1));
4999 895 : insert_before (g);
5000 895 : g = gimple_build_assign (this_ovf, GT_EXPR,
5001 : gimple_assign_lhs (g),
5002 : build_int_cst (m_limb_type, 1));
5003 : }
5004 : else
5005 2463 : g = gimple_build_assign (this_ovf, NE_EXPR, l, cmp);
5006 3358 : insert_before (g);
5007 3358 : if (ovf == NULL_TREE)
5008 : ovf = this_ovf;
5009 : else
5010 : {
5011 1064 : tree b = make_ssa_name (boolean_type_node);
5012 1064 : g = gimple_build_assign (b, BIT_IOR_EXPR, ovf, this_ovf);
5013 1064 : insert_before (g);
5014 1064 : ovf = b;
5015 : }
5016 : }
5017 : }
5018 2364 : else if (startlimb < fin)
5019 : {
5020 854 : if (m_first && startlimb + 2 < fin)
5021 : {
5022 324 : tree data_out;
5023 324 : ovf = prepare_data_in_out (boolean_false_node, idx, &data_out);
5024 324 : ovf_out = m_data.pop ();
5025 324 : m_data.pop ();
5026 324 : if (!check_zero)
5027 : {
5028 169 : cmp = prepare_data_in_out (cmp, idx, &data_out);
5029 169 : cmp_out = m_data.pop ();
5030 169 : m_data.pop ();
5031 : }
5032 : }
5033 854 : if (i != 0 || startlimb != fin - 1)
5034 : {
5035 839 : tree_code cmp_code;
5036 839 : bool single_comparison
5037 839 : = (startlimb + 2 >= fin || (startlimb & 1) != (i & 1));
5038 : if (!single_comparison)
5039 : cmp_code = GE_EXPR;
5040 515 : else if ((startlimb & 1) == (i & 1))
5041 : cmp_code = EQ_EXPR;
5042 : else
5043 412 : cmp_code = GT_EXPR;
5044 839 : if (bitint_big_endian)
5045 0 : g = gimple_build_cond (swap_tree_comparison (cmp_code),
5046 0 : idx, size_int (nelts - 1
5047 : - startlimb),
5048 : NULL_TREE, NULL_TREE);
5049 : else
5050 839 : g = gimple_build_cond (cmp_code, idx, size_int (startlimb),
5051 : NULL_TREE, NULL_TREE);
5052 839 : edge edge_true_true, edge_true_false, edge_false;
5053 839 : gimple *g2 = NULL;
5054 839 : if (!single_comparison)
5055 324 : g2 = gimple_build_cond (NE_EXPR, idx,
5056 324 : size_int (bitint_big_endian
5057 : ? nelts - 1 - startlimb
5058 : : startlimb),
5059 : NULL_TREE, NULL_TREE);
5060 839 : if_then_if_then_else (g, g2, profile_probability::likely (),
5061 : profile_probability::likely (),
5062 : edge_true_true, edge_true_false,
5063 : edge_false);
5064 839 : unsigned tidx = startlimb + (cmp_code == GT_EXPR);
5065 839 : tree l = arith_overflow_extract_bits (start, end, rhs, tidx,
5066 : check_zero);
5067 839 : tree this_ovf = make_ssa_name (boolean_type_node);
5068 839 : if (cmp_code != GT_EXPR && !check_zero)
5069 : {
5070 173 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5071 : PLUS_EXPR, l,
5072 : build_int_cst (m_limb_type, 1));
5073 173 : insert_before (g);
5074 173 : g = gimple_build_assign (this_ovf, GT_EXPR,
5075 : gimple_assign_lhs (g),
5076 : build_int_cst (m_limb_type, 1));
5077 : }
5078 : else
5079 666 : g = gimple_build_assign (this_ovf, NE_EXPR, l, cmp);
5080 839 : insert_before (g);
5081 839 : if (cmp_code == GT_EXPR)
5082 : {
5083 412 : tree t = make_ssa_name (boolean_type_node);
5084 412 : g = gimple_build_assign (t, BIT_IOR_EXPR, ovf, this_ovf);
5085 412 : insert_before (g);
5086 412 : this_ovf = t;
5087 : }
5088 839 : tree this_ovf2 = NULL_TREE;
5089 839 : if (!single_comparison)
5090 : {
5091 324 : m_gsi = gsi_after_labels (edge_true_true->src);
5092 324 : tree t = make_ssa_name (boolean_type_node);
5093 324 : g = gimple_build_assign (t, NE_EXPR, rhs, cmp);
5094 324 : insert_before (g);
5095 324 : this_ovf2 = make_ssa_name (boolean_type_node);
5096 324 : g = gimple_build_assign (this_ovf2, BIT_IOR_EXPR,
5097 : ovf, t);
5098 324 : insert_before (g);
5099 : }
5100 839 : m_gsi = gsi_after_labels (edge_true_false->dest);
5101 839 : tree t;
5102 839 : if (i == 1 && ovf_out)
5103 : t = ovf_out;
5104 : else
5105 515 : t = make_ssa_name (boolean_type_node);
5106 839 : gphi *phi = create_phi_node (t, edge_true_false->dest);
5107 839 : add_phi_arg (phi, this_ovf, edge_true_false,
5108 : UNKNOWN_LOCATION);
5109 839 : add_phi_arg (phi, ovf ? ovf
5110 : : boolean_false_node, edge_false,
5111 : UNKNOWN_LOCATION);
5112 839 : if (edge_true_true)
5113 324 : add_phi_arg (phi, this_ovf2, edge_true_true,
5114 : UNKNOWN_LOCATION);
5115 839 : ovf = t;
5116 839 : if (!check_zero && cmp_code != GT_EXPR)
5117 : {
5118 173 : t = cmp_out ? cmp_out : make_ssa_name (m_limb_type);
5119 173 : phi = create_phi_node (t, edge_true_false->dest);
5120 173 : add_phi_arg (phi, l, edge_true_false, UNKNOWN_LOCATION);
5121 173 : add_phi_arg (phi, cmp, edge_false, UNKNOWN_LOCATION);
5122 173 : if (edge_true_true)
5123 169 : add_phi_arg (phi, cmp, edge_true_true,
5124 : UNKNOWN_LOCATION);
5125 : cmp = t;
5126 : }
5127 : }
5128 : }
5129 : }
5130 :
5131 9135 : if (var || obj)
5132 : {
5133 8907 : if (tree_fits_uhwi_p (idx)
5134 6661 : && (bitint_big_endian
5135 6661 : ? nelts - 1 - tree_to_uhwi (idx)
5136 6661 : : tree_to_uhwi (idx)) >= prec_limbs)
5137 : ;
5138 7519 : else if (!tree_fits_uhwi_p (idx)
5139 2246 : && (unsigned) prec < (fin - (i == 0)) * limb_prec)
5140 : {
5141 1458 : bool single_comparison
5142 729 : = (((unsigned) prec % limb_prec) == 0
5143 571 : || prec_limbs + 1 >= fin
5144 1231 : || (prec_limbs & 1) == (i & 1));
5145 729 : if (bitint_big_endian)
5146 0 : g = gimple_build_cond (GE_EXPR, idx,
5147 0 : size_int (nelts - prec_limbs),
5148 : NULL_TREE, NULL_TREE);
5149 : else
5150 729 : g = gimple_build_cond (LE_EXPR, idx, size_int (prec_limbs - 1),
5151 : NULL_TREE, NULL_TREE);
5152 729 : gimple *g2 = NULL;
5153 729 : if (!single_comparison)
5154 251 : g2 = gimple_build_cond (EQ_EXPR, idx,
5155 251 : size_int (bitint_big_endian
5156 : ? nelts - prec_limbs
5157 : : prec_limbs - 1),
5158 : NULL_TREE, NULL_TREE);
5159 729 : edge edge_true_true, edge_true_false, edge_false;
5160 729 : if_then_if_then_else (g, g2, profile_probability::likely (),
5161 : profile_probability::unlikely (),
5162 : edge_true_true, edge_true_false,
5163 : edge_false);
5164 729 : tree idxl = idx;
5165 729 : if (bitint_big_endian && prec_limbs != nelts)
5166 : {
5167 0 : HOST_WIDE_INT diff = ((HOST_WIDE_INT) prec_limbs
5168 0 : - (HOST_WIDE_INT) nelts);
5169 0 : if (tree_fits_uhwi_p (idx))
5170 0 : idxl = size_int (tree_to_uhwi (idx) + diff);
5171 : else
5172 : {
5173 0 : idxl = make_ssa_name (sizetype);
5174 0 : g = gimple_build_assign (idxl, PLUS_EXPR, idx,
5175 0 : size_int (diff));
5176 0 : insert_before (g);
5177 : }
5178 : }
5179 1082 : tree l = limb_access (type, var ? var : obj, idxl, true);
5180 729 : g = gimple_build_assign (l, rhs);
5181 729 : insert_before (g);
5182 729 : if (!single_comparison)
5183 : {
5184 251 : m_gsi = gsi_after_labels (edge_true_true->src);
5185 251 : tree plm1idx = size_int (bitint_big_endian
5186 : ? 0 : prec_limbs - 1);
5187 251 : tree plm1type = limb_access_type (type, plm1idx);
5188 251 : l = limb_access (type, var ? var : obj, plm1idx, true);
5189 251 : if (!useless_type_conversion_p (plm1type, TREE_TYPE (rhs)))
5190 251 : rhs = add_cast (plm1type, rhs);
5191 251 : if (!useless_type_conversion_p (TREE_TYPE (l),
5192 251 : TREE_TYPE (rhs)))
5193 251 : rhs = add_cast (TREE_TYPE (l), rhs);
5194 251 : g = gimple_build_assign (l, rhs);
5195 251 : insert_before (g);
5196 : }
5197 729 : m_gsi = gsi_after_labels (edge_true_false->dest);
5198 729 : }
5199 : else
5200 : {
5201 6790 : tree idxl = idx;
5202 6790 : if (bitint_big_endian && prec_limbs != nelts)
5203 : {
5204 0 : HOST_WIDE_INT diff = ((HOST_WIDE_INT) prec_limbs
5205 0 : - (HOST_WIDE_INT) nelts);
5206 0 : if (tree_fits_uhwi_p (idx))
5207 0 : idxl = size_int (tree_to_uhwi (idx) + diff);
5208 : else
5209 : {
5210 0 : idxl = make_ssa_name (sizetype);
5211 0 : g = gimple_build_assign (idxl, PLUS_EXPR, idx,
5212 0 : size_int (diff));
5213 0 : insert_before (g);
5214 : }
5215 : }
5216 13551 : tree l = limb_access (type, var ? var : obj, idxl, true);
5217 6790 : if (bitint_extended && tree_fits_uhwi_p (idxl))
5218 : {
5219 0 : tree atype = limb_access_type (type, idxl);
5220 0 : if (!useless_type_conversion_p (atype, TREE_TYPE (rhs)))
5221 0 : rhs = add_cast (atype, rhs);
5222 : }
5223 6790 : if (!useless_type_conversion_p (TREE_TYPE (l), TREE_TYPE (rhs)))
5224 0 : rhs = add_cast (TREE_TYPE (l), rhs);
5225 6790 : g = gimple_build_assign (l, rhs);
5226 6790 : insert_before (g);
5227 : }
5228 : }
5229 9135 : m_first = false;
5230 9135 : if (kind == bitint_prec_huge && i <= 1)
5231 : {
5232 2364 : if (i == 0)
5233 : {
5234 1182 : idx = make_ssa_name (sizetype);
5235 1182 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
5236 : bitint_big_endian
5237 0 : ? size_int (-1) : size_one_node);
5238 1182 : insert_before (g);
5239 : }
5240 : else
5241 : {
5242 1182 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
5243 2364 : size_int (bitint_big_endian ? -2 : 2));
5244 1182 : insert_before (g);
5245 1182 : if (bitint_big_endian)
5246 0 : g = gimple_build_cond (NE_EXPR, idx_first,
5247 0 : size_int (nelts + 1 - fin),
5248 : NULL_TREE, NULL_TREE);
5249 : else
5250 1182 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (fin),
5251 : NULL_TREE, NULL_TREE);
5252 1182 : insert_before (g);
5253 1182 : m_gsi = gsi_for_stmt (final_stmt);
5254 1182 : m_bb = NULL;
5255 : }
5256 : }
5257 : }
5258 :
5259 2721 : finish_arith_overflow (var, obj, type, ovf, lhs, orig_obj, stmt,
5260 : prec_limbs, code);
5261 : }
5262 :
5263 : /* Lower a .MUL_OVERFLOW call with at least one large/huge _BitInt
5264 : argument or return type _Complex large/huge _BitInt. */
5265 :
5266 : void
5267 1460 : bitint_large_huge::lower_mul_overflow (tree obj, gimple *stmt)
5268 : {
5269 1460 : tree arg0 = gimple_call_arg (stmt, 0);
5270 1460 : tree arg1 = gimple_call_arg (stmt, 1);
5271 1460 : tree lhs = gimple_call_lhs (stmt);
5272 1460 : if (!lhs)
5273 : {
5274 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5275 0 : gsi_remove (&gsi, true);
5276 0 : return;
5277 : }
5278 1460 : gimple *final_stmt = gsi_stmt (m_gsi);
5279 1460 : tree type = TREE_TYPE (lhs);
5280 1460 : if (TREE_CODE (type) == COMPLEX_TYPE)
5281 1451 : type = TREE_TYPE (type);
5282 1460 : int prec = TYPE_PRECISION (type), prec0, prec1;
5283 1460 : arg0 = handle_operand_addr (arg0, stmt, NULL, &prec0);
5284 1460 : arg1 = handle_operand_addr (arg1, stmt, NULL, &prec1);
5285 1460 : int prec2 = ((prec0 < 0 ? -prec0 : prec0)
5286 1460 : + (prec1 < 0 ? -prec1 : prec1));
5287 1460 : if (prec0 == 1 || prec1 == 1)
5288 25 : --prec2;
5289 1460 : tree var = NULL_TREE;
5290 1460 : tree orig_obj = obj;
5291 1460 : bool force_var = false;
5292 1460 : if (obj == NULL_TREE
5293 869 : && BITINT_TYPE_P (type)
5294 863 : && bitint_precision_kind (type) >= bitint_prec_large
5295 839 : && m_names
5296 2287 : && bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
5297 : {
5298 827 : int part = var_to_partition (m_map, lhs);
5299 827 : gcc_assert (m_vars[part] != NULL_TREE);
5300 827 : obj = m_vars[part];
5301 827 : if (TREE_TYPE (lhs) == type)
5302 1 : orig_obj = obj;
5303 : }
5304 633 : else if (obj != NULL_TREE && DECL_P (obj))
5305 : {
5306 1749 : for (int i = 0; i < 2; ++i)
5307 : {
5308 1166 : tree arg = i ? arg1 : arg0;
5309 1166 : if (TREE_CODE (arg) == ADDR_EXPR)
5310 1166 : arg = TREE_OPERAND (arg, 0);
5311 1166 : if (get_base_address (arg) == obj)
5312 : {
5313 : force_var = true;
5314 : break;
5315 : }
5316 : }
5317 : }
5318 1460 : if (obj == NULL_TREE
5319 1460 : || force_var
5320 1418 : || !BITINT_TYPE_P (type)
5321 1418 : || bitint_precision_kind (type) < bitint_prec_large
5322 3704 : || prec2 > (CEIL (prec, limb_prec) * limb_prec * (orig_obj ? 1 : 2)))
5323 : {
5324 554 : unsigned HOST_WIDE_INT nelts = CEIL (MAX (prec, prec2), limb_prec);
5325 554 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5326 554 : var = create_tmp_var (atype);
5327 : }
5328 1460 : tree addr = build_fold_addr_expr (var ? var : obj);
5329 1460 : addr = force_gimple_operand_gsi (&m_gsi, addr, true,
5330 : NULL_TREE, true, GSI_SAME_STMT);
5331 1460 : tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
5332 1460 : gimple *g
5333 2920 : = gimple_build_call_internal (IFN_MULBITINT, 6,
5334 : addr, build_int_cst (sitype,
5335 1522 : MAX (prec2, prec)),
5336 1460 : arg0, build_int_cst (sitype, prec0),
5337 1460 : arg1, build_int_cst (sitype, prec1));
5338 1460 : insert_before (g);
5339 :
5340 1460 : unsigned start, end;
5341 1460 : bool check_zero;
5342 1460 : tree ovf = arith_overflow (MULT_EXPR, type, prec, prec0, prec1, prec2,
5343 : &start, &end, &check_zero);
5344 1460 : if (ovf == NULL_TREE)
5345 : {
5346 1460 : unsigned startlimb = start / limb_prec;
5347 1460 : unsigned endlimb = (end - 1) / limb_prec;
5348 1460 : unsigned nelts = CEIL (MAX (prec, prec2), limb_prec);
5349 1460 : unsigned cnt;
5350 1460 : bool use_loop = false;
5351 1460 : if (startlimb == endlimb)
5352 : cnt = 1;
5353 1180 : else if (startlimb + 1 == endlimb)
5354 : cnt = 2;
5355 1009 : else if ((end % limb_prec) == 0)
5356 : {
5357 : cnt = 2;
5358 : use_loop = true;
5359 : }
5360 : else
5361 : {
5362 766 : cnt = 3;
5363 766 : use_loop = startlimb + 2 < endlimb;
5364 : }
5365 766 : if (cnt == 1)
5366 : {
5367 494 : tree l = limb_access (NULL_TREE, var ? var : obj,
5368 280 : size_int (bitint_big_endian
5369 : ? nelts - 1 - startlimb
5370 : : startlimb), true);
5371 280 : g = gimple_build_assign (make_ssa_name (m_limb_type), l);
5372 280 : insert_before (g);
5373 280 : l = arith_overflow_extract_bits (start, end, gimple_assign_lhs (g),
5374 : startlimb, check_zero);
5375 280 : ovf = make_ssa_name (boolean_type_node);
5376 280 : if (check_zero)
5377 240 : g = gimple_build_assign (ovf, NE_EXPR, l,
5378 : build_zero_cst (m_limb_type));
5379 : else
5380 : {
5381 40 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5382 : PLUS_EXPR, l,
5383 : build_int_cst (m_limb_type, 1));
5384 40 : insert_before (g);
5385 40 : g = gimple_build_assign (ovf, GT_EXPR, gimple_assign_lhs (g),
5386 : build_int_cst (m_limb_type, 1));
5387 : }
5388 280 : insert_before (g);
5389 : }
5390 : else
5391 : {
5392 1180 : basic_block edge_bb = NULL;
5393 1180 : gimple_stmt_iterator gsi = m_gsi;
5394 1180 : gsi_prev (&gsi);
5395 1180 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5396 1180 : edge_bb = e->src;
5397 1180 : m_gsi = gsi_end_bb (edge_bb);
5398 :
5399 1180 : tree cmp = build_zero_cst (m_limb_type);
5400 4306 : for (unsigned i = 0; i < cnt; i++)
5401 : {
5402 3126 : tree idx, idx_next = NULL_TREE;
5403 3126 : if (i == 0)
5404 1180 : idx = size_int (bitint_big_endian
5405 : ? nelts - 1 - startlimb : startlimb);
5406 1946 : else if (i == 2)
5407 766 : idx = size_int (bitint_big_endian
5408 : ? nelts - 1 - endlimb : endlimb);
5409 1180 : else if (use_loop)
5410 661 : idx = create_loop (size_int (bitint_big_endian
5411 : ? nelts - startlimb - 2
5412 : : startlimb + 1), &idx_next);
5413 : else
5414 519 : idx = size_int (bitint_big_endian
5415 : ? nelts - startlimb - 2 : startlimb + 1);
5416 4965 : tree l = limb_access (NULL_TREE, var ? var : obj, idx, true);
5417 3126 : g = gimple_build_assign (make_ssa_name (m_limb_type), l);
5418 3126 : insert_before (g);
5419 3126 : l = gimple_assign_lhs (g);
5420 3126 : if (i == 0 || i == 2)
5421 2712 : l = arith_overflow_extract_bits (start, end, l,
5422 : i == 0 ? startlimb : endlimb,
5423 : check_zero);
5424 1946 : if (i == 0 && !check_zero)
5425 : {
5426 450 : cmp = l;
5427 450 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5428 : PLUS_EXPR, l,
5429 : build_int_cst (m_limb_type, 1));
5430 450 : insert_before (g);
5431 450 : g = gimple_build_cond (GT_EXPR, gimple_assign_lhs (g),
5432 : build_int_cst (m_limb_type, 1),
5433 : NULL_TREE, NULL_TREE);
5434 : }
5435 : else
5436 2676 : g = gimple_build_cond (NE_EXPR, l, cmp, NULL_TREE, NULL_TREE);
5437 3126 : insert_before (g);
5438 3126 : edge e1 = split_block (gsi_bb (m_gsi), g);
5439 3126 : e1->flags = EDGE_FALSE_VALUE;
5440 3126 : edge e2 = make_edge (e1->src, gimple_bb (final_stmt),
5441 : EDGE_TRUE_VALUE);
5442 3126 : e1->probability = profile_probability::likely ();
5443 3126 : e2->probability = e1->probability.invert ();
5444 3126 : if (i == 0)
5445 1180 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5446 3126 : m_gsi = gsi_after_labels (e1->dest);
5447 3126 : if (i == 1 && use_loop)
5448 : {
5449 661 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
5450 : bitint_big_endian
5451 0 : ? size_int (-1) : size_one_node);
5452 661 : insert_before (g);
5453 661 : if (bitint_big_endian)
5454 0 : g = gimple_build_cond (NE_EXPR, idx,
5455 0 : size_int (nelts - endlimb
5456 : - (cnt == 2)),
5457 : NULL_TREE, NULL_TREE);
5458 : else
5459 661 : g = gimple_build_cond (NE_EXPR, idx_next,
5460 661 : size_int (endlimb + (cnt == 2)),
5461 : NULL_TREE, NULL_TREE);
5462 661 : insert_before (g);
5463 661 : edge true_edge, false_edge;
5464 661 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
5465 : &true_edge,
5466 : &false_edge);
5467 661 : m_gsi = gsi_after_labels (false_edge->dest);
5468 661 : m_bb = NULL;
5469 : }
5470 : }
5471 :
5472 1180 : ovf = make_ssa_name (boolean_type_node);
5473 1180 : basic_block bb = gimple_bb (final_stmt);
5474 1180 : gphi *phi = create_phi_node (ovf, bb);
5475 1180 : edge e1 = find_edge (gsi_bb (m_gsi), bb);
5476 1180 : edge_iterator ei;
5477 5486 : FOR_EACH_EDGE (e, ei, bb->preds)
5478 : {
5479 4306 : tree val = e == e1 ? boolean_false_node : boolean_true_node;
5480 4306 : add_phi_arg (phi, val, e, UNKNOWN_LOCATION);
5481 : }
5482 1180 : m_gsi = gsi_for_stmt (final_stmt);
5483 : }
5484 : }
5485 :
5486 1460 : finish_arith_overflow (var, obj, type, ovf, lhs, orig_obj, stmt,
5487 1460 : CEIL (MAX (prec, prec2), limb_prec), MULT_EXPR);
5488 : }
5489 :
5490 : /* Lower REALPART_EXPR or IMAGPART_EXPR stmt extracting part of result from
5491 : .{ADD,SUB,MUL}_OVERFLOW call. */
5492 :
5493 : void
5494 5984 : bitint_large_huge::lower_cplxpart_stmt (tree obj, gimple *stmt)
5495 : {
5496 5984 : tree rhs1 = gimple_assign_rhs1 (stmt);
5497 5984 : rhs1 = TREE_OPERAND (rhs1, 0);
5498 5984 : if (obj == NULL_TREE)
5499 : {
5500 5911 : int part = var_to_partition (m_map, gimple_assign_lhs (stmt));
5501 5911 : gcc_assert (m_vars[part] != NULL_TREE);
5502 : obj = m_vars[part];
5503 : }
5504 5984 : if (TREE_CODE (rhs1) == SSA_NAME
5505 5984 : && (m_names == NULL
5506 5983 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
5507 : {
5508 1547 : lower_call (obj, SSA_NAME_DEF_STMT (rhs1));
5509 1547 : return;
5510 : }
5511 4437 : int part = var_to_partition (m_map, rhs1);
5512 4437 : gcc_assert (m_vars[part] != NULL_TREE);
5513 4437 : tree var = m_vars[part];
5514 4437 : unsigned HOST_WIDE_INT nelts
5515 4437 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))) / limb_prec;
5516 4437 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5517 4437 : if (!useless_type_conversion_p (atype, TREE_TYPE (obj)))
5518 96 : obj = build1 (VIEW_CONVERT_EXPR, atype, obj);
5519 4437 : tree off = build_int_cst (build_pointer_type (TREE_TYPE (var)),
5520 4437 : gimple_assign_rhs_code (stmt) == REALPART_EXPR
5521 4437 : ? 0 : nelts * m_limb_size);
5522 4437 : tree v2 = build2 (MEM_REF, atype, build_fold_addr_expr (var), off);
5523 4437 : gimple *g = gimple_build_assign (obj, v2);
5524 4437 : insert_before (g);
5525 : }
5526 :
5527 : /* Lower COMPLEX_EXPR stmt. */
5528 :
5529 : void
5530 18 : bitint_large_huge::lower_complexexpr_stmt (gimple *stmt)
5531 : {
5532 18 : tree lhs = gimple_assign_lhs (stmt);
5533 18 : tree rhs1 = gimple_assign_rhs1 (stmt);
5534 18 : tree rhs2 = gimple_assign_rhs2 (stmt);
5535 18 : int part = var_to_partition (m_map, lhs);
5536 18 : gcc_assert (m_vars[part] != NULL_TREE);
5537 18 : lhs = m_vars[part];
5538 18 : unsigned HOST_WIDE_INT nelts
5539 18 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (rhs1))) / limb_prec;
5540 18 : tree atype = build_array_type_nelts (m_limb_type, nelts);
5541 18 : tree zero = build_zero_cst (build_pointer_type (TREE_TYPE (lhs)));
5542 18 : tree v1 = build2 (MEM_REF, atype, build_fold_addr_expr (lhs), zero);
5543 18 : tree v2;
5544 18 : if (TREE_CODE (rhs1) == SSA_NAME)
5545 : {
5546 18 : part = var_to_partition (m_map, rhs1);
5547 18 : gcc_assert (m_vars[part] != NULL_TREE);
5548 : v2 = m_vars[part];
5549 : }
5550 0 : else if (integer_zerop (rhs1))
5551 0 : v2 = build_zero_cst (atype);
5552 : else
5553 0 : v2 = tree_output_constant_def (rhs1);
5554 18 : if (!useless_type_conversion_p (atype, TREE_TYPE (v2)))
5555 18 : v2 = build1 (VIEW_CONVERT_EXPR, atype, v2);
5556 18 : gimple *g = gimple_build_assign (v1, v2);
5557 18 : insert_before (g);
5558 18 : tree off = fold_convert (build_pointer_type (TREE_TYPE (lhs)),
5559 : TYPE_SIZE_UNIT (atype));
5560 18 : v1 = build2 (MEM_REF, atype, build_fold_addr_expr (lhs), off);
5561 18 : if (TREE_CODE (rhs2) == SSA_NAME)
5562 : {
5563 0 : part = var_to_partition (m_map, rhs2);
5564 0 : gcc_assert (m_vars[part] != NULL_TREE);
5565 : v2 = m_vars[part];
5566 : }
5567 18 : else if (integer_zerop (rhs2))
5568 18 : v2 = build_zero_cst (atype);
5569 : else
5570 0 : v2 = tree_output_constant_def (rhs2);
5571 18 : if (!useless_type_conversion_p (atype, TREE_TYPE (v2)))
5572 0 : v2 = build1 (VIEW_CONVERT_EXPR, atype, v2);
5573 18 : g = gimple_build_assign (v1, v2);
5574 18 : insert_before (g);
5575 18 : }
5576 :
5577 : /* Lower a .{CLZ,CTZ,CLRSB,FFS,PARITY,POPCOUNT} call with one large/huge _BitInt
5578 : argument. */
5579 :
5580 : void
5581 97 : bitint_large_huge::lower_bit_query (gimple *stmt)
5582 : {
5583 97 : tree arg0 = gimple_call_arg (stmt, 0);
5584 97 : tree arg1 = (gimple_call_num_args (stmt) == 2
5585 97 : ? gimple_call_arg (stmt, 1) : NULL_TREE);
5586 97 : tree lhs = gimple_call_lhs (stmt);
5587 97 : gimple *g;
5588 :
5589 97 : if (!lhs)
5590 : {
5591 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5592 0 : gsi_remove (&gsi, true);
5593 0 : return;
5594 : }
5595 97 : tree type = TREE_TYPE (arg0);
5596 97 : gcc_assert (BITINT_TYPE_P (type));
5597 97 : bitint_prec_kind kind = bitint_precision_kind (type);
5598 97 : gcc_assert (kind >= bitint_prec_large);
5599 97 : enum internal_fn ifn = gimple_call_internal_fn (stmt);
5600 97 : enum built_in_function fcode = END_BUILTINS;
5601 97 : gcc_assert (TYPE_PRECISION (unsigned_type_node) == limb_prec
5602 : || TYPE_PRECISION (long_unsigned_type_node) == limb_prec
5603 : || TYPE_PRECISION (long_long_unsigned_type_node) == limb_prec);
5604 97 : switch (ifn)
5605 : {
5606 26 : case IFN_CLZ:
5607 26 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5608 : fcode = BUILT_IN_CLZ;
5609 26 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5610 : fcode = BUILT_IN_CLZL;
5611 : else
5612 0 : fcode = BUILT_IN_CLZLL;
5613 : break;
5614 11 : case IFN_FFS:
5615 : /* .FFS (X) is .CTZ (X, -1) + 1, though under the hood
5616 : we don't add the addend at the end. */
5617 11 : arg1 = integer_zero_node;
5618 : /* FALLTHRU */
5619 39 : case IFN_CTZ:
5620 39 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5621 : fcode = BUILT_IN_CTZ;
5622 39 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5623 : fcode = BUILT_IN_CTZL;
5624 : else
5625 0 : fcode = BUILT_IN_CTZLL;
5626 39 : m_upwards = true;
5627 39 : break;
5628 9 : case IFN_CLRSB:
5629 9 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5630 : fcode = BUILT_IN_CLRSB;
5631 9 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5632 : fcode = BUILT_IN_CLRSBL;
5633 : else
5634 0 : fcode = BUILT_IN_CLRSBLL;
5635 : break;
5636 12 : case IFN_PARITY:
5637 12 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5638 : fcode = BUILT_IN_PARITY;
5639 12 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5640 : fcode = BUILT_IN_PARITYL;
5641 : else
5642 0 : fcode = BUILT_IN_PARITYLL;
5643 12 : m_upwards = true;
5644 12 : break;
5645 11 : case IFN_POPCOUNT:
5646 11 : if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
5647 : fcode = BUILT_IN_POPCOUNT;
5648 11 : else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
5649 : fcode = BUILT_IN_POPCOUNTL;
5650 : else
5651 0 : fcode = BUILT_IN_POPCOUNTLL;
5652 11 : m_upwards = true;
5653 11 : break;
5654 0 : default:
5655 0 : gcc_unreachable ();
5656 : }
5657 97 : tree fndecl = builtin_decl_explicit (fcode), res = NULL_TREE;
5658 97 : unsigned cnt = 0, rem = 0, end = 0, prec = TYPE_PRECISION (type);
5659 97 : unsigned nelts = CEIL (prec, limb_prec);
5660 97 : struct bq_details { edge e; tree val, addend; } *bqp = NULL;
5661 97 : basic_block edge_bb = NULL;
5662 97 : if (m_upwards)
5663 : {
5664 62 : tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
5665 62 : if (kind == bitint_prec_large)
5666 : cnt = nelts;
5667 : else
5668 : {
5669 36 : rem = (prec % (2 * limb_prec));
5670 36 : end = (prec - rem) / limb_prec;
5671 36 : cnt = 2 + CEIL (rem, limb_prec);
5672 36 : idx = idx_first = create_loop (bitint_big_endian
5673 36 : ? size_int (nelts - 1)
5674 : : size_zero_node, &idx_next);
5675 : }
5676 :
5677 62 : if (ifn == IFN_CTZ || ifn == IFN_FFS)
5678 : {
5679 39 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5680 39 : gsi_prev (&gsi);
5681 39 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5682 39 : edge_bb = e->src;
5683 39 : if (kind == bitint_prec_large)
5684 32 : m_gsi = gsi_end_bb (edge_bb);
5685 39 : bqp = XALLOCAVEC (struct bq_details, cnt);
5686 : }
5687 : else
5688 23 : m_after_stmt = stmt;
5689 62 : if (kind != bitint_prec_large)
5690 36 : m_upwards_2limb = end;
5691 :
5692 230 : for (unsigned i = 0; i < cnt; i++)
5693 : {
5694 168 : m_data_cnt = 0;
5695 168 : if (kind == bitint_prec_large)
5696 80 : idx = size_int (bitint_big_endian ? nelts - 1 - i : i);
5697 88 : else if (i >= 2)
5698 16 : idx = size_int (bitint_big_endian
5699 : ? nelts - 1 - end - (i > 2) : end + (i > 2));
5700 :
5701 168 : tree rhs1 = handle_operand (arg0, idx);
5702 168 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
5703 : {
5704 30 : if (!TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5705 5 : rhs1 = add_cast (unsigned_type_for (TREE_TYPE (rhs1)), rhs1);
5706 30 : rhs1 = add_cast (m_limb_type, rhs1);
5707 : }
5708 :
5709 168 : tree in, out, tem;
5710 168 : if (ifn == IFN_PARITY)
5711 33 : in = prepare_data_in_out (build_zero_cst (m_limb_type), idx, &out);
5712 135 : else if (ifn == IFN_FFS)
5713 29 : in = prepare_data_in_out (integer_one_node, idx, &out);
5714 : else
5715 106 : in = prepare_data_in_out (integer_zero_node, idx, &out);
5716 :
5717 168 : switch (ifn)
5718 : {
5719 104 : case IFN_CTZ:
5720 104 : case IFN_FFS:
5721 104 : g = gimple_build_cond (NE_EXPR, rhs1,
5722 : build_zero_cst (m_limb_type),
5723 : NULL_TREE, NULL_TREE);
5724 104 : insert_before (g);
5725 104 : edge e1, e2;
5726 104 : e1 = split_block (gsi_bb (m_gsi), g);
5727 104 : e1->flags = EDGE_FALSE_VALUE;
5728 104 : e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
5729 104 : e1->probability = profile_probability::unlikely ();
5730 104 : e2->probability = e1->probability.invert ();
5731 104 : if (i == 0)
5732 39 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5733 104 : m_gsi = gsi_after_labels (e1->dest);
5734 104 : bqp[i].e = e2;
5735 104 : bqp[i].val = rhs1;
5736 104 : if (tree_fits_uhwi_p (idx))
5737 58 : bqp[i].addend
5738 58 : = build_int_cst (integer_type_node,
5739 : (bitint_big_endian
5740 58 : ? nelts - 1 - tree_to_uhwi (idx)
5741 58 : : tree_to_uhwi (idx)) * limb_prec
5742 58 : + (ifn == IFN_FFS));
5743 : else
5744 : {
5745 46 : bqp[i].addend = in;
5746 46 : if (i == 1)
5747 23 : res = out;
5748 : else
5749 23 : res = make_ssa_name (integer_type_node);
5750 46 : g = gimple_build_assign (res, PLUS_EXPR, in,
5751 : build_int_cst (integer_type_node,
5752 46 : limb_prec));
5753 46 : insert_before (g);
5754 46 : m_data[m_data_cnt] = res;
5755 : }
5756 : break;
5757 33 : case IFN_PARITY:
5758 33 : if (!integer_zerop (in))
5759 : {
5760 28 : if (kind == bitint_prec_huge && i == 1)
5761 7 : res = out;
5762 : else
5763 21 : res = make_ssa_name (m_limb_type);
5764 28 : g = gimple_build_assign (res, BIT_XOR_EXPR, in, rhs1);
5765 28 : insert_before (g);
5766 : }
5767 : else
5768 : res = rhs1;
5769 33 : m_data[m_data_cnt] = res;
5770 33 : break;
5771 31 : case IFN_POPCOUNT:
5772 31 : g = gimple_build_call (fndecl, 1, rhs1);
5773 31 : tem = make_ssa_name (integer_type_node);
5774 31 : gimple_call_set_lhs (g, tem);
5775 31 : insert_before (g);
5776 31 : if (!integer_zerop (in))
5777 : {
5778 26 : if (kind == bitint_prec_huge && i == 1)
5779 6 : res = out;
5780 : else
5781 20 : res = make_ssa_name (integer_type_node);
5782 26 : g = gimple_build_assign (res, PLUS_EXPR, in, tem);
5783 26 : insert_before (g);
5784 : }
5785 : else
5786 : res = tem;
5787 31 : m_data[m_data_cnt] = res;
5788 31 : break;
5789 0 : default:
5790 0 : gcc_unreachable ();
5791 : }
5792 :
5793 168 : m_first = false;
5794 168 : if (kind == bitint_prec_huge && i <= 1)
5795 : {
5796 72 : if (i == 0)
5797 : {
5798 36 : idx = make_ssa_name (sizetype);
5799 36 : g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
5800 : bitint_big_endian
5801 0 : ? size_int (-1) : size_one_node);
5802 36 : insert_before (g);
5803 : }
5804 : else
5805 : {
5806 36 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
5807 72 : size_int (bitint_big_endian
5808 : ? -2 : 2));
5809 36 : insert_before (g);
5810 36 : if (bitint_big_endian)
5811 0 : g = gimple_build_cond (NE_EXPR, idx_first,
5812 0 : size_int (cnt - 1),
5813 : NULL_TREE, NULL_TREE);
5814 : else
5815 36 : g = gimple_build_cond (NE_EXPR, idx_next, size_int (end),
5816 : NULL_TREE, NULL_TREE);
5817 36 : insert_before (g);
5818 36 : if (ifn == IFN_CTZ || ifn == IFN_FFS)
5819 23 : m_gsi = gsi_after_labels (edge_bb);
5820 : else
5821 13 : m_gsi = gsi_for_stmt (stmt);
5822 36 : m_bb = NULL;
5823 : }
5824 : }
5825 : }
5826 : }
5827 : else
5828 : {
5829 35 : tree idx = NULL_TREE, idx_next = NULL_TREE, first = NULL_TREE;
5830 35 : int sub_one = 0;
5831 35 : if (kind == bitint_prec_large)
5832 : cnt = nelts;
5833 : else
5834 : {
5835 18 : rem = prec % limb_prec;
5836 18 : if (rem == 0 && (!TYPE_UNSIGNED (type) || ifn == IFN_CLRSB))
5837 : rem = limb_prec;
5838 18 : end = (prec - rem) / limb_prec;
5839 18 : cnt = 1 + (rem != 0);
5840 18 : if (ifn == IFN_CLRSB)
5841 5 : sub_one = 1;
5842 : }
5843 :
5844 35 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5845 35 : gsi_prev (&gsi);
5846 35 : edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5847 35 : edge_bb = e->src;
5848 35 : m_gsi = gsi_end_bb (edge_bb);
5849 :
5850 35 : if (ifn == IFN_CLZ)
5851 26 : bqp = XALLOCAVEC (struct bq_details, cnt);
5852 : else
5853 : {
5854 9 : gsi = gsi_for_stmt (stmt);
5855 9 : gsi_prev (&gsi);
5856 9 : e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
5857 9 : edge_bb = e->src;
5858 9 : bqp = XALLOCAVEC (struct bq_details, 2 * cnt);
5859 : }
5860 :
5861 116 : for (unsigned i = 0; i < cnt; i++)
5862 : {
5863 81 : m_data_cnt = 0;
5864 81 : if (kind == bitint_prec_large)
5865 51 : idx = size_int (bitint_big_endian ? i : cnt - i - 1);
5866 30 : else if (i == cnt - 1)
5867 18 : idx = create_loop (size_int (bitint_big_endian ? i : end - 1),
5868 : &idx_next);
5869 : else
5870 12 : idx = bitint_big_endian ? size_zero_node : size_int (end);
5871 :
5872 81 : tree rhs1 = handle_operand (arg0, idx);
5873 81 : if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
5874 : {
5875 19 : if (ifn == IFN_CLZ && !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5876 0 : rhs1 = add_cast (unsigned_type_for (TREE_TYPE (rhs1)), rhs1);
5877 19 : else if (ifn == IFN_CLRSB && TYPE_UNSIGNED (TREE_TYPE (rhs1)))
5878 0 : rhs1 = add_cast (signed_type_for (TREE_TYPE (rhs1)), rhs1);
5879 19 : rhs1 = add_cast (m_limb_type, rhs1);
5880 : }
5881 :
5882 81 : if (ifn == IFN_CLZ)
5883 : {
5884 59 : g = gimple_build_cond (NE_EXPR, rhs1,
5885 : build_zero_cst (m_limb_type),
5886 : NULL_TREE, NULL_TREE);
5887 59 : insert_before (g);
5888 59 : edge e1 = split_block (gsi_bb (m_gsi), g);
5889 59 : e1->flags = EDGE_FALSE_VALUE;
5890 59 : edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
5891 59 : e1->probability = profile_probability::unlikely ();
5892 59 : e2->probability = e1->probability.invert ();
5893 59 : if (i == 0)
5894 26 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5895 59 : m_gsi = gsi_after_labels (e1->dest);
5896 59 : bqp[i].e = e2;
5897 59 : bqp[i].val = rhs1;
5898 : }
5899 : else
5900 : {
5901 22 : if (i == 0)
5902 : {
5903 9 : first = rhs1;
5904 9 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5905 : PLUS_EXPR, rhs1,
5906 : build_int_cst (m_limb_type, 1));
5907 9 : insert_before (g);
5908 9 : g = gimple_build_cond (GT_EXPR, gimple_assign_lhs (g),
5909 : build_int_cst (m_limb_type, 1),
5910 : NULL_TREE, NULL_TREE);
5911 9 : insert_before (g);
5912 : }
5913 : else
5914 : {
5915 13 : g = gimple_build_assign (make_ssa_name (m_limb_type),
5916 : BIT_XOR_EXPR, rhs1, first);
5917 13 : insert_before (g);
5918 13 : tree stype = signed_type_for (m_limb_type);
5919 13 : g = gimple_build_cond (LT_EXPR,
5920 : add_cast (stype,
5921 : gimple_assign_lhs (g)),
5922 : build_zero_cst (stype),
5923 : NULL_TREE, NULL_TREE);
5924 13 : insert_before (g);
5925 13 : edge e1 = split_block (gsi_bb (m_gsi), g);
5926 13 : e1->flags = EDGE_FALSE_VALUE;
5927 13 : edge e2 = make_edge (e1->src, gimple_bb (stmt),
5928 : EDGE_TRUE_VALUE);
5929 13 : e1->probability = profile_probability::unlikely ();
5930 13 : e2->probability = e1->probability.invert ();
5931 13 : if (i == 1)
5932 9 : set_immediate_dominator (CDI_DOMINATORS, e2->dest,
5933 : e2->src);
5934 13 : m_gsi = gsi_after_labels (e1->dest);
5935 13 : bqp[2 * i].e = e2;
5936 13 : g = gimple_build_cond (NE_EXPR, rhs1, first,
5937 : NULL_TREE, NULL_TREE);
5938 13 : insert_before (g);
5939 : }
5940 22 : edge e1 = split_block (gsi_bb (m_gsi), g);
5941 22 : e1->flags = EDGE_FALSE_VALUE;
5942 22 : edge e2 = make_edge (e1->src, edge_bb, EDGE_TRUE_VALUE);
5943 22 : e1->probability = profile_probability::unlikely ();
5944 22 : e2->probability = e1->probability.invert ();
5945 22 : if (i == 0)
5946 9 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
5947 22 : m_gsi = gsi_after_labels (e1->dest);
5948 22 : bqp[2 * i + 1].e = e2;
5949 22 : bqp[i].val = rhs1;
5950 : }
5951 81 : if (tree_fits_uhwi_p (idx))
5952 126 : bqp[i].addend
5953 63 : = build_int_cst (integer_type_node,
5954 63 : (int) prec
5955 63 : - (((int) (bitint_big_endian
5956 0 : ? nelts - 1 - tree_to_uhwi (idx)
5957 63 : : tree_to_uhwi (idx)) + 1)
5958 63 : * limb_prec) - sub_one);
5959 : else
5960 : {
5961 18 : tree in, out;
5962 18 : in = build_int_cst (integer_type_node, rem - sub_one);
5963 18 : m_first = true;
5964 18 : in = prepare_data_in_out (in, idx, &out);
5965 18 : out = m_data[m_data_cnt + 1];
5966 18 : bqp[i].addend = in;
5967 18 : g = gimple_build_assign (out, PLUS_EXPR, in,
5968 : build_int_cst (integer_type_node,
5969 18 : limb_prec));
5970 18 : insert_before (g);
5971 18 : m_data[m_data_cnt] = out;
5972 : }
5973 :
5974 81 : m_first = false;
5975 81 : if (kind == bitint_prec_huge && i == cnt - 1)
5976 : {
5977 36 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
5978 : bitint_big_endian
5979 18 : ? size_one_node : size_int (-1));
5980 18 : insert_before (g);
5981 18 : g = gimple_build_cond (NE_EXPR, idx,
5982 : bitint_big_endian
5983 0 : ? size_int (nelts - 1) : size_zero_node,
5984 : NULL_TREE, NULL_TREE);
5985 18 : insert_before (g);
5986 18 : edge true_edge, false_edge;
5987 18 : extract_true_false_edges_from_block (gsi_bb (m_gsi),
5988 : &true_edge, &false_edge);
5989 18 : m_gsi = gsi_after_labels (false_edge->dest);
5990 18 : m_bb = NULL;
5991 : }
5992 : }
5993 : }
5994 97 : switch (ifn)
5995 : {
5996 65 : case IFN_CLZ:
5997 65 : case IFN_CTZ:
5998 65 : case IFN_FFS:
5999 65 : gphi *phi1, *phi2, *phi3;
6000 65 : basic_block bb;
6001 65 : bb = gsi_bb (m_gsi);
6002 65 : remove_edge (find_edge (bb, gimple_bb (stmt)));
6003 65 : phi1 = create_phi_node (make_ssa_name (m_limb_type),
6004 : gimple_bb (stmt));
6005 65 : phi2 = create_phi_node (make_ssa_name (integer_type_node),
6006 : gimple_bb (stmt));
6007 228 : for (unsigned i = 0; i < cnt; i++)
6008 : {
6009 163 : add_phi_arg (phi1, bqp[i].val, bqp[i].e, UNKNOWN_LOCATION);
6010 163 : add_phi_arg (phi2, bqp[i].addend, bqp[i].e, UNKNOWN_LOCATION);
6011 : }
6012 65 : if (arg1 == NULL_TREE)
6013 : {
6014 35 : g = gimple_build_builtin_unreachable (m_loc);
6015 35 : insert_before (g);
6016 : }
6017 65 : m_gsi = gsi_for_stmt (stmt);
6018 65 : g = gimple_build_call (fndecl, 1, gimple_phi_result (phi1));
6019 65 : gimple_call_set_lhs (g, make_ssa_name (integer_type_node));
6020 65 : insert_before (g);
6021 65 : if (arg1 == NULL_TREE)
6022 35 : g = gimple_build_assign (lhs, PLUS_EXPR,
6023 : gimple_phi_result (phi2),
6024 : gimple_call_lhs (g));
6025 : else
6026 : {
6027 30 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6028 : PLUS_EXPR, gimple_phi_result (phi2),
6029 : gimple_call_lhs (g));
6030 30 : insert_before (g);
6031 30 : edge e1 = split_block (gimple_bb (stmt), g);
6032 30 : edge e2 = make_edge (bb, e1->dest, EDGE_FALLTHRU);
6033 30 : e2->probability = profile_probability::always ();
6034 30 : set_immediate_dominator (CDI_DOMINATORS, e1->dest,
6035 : get_immediate_dominator (CDI_DOMINATORS,
6036 : e1->src));
6037 30 : phi3 = create_phi_node (make_ssa_name (integer_type_node), e1->dest);
6038 30 : add_phi_arg (phi3, gimple_assign_lhs (g), e1, UNKNOWN_LOCATION);
6039 30 : add_phi_arg (phi3, arg1, e2, UNKNOWN_LOCATION);
6040 30 : m_gsi = gsi_for_stmt (stmt);
6041 30 : g = gimple_build_assign (lhs, gimple_phi_result (phi3));
6042 : }
6043 65 : gsi_replace (&m_gsi, g, true);
6044 65 : break;
6045 9 : case IFN_CLRSB:
6046 9 : bb = gsi_bb (m_gsi);
6047 9 : remove_edge (find_edge (bb, edge_bb));
6048 9 : edge e;
6049 9 : e = make_edge (bb, gimple_bb (stmt), EDGE_FALLTHRU);
6050 9 : e->probability = profile_probability::always ();
6051 9 : set_immediate_dominator (CDI_DOMINATORS, gimple_bb (stmt),
6052 : get_immediate_dominator (CDI_DOMINATORS,
6053 : edge_bb));
6054 9 : phi1 = create_phi_node (make_ssa_name (m_limb_type),
6055 : edge_bb);
6056 9 : phi2 = create_phi_node (make_ssa_name (integer_type_node),
6057 : edge_bb);
6058 9 : phi3 = create_phi_node (make_ssa_name (integer_type_node),
6059 : gimple_bb (stmt));
6060 31 : for (unsigned i = 0; i < cnt; i++)
6061 : {
6062 22 : add_phi_arg (phi1, bqp[i].val, bqp[2 * i + 1].e, UNKNOWN_LOCATION);
6063 22 : add_phi_arg (phi2, bqp[i].addend, bqp[2 * i + 1].e,
6064 : UNKNOWN_LOCATION);
6065 22 : tree a = bqp[i].addend;
6066 22 : if (i && kind == bitint_prec_large)
6067 8 : a = int_const_binop (PLUS_EXPR, a, integer_minus_one_node);
6068 22 : if (i)
6069 13 : add_phi_arg (phi3, a, bqp[2 * i].e, UNKNOWN_LOCATION);
6070 : }
6071 9 : add_phi_arg (phi3, build_int_cst (integer_type_node, prec - 1), e,
6072 : UNKNOWN_LOCATION);
6073 9 : m_gsi = gsi_after_labels (edge_bb);
6074 9 : g = gimple_build_call (fndecl, 1,
6075 : add_cast (signed_type_for (m_limb_type),
6076 : gimple_phi_result (phi1)));
6077 9 : gimple_call_set_lhs (g, make_ssa_name (integer_type_node));
6078 9 : insert_before (g);
6079 9 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6080 : PLUS_EXPR, gimple_call_lhs (g),
6081 : gimple_phi_result (phi2));
6082 9 : insert_before (g);
6083 9 : if (kind != bitint_prec_large)
6084 : {
6085 5 : g = gimple_build_assign (make_ssa_name (integer_type_node),
6086 : PLUS_EXPR, gimple_assign_lhs (g),
6087 : integer_one_node);
6088 5 : insert_before (g);
6089 : }
6090 9 : add_phi_arg (phi3, gimple_assign_lhs (g),
6091 : find_edge (edge_bb, gimple_bb (stmt)), UNKNOWN_LOCATION);
6092 9 : m_gsi = gsi_for_stmt (stmt);
6093 9 : g = gimple_build_assign (lhs, gimple_phi_result (phi3));
6094 9 : gsi_replace (&m_gsi, g, true);
6095 9 : break;
6096 12 : case IFN_PARITY:
6097 12 : g = gimple_build_call (fndecl, 1, res);
6098 12 : gimple_call_set_lhs (g, lhs);
6099 12 : gsi_replace (&m_gsi, g, true);
6100 12 : break;
6101 11 : case IFN_POPCOUNT:
6102 11 : g = gimple_build_assign (lhs, res);
6103 11 : gsi_replace (&m_gsi, g, true);
6104 11 : break;
6105 0 : default:
6106 0 : gcc_unreachable ();
6107 : }
6108 : }
6109 :
6110 : /* Lower a .{BSWAP,BITREVERSE} call with one large/huge _BitInt argument. */
6111 :
6112 : void
6113 13 : bitint_large_huge::lower_bswap_bitreverse (tree obj, gimple *stmt)
6114 : {
6115 13 : tree arg = gimple_call_arg (stmt, 0);
6116 13 : tree lhs = gimple_call_lhs (stmt);
6117 13 : gimple *g;
6118 :
6119 13 : if (!lhs)
6120 : {
6121 0 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6122 0 : gsi_remove (&gsi, true);
6123 0 : return;
6124 : }
6125 13 : tree type = TREE_TYPE (arg);
6126 13 : gcc_assert (BITINT_TYPE_P (type));
6127 13 : bitint_prec_kind kind = bitint_precision_kind (type);
6128 13 : gcc_assert (kind >= bitint_prec_large);
6129 13 : if (!obj)
6130 : {
6131 0 : int part = var_to_partition (m_map, lhs);
6132 0 : gcc_assert (m_vars[part] != NULL_TREE);
6133 : obj = m_vars[part];
6134 : }
6135 13 : enum internal_fn ifn = gimple_call_internal_fn (stmt);
6136 13 : enum built_in_function bcode = END_BUILTINS;
6137 13 : switch (limb_prec)
6138 : {
6139 0 : case 16:
6140 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP16 : BUILT_IN_BITREVERSE16;
6141 : break;
6142 0 : case 32:
6143 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP32 : BUILT_IN_BITREVERSE32;
6144 : break;
6145 13 : case 64:
6146 13 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP64 : BUILT_IN_BITREVERSE64;
6147 : break;
6148 0 : case 128:
6149 0 : bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP128 : BUILT_IN_BITREVERSE128;
6150 : break;
6151 0 : default:
6152 0 : gcc_unreachable ();
6153 : }
6154 13 : tree fndecl = builtin_decl_explicit (bcode);
6155 13 : unsigned prec = TYPE_PRECISION (type);
6156 26 : tree p = build_int_cst (sizetype,
6157 13 : prec / limb_prec - (prec % limb_prec == 0));
6158 : /* For IFN .BSWAP or .BITREVERSE and
6159 : FN corresponding __builtin_bswapN or __builtin_bitreverseN where N
6160 : is limb_prec, lower
6161 : dst = IFN (src);
6162 : as
6163 : size_t p = prec / limb_prec - (prec % limb_prec == 0);
6164 : if constexpr ((prec % limb_prec) == 0)
6165 : {
6166 : for (idx = 0; idx <= p; ++idx)
6167 : dst[p - idx] = FN (src[idx]);
6168 : }
6169 : else
6170 : {
6171 : unsigned n1 = prec % limb_prec;
6172 : unsigned n2 = limb_prec - n1;
6173 : dst[p] = FN (src[0]) >> n2;
6174 : for (idx = p - 1; (ssize_t) idx >= 0; --idx)
6175 : dst[idx] = FN ((src[p - idx] << n2) | (src[p - idx - 1] >> n1));
6176 : } */
6177 13 : if (prec % limb_prec == 0)
6178 : {
6179 2 : tree idx_next;
6180 2 : tree idx = create_loop (size_zero_node, &idx_next);
6181 2 : tree pmidx = make_ssa_name (sizetype);
6182 2 : g = gimple_build_assign (pmidx, MINUS_EXPR, p, idx);
6183 2 : insert_before (g);
6184 2 : m_data_cnt = 0;
6185 2 : tree t = handle_operand (arg, idx);
6186 2 : m_first = false;
6187 2 : g = gimple_build_call (fndecl, 1, t);
6188 2 : t = make_ssa_name (m_limb_type);
6189 2 : gimple_call_set_lhs (g, t);
6190 2 : insert_before (g);
6191 2 : tree l = limb_access (TREE_TYPE (lhs), obj, pmidx, true);
6192 2 : g = gimple_build_assign (l, t);
6193 2 : insert_before (g);
6194 2 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_one_node);
6195 2 : insert_before (g);
6196 2 : g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
6197 2 : insert_before (g);
6198 : }
6199 : else
6200 : {
6201 11 : tree n1 = build_int_cst (unsigned_type_node, prec % limb_prec);
6202 22 : tree n2 = build_int_cst (unsigned_type_node,
6203 11 : limb_prec - (prec % limb_prec));
6204 11 : m_data_cnt = 0;
6205 11 : tree t = handle_operand (arg, bitint_big_endian ? p : size_zero_node);
6206 11 : m_first = false;
6207 : /* Nothing is needed to bswap 8 bits or bitreverse 1 bit, so just
6208 : mask off higher bits in that case. */
6209 17 : if ((prec % limb_prec) != (ifn == IFN_BSWAP ? 8 : 1))
6210 : {
6211 7 : g = gimple_build_call (fndecl, 1, t);
6212 7 : t = make_ssa_name (m_limb_type);
6213 7 : gimple_call_set_lhs (g, t);
6214 7 : insert_before (g);
6215 7 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
6216 : t, n2);
6217 : }
6218 : else
6219 4 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_AND_EXPR,
6220 : t, build_int_cst (m_limb_type,
6221 : ifn == IFN_BSWAP
6222 6 : ? 0xff : 1));
6223 11 : insert_before (g);
6224 11 : t = gimple_assign_lhs (g);
6225 11 : tree l = limb_access (TREE_TYPE (lhs), obj,
6226 : bitint_big_endian ? size_zero_node : p, true);
6227 11 : g = gimple_build_assign (l, t);
6228 11 : insert_before (g);
6229 11 : tree pm1 = build_int_cst (sizetype, prec / limb_prec - 1);
6230 11 : tree idx_next;
6231 11 : tree idx = create_loop (pm1, &idx_next);
6232 11 : tree pmidx = make_ssa_name (sizetype);
6233 11 : g = gimple_build_assign (pmidx, MINUS_EXPR, p, idx);
6234 11 : insert_before (g);
6235 11 : m_data_cnt = 0;
6236 22 : t = handle_operand (arg, bitint_big_endian ? idx : pmidx);
6237 11 : m_data_cnt = 0;
6238 11 : g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
6239 : t, n2);
6240 11 : insert_before (g);
6241 11 : t = gimple_assign_lhs (g);
6242 11 : tree t2 = make_ssa_name (sizetype);
6243 11 : if (bitint_big_endian)
6244 0 : g = gimple_build_assign (t2, PLUS_EXPR, idx, size_one_node);
6245 : else
6246 11 : g = gimple_build_assign (t2, PLUS_EXPR, pmidx, size_int (-1));
6247 11 : insert_before (g);
6248 11 : t2 = handle_operand (arg, t2);
6249 11 : g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
6250 : t2, n1);
6251 11 : insert_before (g);
6252 11 : t2 = gimple_assign_lhs (g);
6253 11 : g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
6254 : t, t2);
6255 11 : insert_before (g);
6256 11 : t = gimple_assign_lhs (g);
6257 11 : g = gimple_build_call (fndecl, 1, t);
6258 11 : t = make_ssa_name (m_limb_type);
6259 11 : gimple_call_set_lhs (g, t);
6260 11 : insert_before (g);
6261 22 : l = limb_access (TREE_TYPE (lhs), obj,
6262 : bitint_big_endian ? pmidx : idx, true);
6263 11 : g = gimple_build_assign (l, t);
6264 11 : insert_before (g);
6265 11 : g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_int (-1));
6266 11 : insert_before (g);
6267 11 : g = gimple_build_cond (NE_EXPR, idx, size_zero_node, NULL_TREE,
6268 : NULL_TREE);
6269 11 : insert_before (g);
6270 : }
6271 13 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6272 13 : if (bitint_extended == bitint_ext_full
6273 0 : && abi_limb_prec > limb_prec
6274 0 : && (CEIL (prec, abi_limb_prec) * abi_limb_prec
6275 0 : > CEIL (prec, limb_prec) * limb_prec))
6276 : {
6277 0 : m_gsi = gsi;
6278 0 : tree p2 = build_int_cst (sizetype,
6279 : CEIL (prec, abi_limb_prec)
6280 0 : * abi_limb_prec / limb_prec - 1);
6281 0 : tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
6282 0 : g = gimple_build_assign (l, build_zero_cst (m_limb_type));
6283 0 : insert_before (g);
6284 : }
6285 : }
6286 :
6287 : /* Lower a call statement with one or more large/huge _BitInt
6288 : arguments or large/huge _BitInt return value. */
6289 :
6290 : void
6291 8787 : bitint_large_huge::lower_call (tree obj, gimple *stmt)
6292 : {
6293 8787 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
6294 8787 : unsigned int nargs = gimple_call_num_args (stmt);
6295 8787 : if (gimple_call_internal_p (stmt))
6296 4291 : switch (gimple_call_internal_fn (stmt))
6297 : {
6298 2721 : case IFN_ADD_OVERFLOW:
6299 2721 : case IFN_SUB_OVERFLOW:
6300 2721 : case IFN_UBSAN_CHECK_ADD:
6301 2721 : case IFN_UBSAN_CHECK_SUB:
6302 2721 : lower_addsub_overflow (obj, stmt);
6303 7012 : return;
6304 1460 : case IFN_MUL_OVERFLOW:
6305 1460 : case IFN_UBSAN_CHECK_MUL:
6306 1460 : lower_mul_overflow (obj, stmt);
6307 1460 : return;
6308 97 : case IFN_CLZ:
6309 97 : case IFN_CTZ:
6310 97 : case IFN_CLRSB:
6311 97 : case IFN_FFS:
6312 97 : case IFN_PARITY:
6313 97 : case IFN_POPCOUNT:
6314 97 : lower_bit_query (stmt);
6315 97 : return;
6316 13 : case IFN_BSWAP:
6317 13 : case IFN_BITREVERSE:
6318 13 : lower_bswap_bitreverse (obj, stmt);
6319 13 : return;
6320 : default:
6321 : break;
6322 : }
6323 4496 : bool returns_twice = (gimple_call_flags (stmt) & ECF_RETURNS_TWICE) != 0;
6324 10120 : for (unsigned int i = 0; i < nargs; ++i)
6325 : {
6326 5624 : tree arg = gimple_call_arg (stmt, i);
6327 8913 : if (TREE_CODE (arg) != SSA_NAME
6328 2432 : || !BITINT_TYPE_P (TREE_TYPE (arg))
6329 7994 : || bitint_precision_kind (TREE_TYPE (arg)) <= bitint_prec_middle)
6330 3289 : continue;
6331 2335 : if (SSA_NAME_IS_DEFAULT_DEF (arg)
6332 2335 : && (!SSA_NAME_VAR (arg) || VAR_P (SSA_NAME_VAR (arg))))
6333 : {
6334 1 : tree var = create_tmp_reg (TREE_TYPE (arg));
6335 1 : arg = get_or_create_ssa_default_def (cfun, var);
6336 : }
6337 : else
6338 : {
6339 2334 : int p = var_to_partition (m_map, arg);
6340 2334 : tree v = m_vars[p];
6341 2334 : gcc_assert (v != NULL_TREE);
6342 2334 : if (!types_compatible_p (TREE_TYPE (arg), TREE_TYPE (v)))
6343 2314 : v = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (arg), v);
6344 2334 : arg = make_ssa_name (TREE_TYPE (arg));
6345 2334 : gimple *g = gimple_build_assign (arg, v);
6346 2334 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
6347 2334 : if (returns_twice && bb_has_abnormal_pred (gimple_bb (stmt)))
6348 : {
6349 11 : m_returns_twice_calls.safe_push (stmt);
6350 11 : returns_twice = false;
6351 : }
6352 : }
6353 2335 : gimple_call_set_arg (stmt, i, arg);
6354 2335 : if (m_preserved == NULL)
6355 404 : m_preserved = BITMAP_ALLOC (NULL);
6356 2335 : bitmap_set_bit (m_preserved, SSA_NAME_VERSION (arg));
6357 : }
6358 4496 : tree lhs = gimple_call_lhs (stmt);
6359 4496 : if (lhs
6360 4365 : && TREE_CODE (lhs) == SSA_NAME
6361 4365 : && BITINT_TYPE_P (TREE_TYPE (lhs))
6362 8791 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large)
6363 : {
6364 4269 : int p = var_to_partition (m_map, lhs);
6365 4269 : tree v = m_vars[p];
6366 4269 : gcc_assert (v != NULL_TREE);
6367 4269 : if (!types_compatible_p (TREE_TYPE (lhs), TREE_TYPE (v)))
6368 4269 : v = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), v);
6369 4269 : gimple_call_set_lhs (stmt, v);
6370 4269 : SSA_NAME_DEF_STMT (lhs) = gimple_build_nop ();
6371 : }
6372 4496 : update_stmt (stmt);
6373 : }
6374 :
6375 : /* Lower __asm STMT which involves large/huge _BitInt values. */
6376 :
6377 : void
6378 3 : bitint_large_huge::lower_asm (gimple *stmt)
6379 : {
6380 3 : gasm *g = as_a <gasm *> (stmt);
6381 3 : unsigned noutputs = gimple_asm_noutputs (g);
6382 3 : unsigned ninputs = gimple_asm_ninputs (g);
6383 :
6384 5 : for (unsigned i = 0; i < noutputs; ++i)
6385 : {
6386 2 : tree t = gimple_asm_output_op (g, i);
6387 2 : tree s = TREE_VALUE (t);
6388 2 : if (TREE_CODE (s) == SSA_NAME
6389 1 : && BITINT_TYPE_P (TREE_TYPE (s))
6390 3 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
6391 : {
6392 1 : int part = var_to_partition (m_map, s);
6393 1 : gcc_assert (m_vars[part] != NULL_TREE);
6394 1 : TREE_VALUE (t) = m_vars[part];
6395 : }
6396 : }
6397 8 : for (unsigned i = 0; i < ninputs; ++i)
6398 : {
6399 5 : tree t = gimple_asm_input_op (g, i);
6400 5 : tree s = TREE_VALUE (t);
6401 5 : if (TREE_CODE (s) == SSA_NAME
6402 4 : && BITINT_TYPE_P (TREE_TYPE (s))
6403 9 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
6404 : {
6405 4 : if (SSA_NAME_IS_DEFAULT_DEF (s)
6406 4 : && (!SSA_NAME_VAR (s) || VAR_P (SSA_NAME_VAR (s))))
6407 : {
6408 1 : TREE_VALUE (t) = create_tmp_var (TREE_TYPE (s), "bitint");
6409 1 : mark_addressable (TREE_VALUE (t));
6410 : }
6411 : else
6412 : {
6413 3 : int part = var_to_partition (m_map, s);
6414 3 : gcc_assert (m_vars[part] != NULL_TREE);
6415 3 : TREE_VALUE (t) = m_vars[part];
6416 : }
6417 : }
6418 : }
6419 3 : update_stmt (stmt);
6420 3 : }
6421 :
6422 : /* Lower statement STMT which involves large/huge _BitInt values
6423 : into code accessing individual limbs. */
6424 :
6425 : void
6426 42634 : bitint_large_huge::lower_stmt (gimple *stmt)
6427 : {
6428 42634 : m_first = true;
6429 42634 : m_lhs = NULL_TREE;
6430 42634 : m_data.truncate (0);
6431 42634 : m_data_cnt = 0;
6432 42634 : m_gsi = gsi_for_stmt (stmt);
6433 42634 : m_after_stmt = NULL;
6434 42634 : m_bb = NULL;
6435 42634 : m_init_gsi = m_gsi;
6436 42634 : gsi_prev (&m_init_gsi);
6437 42634 : m_preheader_bb = NULL;
6438 42634 : m_upwards_2limb = 0;
6439 42634 : m_upwards = false;
6440 42634 : m_var_msb = false;
6441 42634 : m_cast_conditional = false;
6442 42634 : m_bitfld_load = 0;
6443 42634 : m_loc = gimple_location (stmt);
6444 42634 : if (is_gimple_call (stmt))
6445 : {
6446 7112 : lower_call (NULL_TREE, stmt);
6447 7112 : return;
6448 : }
6449 35522 : if (gimple_code (stmt) == GIMPLE_ASM)
6450 : {
6451 3 : lower_asm (stmt);
6452 3 : return;
6453 : }
6454 35519 : tree lhs = NULL_TREE, cmp_op1 = NULL_TREE, cmp_op2 = NULL_TREE;
6455 35519 : tree_code cmp_code = comparison_op (stmt, &cmp_op1, &cmp_op2);
6456 35519 : bool eq_p = (cmp_code == EQ_EXPR || cmp_code == NE_EXPR);
6457 35519 : bool mergeable_cast_p = false;
6458 35519 : bool final_cast_p = false;
6459 35519 : if (gimple_assign_cast_p (stmt))
6460 : {
6461 5416 : lhs = gimple_assign_lhs (stmt);
6462 5416 : tree rhs1 = gimple_assign_rhs1 (stmt);
6463 5416 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
6464 46 : rhs1 = TREE_OPERAND (rhs1, 0);
6465 9609 : if (BITINT_TYPE_P (TREE_TYPE (lhs))
6466 1223 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
6467 6603 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1)))
6468 : mergeable_cast_p = true;
6469 4519 : else if (BITINT_TYPE_P (TREE_TYPE (rhs1))
6470 4229 : && bitint_precision_kind (TREE_TYPE (rhs1)) >= bitint_prec_large
6471 8603 : && (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6472 36 : || POINTER_TYPE_P (TREE_TYPE (lhs))
6473 35 : || gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR))
6474 : {
6475 4229 : final_cast_p = true;
6476 4229 : if (((TREE_CODE (TREE_TYPE (lhs)) == INTEGER_TYPE
6477 536 : && TYPE_PRECISION (TREE_TYPE (lhs)) > MAX_FIXED_MODE_SIZE)
6478 4229 : || (!INTEGRAL_TYPE_P (TREE_TYPE (lhs))
6479 36 : && !POINTER_TYPE_P (TREE_TYPE (lhs))))
6480 4264 : && gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)
6481 : {
6482 : /* Handle VIEW_CONVERT_EXPRs to not generally supported
6483 : huge INTEGER_TYPEs like uint256_t or uint512_t. These
6484 : are usually emitted from memcpy folding and backends
6485 : support moves with them but that is usually it.
6486 : Similarly handle VCEs to vector/complex types etc. */
6487 35 : gcc_assert (TREE_CODE (rhs1) == SSA_NAME);
6488 35 : if (SSA_NAME_IS_DEFAULT_DEF (rhs1)
6489 35 : && (!SSA_NAME_VAR (rhs1) || VAR_P (SSA_NAME_VAR (rhs1))))
6490 : {
6491 0 : tree var = create_tmp_reg (TREE_TYPE (lhs));
6492 0 : rhs1 = get_or_create_ssa_default_def (cfun, var);
6493 0 : gimple_assign_set_rhs1 (stmt, rhs1);
6494 0 : gimple_assign_set_rhs_code (stmt, SSA_NAME);
6495 : }
6496 35 : else if (m_names == NULL
6497 35 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))
6498 : {
6499 0 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6500 0 : gcc_assert (gimple_assign_load_p (g));
6501 0 : tree mem = gimple_assign_rhs1 (g);
6502 0 : tree ltype = TREE_TYPE (lhs);
6503 0 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (mem));
6504 0 : if (as != TYPE_ADDR_SPACE (ltype))
6505 0 : ltype
6506 0 : = build_qualified_type (ltype,
6507 0 : TYPE_QUALS (ltype)
6508 0 : | ENCODE_QUAL_ADDR_SPACE (as));
6509 0 : rhs1 = build1 (VIEW_CONVERT_EXPR, ltype, unshare_expr (mem));
6510 0 : gimple_assign_set_rhs1 (stmt, rhs1);
6511 : }
6512 : else
6513 : {
6514 35 : int part = var_to_partition (m_map, rhs1);
6515 35 : gcc_assert (m_vars[part] != NULL_TREE);
6516 35 : rhs1 = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs),
6517 : m_vars[part]);
6518 35 : gimple_assign_set_rhs1 (stmt, rhs1);
6519 : }
6520 35 : update_stmt (stmt);
6521 35 : return;
6522 : }
6523 4194 : if (TREE_CODE (rhs1) == SSA_NAME
6524 4194 : && (m_names == NULL
6525 4157 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
6526 : {
6527 1709 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6528 1709 : if (is_gimple_assign (g)
6529 1709 : && gimple_assign_rhs_code (g) == IMAGPART_EXPR)
6530 : {
6531 1638 : tree rhs2 = TREE_OPERAND (gimple_assign_rhs1 (g), 0);
6532 1638 : if (TREE_CODE (rhs2) == SSA_NAME
6533 1638 : && (m_names == NULL
6534 1601 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs2))))
6535 : {
6536 1638 : g = SSA_NAME_DEF_STMT (rhs2);
6537 1638 : int ovf = optimizable_arith_overflow (g);
6538 1638 : if (ovf == 2)
6539 : /* If .{ADD,SUB,MUL}_OVERFLOW has both REALPART_EXPR
6540 : and IMAGPART_EXPR uses, where the latter is cast to
6541 : non-_BitInt, it will be optimized when handling
6542 : the REALPART_EXPR. */
6543 : return;
6544 91 : if (ovf == 1)
6545 : {
6546 91 : lower_call (NULL_TREE, g);
6547 91 : return;
6548 : }
6549 : }
6550 : }
6551 : }
6552 : }
6553 145 : else if (BITINT_TYPE_P (TREE_TYPE (lhs))
6554 145 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
6555 145 : && !INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
6556 145 : && !POINTER_TYPE_P (TREE_TYPE (rhs1))
6557 290 : && gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)
6558 : {
6559 10 : int part = var_to_partition (m_map, lhs);
6560 10 : gcc_assert (m_vars[part] != NULL_TREE);
6561 10 : lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (rhs1), m_vars[part]);
6562 10 : insert_before (gimple_build_assign (lhs, rhs1));
6563 10 : return;
6564 : }
6565 : }
6566 33836 : if (gimple_store_p (stmt))
6567 : {
6568 8949 : tree rhs1 = gimple_assign_rhs1 (stmt);
6569 8949 : if (TREE_CODE (rhs1) == SSA_NAME
6570 8949 : && (m_names == NULL
6571 7906 : || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
6572 : {
6573 1610 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
6574 1610 : m_loc = gimple_location (g);
6575 1610 : lhs = gimple_assign_lhs (stmt);
6576 1610 : if (is_gimple_assign (g) && !mergeable_op (g))
6577 635 : switch (gimple_assign_rhs_code (g))
6578 : {
6579 119 : case LSHIFT_EXPR:
6580 119 : case RSHIFT_EXPR:
6581 119 : lower_shift_stmt (lhs, g);
6582 478 : handled:
6583 478 : m_gsi = gsi_for_stmt (stmt);
6584 478 : unlink_stmt_vdef (stmt);
6585 956 : release_ssa_name (gimple_vdef (stmt));
6586 478 : gsi_remove (&m_gsi, true);
6587 478 : return;
6588 205 : case MULT_EXPR:
6589 205 : case TRUNC_DIV_EXPR:
6590 205 : case EXACT_DIV_EXPR:
6591 205 : case TRUNC_MOD_EXPR:
6592 205 : lower_muldiv_stmt (lhs, g);
6593 205 : goto handled;
6594 44 : case FIX_TRUNC_EXPR:
6595 44 : lower_float_conv_stmt (lhs, g);
6596 44 : goto handled;
6597 73 : case REALPART_EXPR:
6598 73 : case IMAGPART_EXPR:
6599 73 : lower_cplxpart_stmt (lhs, g);
6600 73 : goto handled;
6601 8 : case VIEW_CONVERT_EXPR:
6602 8 : {
6603 8 : tree rhs1 = gimple_assign_rhs1 (g);
6604 8 : rhs1 = TREE_OPERAND (rhs1, 0);
6605 8 : if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
6606 7 : && !POINTER_TYPE_P (TREE_TYPE (rhs1)))
6607 : {
6608 7 : tree ltype = TREE_TYPE (rhs1);
6609 7 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (lhs));
6610 7 : ltype
6611 14 : = build_qualified_type (ltype,
6612 7 : TYPE_QUALS (TREE_TYPE (lhs))
6613 7 : | ENCODE_QUAL_ADDR_SPACE (as));
6614 7 : lhs = build1 (VIEW_CONVERT_EXPR, ltype, lhs);
6615 7 : gimple_assign_set_lhs (stmt, lhs);
6616 7 : gimple_assign_set_rhs1 (stmt, rhs1);
6617 7 : gimple_assign_set_rhs_code (stmt, TREE_CODE (rhs1));
6618 7 : update_stmt (stmt);
6619 7 : return;
6620 : }
6621 : }
6622 : break;
6623 : default:
6624 : break;
6625 : }
6626 975 : else if (optimizable_arith_overflow (g) == 3)
6627 : {
6628 24 : lower_call (lhs, g);
6629 24 : goto handled;
6630 : }
6631 951 : else if (is_gimple_call (g) && gimple_call_internal_p (g))
6632 13 : switch (gimple_call_internal_fn (g))
6633 : {
6634 13 : case IFN_BSWAP:
6635 13 : case IFN_BITREVERSE:
6636 13 : lower_call (lhs, g);
6637 13 : goto handled;
6638 : default:
6639 : break;
6640 : }
6641 :
6642 1125 : m_loc = gimple_location (stmt);
6643 : }
6644 : }
6645 33351 : if (mergeable_op (stmt)
6646 22009 : || gimple_store_p (stmt)
6647 22009 : || gimple_assign_load_p (stmt)
6648 : || eq_p
6649 17203 : || mergeable_cast_p
6650 43451 : || (is_gimple_assign (stmt)
6651 9712 : && gimple_assign_rhs_code (stmt) == PAREN_EXPR))
6652 : {
6653 23253 : lhs = lower_mergeable_stmt (stmt, cmp_code, cmp_op1, cmp_op2);
6654 23253 : if (!eq_p)
6655 : return;
6656 : }
6657 10098 : else if (cmp_code != ERROR_MARK)
6658 722 : lhs = lower_comparison_stmt (stmt, cmp_code, cmp_op1, cmp_op2);
6659 16624 : if (cmp_code != ERROR_MARK)
6660 : {
6661 7248 : if (gimple_code (stmt) == GIMPLE_COND)
6662 : {
6663 6482 : gcond *cstmt = as_a <gcond *> (stmt);
6664 6482 : gimple_cond_set_lhs (cstmt, lhs);
6665 6482 : gimple_cond_set_rhs (cstmt, boolean_false_node);
6666 6482 : gimple_cond_set_code (cstmt, cmp_code);
6667 6482 : update_stmt (stmt);
6668 6482 : return;
6669 : }
6670 766 : if (gimple_assign_rhs_code (stmt) == COND_EXPR)
6671 : {
6672 0 : tree cond = build2 (cmp_code, boolean_type_node, lhs,
6673 : boolean_false_node);
6674 0 : gimple_assign_set_rhs1 (stmt, cond);
6675 0 : lhs = gimple_assign_lhs (stmt);
6676 0 : gcc_assert (!BITINT_TYPE_P (TREE_TYPE (lhs))
6677 : || (bitint_precision_kind (TREE_TYPE (lhs))
6678 : <= bitint_prec_middle));
6679 0 : update_stmt (stmt);
6680 0 : return;
6681 : }
6682 766 : gimple_assign_set_rhs1 (stmt, lhs);
6683 766 : gimple_assign_set_rhs2 (stmt, boolean_false_node);
6684 766 : gimple_assign_set_rhs_code (stmt, cmp_code);
6685 766 : update_stmt (stmt);
6686 766 : return;
6687 : }
6688 9376 : if (final_cast_p)
6689 : {
6690 2556 : tree lhs_type = TREE_TYPE (lhs);
6691 : /* Add support for 3 or more limbs filled in from normal integral
6692 : type if this assert fails. If no target chooses limb mode smaller
6693 : than half of largest supported normal integral type, this will not
6694 : be needed. */
6695 2556 : gcc_assert (TYPE_PRECISION (lhs_type) <= 2 * limb_prec);
6696 2556 : gimple *g;
6697 2520 : if ((BITINT_TYPE_P (lhs_type)
6698 36 : && bitint_precision_kind (lhs_type) == bitint_prec_middle)
6699 5097 : || POINTER_TYPE_P (lhs_type))
6700 16 : lhs_type = build_nonstandard_integer_type (TYPE_PRECISION (lhs_type),
6701 16 : TYPE_UNSIGNED (lhs_type));
6702 2556 : m_data_cnt = 0;
6703 2556 : tree rhs1 = gimple_assign_rhs1 (stmt);
6704 2556 : unsigned int prec = TYPE_PRECISION (TREE_TYPE (rhs1));
6705 2556 : unsigned int cnt = CEIL (prec, limb_prec);
6706 2556 : tree r1 = handle_operand (rhs1, size_int (bitint_big_endian
6707 : ? cnt - 1 : 0));
6708 2556 : if (!useless_type_conversion_p (lhs_type, TREE_TYPE (r1)))
6709 2449 : r1 = add_cast (lhs_type, r1);
6710 2556 : if (TYPE_PRECISION (lhs_type) > limb_prec)
6711 : {
6712 70 : m_data_cnt = 0;
6713 70 : m_first = false;
6714 70 : tree r2 = handle_operand (rhs1, size_int (bitint_big_endian
6715 : ? cnt - 2 : 1));
6716 70 : r2 = add_cast (lhs_type, r2);
6717 70 : g = gimple_build_assign (make_ssa_name (lhs_type), LSHIFT_EXPR, r2,
6718 : build_int_cst (unsigned_type_node,
6719 70 : limb_prec));
6720 70 : insert_before (g);
6721 70 : g = gimple_build_assign (make_ssa_name (lhs_type), BIT_IOR_EXPR, r1,
6722 : gimple_assign_lhs (g));
6723 70 : insert_before (g);
6724 70 : r1 = gimple_assign_lhs (g);
6725 : }
6726 2556 : if (lhs_type != TREE_TYPE (lhs))
6727 16 : g = gimple_build_assign (lhs, NOP_EXPR, r1);
6728 : else
6729 2540 : g = gimple_build_assign (lhs, r1);
6730 2556 : gsi_replace (&m_gsi, g, true);
6731 2556 : return;
6732 : }
6733 6820 : if (is_gimple_assign (stmt))
6734 6820 : switch (gimple_assign_rhs_code (stmt))
6735 : {
6736 465 : case LSHIFT_EXPR:
6737 465 : case RSHIFT_EXPR:
6738 465 : lower_shift_stmt (NULL_TREE, stmt);
6739 465 : return;
6740 151 : case MULT_EXPR:
6741 151 : case TRUNC_DIV_EXPR:
6742 151 : case EXACT_DIV_EXPR:
6743 151 : case TRUNC_MOD_EXPR:
6744 151 : lower_muldiv_stmt (NULL_TREE, stmt);
6745 151 : return;
6746 275 : case FIX_TRUNC_EXPR:
6747 275 : case FLOAT_EXPR:
6748 275 : lower_float_conv_stmt (NULL_TREE, stmt);
6749 275 : return;
6750 5911 : case REALPART_EXPR:
6751 5911 : case IMAGPART_EXPR:
6752 5911 : lower_cplxpart_stmt (NULL_TREE, stmt);
6753 5911 : return;
6754 18 : case COMPLEX_EXPR:
6755 18 : lower_complexexpr_stmt (stmt);
6756 18 : return;
6757 : default:
6758 : break;
6759 : }
6760 0 : gcc_unreachable ();
6761 : }
6762 :
6763 : /* Helper for walk_non_aliased_vuses. Determine if we arrived at
6764 : the desired memory state. */
6765 :
6766 : void *
6767 2186 : vuse_eq (ao_ref *, tree vuse1, void *data)
6768 : {
6769 2186 : tree vuse2 = (tree) data;
6770 2186 : if (vuse1 == vuse2)
6771 820 : return data;
6772 :
6773 : return NULL;
6774 : }
6775 :
6776 : /* Return true if STMT uses a library function and needs to take
6777 : address of its inputs. We need to avoid bit-fields in those
6778 : cases. Similarly, we need to avoid overlap between destination
6779 : and source limb arrays. */
6780 :
6781 : bool
6782 15079 : stmt_needs_operand_addr (gimple *stmt)
6783 : {
6784 15079 : if (is_gimple_assign (stmt))
6785 10279 : switch (gimple_assign_rhs_code (stmt))
6786 : {
6787 585 : case MULT_EXPR:
6788 585 : case TRUNC_DIV_EXPR:
6789 585 : case EXACT_DIV_EXPR:
6790 585 : case TRUNC_MOD_EXPR:
6791 585 : case FLOAT_EXPR:
6792 585 : return true;
6793 : default:
6794 : break;
6795 : }
6796 4800 : else if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
6797 46 : switch (gimple_call_internal_fn (stmt))
6798 : {
6799 17 : case IFN_MUL_OVERFLOW:
6800 17 : case IFN_UBSAN_CHECK_MUL:
6801 17 : return true;
6802 : /* These two actually don't take address, but reshuffle
6803 : all bytes or bits, so need similar treatment. */
6804 4 : case IFN_BSWAP:
6805 4 : case IFN_BITREVERSE:
6806 4 : return true;
6807 : default:
6808 : break;
6809 : }
6810 : return false;
6811 : }
6812 :
6813 : /* Dominator walker used to discover which large/huge _BitInt
6814 : loads could be sunk into all their uses. */
6815 :
6816 624 : class bitint_dom_walker : public dom_walker
6817 : {
6818 : public:
6819 312 : bitint_dom_walker (bitmap names, bitmap loads)
6820 624 : : dom_walker (CDI_DOMINATORS), m_names (names), m_loads (loads) {}
6821 :
6822 : edge before_dom_children (basic_block) final override;
6823 :
6824 : private:
6825 : bitmap m_names, m_loads;
6826 : };
6827 :
6828 : edge
6829 4508 : bitint_dom_walker::before_dom_children (basic_block bb)
6830 : {
6831 4508 : gphi *phi = get_virtual_phi (bb);
6832 4508 : tree vop;
6833 4508 : if (phi)
6834 794 : vop = gimple_phi_result (phi);
6835 3714 : else if (bb == ENTRY_BLOCK_PTR_FOR_FN (cfun))
6836 : vop = NULL_TREE;
6837 : else
6838 3402 : vop = (tree) get_immediate_dominator (CDI_DOMINATORS, bb)->aux;
6839 :
6840 4508 : auto_vec<tree, 16> worklist;
6841 9016 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
6842 19976 : !gsi_end_p (gsi); gsi_next (&gsi))
6843 : {
6844 15468 : gimple *stmt = gsi_stmt (gsi);
6845 15468 : if (is_gimple_debug (stmt))
6846 2764 : continue;
6847 :
6848 15513 : if (!vop && gimple_vuse (stmt))
6849 : vop = gimple_vuse (stmt);
6850 :
6851 15079 : tree cvop = vop;
6852 28300 : if (gimple_vdef (stmt))
6853 15079 : vop = gimple_vdef (stmt);
6854 :
6855 15079 : tree lhs = gimple_get_lhs (stmt);
6856 17454 : if (lhs
6857 10954 : && TREE_CODE (lhs) == SSA_NAME
6858 8733 : && BITINT_TYPE_P (TREE_TYPE (lhs))
6859 5860 : && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
6860 20791 : && !bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
6861 : /* If lhs of stmt is large/huge _BitInt SSA_NAME not in m_names,
6862 : it means it will be handled in a loop or straight line code
6863 : at the location of its (ultimate) immediate use, so for
6864 : vop checking purposes check these only at the ultimate
6865 : immediate use. */
6866 2375 : continue;
6867 :
6868 12704 : ssa_op_iter oi;
6869 12704 : use_operand_p use_p;
6870 21536 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, oi, SSA_OP_USE)
6871 : {
6872 8832 : tree s = USE_FROM_PTR (use_p);
6873 14246 : if (BITINT_TYPE_P (TREE_TYPE (s))
6874 8832 : && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
6875 3263 : worklist.safe_push (s);
6876 : }
6877 :
6878 12704 : bool needs_operand_addr = stmt_needs_operand_addr (stmt);
6879 31504 : while (worklist.length () > 0)
6880 : {
6881 6096 : tree s = worklist.pop ();
6882 :
6883 6096 : if (!bitmap_bit_p (m_names, SSA_NAME_VERSION (s)))
6884 : {
6885 2375 : gimple *g = SSA_NAME_DEF_STMT (s);
6886 2375 : needs_operand_addr |= stmt_needs_operand_addr (g);
6887 5447 : FOR_EACH_SSA_USE_OPERAND (use_p, g, oi, SSA_OP_USE)
6888 : {
6889 3072 : tree s2 = USE_FROM_PTR (use_p);
6890 3262 : if (BITINT_TYPE_P (TREE_TYPE (s2))
6891 3072 : && (bitint_precision_kind (TREE_TYPE (s2))
6892 : >= bitint_prec_large))
6893 2833 : worklist.safe_push (s2);
6894 : }
6895 3094 : continue;
6896 2375 : }
6897 3721 : if (!SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
6898 3721 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (s)))
6899 : {
6900 226 : tree rhs = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s));
6901 396 : if (TREE_CODE (rhs) == SSA_NAME
6902 226 : && bitmap_bit_p (m_loads, SSA_NAME_VERSION (rhs)))
6903 : s = rhs;
6904 : else
6905 170 : continue;
6906 : }
6907 3495 : else if (!bitmap_bit_p (m_loads, SSA_NAME_VERSION (s)))
6908 547 : continue;
6909 :
6910 3004 : gimple *g = SSA_NAME_DEF_STMT (s);
6911 3004 : tree rhs1 = gimple_assign_rhs1 (g);
6912 3004 : if (needs_operand_addr
6913 218 : && TREE_CODE (rhs1) == COMPONENT_REF
6914 3021 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (rhs1, 1)))
6915 : {
6916 4 : tree fld = TREE_OPERAND (rhs1, 1);
6917 : /* For little-endian, we can allow as inputs bit-fields
6918 : which start at a limb boundary. */
6919 6 : if (!bitint_big_endian
6920 4 : && DECL_OFFSET_ALIGN (fld) >= TYPE_ALIGN (TREE_TYPE (rhs1))
6921 4 : && tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld))
6922 4 : && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
6923 4 : % limb_prec) == 0)
6924 : ;
6925 : else
6926 : {
6927 2 : bitmap_clear_bit (m_loads, SSA_NAME_VERSION (s));
6928 2 : continue;
6929 : }
6930 : }
6931 :
6932 3002 : ao_ref ref;
6933 3002 : ao_ref_init (&ref, rhs1);
6934 3002 : tree lvop = gimple_vuse (g);
6935 3002 : unsigned limit = 64;
6936 3002 : tree vuse = cvop;
6937 3002 : if (vop != cvop
6938 1334 : && is_gimple_assign (stmt)
6939 1332 : && gimple_store_p (stmt)
6940 4334 : && (needs_operand_addr
6941 1139 : || !operand_equal_p (lhs, gimple_assign_rhs1 (g), 0)))
6942 : vuse = vop;
6943 3002 : if (vuse != lvop
6944 3002 : && walk_non_aliased_vuses (&ref, vuse, false, vuse_eq,
6945 : NULL, NULL, NULL, limit, lvop) == NULL)
6946 528 : bitmap_clear_bit (m_loads, SSA_NAME_VERSION (s));
6947 : }
6948 : }
6949 :
6950 4508 : bb->aux = (void *) vop;
6951 4508 : return NULL;
6952 4508 : }
6953 :
6954 : }
6955 :
6956 : /* Replacement for normal processing of STMT in tree-ssa-coalesce.cc
6957 : build_ssa_conflict_graph.
6958 : The differences are:
6959 : 1) don't process assignments with large/huge _BitInt lhs not in NAMES
6960 : 2) for large/huge _BitInt multiplication/division/modulo process def
6961 : only after processing uses rather than before to make uses conflict
6962 : with the definition
6963 : 3) for large/huge _BitInt uses not in NAMES mark the uses of their
6964 : SSA_NAME_DEF_STMT (recursively), because those uses will be sunk into
6965 : the final statement. */
6966 :
6967 : void
6968 83589 : build_bitint_stmt_ssa_conflicts (gimple *stmt, live_track *live,
6969 : ssa_conflicts *graph, bitmap names,
6970 : void (*def) (live_track *, tree,
6971 : ssa_conflicts *),
6972 : void (*use) (live_track *, tree),
6973 : void (*clear) (live_track *, tree))
6974 : {
6975 83589 : bool muldiv_p = false;
6976 83589 : tree lhs = NULL_TREE;
6977 83589 : if (is_gimple_assign (stmt))
6978 : {
6979 45886 : lhs = gimple_assign_lhs (stmt);
6980 45886 : if (TREE_CODE (lhs) == SSA_NAME)
6981 : {
6982 33117 : tree type = TREE_TYPE (lhs);
6983 33117 : if (TREE_CODE (type) == COMPLEX_TYPE)
6984 63 : type = TREE_TYPE (type);
6985 12675 : if (BITINT_TYPE_P (type)
6986 33122 : && bitint_precision_kind (type) >= bitint_prec_large)
6987 : {
6988 19832 : if (!bitmap_bit_p (names, SSA_NAME_VERSION (lhs)))
6989 4846 : return;
6990 :
6991 : /* A copy between 2 partitions does not introduce an interference
6992 : by itself. If they did, you would never be able to coalesce
6993 : two things which are copied. If the two variables really do
6994 : conflict, they will conflict elsewhere in the program.
6995 :
6996 : This is handled by simply removing the SRC of the copy from
6997 : the live list, and processing the stmt normally.
6998 :
6999 : Don't do this if lhs is not in names though, in such cases
7000 : it is actually used at some point later in the basic
7001 : block. */
7002 14986 : if (gimple_assign_copy_p (stmt))
7003 : {
7004 1667 : tree rhs1 = gimple_assign_rhs1 (stmt);
7005 1667 : if (TREE_CODE (rhs1) == SSA_NAME)
7006 31 : clear (live, rhs1);
7007 : }
7008 :
7009 14986 : switch (gimple_assign_rhs_code (stmt))
7010 : {
7011 151 : case MULT_EXPR:
7012 151 : case TRUNC_DIV_EXPR:
7013 151 : case EXACT_DIV_EXPR:
7014 151 : case TRUNC_MOD_EXPR:
7015 151 : muldiv_p = true;
7016 : default:
7017 : break;
7018 : }
7019 : }
7020 : }
7021 : }
7022 37703 : else if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
7023 4245 : switch (gimple_call_internal_fn (stmt))
7024 : {
7025 4134 : case IFN_ADD_OVERFLOW:
7026 4134 : case IFN_SUB_OVERFLOW:
7027 4134 : case IFN_UBSAN_CHECK_ADD:
7028 4134 : case IFN_UBSAN_CHECK_SUB:
7029 4134 : case IFN_MUL_OVERFLOW:
7030 4134 : case IFN_UBSAN_CHECK_MUL:
7031 4134 : if (bitint_big_endian)
7032 : {
7033 0 : lhs = gimple_call_lhs (stmt);
7034 0 : if (lhs)
7035 78743 : muldiv_p = true;
7036 : }
7037 : break;
7038 11 : case IFN_BSWAP:
7039 11 : case IFN_BITREVERSE:
7040 11 : lhs = gimple_call_lhs (stmt);
7041 11 : if (lhs)
7042 78743 : muldiv_p = true;
7043 : break;
7044 : default:
7045 : break;
7046 : }
7047 :
7048 157486 : auto_vec<tree, 16> worklist;
7049 78743 : ssa_op_iter iter;
7050 78743 : tree var;
7051 : /* On little-endian, mergeable ops process limbs from 0 up so except
7052 : for multiplication/division/modulo there is no risk in using the
7053 : same underlying variable for lhs and some operand, even when casts
7054 : are involved, the lhs limb is stored only after processing the source
7055 : limbs with the same index.
7056 : For multiplication/division/modulo, the libgcc library function requires
7057 : no aliasing between result and sources.
7058 : On big-endian, even mergeable ops limb processing can be problematic
7059 : though, because it can apply various index corrections e.g. when there
7060 : is a cast from operand with different number of limbs. So, make the
7061 : lhs conflict with all the operands which are (for now virtually) used on
7062 : the current stmt if there is any mismatch in the number of limbs between
7063 : operands and the lhs. */
7064 78743 : if (bitint_big_endian && lhs && !muldiv_p)
7065 : {
7066 0 : tree ltype = TREE_TYPE (lhs);
7067 0 : if (TREE_CODE (ltype) == COMPLEX_TYPE)
7068 : muldiv_p = true;
7069 0 : else if (TREE_CODE (lhs) == SSA_NAME
7070 0 : && BITINT_TYPE_P (ltype)
7071 0 : && bitint_precision_kind (ltype) >= bitint_prec_large)
7072 : {
7073 0 : unsigned lnelts = CEIL (TYPE_PRECISION (ltype), limb_prec);
7074 0 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_USE)
7075 : {
7076 0 : tree type = TREE_TYPE (var);
7077 0 : if (TREE_CODE (type) == COMPLEX_TYPE)
7078 0 : type = TREE_TYPE (type);
7079 0 : if (BITINT_TYPE_P (type)
7080 0 : && bitint_precision_kind (type) >= bitint_prec_large)
7081 : {
7082 0 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7083 : {
7084 0 : unsigned nelts = CEIL (TYPE_PRECISION (type), limb_prec);
7085 0 : if (TREE_CODE (TREE_TYPE (var)) == COMPLEX_TYPE
7086 0 : || lnelts != nelts)
7087 : {
7088 0 : muldiv_p = true;
7089 : break;
7090 : }
7091 : }
7092 : else
7093 0 : worklist.safe_push (var);
7094 : }
7095 : }
7096 :
7097 0 : while (!muldiv_p && worklist.length () > 0)
7098 : {
7099 0 : tree s = worklist.pop ();
7100 0 : FOR_EACH_SSA_TREE_OPERAND (var, SSA_NAME_DEF_STMT (s), iter,
7101 : SSA_OP_USE)
7102 : {
7103 0 : tree type = TREE_TYPE (var);
7104 0 : if (TREE_CODE (type) == COMPLEX_TYPE)
7105 0 : type = TREE_TYPE (type);
7106 0 : if (BITINT_TYPE_P (type)
7107 0 : && bitint_precision_kind (type) >= bitint_prec_large)
7108 : {
7109 0 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7110 : {
7111 0 : unsigned nelts = CEIL (TYPE_PRECISION (type),
7112 : limb_prec);
7113 0 : if (TREE_CODE (TREE_TYPE (var)) == COMPLEX_TYPE
7114 0 : || lnelts != nelts)
7115 : {
7116 : muldiv_p = true;
7117 : break;
7118 : }
7119 : }
7120 : else
7121 0 : worklist.safe_push (var);
7122 : }
7123 : }
7124 : }
7125 0 : worklist.truncate (0);
7126 : }
7127 : }
7128 :
7129 78743 : if (!muldiv_p)
7130 : {
7131 : /* For stmts with more than one SSA_NAME definition pretend all the
7132 : SSA_NAME outputs but the first one are live at this point, so
7133 : that conflicts are added in between all those even when they are
7134 : actually not really live after the asm, because expansion might
7135 : copy those into pseudos after the asm and if multiple outputs
7136 : share the same partition, it might overwrite those that should
7137 : be live. E.g.
7138 : asm volatile (".." : "=r" (a) : "=r" (b) : "0" (a), "1" (a));
7139 : return a;
7140 : See PR70593. */
7141 78581 : bool first = true;
7142 115750 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
7143 37169 : if (first)
7144 : first = false;
7145 : else
7146 0 : use (live, var);
7147 :
7148 115750 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
7149 37169 : def (live, var, graph);
7150 : }
7151 :
7152 134258 : FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_USE)
7153 : {
7154 55515 : tree type = TREE_TYPE (var);
7155 55515 : if (TREE_CODE (type) == COMPLEX_TYPE)
7156 6287 : type = TREE_TYPE (type);
7157 18686 : if (BITINT_TYPE_P (type)
7158 55532 : && bitint_precision_kind (type) >= bitint_prec_large)
7159 : {
7160 35850 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7161 30795 : use (live, var);
7162 : else
7163 5055 : worklist.safe_push (var);
7164 : }
7165 : }
7166 :
7167 86786 : while (worklist.length () > 0)
7168 : {
7169 8043 : tree s = worklist.pop ();
7170 16824 : FOR_EACH_SSA_TREE_OPERAND (var, SSA_NAME_DEF_STMT (s), iter, SSA_OP_USE)
7171 : {
7172 8781 : tree type = TREE_TYPE (var);
7173 8781 : if (TREE_CODE (type) == COMPLEX_TYPE)
7174 1673 : type = TREE_TYPE (type);
7175 528 : if (BITINT_TYPE_P (type)
7176 8786 : && bitint_precision_kind (type) >= bitint_prec_large)
7177 : {
7178 8123 : if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
7179 5135 : use (live, var);
7180 : else
7181 2988 : worklist.safe_push (var);
7182 : }
7183 : }
7184 : }
7185 :
7186 78743 : if (muldiv_p)
7187 162 : def (live, lhs, graph);
7188 : }
7189 :
7190 : /* If STMT is .{ADD,SUB,MUL}_OVERFLOW with INTEGER_CST arguments,
7191 : return the largest bitint_prec_kind of them, otherwise return
7192 : bitint_prec_small. */
7193 :
7194 : static bitint_prec_kind
7195 190738 : arith_overflow_arg_kind (gimple *stmt)
7196 : {
7197 190738 : bitint_prec_kind ret = bitint_prec_small;
7198 190738 : if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
7199 88192 : switch (gimple_call_internal_fn (stmt))
7200 : {
7201 : case IFN_ADD_OVERFLOW:
7202 : case IFN_SUB_OVERFLOW:
7203 : case IFN_MUL_OVERFLOW:
7204 224052 : for (int i = 0; i < 2; ++i)
7205 : {
7206 149368 : tree a = gimple_call_arg (stmt, i);
7207 149368 : if (TREE_CODE (a) == INTEGER_CST
7208 149368 : && BITINT_TYPE_P (TREE_TYPE (a)))
7209 : {
7210 5928 : bitint_prec_kind kind = bitint_precision_kind (TREE_TYPE (a));
7211 149368 : ret = MAX (ret, kind);
7212 : }
7213 : }
7214 : break;
7215 : default:
7216 : break;
7217 : }
7218 190738 : return ret;
7219 : }
7220 :
7221 : /* Entry point for _BitInt(N) operation lowering during optimization. */
7222 :
7223 : static unsigned int
7224 1489264 : gimple_lower_bitint (void)
7225 : {
7226 1489264 : small_max_prec = mid_min_prec = large_min_prec = huge_min_prec = 0;
7227 1489264 : limb_prec = abi_limb_prec = 0;
7228 1489264 : bitint_big_endian = false;
7229 :
7230 1489264 : unsigned int i;
7231 62880884 : for (i = 0; i < num_ssa_names; ++i)
7232 : {
7233 61398876 : tree s = ssa_name (i);
7234 61398876 : if (s == NULL)
7235 12803155 : continue;
7236 48595721 : tree type = TREE_TYPE (s);
7237 48595721 : if (TREE_CODE (type) == COMPLEX_TYPE)
7238 : {
7239 180092 : if (arith_overflow_arg_kind (SSA_NAME_DEF_STMT (s))
7240 : != bitint_prec_small)
7241 : break;
7242 179979 : type = TREE_TYPE (type);
7243 : }
7244 48579513 : if (BITINT_TYPE_P (type)
7245 48595615 : && bitint_precision_kind (type) != bitint_prec_small)
7246 : break;
7247 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
7248 : into memory. Such functions could have no large/huge SSA_NAMEs. */
7249 48588530 : if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
7250 : {
7251 21185996 : gimple *g = SSA_NAME_DEF_STMT (s);
7252 21185996 : if (is_gimple_assign (g) && gimple_store_p (g))
7253 : {
7254 10732321 : tree t = gimple_assign_rhs1 (g);
7255 21462204 : if (BITINT_TYPE_P (TREE_TYPE (t))
7256 10732321 : && (bitint_precision_kind (TREE_TYPE (t))
7257 : >= bitint_prec_large))
7258 : break;
7259 : }
7260 : }
7261 : /* Similarly, e.g. with -frounding-math casts from _BitInt INTEGER_CSTs
7262 : to floating point types need to be rewritten. */
7263 27402534 : else if (SCALAR_FLOAT_TYPE_P (type))
7264 : {
7265 2299521 : gimple *g = SSA_NAME_DEF_STMT (s);
7266 2299521 : if (is_gimple_assign (g) && gimple_assign_rhs_code (g) == FLOAT_EXPR)
7267 : {
7268 127794 : tree t = gimple_assign_rhs1 (g);
7269 127794 : if (TREE_CODE (t) == INTEGER_CST
7270 110 : && BITINT_TYPE_P (TREE_TYPE (t))
7271 127795 : && (bitint_precision_kind (TREE_TYPE (t))
7272 : != bitint_prec_small))
7273 : break;
7274 : }
7275 : }
7276 : }
7277 2978528 : if (i == num_ssa_names)
7278 : return 0;
7279 :
7280 7256 : basic_block bb;
7281 7256 : auto_vec<gimple *, 4> switch_statements;
7282 45780 : FOR_EACH_BB_FN (bb, cfun)
7283 : {
7284 114834 : if (gswitch *swtch = safe_dyn_cast <gswitch *> (*gsi_last_bb (bb)))
7285 : {
7286 23 : tree idx = gimple_switch_index (swtch);
7287 31 : if (!BITINT_TYPE_P (TREE_TYPE (idx))
7288 23 : || bitint_precision_kind (TREE_TYPE (idx)) < bitint_prec_large)
7289 12 : continue;
7290 :
7291 11 : if (optimize)
7292 6 : group_case_labels_stmt (swtch);
7293 11 : if (gimple_switch_num_labels (swtch) == 1)
7294 : {
7295 0 : single_succ_edge (bb)->flags |= EDGE_FALLTHRU;
7296 0 : gimple_stmt_iterator gsi = gsi_for_stmt (swtch);
7297 0 : gsi_remove (&gsi, true);
7298 : }
7299 : else
7300 11 : switch_statements.safe_push (swtch);
7301 : }
7302 : }
7303 :
7304 7256 : if (!switch_statements.is_empty ())
7305 : {
7306 11 : bool expanded = false;
7307 11 : gimple *stmt;
7308 11 : unsigned int j;
7309 11 : i = 0;
7310 22 : FOR_EACH_VEC_ELT (switch_statements, j, stmt)
7311 : {
7312 11 : gswitch *swtch = as_a<gswitch *> (stmt);
7313 11 : tree_switch_conversion::switch_decision_tree dt (swtch);
7314 11 : expanded |= dt.analyze_switch_statement ();
7315 11 : }
7316 :
7317 11 : if (expanded)
7318 : {
7319 11 : free_dominance_info (CDI_DOMINATORS);
7320 11 : free_dominance_info (CDI_POST_DOMINATORS);
7321 11 : mark_virtual_operands_for_renaming (cfun);
7322 11 : cleanup_tree_cfg (TODO_update_ssa);
7323 : }
7324 : }
7325 :
7326 7256 : struct bitint_large_huge large_huge;
7327 7256 : bool has_large_huge_parm_result = false;
7328 7256 : bool has_large_huge = false;
7329 7256 : unsigned int ret = 0, first_large_huge = ~0U;
7330 7256 : bool edge_insertions = false;
7331 125435 : for (; i < num_ssa_names; ++i)
7332 : {
7333 118179 : tree s = ssa_name (i);
7334 118179 : if (s == NULL)
7335 2730 : continue;
7336 115449 : tree type = TREE_TYPE (s);
7337 115449 : if (TREE_CODE (type) == COMPLEX_TYPE)
7338 : {
7339 5338 : if (arith_overflow_arg_kind (SSA_NAME_DEF_STMT (s))
7340 : >= bitint_prec_large)
7341 1957 : has_large_huge = true;
7342 5338 : type = TREE_TYPE (type);
7343 : }
7344 69693 : if (BITINT_TYPE_P (type)
7345 115475 : && bitint_precision_kind (type) >= bitint_prec_large)
7346 : {
7347 35770 : if (first_large_huge == ~0U)
7348 5713 : first_large_huge = i;
7349 35770 : gimple *stmt = SSA_NAME_DEF_STMT (s), *g;
7350 35770 : gimple_stmt_iterator gsi;
7351 35770 : tree_code rhs_code;
7352 : /* Unoptimize certain constructs to simpler alternatives to
7353 : avoid having to lower all of them. */
7354 35770 : if (is_gimple_assign (stmt) && gimple_bb (stmt))
7355 22095 : switch (rhs_code = gimple_assign_rhs_code (stmt))
7356 : {
7357 : default:
7358 : break;
7359 356 : case MULT_EXPR:
7360 356 : case TRUNC_DIV_EXPR:
7361 356 : case EXACT_DIV_EXPR:
7362 356 : case TRUNC_MOD_EXPR:
7363 356 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s))
7364 : {
7365 2 : location_t loc = gimple_location (stmt);
7366 2 : gsi = gsi_for_stmt (stmt);
7367 2 : tree rhs1 = gimple_assign_rhs1 (stmt);
7368 2 : tree rhs2 = gimple_assign_rhs2 (stmt);
7369 : /* For multiplication and division with (ab)
7370 : lhs and one or both operands force the operands
7371 : into new SSA_NAMEs to avoid coalescing failures. */
7372 2 : if (TREE_CODE (rhs1) == SSA_NAME
7373 2 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1))
7374 : {
7375 2 : first_large_huge = 0;
7376 2 : tree t = make_ssa_name (TREE_TYPE (rhs1));
7377 2 : g = gimple_build_assign (t, SSA_NAME, rhs1);
7378 2 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7379 2 : gimple_set_location (g, loc);
7380 2 : gimple_assign_set_rhs1 (stmt, t);
7381 2 : if (rhs1 == rhs2)
7382 : {
7383 0 : gimple_assign_set_rhs2 (stmt, t);
7384 0 : rhs2 = t;
7385 : }
7386 2 : update_stmt (stmt);
7387 : }
7388 2 : if (TREE_CODE (rhs2) == SSA_NAME
7389 2 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs2))
7390 : {
7391 0 : first_large_huge = 0;
7392 0 : tree t = make_ssa_name (TREE_TYPE (rhs2));
7393 0 : g = gimple_build_assign (t, SSA_NAME, rhs2);
7394 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7395 0 : gimple_set_location (g, loc);
7396 0 : gimple_assign_set_rhs2 (stmt, t);
7397 0 : update_stmt (stmt);
7398 : }
7399 : }
7400 : break;
7401 3 : case LROTATE_EXPR:
7402 3 : case RROTATE_EXPR:
7403 3 : {
7404 3 : first_large_huge = 0;
7405 3 : location_t loc = gimple_location (stmt);
7406 3 : gsi = gsi_for_stmt (stmt);
7407 3 : tree rhs1 = gimple_assign_rhs1 (stmt);
7408 3 : tree type = TREE_TYPE (rhs1);
7409 3 : tree n = gimple_assign_rhs2 (stmt), m;
7410 3 : tree p = build_int_cst (TREE_TYPE (n),
7411 3 : TYPE_PRECISION (type));
7412 3 : if (TREE_CODE (n) == INTEGER_CST)
7413 : {
7414 0 : if (integer_zerop (n))
7415 : m = n;
7416 : else
7417 0 : m = fold_build2 (MINUS_EXPR, TREE_TYPE (n), p, n);
7418 : }
7419 : else
7420 : {
7421 3 : tree tem = make_ssa_name (TREE_TYPE (n));
7422 3 : g = gimple_build_assign (tem, MINUS_EXPR, p, n);
7423 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7424 3 : gimple_set_location (g, loc);
7425 3 : m = make_ssa_name (TREE_TYPE (n));
7426 3 : g = gimple_build_assign (m, TRUNC_MOD_EXPR, tem, p);
7427 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7428 3 : gimple_set_location (g, loc);
7429 : }
7430 3 : if (!TYPE_UNSIGNED (type))
7431 : {
7432 0 : tree utype = build_bitint_type (TYPE_PRECISION (type),
7433 : 1);
7434 0 : if (TREE_CODE (rhs1) == INTEGER_CST)
7435 0 : rhs1 = fold_convert (utype, rhs1);
7436 : else
7437 : {
7438 0 : tree t = make_ssa_name (type);
7439 0 : g = gimple_build_assign (t, NOP_EXPR, rhs1);
7440 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7441 0 : gimple_set_location (g, loc);
7442 : }
7443 : }
7444 4 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7445 : rhs_code == LROTATE_EXPR
7446 : ? LSHIFT_EXPR : RSHIFT_EXPR,
7447 : rhs1, n);
7448 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7449 3 : gimple_set_location (g, loc);
7450 3 : tree op1 = gimple_assign_lhs (g);
7451 4 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7452 : rhs_code == LROTATE_EXPR
7453 : ? RSHIFT_EXPR : LSHIFT_EXPR,
7454 : rhs1, m);
7455 3 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7456 3 : gimple_set_location (g, loc);
7457 3 : tree op2 = gimple_assign_lhs (g);
7458 3 : tree lhs = gimple_assign_lhs (stmt);
7459 3 : if (!TYPE_UNSIGNED (type))
7460 : {
7461 0 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (op1)),
7462 : BIT_IOR_EXPR, op1, op2);
7463 0 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7464 0 : gimple_set_location (g, loc);
7465 0 : g = gimple_build_assign (lhs, NOP_EXPR,
7466 : gimple_assign_lhs (g));
7467 : }
7468 : else
7469 3 : g = gimple_build_assign (lhs, BIT_IOR_EXPR, op1, op2);
7470 3 : gsi_replace (&gsi, g, true);
7471 3 : gimple_set_location (g, loc);
7472 : }
7473 3 : break;
7474 21 : case ABS_EXPR:
7475 21 : case ABSU_EXPR:
7476 21 : case MIN_EXPR:
7477 21 : case MAX_EXPR:
7478 21 : case COND_EXPR:
7479 21 : first_large_huge = 0;
7480 21 : gsi = gsi_for_stmt (stmt);
7481 21 : tree lhs = gimple_assign_lhs (stmt);
7482 21 : tree rhs1 = gimple_assign_rhs1 (stmt), rhs2 = NULL_TREE;
7483 21 : location_t loc = gimple_location (stmt);
7484 21 : if (rhs_code == ABS_EXPR)
7485 4 : g = gimple_build_cond (LT_EXPR, rhs1,
7486 4 : build_zero_cst (TREE_TYPE (rhs1)),
7487 : NULL_TREE, NULL_TREE);
7488 17 : else if (rhs_code == ABSU_EXPR)
7489 : {
7490 8 : rhs2 = make_ssa_name (TREE_TYPE (lhs));
7491 8 : g = gimple_build_assign (rhs2, NOP_EXPR, rhs1);
7492 8 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7493 8 : gimple_set_location (g, loc);
7494 8 : g = gimple_build_cond (LT_EXPR, rhs1,
7495 8 : build_zero_cst (TREE_TYPE (rhs1)),
7496 : NULL_TREE, NULL_TREE);
7497 8 : rhs1 = rhs2;
7498 : }
7499 9 : else if (rhs_code == MIN_EXPR || rhs_code == MAX_EXPR)
7500 : {
7501 9 : rhs2 = gimple_assign_rhs2 (stmt);
7502 9 : if (TREE_CODE (rhs1) == INTEGER_CST)
7503 0 : std::swap (rhs1, rhs2);
7504 9 : g = gimple_build_cond (LT_EXPR, rhs1, rhs2,
7505 : NULL_TREE, NULL_TREE);
7506 9 : if (rhs_code == MAX_EXPR)
7507 5 : std::swap (rhs1, rhs2);
7508 : }
7509 : else
7510 : {
7511 0 : g = gimple_build_cond (NE_EXPR, rhs1,
7512 0 : build_zero_cst (TREE_TYPE (rhs1)),
7513 : NULL_TREE, NULL_TREE);
7514 0 : rhs1 = gimple_assign_rhs2 (stmt);
7515 0 : rhs2 = gimple_assign_rhs3 (stmt);
7516 : }
7517 21 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7518 21 : gimple_set_location (g, loc);
7519 21 : edge e1 = split_block (gsi_bb (gsi), g);
7520 21 : edge e2 = split_block (e1->dest, (gimple *) NULL);
7521 21 : edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
7522 21 : e3->probability = profile_probability::even ();
7523 21 : e1->flags = EDGE_TRUE_VALUE;
7524 21 : e1->probability = e3->probability.invert ();
7525 21 : if (dom_info_available_p (CDI_DOMINATORS))
7526 13 : set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
7527 21 : if (rhs_code == ABS_EXPR || rhs_code == ABSU_EXPR)
7528 : {
7529 12 : gsi = gsi_after_labels (e1->dest);
7530 12 : g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
7531 : NEGATE_EXPR, rhs1);
7532 12 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
7533 12 : gimple_set_location (g, loc);
7534 12 : rhs2 = gimple_assign_lhs (g);
7535 12 : std::swap (rhs1, rhs2);
7536 : }
7537 21 : gsi = gsi_for_stmt (stmt);
7538 21 : gsi_remove (&gsi, true);
7539 21 : gphi *phi = create_phi_node (lhs, e2->dest);
7540 21 : add_phi_arg (phi, rhs1, e2, UNKNOWN_LOCATION);
7541 21 : add_phi_arg (phi, rhs2, e3, UNKNOWN_LOCATION);
7542 21 : break;
7543 : }
7544 : }
7545 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
7546 : into memory. Such functions could have no large/huge SSA_NAMEs. */
7547 79679 : else if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
7548 : {
7549 50806 : gimple *g = SSA_NAME_DEF_STMT (s);
7550 50806 : if (is_gimple_assign (g) && gimple_store_p (g))
7551 : {
7552 16316 : tree t = gimple_assign_rhs1 (g);
7553 18493 : if (BITINT_TYPE_P (TREE_TYPE (t))
7554 16316 : && (bitint_precision_kind (TREE_TYPE (t))
7555 : >= bitint_prec_large))
7556 : has_large_huge = true;
7557 : }
7558 : }
7559 : /* Similarly, e.g. with -frounding-math casts from _BitInt INTEGER_CSTs
7560 : to floating point types need to be rewritten. */
7561 28873 : else if (SCALAR_FLOAT_TYPE_P (type))
7562 : {
7563 686 : gimple *g = SSA_NAME_DEF_STMT (s);
7564 686 : if (is_gimple_assign (g) && gimple_assign_rhs_code (g) == FLOAT_EXPR)
7565 : {
7566 179 : tree t = gimple_assign_rhs1 (g);
7567 179 : if (TREE_CODE (t) == INTEGER_CST
7568 1 : && BITINT_TYPE_P (TREE_TYPE (t))
7569 180 : && (bitint_precision_kind (TREE_TYPE (t))
7570 : >= bitint_prec_large))
7571 : has_large_huge = true;
7572 : }
7573 : }
7574 : }
7575 102124 : for (i = first_large_huge; i < num_ssa_names; ++i)
7576 : {
7577 94868 : tree s = ssa_name (i);
7578 94868 : if (s == NULL)
7579 2421 : continue;
7580 92447 : tree type = TREE_TYPE (s);
7581 92447 : if (TREE_CODE (type) == COMPLEX_TYPE)
7582 4109 : type = TREE_TYPE (type);
7583 56055 : if (BITINT_TYPE_P (type)
7584 92471 : && bitint_precision_kind (type) >= bitint_prec_large)
7585 : {
7586 35774 : use_operand_p use_p;
7587 35774 : gimple *use_stmt;
7588 35774 : has_large_huge = true;
7589 37436 : if (optimize
7590 52486 : && optimizable_arith_overflow (SSA_NAME_DEF_STMT (s)))
7591 6424 : continue;
7592 : /* Ignore large/huge _BitInt SSA_NAMEs which have single use in
7593 : the same bb and could be handled in the same loop with the
7594 : immediate use. */
7595 34112 : if (optimize
7596 15050 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
7597 15028 : && single_imm_use (s, &use_p, &use_stmt)
7598 48572 : && gimple_bb (SSA_NAME_DEF_STMT (s)) == gimple_bb (use_stmt))
7599 : {
7600 10174 : if (mergeable_op (SSA_NAME_DEF_STMT (s)))
7601 : {
7602 2146 : if (mergeable_op (use_stmt))
7603 1868 : continue;
7604 278 : tree_code cmp_code = comparison_op (use_stmt, NULL, NULL);
7605 278 : if (cmp_code == EQ_EXPR || cmp_code == NE_EXPR)
7606 26 : continue;
7607 252 : if (gimple_assign_cast_p (use_stmt))
7608 : {
7609 109 : tree lhs = gimple_assign_lhs (use_stmt);
7610 218 : if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
7611 : /* Don't merge with VIEW_CONVERT_EXPRs to
7612 : huge INTEGER_TYPEs used sometimes in memcpy
7613 : expansion. */
7614 206 : && (TREE_CODE (TREE_TYPE (lhs)) != INTEGER_TYPE
7615 0 : || (TYPE_PRECISION (TREE_TYPE (lhs))
7616 0 : <= MAX_FIXED_MODE_SIZE)))
7617 97 : continue;
7618 : }
7619 143 : else if (gimple_store_p (use_stmt)
7620 0 : && is_gimple_assign (use_stmt)
7621 0 : && !gimple_has_volatile_ops (use_stmt)
7622 143 : && !stmt_ends_bb_p (use_stmt))
7623 0 : continue;
7624 : }
7625 8183 : if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (s)))
7626 : {
7627 827 : tree rhs1 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s));
7628 827 : if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
7629 : {
7630 20 : rhs1 = TREE_OPERAND (rhs1, 0);
7631 20 : if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
7632 16 : && !POINTER_TYPE_P (TREE_TYPE (rhs1))
7633 16 : && gimple_store_p (use_stmt))
7634 7 : continue;
7635 : }
7636 1640 : if (INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
7637 729 : && ((is_gimple_assign (use_stmt)
7638 682 : && (gimple_assign_rhs_code (use_stmt)
7639 : != COMPLEX_EXPR))
7640 47 : || gimple_code (use_stmt) == GIMPLE_COND)
7641 719 : && (!gimple_store_p (use_stmt)
7642 122 : || (is_gimple_assign (use_stmt)
7643 122 : && !gimple_has_volatile_ops (use_stmt)
7644 122 : && !stmt_ends_bb_p (use_stmt)))
7645 1539 : && (TREE_CODE (rhs1) != SSA_NAME
7646 719 : || !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)))
7647 : {
7648 719 : if (is_gimple_assign (use_stmt))
7649 682 : switch (gimple_assign_rhs_code (use_stmt))
7650 : {
7651 54 : case TRUNC_DIV_EXPR:
7652 54 : case EXACT_DIV_EXPR:
7653 54 : case TRUNC_MOD_EXPR:
7654 54 : case FLOAT_EXPR:
7655 : /* For division, modulo and casts to floating
7656 : point, avoid representing unsigned operands
7657 : using negative prec if they were sign-extended
7658 : from narrower precision. */
7659 54 : if (TYPE_UNSIGNED (TREE_TYPE (s))
7660 28 : && !TYPE_UNSIGNED (TREE_TYPE (rhs1))
7661 63 : && (TYPE_PRECISION (TREE_TYPE (s))
7662 9 : > TYPE_PRECISION (TREE_TYPE (rhs1))))
7663 8 : goto force_name;
7664 : /* FALLTHRU */
7665 107 : case MULT_EXPR:
7666 116 : if (!BITINT_TYPE_P (TREE_TYPE (rhs1))
7667 107 : || (bitint_precision_kind (TREE_TYPE (rhs1))
7668 : < bitint_prec_large))
7669 42 : continue;
7670 : /* Uses which use handle_operand_addr can't
7671 : deal with nested casts. */
7672 65 : if (TREE_CODE (rhs1) == SSA_NAME
7673 65 : && gimple_assign_cast_p
7674 65 : (SSA_NAME_DEF_STMT (rhs1))
7675 43 : && has_single_use (rhs1)
7676 108 : && (gimple_bb (SSA_NAME_DEF_STMT (rhs1))
7677 43 : == gimple_bb (SSA_NAME_DEF_STMT (s))))
7678 43 : goto force_name;
7679 : break;
7680 0 : case VIEW_CONVERT_EXPR:
7681 0 : {
7682 0 : tree lhs = gimple_assign_lhs (use_stmt);
7683 : /* Don't merge with VIEW_CONVERT_EXPRs to
7684 : non-integral types. */
7685 0 : if (!INTEGRAL_TYPE_P (TREE_TYPE (lhs)))
7686 0 : goto force_name;
7687 : /* Don't merge with VIEW_CONVERT_EXPRs to
7688 : huge INTEGER_TYPEs used sometimes in memcpy
7689 : expansion. */
7690 0 : if (TREE_CODE (TREE_TYPE (lhs)) == INTEGER_TYPE
7691 0 : && (TYPE_PRECISION (TREE_TYPE (lhs))
7692 0 : > MAX_FIXED_MODE_SIZE))
7693 0 : goto force_name;
7694 : }
7695 : break;
7696 : default:
7697 : break;
7698 : }
7699 802 : if (!BITINT_TYPE_P (TREE_TYPE (rhs1))
7700 626 : || (bitint_precision_kind (TREE_TYPE (rhs1))
7701 : < bitint_prec_large))
7702 224 : continue;
7703 402 : if ((TYPE_PRECISION (TREE_TYPE (rhs1))
7704 402 : >= TYPE_PRECISION (TREE_TYPE (s)))
7705 402 : && mergeable_op (use_stmt))
7706 60 : continue;
7707 : /* Prevent merging a widening non-mergeable cast
7708 : on result of some narrower mergeable op
7709 : together with later mergeable operations. E.g.
7710 : result of _BitInt(223) addition shouldn't be
7711 : sign-extended to _BitInt(513) and have another
7712 : _BitInt(513) added to it, as handle_plus_minus
7713 : with its PHI node handling inside of handle_cast
7714 : will not work correctly. An exception is if
7715 : use_stmt is a store, this is handled directly
7716 : in lower_mergeable_stmt. */
7717 677 : if (TREE_CODE (rhs1) != SSA_NAME
7718 342 : || !has_single_use (rhs1)
7719 256 : || (gimple_bb (SSA_NAME_DEF_STMT (rhs1))
7720 256 : != gimple_bb (SSA_NAME_DEF_STMT (s)))
7721 207 : || !mergeable_op (SSA_NAME_DEF_STMT (rhs1))
7722 420 : || gimple_store_p (use_stmt))
7723 335 : continue;
7724 7 : if ((TYPE_PRECISION (TREE_TYPE (rhs1))
7725 7 : < TYPE_PRECISION (TREE_TYPE (s)))
7726 9 : && gimple_assign_cast_p (SSA_NAME_DEF_STMT (rhs1)))
7727 : {
7728 : /* Another exception is if the widening cast is
7729 : from mergeable same precision cast from something
7730 : not mergeable. */
7731 0 : tree rhs2
7732 0 : = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (rhs1));
7733 0 : if (BITINT_TYPE_P (TREE_TYPE (rhs2))
7734 0 : && (TYPE_PRECISION (TREE_TYPE (rhs1))
7735 0 : == TYPE_PRECISION (TREE_TYPE (rhs2))))
7736 : {
7737 0 : if (TREE_CODE (rhs2) != SSA_NAME
7738 0 : || !has_single_use (rhs2)
7739 0 : || (gimple_bb (SSA_NAME_DEF_STMT (rhs2))
7740 0 : != gimple_bb (SSA_NAME_DEF_STMT (s)))
7741 0 : || !mergeable_op (SSA_NAME_DEF_STMT (rhs2)))
7742 0 : continue;
7743 : }
7744 : }
7745 : }
7746 : }
7747 7464 : if (is_gimple_assign (SSA_NAME_DEF_STMT (s)))
7748 5379 : switch (gimple_assign_rhs_code (SSA_NAME_DEF_STMT (s)))
7749 : {
7750 1780 : case REALPART_EXPR:
7751 1780 : case IMAGPART_EXPR:
7752 1780 : {
7753 1780 : gimple *ds = SSA_NAME_DEF_STMT (s);
7754 1780 : tree rhs1 = gimple_assign_rhs1 (ds);
7755 1780 : rhs1 = TREE_OPERAND (rhs1, 0);
7756 1780 : if (TREE_CODE (rhs1) == SSA_NAME)
7757 : {
7758 1780 : gimple *g = SSA_NAME_DEF_STMT (rhs1);
7759 1780 : if (optimizable_arith_overflow (g))
7760 : {
7761 1648 : if (gimple_assign_rhs_code (ds) == IMAGPART_EXPR)
7762 1638 : continue;
7763 10 : if (gimple_store_p (use_stmt))
7764 : {
7765 : /* Punt if the cast use of IMAGPART_EXPR stmt
7766 : appears before the store use_stmt, because
7767 : optimizable arith overflow can't be
7768 : lowered at the store location in that case.
7769 : See PR121828. */
7770 10 : gimple_stmt_iterator gsi
7771 10 : = gsi_for_stmt (use_stmt);
7772 10 : unsigned int cnt = 0;
7773 12 : do
7774 : {
7775 12 : gsi_prev_nondebug (&gsi);
7776 12 : if (gsi_end_p (gsi))
7777 : break;
7778 12 : gimple *g2 = gsi_stmt (gsi);
7779 12 : if (g2 == ds)
7780 : break;
7781 3 : if (++cnt == 64)
7782 : break;
7783 3 : if (!gimple_assign_cast_p (g2))
7784 2 : continue;
7785 1 : tree rhs2 = gimple_assign_rhs1 (g2);
7786 1 : if (TREE_CODE (rhs2) != SSA_NAME)
7787 0 : continue;
7788 1 : gimple *g3 = SSA_NAME_DEF_STMT (rhs2);
7789 1 : if (!is_gimple_assign (g3))
7790 0 : continue;
7791 1 : if (gimple_assign_rhs_code (g3)
7792 : != IMAGPART_EXPR)
7793 0 : continue;
7794 1 : rhs2 = gimple_assign_rhs1 (g3);
7795 1 : rhs2 = TREE_OPERAND (rhs2, 0);
7796 1 : if (rhs2 != rhs1)
7797 0 : continue;
7798 : cnt = 64;
7799 : break;
7800 : }
7801 : while (1);
7802 10 : if (cnt == 64)
7803 : break;
7804 : }
7805 : }
7806 : }
7807 : }
7808 : /* FALLTHRU */
7809 819 : case LSHIFT_EXPR:
7810 819 : case RSHIFT_EXPR:
7811 819 : case MULT_EXPR:
7812 819 : case TRUNC_DIV_EXPR:
7813 819 : case EXACT_DIV_EXPR:
7814 819 : case TRUNC_MOD_EXPR:
7815 819 : case FIX_TRUNC_EXPR:
7816 819 : if (gimple_store_p (use_stmt)
7817 444 : && is_gimple_assign (use_stmt)
7818 444 : && !gimple_has_volatile_ops (use_stmt)
7819 1263 : && !stmt_ends_bb_p (use_stmt))
7820 : {
7821 444 : tree lhs = gimple_assign_lhs (use_stmt);
7822 : /* As multiply/division passes address of the lhs
7823 : to library function and that assumes it can extend
7824 : it to whole number of limbs, avoid merging those
7825 : with bit-field stores. Don't allow it for
7826 : shifts etc. either, so that the bit-field store
7827 : handling doesn't have to be done everywhere. */
7828 444 : if (TREE_CODE (lhs) == COMPONENT_REF
7829 444 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
7830 : break;
7831 441 : continue;
7832 441 : }
7833 : break;
7834 : default:
7835 : break;
7836 : }
7837 2085 : else if (is_gimple_call (SSA_NAME_DEF_STMT (s))
7838 2085 : && gimple_call_internal_p (SSA_NAME_DEF_STMT (s)))
7839 15 : switch (gimple_call_internal_fn (SSA_NAME_DEF_STMT (s)))
7840 : {
7841 13 : case IFN_BSWAP:
7842 13 : case IFN_BITREVERSE:
7843 13 : if (gimple_store_p (use_stmt)
7844 13 : && is_gimple_assign (use_stmt)
7845 13 : && !gimple_has_volatile_ops (use_stmt)
7846 26 : && !stmt_ends_bb_p (use_stmt))
7847 : {
7848 13 : tree lhs = gimple_assign_lhs (use_stmt);
7849 13 : if (TREE_CODE (lhs) == COMPONENT_REF
7850 13 : && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
7851 : break;
7852 13 : continue;
7853 13 : }
7854 : break;
7855 : default:
7856 : break;
7857 : }
7858 : }
7859 :
7860 : /* Also ignore uninitialized uses. */
7861 29310 : if (SSA_NAME_IS_DEFAULT_DEF (s)
7862 29310 : && (!SSA_NAME_VAR (s) || VAR_P (SSA_NAME_VAR (s))))
7863 11 : continue;
7864 :
7865 29350 : force_name:
7866 29350 : if (!large_huge.m_names)
7867 5612 : large_huge.m_names = BITMAP_ALLOC (NULL);
7868 29350 : bitmap_set_bit (large_huge.m_names, SSA_NAME_VERSION (s));
7869 29350 : if (has_single_use (s))
7870 : {
7871 : tree s2 = s;
7872 : /* The coalescing hook special cases SSA_NAME copies.
7873 : Make sure not to mark in m_single_use_names single
7874 : use SSA_NAMEs copied from non-single use SSA_NAMEs. */
7875 25694 : while (gimple_assign_copy_p (SSA_NAME_DEF_STMT (s2)))
7876 : {
7877 919 : s2 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s2));
7878 919 : if (TREE_CODE (s2) != SSA_NAME)
7879 : break;
7880 27 : if (!has_single_use (s2))
7881 : {
7882 : s2 = NULL_TREE;
7883 : break;
7884 : }
7885 : }
7886 25687 : if (s2)
7887 : {
7888 25667 : if (!large_huge.m_single_use_names)
7889 5498 : large_huge.m_single_use_names = BITMAP_ALLOC (NULL);
7890 25667 : bitmap_set_bit (large_huge.m_single_use_names,
7891 25667 : SSA_NAME_VERSION (s));
7892 : }
7893 : }
7894 29350 : if (SSA_NAME_VAR (s)
7895 7207 : && ((TREE_CODE (SSA_NAME_VAR (s)) == PARM_DECL
7896 5362 : && SSA_NAME_IS_DEFAULT_DEF (s))
7897 1882 : || TREE_CODE (SSA_NAME_VAR (s)) == RESULT_DECL))
7898 : has_large_huge_parm_result = true;
7899 29350 : if (optimize
7900 10288 : && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
7901 10266 : && gimple_assign_load_p (SSA_NAME_DEF_STMT (s))
7902 6062 : && !gimple_has_volatile_ops (SSA_NAME_DEF_STMT (s))
7903 32332 : && !stmt_ends_bb_p (SSA_NAME_DEF_STMT (s)))
7904 : {
7905 2982 : use_operand_p use_p;
7906 2982 : imm_use_iterator iter;
7907 2982 : bool optimizable_load = true;
7908 9010 : FOR_EACH_IMM_USE_FAST (use_p, iter, s)
7909 : {
7910 3144 : gimple *use_stmt = USE_STMT (use_p);
7911 3144 : if (is_gimple_debug (use_stmt))
7912 0 : continue;
7913 3144 : if (is_gimple_call (use_stmt)
7914 3144 : && gimple_call_internal_p (use_stmt))
7915 66 : switch (gimple_call_internal_fn (use_stmt))
7916 : {
7917 10 : case IFN_CLZ:
7918 10 : case IFN_CTZ:
7919 10 : case IFN_CLRSB:
7920 10 : case IFN_FFS:
7921 10 : case IFN_PARITY:
7922 10 : case IFN_POPCOUNT:
7923 10 : case IFN_BSWAP:
7924 10 : case IFN_BITREVERSE:
7925 10 : continue;
7926 : default:
7927 : break;
7928 : }
7929 3134 : if (gimple_code (use_stmt) == GIMPLE_PHI
7930 3122 : || is_gimple_call (use_stmt)
7931 3037 : || gimple_code (use_stmt) == GIMPLE_ASM
7932 6170 : || (is_gimple_assign (use_stmt)
7933 1902 : && (gimple_assign_rhs_code (use_stmt)
7934 : == COMPLEX_EXPR)))
7935 : {
7936 : optimizable_load = false;
7937 : break;
7938 : }
7939 2982 : }
7940 :
7941 2982 : ssa_op_iter oi;
7942 4027 : FOR_EACH_SSA_USE_OPERAND (use_p, SSA_NAME_DEF_STMT (s),
7943 : oi, SSA_OP_USE)
7944 : {
7945 1045 : tree s2 = USE_FROM_PTR (use_p);
7946 1045 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s2))
7947 : {
7948 : optimizable_load = false;
7949 : break;
7950 : }
7951 : }
7952 :
7953 2982 : if (optimizable_load && !stmt_ends_bb_p (SSA_NAME_DEF_STMT (s)))
7954 : {
7955 2884 : if (!large_huge.m_loads)
7956 312 : large_huge.m_loads = BITMAP_ALLOC (NULL);
7957 2884 : bitmap_set_bit (large_huge.m_loads, SSA_NAME_VERSION (s));
7958 : }
7959 : }
7960 : }
7961 : /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
7962 : into memory. Such functions could have no large/huge SSA_NAMEs. */
7963 56673 : else if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
7964 : {
7965 40032 : gimple *g = SSA_NAME_DEF_STMT (s);
7966 40032 : if (is_gimple_assign (g) && gimple_store_p (g))
7967 : {
7968 13676 : tree t = gimple_assign_rhs1 (g);
7969 15549 : if (BITINT_TYPE_P (TREE_TYPE (t))
7970 13676 : && bitint_precision_kind (TREE_TYPE (t)) >= bitint_prec_large)
7971 : has_large_huge = true;
7972 : }
7973 : }
7974 : }
7975 :
7976 7256 : if (large_huge.m_names || has_large_huge)
7977 : {
7978 5788 : ret = TODO_update_ssa_only_virtuals | TODO_cleanup_cfg;
7979 5788 : calculate_dominance_info (CDI_DOMINATORS);
7980 5788 : if (optimize)
7981 3017 : enable_ranger (cfun);
7982 5788 : if (large_huge.m_loads)
7983 : {
7984 312 : basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
7985 312 : entry->aux = NULL;
7986 624 : bitint_dom_walker (large_huge.m_names,
7987 312 : large_huge.m_loads).walk (entry);
7988 312 : bitmap_and_compl_into (large_huge.m_names, large_huge.m_loads);
7989 312 : clear_aux_for_blocks ();
7990 312 : BITMAP_FREE (large_huge.m_loads);
7991 : }
7992 5788 : large_huge.m_limb_type = build_nonstandard_integer_type (limb_prec, 1);
7993 5788 : large_huge.m_limb_size
7994 5788 : = tree_to_uhwi (TYPE_SIZE_UNIT (large_huge.m_limb_type));
7995 : }
7996 7256 : if (large_huge.m_names)
7997 : {
7998 5612 : large_huge.m_map
7999 11224 : = init_var_map (num_ssa_names, NULL, large_huge.m_names);
8000 5612 : coalesce_ssa_name (large_huge.m_map);
8001 5612 : partition_view_normal (large_huge.m_map);
8002 5612 : if (dump_file && (dump_flags & TDF_DETAILS))
8003 : {
8004 0 : fprintf (dump_file, "After Coalescing:\n");
8005 0 : dump_var_map (dump_file, large_huge.m_map);
8006 : }
8007 5612 : large_huge.m_vars
8008 5612 : = XCNEWVEC (tree, num_var_partitions (large_huge.m_map));
8009 5612 : bitmap_iterator bi;
8010 5612 : if (has_large_huge_parm_result)
8011 19547 : EXECUTE_IF_SET_IN_BITMAP (large_huge.m_names, 0, i, bi)
8012 : {
8013 15385 : tree s = ssa_name (i);
8014 15385 : if (SSA_NAME_VAR (s)
8015 6123 : && ((TREE_CODE (SSA_NAME_VAR (s)) == PARM_DECL
8016 5362 : && SSA_NAME_IS_DEFAULT_DEF (s))
8017 798 : || TREE_CODE (SSA_NAME_VAR (s)) == RESULT_DECL))
8018 : {
8019 5325 : int p = var_to_partition (large_huge.m_map, s);
8020 5325 : if (large_huge.m_vars[p] == NULL_TREE)
8021 : {
8022 5325 : large_huge.m_vars[p] = SSA_NAME_VAR (s);
8023 5325 : mark_addressable (SSA_NAME_VAR (s));
8024 : }
8025 : }
8026 : }
8027 5612 : tree atype = NULL_TREE;
8028 5612 : if (dump_file && (dump_flags & TDF_DETAILS))
8029 0 : fprintf (dump_file, "Mapping SSA_NAMEs to decls:\n");
8030 32608 : EXECUTE_IF_SET_IN_BITMAP (large_huge.m_names, 0, i, bi)
8031 : {
8032 26996 : tree s = ssa_name (i);
8033 26996 : int p = var_to_partition (large_huge.m_map, s);
8034 26996 : if (large_huge.m_vars[p] == NULL_TREE)
8035 : {
8036 18821 : if (atype == NULL_TREE
8037 32900 : || !tree_int_cst_equal (TYPE_SIZE (atype),
8038 14079 : TYPE_SIZE (TREE_TYPE (s))))
8039 : {
8040 7691 : unsigned HOST_WIDE_INT nelts
8041 7691 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (s))) / limb_prec;
8042 7691 : atype = build_array_type_nelts (large_huge.m_limb_type,
8043 7691 : nelts);
8044 : }
8045 18821 : large_huge.m_vars[p] = create_tmp_var (atype, "bitint");
8046 18821 : mark_addressable (large_huge.m_vars[p]);
8047 : }
8048 26996 : if (dump_file && (dump_flags & TDF_DETAILS))
8049 : {
8050 0 : print_generic_expr (dump_file, s, TDF_SLIM);
8051 0 : fprintf (dump_file, " -> ");
8052 0 : print_generic_expr (dump_file, large_huge.m_vars[p], TDF_SLIM);
8053 0 : fprintf (dump_file, "\n");
8054 : }
8055 : }
8056 : }
8057 :
8058 46075 : FOR_EACH_BB_REVERSE_FN (bb, cfun)
8059 : {
8060 38819 : gimple_stmt_iterator prev;
8061 196160 : for (gimple_stmt_iterator gsi = gsi_last_bb (bb); !gsi_end_p (gsi);
8062 118522 : gsi = prev)
8063 : {
8064 118522 : prev = gsi;
8065 118522 : gsi_prev (&prev);
8066 118522 : ssa_op_iter iter;
8067 118522 : gimple *stmt = gsi_stmt (gsi);
8068 118522 : if (is_gimple_debug (stmt))
8069 80593 : continue;
8070 113817 : bitint_prec_kind kind = bitint_prec_small;
8071 113817 : tree t;
8072 342519 : FOR_EACH_SSA_TREE_OPERAND (t, stmt, iter, SSA_OP_ALL_OPERANDS)
8073 228702 : if (BITINT_TYPE_P (TREE_TYPE (t)))
8074 : {
8075 78669 : bitint_prec_kind this_kind
8076 78669 : = bitint_precision_kind (TREE_TYPE (t));
8077 228910 : kind = MAX (kind, this_kind);
8078 : }
8079 113817 : if (is_gimple_assign (stmt) && gimple_store_p (stmt))
8080 : {
8081 16338 : t = gimple_assign_rhs1 (stmt);
8082 16338 : if (BITINT_TYPE_P (TREE_TYPE (t)))
8083 : {
8084 14143 : bitint_prec_kind this_kind
8085 14143 : = bitint_precision_kind (TREE_TYPE (t));
8086 14143 : kind = MAX (kind, this_kind);
8087 : }
8088 : }
8089 113817 : if (is_gimple_assign (stmt)
8090 113817 : && gimple_assign_rhs_code (stmt) == FLOAT_EXPR)
8091 : {
8092 181 : t = gimple_assign_rhs1 (stmt);
8093 210 : if (BITINT_TYPE_P (TREE_TYPE (t))
8094 183 : && TREE_CODE (t) == INTEGER_CST)
8095 : {
8096 1 : bitint_prec_kind this_kind
8097 1 : = bitint_precision_kind (TREE_TYPE (t));
8098 1 : kind = MAX (kind, this_kind);
8099 : }
8100 : }
8101 113817 : if (is_gimple_call (stmt))
8102 : {
8103 25649 : t = gimple_call_lhs (stmt);
8104 25649 : if (t && TREE_CODE (TREE_TYPE (t)) == COMPLEX_TYPE)
8105 : {
8106 5308 : bitint_prec_kind this_kind = arith_overflow_arg_kind (stmt);
8107 5308 : kind = MAX (kind, this_kind);
8108 5308 : if (BITINT_TYPE_P (TREE_TYPE (TREE_TYPE (t))))
8109 : {
8110 5188 : this_kind
8111 5188 : = bitint_precision_kind (TREE_TYPE (TREE_TYPE (t)));
8112 5188 : kind = MAX (kind, this_kind);
8113 : }
8114 : }
8115 : }
8116 113495 : if (kind == bitint_prec_small)
8117 44645 : continue;
8118 69172 : switch (gimple_code (stmt))
8119 : {
8120 11067 : case GIMPLE_CALL:
8121 : /* For now. We'll need to handle some internal functions and
8122 : perhaps some builtins. */
8123 11067 : if (kind == bitint_prec_middle)
8124 2280 : continue;
8125 : break;
8126 4 : case GIMPLE_ASM:
8127 4 : if (kind == bitint_prec_middle)
8128 1 : continue;
8129 : break;
8130 1124 : case GIMPLE_RETURN:
8131 1124 : continue;
8132 48370 : case GIMPLE_ASSIGN:
8133 48370 : if (gimple_clobber_p (stmt))
8134 3509 : continue;
8135 44861 : if (kind >= bitint_prec_large)
8136 : break;
8137 8732 : if (gimple_assign_single_p (stmt))
8138 : /* No need to lower copies, loads or stores. */
8139 5788 : continue;
8140 2944 : if (gimple_assign_cast_p (stmt))
8141 : {
8142 2379 : tree lhs = gimple_assign_lhs (stmt);
8143 2379 : tree rhs = gimple_assign_rhs1 (stmt);
8144 4758 : if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
8145 2379 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs))
8146 4752 : && (TYPE_PRECISION (TREE_TYPE (lhs))
8147 2373 : == TYPE_PRECISION (TREE_TYPE (rhs))))
8148 : /* No need to lower casts to same precision. */
8149 28 : continue;
8150 : }
8151 : break;
8152 : default:
8153 : break;
8154 1124 : }
8155 :
8156 11523 : if (kind == bitint_prec_middle)
8157 : {
8158 5041 : tree type = NULL_TREE;
8159 : /* Middle _BitInt(N) is rewritten to casts to INTEGER_TYPEs
8160 : with the same precision and back. */
8161 5041 : unsigned int nops = gimple_num_ops (stmt);
8162 16928 : for (unsigned int i = is_gimple_assign (stmt) ? 1 : 0;
8163 16928 : i < nops; ++i)
8164 11887 : if (tree op = gimple_op (stmt, i))
8165 : {
8166 7645 : tree nop = maybe_cast_middle_bitint (&gsi, op, type);
8167 7645 : if (nop != op)
8168 6765 : gimple_set_op (stmt, i, nop);
8169 880 : else if (COMPARISON_CLASS_P (op))
8170 : {
8171 0 : TREE_OPERAND (op, 0)
8172 0 : = maybe_cast_middle_bitint (&gsi,
8173 0 : TREE_OPERAND (op, 0),
8174 : type);
8175 0 : TREE_OPERAND (op, 1)
8176 0 : = maybe_cast_middle_bitint (&gsi,
8177 0 : TREE_OPERAND (op, 1),
8178 : type);
8179 : }
8180 880 : else if (TREE_CODE (op) == CASE_LABEL_EXPR)
8181 : {
8182 24 : CASE_LOW (op)
8183 24 : = maybe_cast_middle_bitint (&gsi, CASE_LOW (op),
8184 : type);
8185 48 : CASE_HIGH (op)
8186 48 : = maybe_cast_middle_bitint (&gsi, CASE_HIGH (op),
8187 : type);
8188 : }
8189 : }
8190 5041 : if (tree lhs = gimple_get_lhs (stmt))
8191 4464 : if (BITINT_TYPE_P (TREE_TYPE (lhs))
8192 2916 : && (bitint_precision_kind (TREE_TYPE (lhs))
8193 : == bitint_prec_middle))
8194 : {
8195 1367 : int prec = TYPE_PRECISION (TREE_TYPE (lhs));
8196 1367 : int uns = TYPE_UNSIGNED (TREE_TYPE (lhs));
8197 1367 : type = build_nonstandard_integer_type (prec, uns);
8198 1367 : tree lhs2 = make_ssa_name (type);
8199 1367 : gimple_set_lhs (stmt, lhs2);
8200 1367 : gimple *g = gimple_build_assign (lhs, NOP_EXPR, lhs2);
8201 1367 : if (stmt_ends_bb_p (stmt))
8202 : {
8203 4 : edge e = find_fallthru_edge (gsi_bb (gsi)->succs);
8204 4 : gsi_insert_on_edge (e, g);
8205 4 : edge_insertions = true;
8206 : }
8207 : else
8208 1363 : gsi_insert_after (&gsi, g, GSI_SAME_STMT);
8209 : }
8210 5041 : update_stmt (stmt);
8211 5041 : continue;
8212 5041 : }
8213 :
8214 51401 : if (tree lhs = gimple_get_lhs (stmt))
8215 44785 : if (TREE_CODE (lhs) == SSA_NAME)
8216 : {
8217 35836 : tree type = TREE_TYPE (lhs);
8218 35836 : if (TREE_CODE (type) == COMPLEX_TYPE)
8219 4184 : type = TREE_TYPE (type);
8220 5385 : if (BITINT_TYPE_P (type)
8221 30463 : && bitint_precision_kind (type) >= bitint_prec_large
8222 66097 : && (large_huge.m_names == NULL
8223 30106 : || !bitmap_bit_p (large_huge.m_names,
8224 30106 : SSA_NAME_VERSION (lhs))))
8225 8767 : continue;
8226 : }
8227 :
8228 42634 : large_huge.lower_stmt (stmt);
8229 : }
8230 :
8231 38819 : tree atype = NULL_TREE;
8232 47361 : for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
8233 8542 : gsi_next (&gsi))
8234 : {
8235 8542 : gphi *phi = gsi.phi ();
8236 8542 : tree lhs = gimple_phi_result (phi);
8237 16900 : if (!BITINT_TYPE_P (TREE_TYPE (lhs))
8238 8542 : || bitint_precision_kind (TREE_TYPE (lhs)) < bitint_prec_large)
8239 8399 : continue;
8240 143 : int p1 = var_to_partition (large_huge.m_map, lhs);
8241 143 : gcc_assert (large_huge.m_vars[p1] != NULL_TREE);
8242 : tree v1 = large_huge.m_vars[p1];
8243 544 : for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
8244 : {
8245 401 : tree arg = gimple_phi_arg_def (phi, i);
8246 401 : edge e = gimple_phi_arg_edge (phi, i);
8247 401 : gimple *g;
8248 401 : switch (TREE_CODE (arg))
8249 : {
8250 74 : case INTEGER_CST:
8251 74 : if (integer_zerop (arg) && VAR_P (v1))
8252 : {
8253 45 : tree zero = build_zero_cst (TREE_TYPE (v1));
8254 45 : g = gimple_build_assign (v1, zero);
8255 45 : gsi_insert_on_edge (e, g);
8256 45 : edge_insertions = true;
8257 136 : break;
8258 : }
8259 29 : int ext;
8260 29 : unsigned int min_prec, prec, rem;
8261 29 : tree c;
8262 29 : prec = TYPE_PRECISION (TREE_TYPE (arg));
8263 29 : rem = prec % (2 * limb_prec);
8264 29 : min_prec = bitint_min_cst_precision (arg, ext);
8265 29 : if (min_prec > prec - rem - 2 * limb_prec
8266 12 : && min_prec > (unsigned) limb_prec)
8267 : /* Constant which has enough significant bits that it
8268 : isn't worth trying to save .rodata space by extending
8269 : from smaller number. */
8270 : min_prec = prec;
8271 : else
8272 : {
8273 20 : min_prec = CEIL (min_prec, limb_prec) * limb_prec;
8274 20 : if (min_prec > (unsigned) limb_prec
8275 3 : && abi_limb_prec > limb_prec)
8276 : {
8277 : /* For targets with ABI limb precision higher than
8278 : limb precision round to ABI limb precision,
8279 : otherwise c can contain padding bits. */
8280 0 : min_prec
8281 0 : = CEIL (min_prec, abi_limb_prec) * abi_limb_prec;
8282 0 : if (min_prec > prec - rem - 2 * limb_prec)
8283 9 : min_prec = prec;
8284 : }
8285 : }
8286 29 : if (min_prec == 0)
8287 : c = NULL_TREE;
8288 26 : else if (min_prec == prec)
8289 9 : c = tree_output_constant_def (arg);
8290 17 : else if (min_prec == (unsigned) limb_prec)
8291 14 : c = fold_convert (large_huge.m_limb_type, arg);
8292 : else
8293 : {
8294 3 : tree ctype = build_bitint_type (min_prec, 1);
8295 3 : c = tree_output_constant_def (fold_convert (ctype, arg));
8296 : }
8297 26 : if (c)
8298 : {
8299 26 : if (VAR_P (v1) && min_prec == prec)
8300 : {
8301 8 : tree v2 = build1 (VIEW_CONVERT_EXPR,
8302 8 : TREE_TYPE (v1), c);
8303 8 : g = gimple_build_assign (v1, v2);
8304 8 : gsi_insert_on_edge (e, g);
8305 8 : edge_insertions = true;
8306 8 : break;
8307 : }
8308 18 : if (TREE_CODE (TREE_TYPE (c)) == INTEGER_TYPE)
8309 : {
8310 14 : if (bitint_big_endian)
8311 : {
8312 0 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8313 0 : tree sz1 = TYPE_SIZE_UNIT (TREE_TYPE (v1));
8314 0 : tree sz2 = TYPE_SIZE_UNIT (TREE_TYPE (c));
8315 0 : tree off = build_int_cst (ptype,
8316 0 : tree_to_uhwi (sz1)
8317 0 : - tree_to_uhwi (sz2));
8318 0 : tree vd = build2 (MEM_REF, TREE_TYPE (c),
8319 : build_fold_addr_expr (v1),
8320 : off);
8321 0 : g = gimple_build_assign (vd, c);
8322 : }
8323 : else
8324 14 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8325 14 : TREE_TYPE (c),
8326 : v1), c);
8327 : }
8328 : else
8329 : {
8330 4 : unsigned HOST_WIDE_INT nelts
8331 4 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (c)))
8332 4 : / limb_prec;
8333 4 : tree vtype
8334 8 : = build_array_type_nelts (large_huge.m_limb_type,
8335 4 : nelts);
8336 4 : tree vd;
8337 4 : if (bitint_big_endian)
8338 : {
8339 0 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8340 0 : tree sz1 = TYPE_SIZE_UNIT (TREE_TYPE (v1));
8341 0 : tree sz2 = TYPE_SIZE_UNIT (vtype);
8342 0 : tree off = build_int_cst (ptype,
8343 0 : tree_to_uhwi (sz1)
8344 0 : - tree_to_uhwi (sz2));
8345 0 : vd = build2 (MEM_REF, vtype,
8346 : build_fold_addr_expr (v1), off);
8347 : }
8348 : else
8349 4 : vd = build1 (VIEW_CONVERT_EXPR, vtype, v1);
8350 4 : g = gimple_build_assign (vd,
8351 : build1 (VIEW_CONVERT_EXPR,
8352 : vtype, c));
8353 : }
8354 18 : gsi_insert_on_edge (e, g);
8355 18 : if (min_prec == prec)
8356 : {
8357 : edge_insertions = true;
8358 : break;
8359 : }
8360 : }
8361 20 : if (ext == 0)
8362 : {
8363 14 : unsigned HOST_WIDE_INT nelts
8364 14 : = (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (v1)))
8365 14 : - min_prec) / limb_prec;
8366 14 : tree vtype
8367 28 : = build_array_type_nelts (large_huge.m_limb_type,
8368 14 : nelts);
8369 14 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8370 14 : tree off;
8371 14 : if (c && !bitint_big_endian)
8372 13 : off = fold_convert (ptype,
8373 : TYPE_SIZE_UNIT (TREE_TYPE (c)));
8374 : else
8375 1 : off = build_zero_cst (ptype);
8376 14 : tree vd = build2 (MEM_REF, vtype,
8377 : build_fold_addr_expr (v1), off);
8378 14 : g = gimple_build_assign (vd, build_zero_cst (vtype));
8379 : }
8380 : else
8381 : {
8382 6 : tree vd = v1;
8383 6 : if (c && !bitint_big_endian)
8384 : {
8385 4 : tree ptype = build_pointer_type (TREE_TYPE (v1));
8386 4 : tree off
8387 4 : = fold_convert (ptype,
8388 : TYPE_SIZE_UNIT (TREE_TYPE (c)));
8389 4 : vd = build2 (MEM_REF, large_huge.m_limb_type,
8390 : build_fold_addr_expr (v1), off);
8391 : }
8392 6 : vd = build_fold_addr_expr (vd);
8393 6 : unsigned HOST_WIDE_INT nbytes
8394 6 : = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (v1)));
8395 6 : if (c)
8396 4 : nbytes
8397 4 : -= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (c)));
8398 6 : tree fn = builtin_decl_implicit (BUILT_IN_MEMSET);
8399 6 : g = gimple_build_call (fn, 3, vd,
8400 : integer_minus_one_node,
8401 : build_int_cst (sizetype,
8402 6 : nbytes));
8403 : }
8404 20 : gsi_insert_on_edge (e, g);
8405 20 : edge_insertions = true;
8406 20 : break;
8407 0 : default:
8408 0 : gcc_unreachable ();
8409 327 : case SSA_NAME:
8410 327 : if (gimple_code (SSA_NAME_DEF_STMT (arg)) == GIMPLE_NOP)
8411 : {
8412 9 : if (large_huge.m_names == NULL
8413 18 : || !bitmap_bit_p (large_huge.m_names,
8414 9 : SSA_NAME_VERSION (arg)))
8415 310 : continue;
8416 : }
8417 327 : int p2 = var_to_partition (large_huge.m_map, arg);
8418 327 : if (p1 == p2)
8419 310 : continue;
8420 17 : gcc_assert (large_huge.m_vars[p2] != NULL_TREE);
8421 17 : tree v2 = large_huge.m_vars[p2];
8422 17 : if (VAR_P (v1) && VAR_P (v2))
8423 17 : g = gimple_build_assign (v1, v2);
8424 0 : else if (VAR_P (v1))
8425 0 : g = gimple_build_assign (v1, build1 (VIEW_CONVERT_EXPR,
8426 0 : TREE_TYPE (v1), v2));
8427 0 : else if (VAR_P (v2))
8428 0 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8429 0 : TREE_TYPE (v2), v1), v2);
8430 : else
8431 : {
8432 0 : if (atype == NULL_TREE
8433 0 : || !tree_int_cst_equal (TYPE_SIZE (atype),
8434 0 : TYPE_SIZE (TREE_TYPE (lhs))))
8435 : {
8436 0 : unsigned HOST_WIDE_INT nelts
8437 0 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (lhs)))
8438 0 : / limb_prec;
8439 0 : atype
8440 0 : = build_array_type_nelts (large_huge.m_limb_type,
8441 0 : nelts);
8442 : }
8443 0 : g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
8444 : atype, v1),
8445 : build1 (VIEW_CONVERT_EXPR,
8446 : atype, v2));
8447 : }
8448 17 : gsi_insert_on_edge (e, g);
8449 17 : edge_insertions = true;
8450 17 : break;
8451 : }
8452 : }
8453 : }
8454 : }
8455 :
8456 7256 : if (large_huge.m_names || has_large_huge)
8457 : {
8458 5788 : gimple *nop = NULL;
8459 383620 : for (i = 0; i < num_ssa_names; ++i)
8460 : {
8461 377832 : tree s = ssa_name (i);
8462 377832 : if (s == NULL_TREE)
8463 15361 : continue;
8464 362471 : tree type = TREE_TYPE (s);
8465 362471 : if (TREE_CODE (type) == COMPLEX_TYPE)
8466 17445 : type = TREE_TYPE (type);
8467 322984 : if (BITINT_TYPE_P (type)
8468 362498 : && bitint_precision_kind (type) >= bitint_prec_large)
8469 : {
8470 40444 : if (large_huge.m_preserved
8471 44918 : && bitmap_bit_p (large_huge.m_preserved,
8472 6809 : SSA_NAME_VERSION (s)))
8473 2335 : continue;
8474 35774 : gimple *g = SSA_NAME_DEF_STMT (s);
8475 35774 : if (gimple_code (g) == GIMPLE_NOP)
8476 : {
8477 9605 : if (SSA_NAME_VAR (s))
8478 5346 : set_ssa_default_def (cfun, SSA_NAME_VAR (s), NULL_TREE);
8479 9605 : release_ssa_name (s);
8480 9605 : continue;
8481 : }
8482 26169 : if (gimple_bb (g) == NULL)
8483 : {
8484 0 : release_ssa_name (s);
8485 0 : continue;
8486 : }
8487 26169 : if (gimple_code (g) != GIMPLE_ASM)
8488 : {
8489 26168 : gimple_stmt_iterator gsi = gsi_for_stmt (g);
8490 26168 : bool save_vta = flag_var_tracking_assignments;
8491 26168 : flag_var_tracking_assignments = false;
8492 26168 : gsi_remove (&gsi, true);
8493 26168 : flag_var_tracking_assignments = save_vta;
8494 : }
8495 26169 : if (nop == NULL)
8496 4847 : nop = gimple_build_nop ();
8497 26169 : SSA_NAME_DEF_STMT (s) = nop;
8498 26169 : release_ssa_name (s);
8499 : }
8500 : }
8501 5788 : if (optimize)
8502 3017 : disable_ranger (cfun);
8503 : }
8504 :
8505 7256 : if (edge_insertions)
8506 33 : gsi_commit_edge_inserts ();
8507 :
8508 : /* Fix up arguments of ECF_RETURNS_TWICE calls. Those were temporarily
8509 : inserted before the call, but that is invalid IL, so move them to the
8510 : right place and add corresponding PHIs. */
8511 7256 : if (!large_huge.m_returns_twice_calls.is_empty ())
8512 : {
8513 9 : auto_vec<gimple *, 16> arg_stmts;
8514 29 : while (!large_huge.m_returns_twice_calls.is_empty ())
8515 : {
8516 11 : gimple *stmt = large_huge.m_returns_twice_calls.pop ();
8517 11 : gimple_stmt_iterator gsi = gsi_after_labels (gimple_bb (stmt));
8518 36 : while (gsi_stmt (gsi) != stmt)
8519 : {
8520 25 : if (is_gimple_debug (gsi_stmt (gsi)))
8521 2 : gsi_next (&gsi);
8522 : else
8523 : {
8524 23 : arg_stmts.safe_push (gsi_stmt (gsi));
8525 23 : gsi_remove (&gsi, false);
8526 : }
8527 : }
8528 11 : gimple *g;
8529 11 : basic_block bb = NULL;
8530 11 : edge e = NULL, ead = NULL;
8531 34 : FOR_EACH_VEC_ELT (arg_stmts, i, g)
8532 : {
8533 23 : gsi_safe_insert_before (&gsi, g);
8534 23 : if (i == 0)
8535 : {
8536 11 : bb = gimple_bb (stmt);
8537 11 : gcc_checking_assert (EDGE_COUNT (bb->preds) == 2);
8538 11 : e = EDGE_PRED (bb, 0);
8539 11 : ead = EDGE_PRED (bb, 1);
8540 11 : if ((ead->flags & EDGE_ABNORMAL) == 0)
8541 0 : std::swap (e, ead);
8542 11 : gcc_checking_assert ((e->flags & EDGE_ABNORMAL) == 0
8543 : && (ead->flags & EDGE_ABNORMAL));
8544 : }
8545 23 : tree lhs = gimple_assign_lhs (g);
8546 23 : tree arg = lhs;
8547 23 : gphi *phi = create_phi_node (copy_ssa_name (arg), bb);
8548 23 : add_phi_arg (phi, arg, e, UNKNOWN_LOCATION);
8549 23 : tree var = create_tmp_reg (TREE_TYPE (arg));
8550 23 : suppress_warning (var, OPT_Wuninitialized);
8551 23 : arg = get_or_create_ssa_default_def (cfun, var);
8552 23 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg) = 1;
8553 23 : add_phi_arg (phi, arg, ead, UNKNOWN_LOCATION);
8554 23 : arg = gimple_phi_result (phi);
8555 23 : SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg) = 1;
8556 23 : imm_use_iterator iter;
8557 23 : gimple *use_stmt;
8558 92 : FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
8559 : {
8560 46 : if (use_stmt == phi)
8561 23 : continue;
8562 23 : gcc_checking_assert (use_stmt == stmt);
8563 23 : use_operand_p use_p;
8564 69 : FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
8565 23 : SET_USE (use_p, arg);
8566 23 : }
8567 : }
8568 11 : update_stmt (stmt);
8569 11 : arg_stmts.truncate (0);
8570 : }
8571 9 : }
8572 :
8573 7256 : return ret;
8574 7256 : }
8575 :
8576 : namespace {
8577 :
8578 : const pass_data pass_data_lower_bitint =
8579 : {
8580 : GIMPLE_PASS, /* type */
8581 : "bitintlower", /* name */
8582 : OPTGROUP_NONE, /* optinfo_flags */
8583 : TV_NONE, /* tv_id */
8584 : PROP_ssa, /* properties_required */
8585 : PROP_gimple_lbitint, /* properties_provided */
8586 : 0, /* properties_destroyed */
8587 : 0, /* todo_flags_start */
8588 : 0, /* todo_flags_finish */
8589 : };
8590 :
8591 : class pass_lower_bitint : public gimple_opt_pass
8592 : {
8593 : public:
8594 597656 : pass_lower_bitint (gcc::context *ctxt)
8595 1195312 : : gimple_opt_pass (pass_data_lower_bitint, ctxt)
8596 : {}
8597 :
8598 : /* opt_pass methods: */
8599 298828 : opt_pass * clone () final override { return new pass_lower_bitint (m_ctxt); }
8600 1042216 : unsigned int execute (function *) final override
8601 : {
8602 1042216 : return gimple_lower_bitint ();
8603 : }
8604 :
8605 : }; // class pass_lower_bitint
8606 :
8607 : } // anon namespace
8608 :
8609 : gimple_opt_pass *
8610 298828 : make_pass_lower_bitint (gcc::context *ctxt)
8611 : {
8612 298828 : return new pass_lower_bitint (ctxt);
8613 : }
8614 :
8615 :
8616 : namespace {
8617 :
8618 : const pass_data pass_data_lower_bitint_O0 =
8619 : {
8620 : GIMPLE_PASS, /* type */
8621 : "bitintlower0", /* name */
8622 : OPTGROUP_NONE, /* optinfo_flags */
8623 : TV_NONE, /* tv_id */
8624 : PROP_cfg, /* properties_required */
8625 : PROP_gimple_lbitint, /* properties_provided */
8626 : 0, /* properties_destroyed */
8627 : 0, /* todo_flags_start */
8628 : 0, /* todo_flags_finish */
8629 : };
8630 :
8631 : class pass_lower_bitint_O0 : public gimple_opt_pass
8632 : {
8633 : public:
8634 298828 : pass_lower_bitint_O0 (gcc::context *ctxt)
8635 597656 : : gimple_opt_pass (pass_data_lower_bitint_O0, ctxt)
8636 : {}
8637 :
8638 : /* opt_pass methods: */
8639 1489159 : bool gate (function *fun) final override
8640 : {
8641 : /* With errors, normal optimization passes are not run. If we don't
8642 : lower bitint operations at all, rtl expansion will abort. */
8643 1489159 : return !(fun->curr_properties & PROP_gimple_lbitint);
8644 : }
8645 :
8646 447048 : unsigned int execute (function *) final override
8647 : {
8648 447048 : return gimple_lower_bitint ();
8649 : }
8650 :
8651 : }; // class pass_lower_bitint_O0
8652 :
8653 : } // anon namespace
8654 :
8655 : gimple_opt_pass *
8656 298828 : make_pass_lower_bitint_O0 (gcc::context *ctxt)
8657 : {
8658 298828 : return new pass_lower_bitint_O0 (ctxt);
8659 : }
|
