Line data Source code
1 : /* Instruction scheduling pass.
2 : Copyright (C) 1992-2026 Free Software Foundation, Inc.
3 : Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
4 : and currently maintained by, Jim Wilson (wilson@cygnus.com)
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : /* Instruction scheduling pass. This file, along with sched-deps.cc,
23 : contains the generic parts. The actual entry point for
24 : the normal instruction scheduling pass is found in sched-rgn.cc.
25 :
26 : We compute insn priorities based on data dependencies. Flow
27 : analysis only creates a fraction of the data-dependencies we must
28 : observe: namely, only those dependencies which the combiner can be
29 : expected to use. For this pass, we must therefore create the
30 : remaining dependencies we need to observe: register dependencies,
31 : memory dependencies, dependencies to keep function calls in order,
32 : and the dependence between a conditional branch and the setting of
33 : condition codes are all dealt with here.
34 :
35 : The scheduler first traverses the data flow graph, starting with
36 : the last instruction, and proceeding to the first, assigning values
37 : to insn_priority as it goes. This sorts the instructions
38 : topologically by data dependence.
39 :
40 : Once priorities have been established, we order the insns using
41 : list scheduling. This works as follows: starting with a list of
42 : all the ready insns, and sorted according to priority number, we
43 : schedule the insn from the end of the list by placing its
44 : predecessors in the list according to their priority order. We
45 : consider this insn scheduled by setting the pointer to the "end" of
46 : the list to point to the previous insn. When an insn has no
47 : predecessors, we either queue it until sufficient time has elapsed
48 : or add it to the ready list. As the instructions are scheduled or
49 : when stalls are introduced, the queue advances and dumps insns into
50 : the ready list. When all insns down to the lowest priority have
51 : been scheduled, the critical path of the basic block has been made
52 : as short as possible. The remaining insns are then scheduled in
53 : remaining slots.
54 :
55 : The following list shows the order in which we want to break ties
56 : among insns in the ready list:
57 :
58 : 1. choose insn with the longest path to end of bb, ties
59 : broken by
60 : 2. choose insn with least contribution to register pressure,
61 : ties broken by
62 : 3. prefer in-block upon interblock motion, ties broken by
63 : 4. prefer useful upon speculative motion, ties broken by
64 : 5. choose insn with largest control flow probability, ties
65 : broken by
66 : 6. choose insn with the least dependences upon the previously
67 : scheduled insn, or finally
68 : 7 choose the insn which has the most insns dependent on it.
69 : 8. choose insn with lowest UID.
70 :
71 : Memory references complicate matters. Only if we can be certain
72 : that memory references are not part of the data dependency graph
73 : (via true, anti, or output dependence), can we move operations past
74 : memory references. To first approximation, reads can be done
75 : independently, while writes introduce dependencies. Better
76 : approximations will yield fewer dependencies.
77 :
78 : Before reload, an extended analysis of interblock data dependences
79 : is required for interblock scheduling. This is performed in
80 : compute_block_dependences ().
81 :
82 : Dependencies set up by memory references are treated in exactly the
83 : same way as other dependencies, by using insn backward dependences
84 : INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences
85 : INSN_FORW_DEPS for the purpose of forward list scheduling.
86 :
87 : Having optimized the critical path, we may have also unduly
88 : extended the lifetimes of some registers. If an operation requires
89 : that constants be loaded into registers, it is certainly desirable
90 : to load those constants as early as necessary, but no earlier.
91 : I.e., it will not do to load up a bunch of registers at the
92 : beginning of a basic block only to use them at the end, if they
93 : could be loaded later, since this may result in excessive register
94 : utilization.
95 :
96 : Note that since branches are never in basic blocks, but only end
97 : basic blocks, this pass will not move branches. But that is ok,
98 : since we can use GNU's delayed branch scheduling pass to take care
99 : of this case.
100 :
101 : Also note that no further optimizations based on algebraic
102 : identities are performed, so this pass would be a good one to
103 : perform instruction splitting, such as breaking up a multiply
104 : instruction into shifts and adds where that is profitable.
105 :
106 : Given the memory aliasing analysis that this pass should perform,
107 : it should be possible to remove redundant stores to memory, and to
108 : load values from registers instead of hitting memory.
109 :
110 : Before reload, speculative insns are moved only if a 'proof' exists
111 : that no exception will be caused by this, and if no live registers
112 : exist that inhibit the motion (live registers constraints are not
113 : represented by data dependence edges).
114 :
115 : This pass must update information that subsequent passes expect to
116 : be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths,
117 : reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END.
118 :
119 : The information in the line number notes is carefully retained by
120 : this pass. Notes that refer to the starting and ending of
121 : exception regions are also carefully retained by this pass. All
122 : other NOTE insns are grouped in their same relative order at the
123 : beginning of basic blocks and regions that have been scheduled. */
124 :
125 : #include "config.h"
126 : #include "system.h"
127 : #include "coretypes.h"
128 : #include "backend.h"
129 : #include "target.h"
130 : #include "rtl.h"
131 : #include "cfghooks.h"
132 : #include "df.h"
133 : #include "memmodel.h"
134 : #include "tm_p.h"
135 : #include "insn-config.h"
136 : #include "regs.h"
137 : #include "ira.h"
138 : #include "recog.h"
139 : #include "insn-attr.h"
140 : #include "cfgrtl.h"
141 : #include "cfgbuild.h"
142 : #include "sched-int.h"
143 : #include "common/common-target.h"
144 : #include "dbgcnt.h"
145 : #include "cfgloop.h"
146 : #include "dumpfile.h"
147 : #include "print-rtl.h"
148 : #include "function-abi.h"
149 :
150 : #ifdef INSN_SCHEDULING
151 :
152 : /* True if we do register pressure relief through live-range
153 : shrinkage. */
154 : static bool live_range_shrinkage_p;
155 :
156 : /* Switch on live range shrinkage. */
157 : void
158 35 : initialize_live_range_shrinkage (void)
159 : {
160 35 : live_range_shrinkage_p = true;
161 35 : }
162 :
163 : /* Switch off live range shrinkage. */
164 : void
165 35 : finish_live_range_shrinkage (void)
166 : {
167 35 : live_range_shrinkage_p = false;
168 35 : }
169 :
170 : /* issue_rate is the number of insns that can be scheduled in the same
171 : machine cycle. It can be defined in the config/mach/mach.h file,
172 : otherwise we set it to 1. */
173 :
174 : int issue_rate;
175 :
176 : /* This can be set to true by a backend if the scheduler should not
177 : enable a DCE pass. */
178 : bool sched_no_dce;
179 :
180 : /* The current initiation interval used when modulo scheduling. */
181 : static int modulo_ii;
182 :
183 : /* The maximum number of stages we are prepared to handle. */
184 : static int modulo_max_stages;
185 :
186 : /* The number of insns that exist in each iteration of the loop. We use this
187 : to detect when we've scheduled all insns from the first iteration. */
188 : static int modulo_n_insns;
189 :
190 : /* The current count of insns in the first iteration of the loop that have
191 : already been scheduled. */
192 : static int modulo_insns_scheduled;
193 :
194 : /* The maximum uid of insns from the first iteration of the loop. */
195 : static int modulo_iter0_max_uid;
196 :
197 : /* The number of times we should attempt to backtrack when modulo scheduling.
198 : Decreased each time we have to backtrack. */
199 : static int modulo_backtracks_left;
200 :
201 : /* The stage in which the last insn from the original loop was
202 : scheduled. */
203 : static int modulo_last_stage;
204 :
205 : /* sched-verbose controls the amount of debugging output the
206 : scheduler prints. It is controlled by -fsched-verbose=N:
207 : N=0: no debugging output.
208 : N=1: default value.
209 : N=2: bb's probabilities, detailed ready list info, unit/insn info.
210 : N=3: rtl at abort point, control-flow, regions info.
211 : N=5: dependences info. */
212 : int sched_verbose = 0;
213 :
214 : /* Debugging file. All printouts are sent to dump. */
215 : FILE *sched_dump = 0;
216 :
217 : /* This is a placeholder for the scheduler parameters common
218 : to all schedulers. */
219 : struct common_sched_info_def *common_sched_info;
220 :
221 : #define INSN_TICK(INSN) (HID (INSN)->tick)
222 : #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick)
223 : #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate)
224 : #define INTER_TICK(INSN) (HID (INSN)->inter_tick)
225 : #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn)
226 : #define SHADOW_P(INSN) (HID (INSN)->shadow_p)
227 : #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec)
228 : /* Cached cost of the instruction. Use insn_sched_cost to get cost of the
229 : insn. -1 here means that the field is not initialized. */
230 : #define INSN_COST(INSN) (HID (INSN)->cost)
231 :
232 : /* If INSN_TICK of an instruction is equal to INVALID_TICK,
233 : then it should be recalculated from scratch. */
234 : #define INVALID_TICK (-(max_insn_queue_index + 1))
235 : /* The minimal value of the INSN_TICK of an instruction. */
236 : #define MIN_TICK (-max_insn_queue_index)
237 :
238 : /* Original order of insns in the ready list.
239 : Used to keep order of normal insns while separating DEBUG_INSNs. */
240 : #define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
241 :
242 : /* The deciding reason for INSN's place in the ready list. */
243 : #define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
244 :
245 : /* List of important notes we must keep around. This is a pointer to the
246 : last element in the list. */
247 : rtx_insn *note_list;
248 :
249 : static struct spec_info_def spec_info_var;
250 : /* Description of the speculative part of the scheduling.
251 : If NULL - no speculation. */
252 : spec_info_t spec_info = NULL;
253 :
254 : /* True, if recovery block was added during scheduling of current block.
255 : Used to determine, if we need to fix INSN_TICKs. */
256 : static bool haifa_recovery_bb_recently_added_p;
257 :
258 : /* True, if recovery block was added during this scheduling pass.
259 : Used to determine if we should have empty memory pools of dependencies
260 : after finishing current region. */
261 : bool haifa_recovery_bb_ever_added_p;
262 :
263 : /* Counters of different types of speculative instructions. */
264 : static int nr_begin_data, nr_be_in_data, nr_begin_control, nr_be_in_control;
265 :
266 : /* Array used in {unlink, restore}_bb_notes. */
267 : static rtx_insn **bb_header = 0;
268 :
269 : /* Basic block after which recovery blocks will be created. */
270 : static basic_block before_recovery;
271 :
272 : /* Basic block just before the EXIT_BLOCK and after recovery, if we have
273 : created it. */
274 : basic_block after_recovery;
275 :
276 : /* FALSE if we add bb to another region, so we don't need to initialize it. */
277 : bool adding_bb_to_current_region_p = true;
278 :
279 : /* Queues, etc. */
280 :
281 : /* An instruction is ready to be scheduled when all insns preceding it
282 : have already been scheduled. It is important to ensure that all
283 : insns which use its result will not be executed until its result
284 : has been computed. An insn is maintained in one of four structures:
285 :
286 : (P) the "Pending" set of insns which cannot be scheduled until
287 : their dependencies have been satisfied.
288 : (Q) the "Queued" set of insns that can be scheduled when sufficient
289 : time has passed.
290 : (R) the "Ready" list of unscheduled, uncommitted insns.
291 : (S) the "Scheduled" list of insns.
292 :
293 : Initially, all insns are either "Pending" or "Ready" depending on
294 : whether their dependencies are satisfied.
295 :
296 : Insns move from the "Ready" list to the "Scheduled" list as they
297 : are committed to the schedule. As this occurs, the insns in the
298 : "Pending" list have their dependencies satisfied and move to either
299 : the "Ready" list or the "Queued" set depending on whether
300 : sufficient time has passed to make them ready. As time passes,
301 : insns move from the "Queued" set to the "Ready" list.
302 :
303 : The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the
304 : unscheduled insns, i.e., those that are ready, queued, and pending.
305 : The "Queued" set (Q) is implemented by the variable `insn_queue'.
306 : The "Ready" list (R) is implemented by the variables `ready' and
307 : `n_ready'.
308 : The "Scheduled" list (S) is the new insn chain built by this pass.
309 :
310 : The transition (R->S) is implemented in the scheduling loop in
311 : `schedule_block' when the best insn to schedule is chosen.
312 : The transitions (P->R and P->Q) are implemented in `schedule_insn' as
313 : insns move from the ready list to the scheduled list.
314 : The transition (Q->R) is implemented in 'queue_to_insn' as time
315 : passes or stalls are introduced. */
316 :
317 : /* Implement a circular buffer to delay instructions until sufficient
318 : time has passed. For the new pipeline description interface,
319 : MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less
320 : than maximal time of instruction execution computed by genattr.cc on
321 : the base maximal time of functional unit reservations and getting a
322 : result. This is the longest time an insn may be queued. */
323 :
324 : static rtx_insn_list **insn_queue;
325 : static int q_ptr = 0;
326 : static int q_size = 0;
327 : #define NEXT_Q(X) (((X)+1) & max_insn_queue_index)
328 : #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index)
329 :
330 : #define QUEUE_SCHEDULED (-3)
331 : #define QUEUE_NOWHERE (-2)
332 : #define QUEUE_READY (-1)
333 : /* QUEUE_SCHEDULED - INSN is scheduled.
334 : QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in
335 : queue or ready list.
336 : QUEUE_READY - INSN is in ready list.
337 : N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. */
338 :
339 : #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index)
340 :
341 : /* The following variable value refers for all current and future
342 : reservations of the processor units. */
343 : state_t curr_state;
344 :
345 : /* The following variable value is size of memory representing all
346 : current and future reservations of the processor units. */
347 : size_t dfa_state_size;
348 :
349 : /* The following array is used to find the best insn from ready when
350 : the automaton pipeline interface is used. */
351 : signed char *ready_try = NULL;
352 :
353 : /* The ready list. */
354 : struct ready_list ready = {NULL, 0, 0, 0, 0};
355 :
356 : /* The pointer to the ready list (to be removed). */
357 : static struct ready_list *readyp = &ready;
358 :
359 : /* Scheduling clock. */
360 : static int clock_var;
361 :
362 : /* Clock at which the previous instruction was issued. */
363 : static int last_clock_var;
364 :
365 : /* Set to true if, when queuing a shadow insn, we discover that it would be
366 : scheduled too late. */
367 : static bool must_backtrack;
368 :
369 : /* The following variable value is number of essential insns issued on
370 : the current cycle. An insn is essential one if it changes the
371 : processors state. */
372 : int cycle_issued_insns;
373 :
374 : /* This records the actual schedule. It is built up during the main phase
375 : of schedule_block, and afterwards used to reorder the insns in the RTL. */
376 : static vec<rtx_insn *> scheduled_insns;
377 :
378 : static int may_trap_exp (const_rtx, int);
379 :
380 : /* Nonzero iff the address is comprised from at most 1 register. */
381 : #define CONST_BASED_ADDRESS_P(x) \
382 : (REG_P (x) \
383 : || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS \
384 : || (GET_CODE (x) == LO_SUM)) \
385 : && (CONSTANT_P (XEXP (x, 0)) \
386 : || CONSTANT_P (XEXP (x, 1)))))
387 :
388 : /* Returns a class that insn with GET_DEST(insn)=x may belong to,
389 : as found by analyzing insn's expression. */
390 :
391 :
392 : static int haifa_luid_for_non_insn (rtx x);
393 :
394 : /* Haifa version of sched_info hooks common to all headers. */
395 : const struct common_sched_info_def haifa_common_sched_info =
396 : {
397 : NULL, /* fix_recovery_cfg */
398 : NULL, /* add_block */
399 : NULL, /* estimate_number_of_insns */
400 : haifa_luid_for_non_insn, /* luid_for_non_insn */
401 : SCHED_PASS_UNKNOWN /* sched_pass_id */
402 : };
403 :
404 : /* Mapping from instruction UID to its Logical UID. */
405 : vec<int> sched_luids;
406 :
407 : /* Next LUID to assign to an instruction. */
408 : int sched_max_luid = 1;
409 :
410 : /* Haifa Instruction Data. */
411 : vec<haifa_insn_data_def> h_i_d;
412 :
413 : void (* sched_init_only_bb) (basic_block, basic_block);
414 :
415 : /* Split block function. Different schedulers might use different functions
416 : to handle their internal data consistent. */
417 : basic_block (* sched_split_block) (basic_block, rtx);
418 :
419 : /* Create empty basic block after the specified block. */
420 : basic_block (* sched_create_empty_bb) (basic_block);
421 :
422 : /* Return the number of cycles until INSN is expected to be ready.
423 : Return zero if it already is. */
424 : static int
425 27322 : insn_delay (rtx_insn *insn)
426 : {
427 27322 : return MAX (INSN_TICK (insn) - clock_var, 0);
428 : }
429 :
430 : static int
431 28052 : may_trap_exp (const_rtx x, int is_store)
432 : {
433 28052 : enum rtx_code code;
434 :
435 28052 : if (x == 0)
436 : return TRAP_FREE;
437 28052 : code = GET_CODE (x);
438 28052 : if (is_store)
439 : {
440 5744 : if (code == MEM && may_trap_p (x))
441 : return TRAP_RISKY;
442 : else
443 5492 : return TRAP_FREE;
444 : }
445 22308 : if (code == MEM)
446 : {
447 : /* The insn uses memory: a volatile load. */
448 1662 : if (MEM_VOLATILE_P (x))
449 : return IRISKY;
450 : /* An exception-free load. */
451 1640 : if (!may_trap_p (x))
452 : return IFREE;
453 : /* A load with 1 base register, to be further checked. */
454 402 : if (CONST_BASED_ADDRESS_P (XEXP (x, 0)))
455 : return PFREE_CANDIDATE;
456 : /* No info on the load, to be further checked. */
457 26 : return PRISKY_CANDIDATE;
458 : }
459 : else
460 : {
461 20646 : const char *fmt;
462 20646 : int i, insn_class = TRAP_FREE;
463 :
464 : /* Neither store nor load, check if it may cause a trap. */
465 20646 : if (may_trap_p (x))
466 : return TRAP_RISKY;
467 : /* Recursive step: walk the insn... */
468 20048 : fmt = GET_RTX_FORMAT (code);
469 45802 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
470 : {
471 25758 : if (fmt[i] == 'e')
472 : {
473 12374 : int tmp_class = may_trap_exp (XEXP (x, i), is_store);
474 12374 : insn_class = WORST_CLASS (insn_class, tmp_class);
475 : }
476 13384 : else if (fmt[i] == 'E')
477 : {
478 : int j;
479 308 : for (j = 0; j < XVECLEN (x, i); j++)
480 : {
481 266 : int tmp_class = may_trap_exp (XVECEXP (x, i, j), is_store);
482 266 : insn_class = WORST_CLASS (insn_class, tmp_class);
483 266 : if (insn_class == TRAP_RISKY || insn_class == IRISKY)
484 : break;
485 : }
486 : }
487 25758 : if (insn_class == TRAP_RISKY || insn_class == IRISKY)
488 : break;
489 : }
490 20048 : return insn_class;
491 : }
492 : }
493 :
494 : /* Classifies rtx X of an insn for the purpose of verifying that X can be
495 : executed speculatively (and consequently the insn can be moved
496 : speculatively), by examining X, returning:
497 : TRAP_RISKY: store, or risky non-load insn (e.g. division by variable).
498 : TRAP_FREE: non-load insn.
499 : IFREE: load from a globally safe location.
500 : IRISKY: volatile load.
501 : PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for
502 : being either PFREE or PRISKY. */
503 :
504 : static int
505 6874 : haifa_classify_rtx (const_rtx x)
506 : {
507 6874 : int tmp_class = TRAP_FREE;
508 6874 : int insn_class = TRAP_FREE;
509 6874 : enum rtx_code code;
510 :
511 6874 : if (GET_CODE (x) == PARALLEL)
512 : {
513 800 : int i, len = XVECLEN (x, 0);
514 :
515 2176 : for (i = len - 1; i >= 0; i--)
516 : {
517 1547 : tmp_class = haifa_classify_rtx (XVECEXP (x, 0, i));
518 1547 : insn_class = WORST_CLASS (insn_class, tmp_class);
519 1547 : if (insn_class == TRAP_RISKY || insn_class == IRISKY)
520 : break;
521 : }
522 : }
523 : else
524 : {
525 6074 : code = GET_CODE (x);
526 6074 : switch (code)
527 : {
528 752 : case CLOBBER:
529 : /* Test if it is a 'store'. */
530 752 : tmp_class = may_trap_exp (XEXP (x, 0), 1);
531 752 : break;
532 4992 : case SET:
533 : /* Test if it is a store. */
534 4992 : tmp_class = may_trap_exp (SET_DEST (x), 1);
535 4992 : if (tmp_class == TRAP_RISKY)
536 : break;
537 : /* Test if it is a load. */
538 4834 : tmp_class =
539 4834 : WORST_CLASS (tmp_class,
540 : may_trap_exp (SET_SRC (x), 0));
541 : break;
542 0 : case COND_EXEC:
543 0 : tmp_class = haifa_classify_rtx (COND_EXEC_CODE (x));
544 0 : if (tmp_class == TRAP_RISKY)
545 : break;
546 0 : tmp_class = WORST_CLASS (tmp_class,
547 : may_trap_exp (COND_EXEC_TEST (x), 0));
548 : break;
549 : case TRAP_IF:
550 169 : tmp_class = TRAP_RISKY;
551 : break;
552 158 : default:;
553 : }
554 : insn_class = tmp_class;
555 : }
556 :
557 6874 : return insn_class;
558 : }
559 :
560 : int
561 5327 : haifa_classify_insn (const_rtx insn)
562 : {
563 5327 : return haifa_classify_rtx (PATTERN (insn));
564 : }
565 :
566 : /* After the scheduler initialization function has been called, this function
567 : can be called to enable modulo scheduling. II is the initiation interval
568 : we should use, it affects the delays for delay_pairs that were recorded as
569 : separated by a given number of stages.
570 :
571 : MAX_STAGES provides us with a limit
572 : after which we give up scheduling; the caller must have unrolled at least
573 : as many copies of the loop body and recorded delay_pairs for them.
574 :
575 : INSNS is the number of real (non-debug) insns in one iteration of
576 : the loop. MAX_UID can be used to test whether an insn belongs to
577 : the first iteration of the loop; all of them have a uid lower than
578 : MAX_UID. */
579 : void
580 0 : set_modulo_params (int ii, int max_stages, int insns, int max_uid)
581 : {
582 0 : modulo_ii = ii;
583 0 : modulo_max_stages = max_stages;
584 0 : modulo_n_insns = insns;
585 0 : modulo_iter0_max_uid = max_uid;
586 0 : modulo_backtracks_left = param_max_modulo_backtrack_attempts;
587 0 : }
588 :
589 : /* A structure to record a pair of insns where the first one is a real
590 : insn that has delay slots, and the second is its delayed shadow.
591 : I1 is scheduled normally and will emit an assembly instruction,
592 : while I2 describes the side effect that takes place at the
593 : transition between cycles CYCLES and (CYCLES + 1) after I1. */
594 : struct delay_pair
595 : {
596 : struct delay_pair *next_same_i1;
597 : rtx_insn *i1, *i2;
598 : int cycles;
599 : /* When doing modulo scheduling, we a delay_pair can also be used to
600 : show that I1 and I2 are the same insn in a different stage. If that
601 : is the case, STAGES will be nonzero. */
602 : int stages;
603 : };
604 :
605 : /* Helpers for delay hashing. */
606 :
607 : struct delay_i1_hasher : nofree_ptr_hash <delay_pair>
608 : {
609 : typedef void *compare_type;
610 : static inline hashval_t hash (const delay_pair *);
611 : static inline bool equal (const delay_pair *, const void *);
612 : };
613 :
614 : /* Returns a hash value for X, based on hashing just I1. */
615 :
616 : inline hashval_t
617 0 : delay_i1_hasher::hash (const delay_pair *x)
618 : {
619 0 : return htab_hash_pointer (x->i1);
620 : }
621 :
622 : /* Return true if I1 of pair X is the same as that of pair Y. */
623 :
624 : inline bool
625 0 : delay_i1_hasher::equal (const delay_pair *x, const void *y)
626 : {
627 0 : return x->i1 == y;
628 : }
629 :
630 : struct delay_i2_hasher : free_ptr_hash <delay_pair>
631 : {
632 : typedef void *compare_type;
633 : static inline hashval_t hash (const delay_pair *);
634 : static inline bool equal (const delay_pair *, const void *);
635 : };
636 :
637 : /* Returns a hash value for X, based on hashing just I2. */
638 :
639 : inline hashval_t
640 0 : delay_i2_hasher::hash (const delay_pair *x)
641 : {
642 0 : return htab_hash_pointer (x->i2);
643 : }
644 :
645 : /* Return true if I2 of pair X is the same as that of pair Y. */
646 :
647 : inline bool
648 0 : delay_i2_hasher::equal (const delay_pair *x, const void *y)
649 : {
650 0 : return x->i2 == y;
651 : }
652 :
653 : /* Two hash tables to record delay_pairs, one indexed by I1 and the other
654 : indexed by I2. */
655 : static hash_table<delay_i1_hasher> *delay_htab;
656 : static hash_table<delay_i2_hasher> *delay_htab_i2;
657 :
658 : /* Called through htab_traverse. Walk the hashtable using I2 as
659 : index, and delete all elements involving an UID higher than
660 : that pointed to by *DATA. */
661 : int
662 0 : haifa_htab_i2_traverse (delay_pair **slot, int *data)
663 : {
664 0 : int maxuid = *data;
665 0 : struct delay_pair *p = *slot;
666 0 : if (INSN_UID (p->i2) >= maxuid || INSN_UID (p->i1) >= maxuid)
667 : {
668 0 : delay_htab_i2->clear_slot (slot);
669 : }
670 0 : return 1;
671 : }
672 :
673 : /* Called through htab_traverse. Walk the hashtable using I2 as
674 : index, and delete all elements involving an UID higher than
675 : that pointed to by *DATA. */
676 : int
677 0 : haifa_htab_i1_traverse (delay_pair **pslot, int *data)
678 : {
679 0 : int maxuid = *data;
680 0 : struct delay_pair *p, *first, **pprev;
681 :
682 0 : if (INSN_UID ((*pslot)->i1) >= maxuid)
683 : {
684 0 : delay_htab->clear_slot (pslot);
685 0 : return 1;
686 : }
687 : pprev = &first;
688 0 : for (p = *pslot; p; p = p->next_same_i1)
689 : {
690 0 : if (INSN_UID (p->i2) < maxuid)
691 : {
692 0 : *pprev = p;
693 0 : pprev = &p->next_same_i1;
694 : }
695 : }
696 0 : *pprev = NULL;
697 0 : if (first == NULL)
698 0 : delay_htab->clear_slot (pslot);
699 : else
700 0 : *pslot = first;
701 : return 1;
702 : }
703 :
704 : /* Discard all delay pairs which involve an insn with an UID higher
705 : than MAX_UID. */
706 : void
707 0 : discard_delay_pairs_above (int max_uid)
708 : {
709 0 : delay_htab->traverse <int *, haifa_htab_i1_traverse> (&max_uid);
710 0 : delay_htab_i2->traverse <int *, haifa_htab_i2_traverse> (&max_uid);
711 0 : }
712 :
713 : /* This function can be called by a port just before it starts the final
714 : scheduling pass. It records the fact that an instruction with delay
715 : slots has been split into two insns, I1 and I2. The first one will be
716 : scheduled normally and initiates the operation. The second one is a
717 : shadow which must follow a specific number of cycles after I1; its only
718 : purpose is to show the side effect that occurs at that cycle in the RTL.
719 : If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN,
720 : while I2 retains the original insn type.
721 :
722 : There are two ways in which the number of cycles can be specified,
723 : involving the CYCLES and STAGES arguments to this function. If STAGES
724 : is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor
725 : which is multiplied by MODULO_II to give the number of cycles. This is
726 : only useful if the caller also calls set_modulo_params to enable modulo
727 : scheduling. */
728 :
729 : void
730 0 : record_delay_slot_pair (rtx_insn *i1, rtx_insn *i2, int cycles, int stages)
731 : {
732 0 : struct delay_pair *p = XNEW (struct delay_pair);
733 0 : struct delay_pair **slot;
734 :
735 0 : p->i1 = i1;
736 0 : p->i2 = i2;
737 0 : p->cycles = cycles;
738 0 : p->stages = stages;
739 :
740 0 : if (!delay_htab)
741 : {
742 0 : delay_htab = new hash_table<delay_i1_hasher> (10);
743 0 : delay_htab_i2 = new hash_table<delay_i2_hasher> (10);
744 : }
745 0 : slot = delay_htab->find_slot_with_hash (i1, htab_hash_pointer (i1), INSERT);
746 0 : p->next_same_i1 = *slot;
747 0 : *slot = p;
748 0 : slot = delay_htab_i2->find_slot (p, INSERT);
749 0 : *slot = p;
750 0 : }
751 :
752 : /* Examine the delay pair hashtable to see if INSN is a shadow for another,
753 : and return the other insn if so. Return NULL otherwise. */
754 : rtx_insn *
755 0 : real_insn_for_shadow (rtx_insn *insn)
756 : {
757 0 : struct delay_pair *pair;
758 :
759 0 : if (!delay_htab)
760 : return NULL;
761 :
762 0 : pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn));
763 0 : if (!pair || pair->stages > 0)
764 : return NULL;
765 0 : return pair->i1;
766 : }
767 :
768 : /* For a pair P of insns, return the fixed distance in cycles from the first
769 : insn after which the second must be scheduled. */
770 : static int
771 0 : pair_delay (struct delay_pair *p)
772 : {
773 0 : if (p->stages == 0)
774 0 : return p->cycles;
775 : else
776 0 : return p->stages * modulo_ii;
777 : }
778 :
779 : /* Given an insn INSN, add a dependence on its delayed shadow if it
780 : has one. Also try to find situations where shadows depend on each other
781 : and add dependencies to the real insns to limit the amount of backtracking
782 : needed. */
783 : void
784 1532 : add_delay_dependencies (rtx_insn *insn)
785 : {
786 1532 : struct delay_pair *pair;
787 1532 : sd_iterator_def sd_it;
788 1532 : dep_t dep;
789 :
790 1532 : if (!delay_htab)
791 1532 : return;
792 :
793 0 : pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn));
794 0 : if (!pair)
795 : return;
796 0 : add_dependence (insn, pair->i1, REG_DEP_ANTI);
797 0 : if (pair->stages)
798 : return;
799 :
800 0 : FOR_EACH_DEP (pair->i2, SD_LIST_BACK, sd_it, dep)
801 : {
802 0 : rtx_insn *pro = DEP_PRO (dep);
803 0 : struct delay_pair *other_pair
804 0 : = delay_htab_i2->find_with_hash (pro, htab_hash_pointer (pro));
805 0 : if (!other_pair || other_pair->stages)
806 0 : continue;
807 0 : if (pair_delay (other_pair) >= pair_delay (pair))
808 : {
809 0 : if (sched_verbose >= 4)
810 : {
811 0 : fprintf (sched_dump, ";;\tadding dependence %d <- %d\n",
812 0 : INSN_UID (other_pair->i1),
813 0 : INSN_UID (pair->i1));
814 0 : fprintf (sched_dump, ";;\tpair1 %d <- %d, cost %d\n",
815 0 : INSN_UID (pair->i1),
816 0 : INSN_UID (pair->i2),
817 : pair_delay (pair));
818 0 : fprintf (sched_dump, ";;\tpair2 %d <- %d, cost %d\n",
819 0 : INSN_UID (other_pair->i1),
820 0 : INSN_UID (other_pair->i2),
821 : pair_delay (other_pair));
822 : }
823 0 : add_dependence (pair->i1, other_pair->i1, REG_DEP_ANTI);
824 : }
825 : }
826 : }
827 :
828 : /* Forward declarations. */
829 :
830 : static int priority (rtx_insn *, bool force_recompute = false);
831 : static int autopref_rank_for_schedule (const rtx_insn *, const rtx_insn *);
832 : static int rank_for_schedule (const void *, const void *);
833 : static void swap_sort (rtx_insn **, int);
834 : static void queue_insn (rtx_insn *, int, const char *);
835 : static int schedule_insn (rtx_insn *);
836 : static void adjust_priority (rtx_insn *);
837 : static void advance_one_cycle (void);
838 : static void extend_h_i_d (void);
839 :
840 :
841 : /* Notes handling mechanism:
842 : =========================
843 : Generally, NOTES are saved before scheduling and restored after scheduling.
844 : The scheduler distinguishes between two types of notes:
845 :
846 : (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes:
847 : Before scheduling a region, a pointer to the note is added to the insn
848 : that follows or precedes it. (This happens as part of the data dependence
849 : computation). After scheduling an insn, the pointer contained in it is
850 : used for regenerating the corresponding note (in reemit_notes).
851 :
852 : (2) All other notes (e.g. INSN_DELETED): Before scheduling a block,
853 : these notes are put in a list (in rm_other_notes() and
854 : unlink_other_notes ()). After scheduling the block, these notes are
855 : inserted at the beginning of the block (in schedule_block()). */
856 :
857 : static void ready_add (struct ready_list *, rtx_insn *, bool);
858 : static rtx_insn *ready_remove_first (struct ready_list *);
859 : static rtx_insn *ready_remove_first_dispatch (struct ready_list *ready);
860 :
861 : static void queue_to_ready (struct ready_list *);
862 : static int early_queue_to_ready (state_t, struct ready_list *);
863 :
864 : /* The following functions are used to implement multi-pass scheduling
865 : on the first cycle. */
866 : static rtx_insn *ready_remove (struct ready_list *, int);
867 : static void ready_remove_insn (rtx_insn *);
868 :
869 : static void fix_inter_tick (rtx_insn *, rtx_insn *);
870 : static int fix_tick_ready (rtx_insn *);
871 : static void change_queue_index (rtx_insn *, int);
872 :
873 : /* The following functions are used to implement scheduling of data/control
874 : speculative instructions. */
875 :
876 : static void extend_h_i_d (void);
877 : static void init_h_i_d (rtx_insn *);
878 : static int haifa_speculate_insn (rtx_insn *, ds_t, rtx *);
879 : static void generate_recovery_code (rtx_insn *);
880 : static void process_insn_forw_deps_be_in_spec (rtx_insn *, rtx_insn *, ds_t);
881 : static void begin_speculative_block (rtx_insn *);
882 : static void add_to_speculative_block (rtx_insn *);
883 : static void init_before_recovery (basic_block *);
884 : static void create_check_block_twin (rtx_insn *, bool);
885 : static void fix_recovery_deps (basic_block);
886 : static bool haifa_change_pattern (rtx_insn *, rtx);
887 : static void dump_new_block_header (int, basic_block, rtx_insn *, rtx_insn *);
888 : static void restore_bb_notes (basic_block);
889 : static void fix_jump_move (rtx_insn *);
890 : static void move_block_after_check (rtx_insn *);
891 : static void move_succs (vec<edge, va_gc> **, basic_block);
892 : static void sched_remove_insn (rtx_insn *);
893 : static void clear_priorities (rtx_insn *, rtx_vec_t *);
894 : static void calc_priorities (const rtx_vec_t &);
895 : static void add_jump_dependencies (rtx_insn *, rtx_insn *);
896 :
897 : #endif /* INSN_SCHEDULING */
898 :
899 : /* Point to state used for the current scheduling pass. */
900 : struct haifa_sched_info *current_sched_info;
901 :
902 : #ifndef INSN_SCHEDULING
903 : void
904 : schedule_insns (void)
905 : {
906 : }
907 : #else
908 :
909 : /* Do register pressure sensitive insn scheduling if the flag is set
910 : up. */
911 : enum sched_pressure_algorithm sched_pressure;
912 :
913 : /* Map regno -> its pressure class. The map defined only when
914 : SCHED_PRESSURE != SCHED_PRESSURE_NONE. */
915 : enum reg_class *sched_regno_pressure_class;
916 :
917 : /* The current register pressure. Only elements corresponding pressure
918 : classes are defined. */
919 : static int curr_reg_pressure[N_REG_CLASSES];
920 :
921 : /* Saved value of the previous array. */
922 : static int saved_reg_pressure[N_REG_CLASSES];
923 :
924 : /* Register living at given scheduling point. */
925 : static bitmap curr_reg_live;
926 :
927 : /* Saved value of the previous array. */
928 : static bitmap saved_reg_live;
929 :
930 : /* Registers mentioned in the current region. */
931 : static bitmap region_ref_regs;
932 :
933 : /* Temporary bitmap used for SCHED_PRESSURE_MODEL. */
934 : static bitmap tmp_bitmap;
935 :
936 : /* Effective number of available registers of a given class (see comment
937 : in sched_pressure_start_bb). */
938 : static int sched_class_regs_num[N_REG_CLASSES];
939 : /* The number of registers that the function would need to save before it
940 : uses them, and the number of fixed_regs. Helpers for calculating of
941 : sched_class_regs_num. */
942 : static int call_saved_regs_num[N_REG_CLASSES];
943 : static int fixed_regs_num[N_REG_CLASSES];
944 :
945 : /* Initiate register pressure relative info for scheduling the current
946 : region. Currently it is only clearing register mentioned in the
947 : current region. */
948 : void
949 415 : sched_init_region_reg_pressure_info (void)
950 : {
951 415 : bitmap_clear (region_ref_regs);
952 415 : }
953 :
954 : /* PRESSURE[CL] describes the pressure on register class CL. Update it
955 : for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO.
956 : LIVE tracks the set of live registers; if it is null, assume that
957 : every birth or death is genuine. */
958 : static inline void
959 49429 : mark_regno_birth_or_death (bitmap live, int *pressure, int regno, bool birth_p)
960 : {
961 49429 : enum reg_class pressure_class;
962 :
963 49429 : pressure_class = sched_regno_pressure_class[regno];
964 49429 : if (regno >= FIRST_PSEUDO_REGISTER)
965 : {
966 41989 : if (pressure_class != NO_REGS)
967 : {
968 41945 : if (birth_p)
969 : {
970 25566 : if (!live || bitmap_set_bit (live, regno))
971 24976 : pressure[pressure_class]
972 24976 : += (ira_reg_class_max_nregs
973 24976 : [pressure_class][PSEUDO_REGNO_MODE (regno)]);
974 : }
975 : else
976 : {
977 16379 : if (!live || bitmap_clear_bit (live, regno))
978 15471 : pressure[pressure_class]
979 15471 : -= (ira_reg_class_max_nregs
980 15471 : [pressure_class][PSEUDO_REGNO_MODE (regno)]);
981 : }
982 : }
983 : }
984 7440 : else if (pressure_class != NO_REGS
985 7440 : && ! TEST_HARD_REG_BIT (ira_no_alloc_regs, regno))
986 : {
987 6211 : if (birth_p)
988 : {
989 4243 : if (!live || bitmap_set_bit (live, regno))
990 4242 : pressure[pressure_class]++;
991 : }
992 : else
993 : {
994 1968 : if (!live || bitmap_clear_bit (live, regno))
995 1968 : pressure[pressure_class]--;
996 : }
997 : }
998 49429 : }
999 :
1000 : /* Initiate current register pressure related info from living
1001 : registers given by LIVE. */
1002 : static void
1003 430 : initiate_reg_pressure_info (bitmap live)
1004 : {
1005 430 : int i;
1006 430 : unsigned int j;
1007 430 : bitmap_iterator bi;
1008 :
1009 2154 : for (i = 0; i < ira_pressure_classes_num; i++)
1010 1724 : curr_reg_pressure[ira_pressure_classes[i]] = 0;
1011 430 : bitmap_clear (curr_reg_live);
1012 4628 : EXECUTE_IF_SET_IN_BITMAP (live, 0, j, bi)
1013 4198 : if (sched_pressure == SCHED_PRESSURE_MODEL
1014 4198 : || current_nr_blocks == 1
1015 4738 : || bitmap_bit_p (region_ref_regs, j))
1016 3970 : mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, j, true);
1017 430 : }
1018 :
1019 : /* Mark registers in X as mentioned in the current region. */
1020 : static void
1021 2580 : setup_ref_regs (rtx x)
1022 : {
1023 2580 : int i, j;
1024 2580 : const RTX_CODE code = GET_CODE (x);
1025 2580 : const char *fmt;
1026 :
1027 2580 : if (REG_P (x))
1028 : {
1029 924 : bitmap_set_range (region_ref_regs, REGNO (x), REG_NREGS (x));
1030 924 : return;
1031 : }
1032 1656 : fmt = GET_RTX_FORMAT (code);
1033 4428 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
1034 2772 : if (fmt[i] == 'e')
1035 1956 : setup_ref_regs (XEXP (x, i));
1036 816 : else if (fmt[i] == 'E')
1037 : {
1038 84 : for (j = 0; j < XVECLEN (x, i); j++)
1039 48 : setup_ref_regs (XVECEXP (x, i, j));
1040 : }
1041 : }
1042 :
1043 : /* Initiate current register pressure related info at the start of
1044 : basic block BB. */
1045 : static void
1046 430 : initiate_bb_reg_pressure_info (basic_block bb)
1047 : {
1048 430 : unsigned int i ATTRIBUTE_UNUSED;
1049 430 : rtx_insn *insn;
1050 :
1051 430 : if (current_nr_blocks > 1)
1052 816 : FOR_BB_INSNS (bb, insn)
1053 780 : if (NONDEBUG_INSN_P (insn))
1054 576 : setup_ref_regs (PATTERN (insn));
1055 430 : initiate_reg_pressure_info (df_get_live_in (bb));
1056 430 : if (bb_has_eh_pred (bb))
1057 0 : for (i = 0; ; ++i)
1058 : {
1059 0 : unsigned int regno = EH_RETURN_DATA_REGNO (i);
1060 :
1061 0 : if (regno == INVALID_REGNUM)
1062 : break;
1063 0 : if (! bitmap_bit_p (df_get_live_in (bb), regno))
1064 0 : mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
1065 : regno, true);
1066 0 : }
1067 430 : }
1068 :
1069 : /* Save current register pressure related info. */
1070 : static void
1071 2579 : save_reg_pressure (void)
1072 : {
1073 2579 : int i;
1074 :
1075 13084 : for (i = 0; i < ira_pressure_classes_num; i++)
1076 10505 : saved_reg_pressure[ira_pressure_classes[i]]
1077 10505 : = curr_reg_pressure[ira_pressure_classes[i]];
1078 2579 : bitmap_copy (saved_reg_live, curr_reg_live);
1079 2579 : }
1080 :
1081 : /* Restore saved register pressure related info. */
1082 : static void
1083 2579 : restore_reg_pressure (void)
1084 : {
1085 2579 : int i;
1086 :
1087 13084 : for (i = 0; i < ira_pressure_classes_num; i++)
1088 10505 : curr_reg_pressure[ira_pressure_classes[i]]
1089 10505 : = saved_reg_pressure[ira_pressure_classes[i]];
1090 2579 : bitmap_copy (curr_reg_live, saved_reg_live);
1091 2579 : }
1092 :
1093 : /* Return TRUE if the register is dying after its USE. */
1094 : static bool
1095 38804 : dying_use_p (struct reg_use_data *use)
1096 : {
1097 38804 : struct reg_use_data *next;
1098 :
1099 46037 : for (next = use->next_regno_use; next != use; next = next->next_regno_use)
1100 17812 : if (NONDEBUG_INSN_P (next->insn)
1101 17812 : && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED)
1102 : return false;
1103 : return true;
1104 : }
1105 :
1106 : /* Print info about the current register pressure and its excess for
1107 : each pressure class. */
1108 : static void
1109 0 : print_curr_reg_pressure (void)
1110 : {
1111 0 : int i;
1112 0 : enum reg_class cl;
1113 :
1114 0 : fprintf (sched_dump, ";;\t");
1115 0 : for (i = 0; i < ira_pressure_classes_num; i++)
1116 : {
1117 0 : cl = ira_pressure_classes[i];
1118 0 : gcc_assert (curr_reg_pressure[cl] >= 0);
1119 0 : fprintf (sched_dump, " %s:%d(%d)", reg_class_names[cl],
1120 : curr_reg_pressure[cl],
1121 0 : curr_reg_pressure[cl] - sched_class_regs_num[cl]);
1122 : }
1123 0 : fprintf (sched_dump, "\n");
1124 0 : }
1125 :
1126 : /* Determine if INSN has a condition that is clobbered if a register
1127 : in SET_REGS is modified. */
1128 : static bool
1129 0 : cond_clobbered_p (rtx_insn *insn, HARD_REG_SET set_regs)
1130 : {
1131 0 : rtx pat = PATTERN (insn);
1132 0 : gcc_assert (GET_CODE (pat) == COND_EXEC);
1133 0 : if (TEST_HARD_REG_BIT (set_regs, REGNO (XEXP (COND_EXEC_TEST (pat), 0))))
1134 : {
1135 0 : sd_iterator_def sd_it;
1136 0 : dep_t dep;
1137 0 : haifa_change_pattern (insn, ORIG_PAT (insn));
1138 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
1139 0 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
1140 0 : TODO_SPEC (insn) = HARD_DEP;
1141 0 : if (sched_verbose >= 2)
1142 0 : fprintf (sched_dump,
1143 : ";;\t\tdequeue insn %s because of clobbered condition\n",
1144 0 : (*current_sched_info->print_insn) (insn, 0));
1145 0 : return true;
1146 : }
1147 :
1148 : return false;
1149 : }
1150 :
1151 : /* This function should be called after modifying the pattern of INSN,
1152 : to update scheduler data structures as needed. */
1153 : static void
1154 665849 : update_insn_after_change (rtx_insn *insn)
1155 : {
1156 665849 : sd_iterator_def sd_it;
1157 665849 : dep_t dep;
1158 :
1159 665849 : dfa_clear_single_insn_cache (insn);
1160 :
1161 665849 : sd_it = sd_iterator_start (insn,
1162 : SD_LIST_FORW | SD_LIST_BACK | SD_LIST_RES_BACK);
1163 7367769 : while (sd_iterator_cond (&sd_it, &dep))
1164 : {
1165 6701920 : DEP_COST (dep) = UNKNOWN_DEP_COST;
1166 6701920 : sd_iterator_next (&sd_it);
1167 : }
1168 :
1169 : /* Invalidate INSN_COST, so it'll be recalculated. */
1170 665849 : INSN_COST (insn) = -1;
1171 : /* Invalidate INSN_TICK, so it'll be recalculated. */
1172 665849 : INSN_TICK (insn) = INVALID_TICK;
1173 :
1174 : /* Invalidate autoprefetch data entry. */
1175 665849 : INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status
1176 665849 : = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
1177 665849 : INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status
1178 665849 : = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
1179 665849 : }
1180 :
1181 :
1182 : /* Two VECs, one to hold dependencies for which pattern replacements
1183 : need to be applied or restored at the start of the next cycle, and
1184 : another to hold an integer that is either one, to apply the
1185 : corresponding replacement, or zero to restore it. */
1186 : static vec<dep_t> next_cycle_replace_deps;
1187 : static vec<int> next_cycle_apply;
1188 :
1189 : static void apply_replacement (dep_t, bool);
1190 : static void restore_pattern (dep_t, bool);
1191 :
1192 : /* Look at the remaining dependencies for insn NEXT, and compute and return
1193 : the TODO_SPEC value we should use for it. This is called after one of
1194 : NEXT's dependencies has been resolved.
1195 : We also perform pattern replacements for predication, and for broken
1196 : replacement dependencies. The latter is only done if FOR_BACKTRACK is
1197 : false. */
1198 :
1199 : static ds_t
1200 311309911 : recompute_todo_spec (rtx_insn *next, bool for_backtrack)
1201 : {
1202 311309911 : ds_t new_ds;
1203 311309911 : sd_iterator_def sd_it;
1204 311309911 : dep_t dep, modify_dep = NULL;
1205 311309911 : int n_spec = 0;
1206 311309911 : int n_control = 0;
1207 311309911 : int n_replace = 0;
1208 311309911 : bool first_p = true;
1209 :
1210 311309911 : if (sd_lists_empty_p (next, SD_LIST_BACK))
1211 : /* NEXT has all its dependencies resolved. */
1212 : return 0;
1213 :
1214 203082601 : if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK))
1215 : return HARD_DEP;
1216 :
1217 : /* If NEXT is intended to sit adjacent to this instruction, we don't
1218 : want to try to break any dependencies. Treat it as a HARD_DEP. */
1219 393858 : if (SCHED_GROUP_P (next))
1220 : return HARD_DEP;
1221 :
1222 : /* Now we've got NEXT with speculative deps only.
1223 : 1. Look at the deps to see what we have to do.
1224 : 2. Check if we can do 'todo'. */
1225 393858 : new_ds = 0;
1226 :
1227 1371208 : FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep)
1228 : {
1229 977350 : rtx_insn *pro = DEP_PRO (dep);
1230 977350 : ds_t ds = DEP_STATUS (dep) & SPECULATIVE;
1231 :
1232 977350 : if (DEBUG_INSN_P (pro) && !DEBUG_INSN_P (next))
1233 134272 : continue;
1234 :
1235 843078 : if (ds)
1236 : {
1237 0 : n_spec++;
1238 0 : if (first_p)
1239 : {
1240 : first_p = false;
1241 :
1242 : new_ds = ds;
1243 : }
1244 : else
1245 0 : new_ds = ds_merge (new_ds, ds);
1246 : }
1247 843078 : else if (DEP_TYPE (dep) == REG_DEP_CONTROL)
1248 : {
1249 0 : if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED)
1250 : {
1251 0 : n_control++;
1252 0 : modify_dep = dep;
1253 : }
1254 0 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
1255 : }
1256 843078 : else if (DEP_REPLACE (dep) != NULL)
1257 : {
1258 843078 : if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED)
1259 : {
1260 843078 : n_replace++;
1261 843078 : modify_dep = dep;
1262 : }
1263 843078 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
1264 : }
1265 : }
1266 :
1267 393858 : if (n_replace > 0 && n_control == 0 && n_spec == 0)
1268 : {
1269 393858 : if (!dbg_cnt (sched_breakdep))
1270 : return HARD_DEP;
1271 1371208 : FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep)
1272 : {
1273 977350 : struct dep_replacement *desc = DEP_REPLACE (dep);
1274 977350 : if (desc != NULL)
1275 : {
1276 843078 : if (desc->insn == next && !for_backtrack)
1277 : {
1278 90957 : gcc_assert (n_replace == 1);
1279 90957 : apply_replacement (dep, true);
1280 : }
1281 843078 : DEP_STATUS (dep) |= DEP_CANCELLED;
1282 : }
1283 : }
1284 : return 0;
1285 : }
1286 :
1287 0 : else if (n_control == 1 && n_replace == 0 && n_spec == 0)
1288 : {
1289 0 : rtx_insn *pro, *other;
1290 0 : rtx new_pat;
1291 0 : rtx cond = NULL_RTX;
1292 0 : bool success;
1293 0 : rtx_insn *prev = NULL;
1294 0 : int i;
1295 0 : unsigned regno;
1296 :
1297 0 : if ((current_sched_info->flags & DO_PREDICATION) == 0
1298 0 : || (ORIG_PAT (next) != NULL_RTX
1299 0 : && PREDICATED_PAT (next) == NULL_RTX))
1300 : return HARD_DEP;
1301 :
1302 0 : pro = DEP_PRO (modify_dep);
1303 0 : other = real_insn_for_shadow (pro);
1304 0 : if (other != NULL_RTX)
1305 0 : pro = other;
1306 :
1307 0 : cond = sched_get_reverse_condition_uncached (pro);
1308 0 : regno = REGNO (XEXP (cond, 0));
1309 :
1310 : /* Find the last scheduled insn that modifies the condition register.
1311 : We can stop looking once we find the insn we depend on through the
1312 : REG_DEP_CONTROL; if the condition register isn't modified after it,
1313 : we know that it still has the right value. */
1314 0 : if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED)
1315 0 : FOR_EACH_VEC_ELT_REVERSE (scheduled_insns, i, prev)
1316 : {
1317 0 : HARD_REG_SET t;
1318 :
1319 0 : find_all_hard_reg_sets (prev, &t, true);
1320 0 : if (TEST_HARD_REG_BIT (t, regno))
1321 0 : return HARD_DEP;
1322 0 : if (prev == pro)
1323 : break;
1324 : }
1325 0 : if (ORIG_PAT (next) == NULL_RTX)
1326 : {
1327 0 : ORIG_PAT (next) = PATTERN (next);
1328 :
1329 0 : new_pat = gen_rtx_COND_EXEC (VOIDmode, cond, PATTERN (next));
1330 0 : success = haifa_change_pattern (next, new_pat);
1331 0 : if (!success)
1332 : return HARD_DEP;
1333 0 : PREDICATED_PAT (next) = new_pat;
1334 : }
1335 0 : else if (PATTERN (next) != PREDICATED_PAT (next))
1336 : {
1337 0 : bool success = haifa_change_pattern (next,
1338 0 : PREDICATED_PAT (next));
1339 0 : gcc_assert (success);
1340 : }
1341 0 : DEP_STATUS (modify_dep) |= DEP_CANCELLED;
1342 0 : return DEP_CONTROL;
1343 : }
1344 :
1345 0 : if (PREDICATED_PAT (next) != NULL_RTX)
1346 : {
1347 0 : int tick = INSN_TICK (next);
1348 0 : bool success = haifa_change_pattern (next,
1349 0 : ORIG_PAT (next));
1350 0 : INSN_TICK (next) = tick;
1351 0 : gcc_assert (success);
1352 : }
1353 :
1354 : /* We can't handle the case where there are both speculative and control
1355 : dependencies, so we return HARD_DEP in such a case. Also fail if
1356 : we have speculative dependencies with not enough points, or more than
1357 : one control dependency. */
1358 0 : if ((n_spec > 0 && (n_control > 0 || n_replace > 0))
1359 0 : || (n_spec > 0
1360 : /* Too few points? */
1361 0 : && ds_weak (new_ds) < spec_info->data_weakness_cutoff)
1362 0 : || n_control > 0
1363 0 : || n_replace > 0)
1364 0 : return HARD_DEP;
1365 :
1366 : return new_ds;
1367 : }
1368 :
1369 : /* Pointer to the last instruction scheduled. */
1370 : static rtx_insn *last_scheduled_insn;
1371 :
1372 : /* Pointer to the last nondebug instruction scheduled within the
1373 : block, or the prev_head of the scheduling block. Used by
1374 : rank_for_schedule, so that insns independent of the last scheduled
1375 : insn will be preferred over dependent instructions. */
1376 : static rtx_insn *last_nondebug_scheduled_insn;
1377 :
1378 : /* Pointer that iterates through the list of unscheduled insns if we
1379 : have a dbg_cnt enabled. It always points at an insn prior to the
1380 : first unscheduled one. */
1381 : static rtx_insn *nonscheduled_insns_begin;
1382 :
1383 : /* Compute cost of executing INSN.
1384 : This is the number of cycles between instruction issue and
1385 : instruction results. */
1386 : int
1387 211578817 : insn_sched_cost (rtx_insn *insn)
1388 : {
1389 211578817 : int cost;
1390 :
1391 211578817 : if (sched_fusion)
1392 : return 0;
1393 :
1394 211578817 : if (sel_sched_p ())
1395 : {
1396 36236 : if (recog_memoized (insn) < 0)
1397 : return 0;
1398 :
1399 35640 : cost = insn_default_latency (insn);
1400 35640 : if (cost < 0)
1401 : cost = 0;
1402 :
1403 35640 : return cost;
1404 : }
1405 :
1406 211542581 : cost = INSN_COST (insn);
1407 :
1408 211542581 : if (cost < 0)
1409 : {
1410 : /* A USE insn, or something else we don't need to
1411 : understand. We can't pass these directly to
1412 : result_ready_cost or insn_default_latency because it will
1413 : trigger a fatal error for unrecognizable insns. */
1414 109204766 : if (recog_memoized (insn) < 0)
1415 : {
1416 49340910 : INSN_COST (insn) = 0;
1417 49340910 : return 0;
1418 : }
1419 : else
1420 : {
1421 59863856 : cost = insn_default_latency (insn);
1422 59863856 : if (cost < 0)
1423 : cost = 0;
1424 :
1425 59863856 : INSN_COST (insn) = cost;
1426 : }
1427 : }
1428 :
1429 : return cost;
1430 : }
1431 :
1432 : /* Compute cost of dependence LINK.
1433 : This is the number of cycles between instruction issue and
1434 : instruction results.
1435 : ??? We also use this function to call recog_memoized on all insns. */
1436 : int
1437 328620314 : dep_cost_1 (dep_t link, dw_t dw)
1438 : {
1439 328620314 : rtx_insn *insn = DEP_PRO (link);
1440 328620314 : rtx_insn *used = DEP_CON (link);
1441 328620314 : int cost;
1442 :
1443 328620314 : if (DEP_COST (link) != UNKNOWN_DEP_COST)
1444 175554171 : return DEP_COST (link);
1445 :
1446 153066143 : if (delay_htab)
1447 : {
1448 0 : struct delay_pair *delay_entry;
1449 0 : delay_entry
1450 0 : = delay_htab_i2->find_with_hash (used, htab_hash_pointer (used));
1451 0 : if (delay_entry)
1452 : {
1453 0 : if (delay_entry->i1 == insn)
1454 : {
1455 0 : DEP_COST (link) = pair_delay (delay_entry);
1456 0 : return DEP_COST (link);
1457 : }
1458 : }
1459 : }
1460 :
1461 : /* A USE insn should never require the value used to be computed.
1462 : This allows the computation of a function's result and parameter
1463 : values to overlap the return and call. We don't care about the
1464 : dependence cost when only decreasing register pressure. */
1465 153066143 : if (recog_memoized (used) < 0)
1466 : {
1467 1416789 : cost = 0;
1468 1416789 : recog_memoized (insn);
1469 : }
1470 : else
1471 : {
1472 151649354 : enum reg_note dep_type = DEP_TYPE (link);
1473 :
1474 151649354 : cost = insn_sched_cost (insn);
1475 :
1476 151649354 : if (INSN_CODE (insn) >= 0)
1477 : {
1478 146579542 : if (dep_type == REG_DEP_ANTI)
1479 : cost = 0;
1480 77543477 : else if (dep_type == REG_DEP_OUTPUT)
1481 : {
1482 23871782 : cost = (insn_default_latency (insn)
1483 23871782 : - insn_default_latency (used));
1484 23871782 : if (cost <= 0)
1485 : cost = 1;
1486 : }
1487 53671695 : else if (bypass_p (insn))
1488 1639 : cost = insn_latency (insn, used);
1489 : }
1490 :
1491 :
1492 151649354 : if (targetm.sched.adjust_cost)
1493 151649354 : cost = targetm.sched.adjust_cost (used, (int) dep_type, insn, cost,
1494 : dw);
1495 :
1496 151649354 : if (cost < 0)
1497 1416789 : cost = 0;
1498 : }
1499 :
1500 153066143 : DEP_COST (link) = cost;
1501 153066143 : return cost;
1502 : }
1503 :
1504 : /* Compute cost of dependence LINK.
1505 : This is the number of cycles between instruction issue and
1506 : instruction results. */
1507 : int
1508 328591449 : dep_cost (dep_t link)
1509 : {
1510 328591449 : return dep_cost_1 (link, 0);
1511 : }
1512 :
1513 : /* Use this sel-sched.cc friendly function in reorder2 instead of increasing
1514 : INSN_PRIORITY explicitly. */
1515 : void
1516 0 : increase_insn_priority (rtx_insn *insn, int amount)
1517 : {
1518 0 : if (!sel_sched_p ())
1519 : {
1520 : /* We're dealing with haifa-sched.cc INSN_PRIORITY. */
1521 0 : if (INSN_PRIORITY_KNOWN (insn))
1522 0 : INSN_PRIORITY (insn) += amount;
1523 : }
1524 : else
1525 : {
1526 : /* In sel-sched.cc INSN_PRIORITY is not kept up to date.
1527 : Use EXPR_PRIORITY instead. */
1528 0 : sel_add_to_insn_priority (insn, amount);
1529 : }
1530 0 : }
1531 :
1532 : /* Return 'true' if DEP should be included in priority calculations. */
1533 : static bool
1534 156107211 : contributes_to_priority_p (dep_t dep)
1535 : {
1536 156107211 : if (DEBUG_INSN_P (DEP_CON (dep))
1537 143205864 : || DEBUG_INSN_P (DEP_PRO (dep)))
1538 : return false;
1539 :
1540 : /* Critical path is meaningful in block boundaries only. */
1541 143205864 : if (!current_sched_info->contributes_to_priority (DEP_CON (dep),
1542 : DEP_PRO (dep)))
1543 : return false;
1544 :
1545 143202189 : if (DEP_REPLACE (dep) != NULL)
1546 : return false;
1547 :
1548 : /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set,
1549 : then speculative instructions will less likely be
1550 : scheduled. That is because the priority of
1551 : their producers will increase, and, thus, the
1552 : producers will more likely be scheduled, thus,
1553 : resolving the dependence. */
1554 142548620 : if (sched_deps_info->generate_spec_deps
1555 0 : && !(spec_info->flags & COUNT_SPEC_IN_CRITICAL_PATH)
1556 0 : && (DEP_STATUS (dep) & SPECULATIVE))
1557 : return false;
1558 :
1559 : return true;
1560 : }
1561 :
1562 : /* Compute the number of nondebug deps in list LIST for INSN. */
1563 : int
1564 655969062 : dep_list_size (rtx_insn *insn, sd_list_types_def list)
1565 : {
1566 655969062 : sd_iterator_def sd_it;
1567 655969062 : dep_t dep;
1568 655969062 : int dbgcount = 0, nodbgcount = 0;
1569 :
1570 655969062 : if (!MAY_HAVE_DEBUG_INSNS)
1571 276572669 : return sd_lists_size (insn, list);
1572 :
1573 : /* TODO: We should split normal and debug insns into separate SD_LIST_*
1574 : sub-lists, and then we'll be able to use something like
1575 : sd_lists_size(insn, list & SD_LIST_NON_DEBUG)
1576 : instead of walking dependencies below. */
1577 :
1578 1334459039 : FOR_EACH_DEP (insn, list, sd_it, dep)
1579 : {
1580 955062646 : if (DEBUG_INSN_P (DEP_CON (dep)))
1581 102663754 : dbgcount++;
1582 852398892 : else if (!DEBUG_INSN_P (DEP_PRO (dep)))
1583 845118247 : nodbgcount++;
1584 : }
1585 :
1586 379396393 : gcc_assert (dbgcount + nodbgcount == sd_lists_size (insn, list));
1587 :
1588 : return nodbgcount;
1589 : }
1590 :
1591 : bool sched_fusion;
1592 :
1593 : /* Compute the priority number for INSN. */
1594 : static int
1595 251776975 : priority (rtx_insn *insn, bool force_recompute)
1596 : {
1597 251776975 : if (! INSN_P (insn))
1598 : return 0;
1599 :
1600 : /* We should not be interested in priority of an already scheduled insn. */
1601 251776975 : gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED);
1602 :
1603 251776975 : if (force_recompute || !INSN_PRIORITY_KNOWN (insn))
1604 : {
1605 109228234 : int this_priority = -1;
1606 :
1607 109228234 : if (sched_fusion)
1608 : {
1609 0 : int this_fusion_priority;
1610 :
1611 0 : targetm.sched.fusion_priority (insn, FUSION_MAX_PRIORITY,
1612 : &this_fusion_priority, &this_priority);
1613 0 : INSN_FUSION_PRIORITY (insn) = this_fusion_priority;
1614 : }
1615 109228234 : else if (dep_list_size (insn, SD_LIST_FORW) == 0)
1616 : /* ??? We should set INSN_PRIORITY to insn_sched_cost when and insn
1617 : has some forward deps but all of them are ignored by
1618 : contributes_to_priority hook. At the moment we set priority of
1619 : such insn to 0. */
1620 59920002 : this_priority = insn_sched_cost (insn);
1621 : else
1622 : {
1623 49308232 : rtx_insn *prev_first, *twin;
1624 49308232 : basic_block rec;
1625 :
1626 : /* For recovery check instructions we calculate priority slightly
1627 : different than that of normal instructions. Instead of walking
1628 : through INSN_FORW_DEPS (check) list, we walk through
1629 : INSN_FORW_DEPS list of each instruction in the corresponding
1630 : recovery block. */
1631 :
1632 : /* Selective scheduling does not define RECOVERY_BLOCK macro. */
1633 49308232 : rec = sel_sched_p () ? NULL : RECOVERY_BLOCK (insn);
1634 49304875 : if (!rec || rec == EXIT_BLOCK_PTR_FOR_FN (cfun))
1635 : {
1636 49308232 : prev_first = PREV_INSN (insn);
1637 49308232 : twin = insn;
1638 : }
1639 : else
1640 : {
1641 0 : prev_first = NEXT_INSN (BB_HEAD (rec));
1642 0 : twin = PREV_INSN (BB_END (rec));
1643 : }
1644 :
1645 49308232 : do
1646 : {
1647 49308232 : sd_iterator_def sd_it;
1648 49308232 : dep_t dep;
1649 :
1650 205415443 : FOR_EACH_DEP (twin, SD_LIST_FORW, sd_it, dep)
1651 : {
1652 156107211 : rtx_insn *next;
1653 156107211 : int next_priority;
1654 :
1655 156107211 : next = DEP_CON (dep);
1656 :
1657 156107211 : if (BLOCK_FOR_INSN (next) != rec)
1658 : {
1659 156107211 : int cost;
1660 :
1661 156107211 : if (!contributes_to_priority_p (dep))
1662 13558591 : continue;
1663 :
1664 142548620 : if (twin == insn)
1665 142548620 : cost = dep_cost (dep);
1666 : else
1667 : {
1668 0 : struct _dep _dep1, *dep1 = &_dep1;
1669 :
1670 0 : init_dep (dep1, insn, next, REG_DEP_ANTI);
1671 :
1672 0 : cost = dep_cost (dep1);
1673 : }
1674 :
1675 142548620 : next_priority = cost + priority (next);
1676 :
1677 142548620 : if (next_priority > this_priority)
1678 101677104 : this_priority = next_priority;
1679 : }
1680 : }
1681 :
1682 49308232 : twin = PREV_INSN (twin);
1683 : }
1684 49308232 : while (twin != prev_first);
1685 : }
1686 :
1687 109228234 : if (this_priority < 0)
1688 : {
1689 9461 : gcc_assert (this_priority == -1);
1690 :
1691 9461 : this_priority = insn_sched_cost (insn);
1692 : }
1693 :
1694 109228234 : INSN_PRIORITY (insn) = this_priority;
1695 109228234 : INSN_PRIORITY_STATUS (insn) = 1;
1696 : }
1697 :
1698 251776975 : return INSN_PRIORITY (insn);
1699 : }
1700 :
1701 : /* Macros and functions for keeping the priority queue sorted, and
1702 : dealing with queuing and dequeuing of instructions. */
1703 :
1704 : /* For each pressure class CL, set DEATH[CL] to the number of registers
1705 : in that class that die in INSN. */
1706 :
1707 : static void
1708 23095 : calculate_reg_deaths (rtx_insn *insn, int *death)
1709 : {
1710 23095 : int i;
1711 23095 : struct reg_use_data *use;
1712 :
1713 126029 : for (i = 0; i < ira_pressure_classes_num; i++)
1714 102934 : death[ira_pressure_classes[i]] = 0;
1715 40369 : for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
1716 17274 : if (dying_use_p (use))
1717 9842 : mark_regno_birth_or_death (0, death, use->regno, true);
1718 23095 : }
1719 :
1720 : /* Setup info about the current register pressure impact of scheduling
1721 : INSN at the current scheduling point. */
1722 : static void
1723 23095 : setup_insn_reg_pressure_info (rtx_insn *insn)
1724 : {
1725 23095 : int i, change, before, after, hard_regno;
1726 23095 : int excess_cost_change;
1727 23095 : machine_mode mode;
1728 23095 : enum reg_class cl;
1729 23095 : struct reg_pressure_data *pressure_info;
1730 23095 : int *max_reg_pressure;
1731 23095 : static int death[N_REG_CLASSES];
1732 :
1733 23095 : gcc_checking_assert (!DEBUG_INSN_P (insn));
1734 :
1735 23095 : excess_cost_change = 0;
1736 23095 : calculate_reg_deaths (insn, death);
1737 23095 : pressure_info = INSN_REG_PRESSURE (insn);
1738 23095 : max_reg_pressure = INSN_MAX_REG_PRESSURE (insn);
1739 23095 : gcc_assert (pressure_info != NULL && max_reg_pressure != NULL);
1740 126029 : for (i = 0; i < ira_pressure_classes_num; i++)
1741 : {
1742 102934 : cl = ira_pressure_classes[i];
1743 102934 : gcc_assert (curr_reg_pressure[cl] >= 0);
1744 102934 : change = (int) pressure_info[i].set_increase - death[cl];
1745 102934 : before = MAX (0, max_reg_pressure[i] - sched_class_regs_num[cl]);
1746 102934 : after = MAX (0, max_reg_pressure[i] + change
1747 : - sched_class_regs_num[cl]);
1748 102934 : hard_regno = ira_class_hard_regs[cl][0];
1749 102934 : gcc_assert (hard_regno >= 0);
1750 102934 : mode = reg_raw_mode[hard_regno];
1751 102934 : excess_cost_change += ((after - before)
1752 102934 : * (ira_memory_move_cost[mode][cl][0]
1753 102934 : + ira_memory_move_cost[mode][cl][1]));
1754 : }
1755 23095 : INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn) = excess_cost_change;
1756 23095 : }
1757 :
1758 : /* This is the first page of code related to SCHED_PRESSURE_MODEL.
1759 : It tries to make the scheduler take register pressure into account
1760 : without introducing too many unnecessary stalls. It hooks into the
1761 : main scheduling algorithm at several points:
1762 :
1763 : - Before scheduling starts, model_start_schedule constructs a
1764 : "model schedule" for the current block. This model schedule is
1765 : chosen solely to keep register pressure down. It does not take the
1766 : target's pipeline or the original instruction order into account,
1767 : except as a tie-breaker. It also doesn't work to a particular
1768 : pressure limit.
1769 :
1770 : This model schedule gives us an idea of what pressure can be
1771 : achieved for the block and gives us an example of a schedule that
1772 : keeps to that pressure. It also makes the final schedule less
1773 : dependent on the original instruction order. This is important
1774 : because the original order can either be "wide" (many values live
1775 : at once, such as in user-scheduled code) or "narrow" (few values
1776 : live at once, such as after loop unrolling, where several
1777 : iterations are executed sequentially).
1778 :
1779 : We do not apply this model schedule to the rtx stream. We simply
1780 : record it in model_schedule. We also compute the maximum pressure,
1781 : MP, that was seen during this schedule.
1782 :
1783 : - Instructions are added to the ready queue even if they require
1784 : a stall. The length of the stall is instead computed as:
1785 :
1786 : MAX (INSN_TICK (INSN) - clock_var, 0)
1787 :
1788 : (= insn_delay). This allows rank_for_schedule to choose between
1789 : introducing a deliberate stall or increasing pressure.
1790 :
1791 : - Before sorting the ready queue, model_set_excess_costs assigns
1792 : a pressure-based cost to each ready instruction in the queue.
1793 : This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE
1794 : (ECC for short) and is effectively measured in cycles.
1795 :
1796 : - rank_for_schedule ranks instructions based on:
1797 :
1798 : ECC (insn) + insn_delay (insn)
1799 :
1800 : then as:
1801 :
1802 : insn_delay (insn)
1803 :
1804 : So, for example, an instruction X1 with an ECC of 1 that can issue
1805 : now will win over an instruction X0 with an ECC of zero that would
1806 : introduce a stall of one cycle. However, an instruction X2 with an
1807 : ECC of 2 that can issue now will lose to both X0 and X1.
1808 :
1809 : - When an instruction is scheduled, model_recompute updates the model
1810 : schedule with the new pressures (some of which might now exceed the
1811 : original maximum pressure MP). model_update_limit_points then searches
1812 : for the new point of maximum pressure, if not already known. */
1813 :
1814 : /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL
1815 : from surrounding debug information. */
1816 : #define MODEL_BAR \
1817 : ";;\t\t+------------------------------------------------------\n"
1818 :
1819 : /* Information about the pressure on a particular register class at a
1820 : particular point of the model schedule. */
1821 : struct model_pressure_data {
1822 : /* The pressure at this point of the model schedule, or -1 if the
1823 : point is associated with an instruction that has already been
1824 : scheduled. */
1825 : int ref_pressure;
1826 :
1827 : /* The maximum pressure during or after this point of the model schedule. */
1828 : int max_pressure;
1829 : };
1830 :
1831 : /* Per-instruction information that is used while building the model
1832 : schedule. Here, "schedule" refers to the model schedule rather
1833 : than the main schedule. */
1834 : struct model_insn_info {
1835 : /* The instruction itself. */
1836 : rtx_insn *insn;
1837 :
1838 : /* If this instruction is in model_worklist, these fields link to the
1839 : previous (higher-priority) and next (lower-priority) instructions
1840 : in the list. */
1841 : struct model_insn_info *prev;
1842 : struct model_insn_info *next;
1843 :
1844 : /* While constructing the schedule, QUEUE_INDEX describes whether an
1845 : instruction has already been added to the schedule (QUEUE_SCHEDULED),
1846 : is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE).
1847 : old_queue records the value that QUEUE_INDEX had before scheduling
1848 : started, so that we can restore it once the schedule is complete. */
1849 : int old_queue;
1850 :
1851 : /* The relative importance of an unscheduled instruction. Higher
1852 : values indicate greater importance. */
1853 : unsigned int model_priority;
1854 :
1855 : /* The length of the longest path of satisfied true dependencies
1856 : that leads to this instruction. */
1857 : unsigned int depth;
1858 :
1859 : /* The length of the longest path of dependencies of any kind
1860 : that leads from this instruction. */
1861 : unsigned int alap;
1862 :
1863 : /* The number of predecessor nodes that must still be scheduled. */
1864 : int unscheduled_preds;
1865 : };
1866 :
1867 : /* Information about the pressure limit for a particular register class.
1868 : This structure is used when applying a model schedule to the main
1869 : schedule. */
1870 : struct model_pressure_limit {
1871 : /* The maximum register pressure seen in the original model schedule. */
1872 : int orig_pressure;
1873 :
1874 : /* The maximum register pressure seen in the current model schedule
1875 : (which excludes instructions that have already been scheduled). */
1876 : int pressure;
1877 :
1878 : /* The point of the current model schedule at which PRESSURE is first
1879 : reached. It is set to -1 if the value needs to be recomputed. */
1880 : int point;
1881 : };
1882 :
1883 : /* Describes a particular way of measuring register pressure. */
1884 : struct model_pressure_group {
1885 : /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. */
1886 : struct model_pressure_limit limits[N_REG_CLASSES];
1887 :
1888 : /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure
1889 : on register class ira_pressure_classes[PCI] at point POINT of the
1890 : current model schedule. A POINT of model_num_insns describes the
1891 : pressure at the end of the schedule. */
1892 : struct model_pressure_data *model;
1893 : };
1894 :
1895 : /* Index POINT gives the instruction at point POINT of the model schedule.
1896 : This array doesn't change during main scheduling. */
1897 : static vec<rtx_insn *> model_schedule;
1898 :
1899 : /* The list of instructions in the model worklist, sorted in order of
1900 : decreasing priority. */
1901 : static struct model_insn_info *model_worklist;
1902 :
1903 : /* Index I describes the instruction with INSN_LUID I. */
1904 : static struct model_insn_info *model_insns;
1905 :
1906 : /* The number of instructions in the model schedule. */
1907 : static int model_num_insns;
1908 :
1909 : /* The index of the first instruction in model_schedule that hasn't yet been
1910 : added to the main schedule, or model_num_insns if all of them have. */
1911 : static int model_curr_point;
1912 :
1913 : /* Describes the pressure before each instruction in the model schedule. */
1914 : static struct model_pressure_group model_before_pressure;
1915 :
1916 : /* The first unused model_priority value (as used in model_insn_info). */
1917 : static unsigned int model_next_priority;
1918 :
1919 :
1920 : /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP
1921 : at point POINT of the model schedule. */
1922 : #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \
1923 : (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)])
1924 :
1925 : /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or
1926 : after point POINT of the model schedule. */
1927 : #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \
1928 : (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure)
1929 :
1930 : /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT
1931 : of the model schedule. */
1932 : #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \
1933 : (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure)
1934 :
1935 : /* Information about INSN that is used when creating the model schedule. */
1936 : #define MODEL_INSN_INFO(INSN) \
1937 : (&model_insns[INSN_LUID (INSN)])
1938 :
1939 : /* The instruction at point POINT of the model schedule. */
1940 : #define MODEL_INSN(POINT) \
1941 : (model_schedule[POINT])
1942 :
1943 :
1944 : /* Return INSN's index in the model schedule, or model_num_insns if it
1945 : doesn't belong to that schedule. */
1946 :
1947 : static int
1948 0 : model_index (rtx_insn *insn)
1949 : {
1950 0 : if (INSN_MODEL_INDEX (insn) == 0)
1951 0 : return model_num_insns;
1952 0 : return INSN_MODEL_INDEX (insn) - 1;
1953 : }
1954 :
1955 : /* Make sure that GROUP->limits is up-to-date for the current point
1956 : of the model schedule. */
1957 :
1958 : static void
1959 0 : model_update_limit_points_in_group (struct model_pressure_group *group)
1960 : {
1961 0 : int pci, max_pressure, point;
1962 :
1963 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
1964 : {
1965 : /* We may have passed the final point at which the pressure in
1966 : group->limits[pci].pressure was reached. Update the limit if so. */
1967 0 : max_pressure = MODEL_MAX_PRESSURE (group, model_curr_point, pci);
1968 0 : group->limits[pci].pressure = max_pressure;
1969 :
1970 : /* Find the point at which MAX_PRESSURE is first reached. We need
1971 : to search in three cases:
1972 :
1973 : - We've already moved past the previous pressure point.
1974 : In this case we search forward from model_curr_point.
1975 :
1976 : - We scheduled the previous point of maximum pressure ahead of
1977 : its position in the model schedule, but doing so didn't bring
1978 : the pressure point earlier. In this case we search forward
1979 : from that previous pressure point.
1980 :
1981 : - Scheduling an instruction early caused the maximum pressure
1982 : to decrease. In this case we will have set the pressure
1983 : point to -1, and we search forward from model_curr_point. */
1984 0 : point = MAX (group->limits[pci].point, model_curr_point);
1985 0 : while (point < model_num_insns
1986 0 : && MODEL_REF_PRESSURE (group, point, pci) < max_pressure)
1987 0 : point++;
1988 0 : group->limits[pci].point = point;
1989 :
1990 0 : gcc_assert (MODEL_REF_PRESSURE (group, point, pci) == max_pressure);
1991 0 : gcc_assert (MODEL_MAX_PRESSURE (group, point, pci) == max_pressure);
1992 : }
1993 0 : }
1994 :
1995 : /* Make sure that all register-pressure limits are up-to-date for the
1996 : current position in the model schedule. */
1997 :
1998 : static void
1999 0 : model_update_limit_points (void)
2000 : {
2001 0 : model_update_limit_points_in_group (&model_before_pressure);
2002 0 : }
2003 :
2004 : /* Return the model_index of the last unscheduled use in chain USE
2005 : outside of USE's instruction. Return -1 if there are no other uses,
2006 : or model_num_insns if the register is live at the end of the block. */
2007 :
2008 : static int
2009 0 : model_last_use_except (struct reg_use_data *use)
2010 : {
2011 0 : struct reg_use_data *next;
2012 0 : int last, index;
2013 :
2014 0 : last = -1;
2015 0 : for (next = use->next_regno_use; next != use; next = next->next_regno_use)
2016 0 : if (NONDEBUG_INSN_P (next->insn)
2017 0 : && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED)
2018 : {
2019 0 : index = model_index (next->insn);
2020 0 : if (index == model_num_insns)
2021 : return model_num_insns;
2022 0 : if (last < index)
2023 0 : last = index;
2024 : }
2025 : return last;
2026 : }
2027 :
2028 : /* An instruction with model_index POINT has just been scheduled, and it
2029 : adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1.
2030 : Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and
2031 : MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. */
2032 :
2033 : static void
2034 0 : model_start_update_pressure (struct model_pressure_group *group,
2035 : int point, int pci, int delta)
2036 : {
2037 0 : int next_max_pressure;
2038 :
2039 0 : if (point == model_num_insns)
2040 : {
2041 : /* The instruction wasn't part of the model schedule; it was moved
2042 : from a different block. Update the pressure for the end of
2043 : the model schedule. */
2044 0 : MODEL_REF_PRESSURE (group, point, pci) += delta;
2045 0 : MODEL_MAX_PRESSURE (group, point, pci) += delta;
2046 : }
2047 : else
2048 : {
2049 : /* Record that this instruction has been scheduled. Nothing now
2050 : changes between POINT and POINT + 1, so get the maximum pressure
2051 : from the latter. If the maximum pressure decreases, the new
2052 : pressure point may be before POINT. */
2053 0 : MODEL_REF_PRESSURE (group, point, pci) = -1;
2054 0 : next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci);
2055 0 : if (MODEL_MAX_PRESSURE (group, point, pci) > next_max_pressure)
2056 : {
2057 0 : MODEL_MAX_PRESSURE (group, point, pci) = next_max_pressure;
2058 0 : if (group->limits[pci].point == point)
2059 0 : group->limits[pci].point = -1;
2060 : }
2061 : }
2062 0 : }
2063 :
2064 : /* Record that scheduling a later instruction has changed the pressure
2065 : at point POINT of the model schedule by DELTA (which might be 0).
2066 : Update GROUP accordingly. Return nonzero if these changes might
2067 : trigger changes to previous points as well. */
2068 :
2069 : static int
2070 0 : model_update_pressure (struct model_pressure_group *group,
2071 : int point, int pci, int delta)
2072 : {
2073 0 : int ref_pressure, max_pressure, next_max_pressure;
2074 :
2075 : /* If POINT hasn't yet been scheduled, update its pressure. */
2076 0 : ref_pressure = MODEL_REF_PRESSURE (group, point, pci);
2077 0 : if (ref_pressure >= 0 && delta != 0)
2078 : {
2079 0 : ref_pressure += delta;
2080 0 : MODEL_REF_PRESSURE (group, point, pci) = ref_pressure;
2081 :
2082 : /* Check whether the maximum pressure in the overall schedule
2083 : has increased. (This means that the MODEL_MAX_PRESSURE of
2084 : every point <= POINT will need to increase too; see below.) */
2085 0 : if (group->limits[pci].pressure < ref_pressure)
2086 0 : group->limits[pci].pressure = ref_pressure;
2087 :
2088 : /* If we are at maximum pressure, and the maximum pressure
2089 : point was previously unknown or later than POINT,
2090 : bring it forward. */
2091 0 : if (group->limits[pci].pressure == ref_pressure
2092 0 : && !IN_RANGE (group->limits[pci].point, 0, point))
2093 0 : group->limits[pci].point = point;
2094 :
2095 : /* If POINT used to be the point of maximum pressure, but isn't
2096 : any longer, we need to recalculate it using a forward walk. */
2097 0 : if (group->limits[pci].pressure > ref_pressure
2098 0 : && group->limits[pci].point == point)
2099 0 : group->limits[pci].point = -1;
2100 : }
2101 :
2102 : /* Update the maximum pressure at POINT. Changes here might also
2103 : affect the maximum pressure at POINT - 1. */
2104 0 : next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci);
2105 0 : max_pressure = MAX (ref_pressure, next_max_pressure);
2106 0 : if (MODEL_MAX_PRESSURE (group, point, pci) != max_pressure)
2107 : {
2108 0 : MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
2109 0 : return 1;
2110 : }
2111 : return 0;
2112 : }
2113 :
2114 : /* INSN has just been scheduled. Update the model schedule accordingly. */
2115 :
2116 : static void
2117 0 : model_recompute (rtx_insn *insn)
2118 : {
2119 0 : struct {
2120 : int last_use;
2121 : int regno;
2122 : } uses[FIRST_PSEUDO_REGISTER + MAX_RECOG_OPERANDS];
2123 0 : struct reg_use_data *use;
2124 0 : struct reg_pressure_data *reg_pressure;
2125 0 : int delta[N_REG_CLASSES];
2126 0 : int pci, point, mix, new_last, cl, ref_pressure, queue;
2127 0 : unsigned int i, num_uses, num_pending_births;
2128 0 : bool print_p;
2129 :
2130 : /* The destinations of INSN were previously live from POINT onwards, but are
2131 : now live from model_curr_point onwards. Set up DELTA accordingly. */
2132 0 : point = model_index (insn);
2133 0 : reg_pressure = INSN_REG_PRESSURE (insn);
2134 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2135 : {
2136 0 : cl = ira_pressure_classes[pci];
2137 0 : delta[cl] = reg_pressure[pci].set_increase;
2138 : }
2139 :
2140 : /* Record which registers previously died at POINT, but which now die
2141 : before POINT. Adjust DELTA so that it represents the effect of
2142 : this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of
2143 : registers that will be born in the range [model_curr_point, POINT). */
2144 0 : num_uses = 0;
2145 0 : num_pending_births = 0;
2146 0 : bitmap_clear (tmp_bitmap);
2147 0 : for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
2148 : {
2149 0 : new_last = model_last_use_except (use);
2150 0 : if (new_last < point
2151 0 : && bitmap_set_bit (tmp_bitmap, use->regno)
2152 : /* df_get_live_in has not necessarily been updated to reflect the
2153 : effect of inter-block movement performed by earlier schedules.
2154 : Cope with stale live-in sets by ignoring registers that are not
2155 : currently assumed to be live. */
2156 0 : && bitmap_bit_p (curr_reg_live, use->regno))
2157 : {
2158 0 : gcc_assert (num_uses < ARRAY_SIZE (uses));
2159 0 : uses[num_uses].last_use = new_last;
2160 0 : uses[num_uses].regno = use->regno;
2161 : /* This register is no longer live after POINT - 1. */
2162 0 : mark_regno_birth_or_death (NULL, delta, use->regno, false);
2163 0 : num_uses++;
2164 0 : if (new_last >= 0)
2165 0 : num_pending_births++;
2166 : }
2167 : }
2168 :
2169 : /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT.
2170 : Also set each group pressure limit for POINT. */
2171 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2172 : {
2173 0 : cl = ira_pressure_classes[pci];
2174 0 : model_start_update_pressure (&model_before_pressure,
2175 : point, pci, delta[cl]);
2176 : }
2177 :
2178 : /* Walk the model schedule backwards, starting immediately before POINT. */
2179 0 : print_p = false;
2180 0 : if (point != model_curr_point)
2181 0 : do
2182 : {
2183 0 : point--;
2184 0 : insn = MODEL_INSN (point);
2185 0 : queue = QUEUE_INDEX (insn);
2186 :
2187 0 : if (queue != QUEUE_SCHEDULED)
2188 : {
2189 : /* DELTA describes the effect of the move on the register pressure
2190 : after POINT. Make it describe the effect on the pressure
2191 : before POINT. */
2192 : i = 0;
2193 0 : while (i < num_uses)
2194 : {
2195 0 : if (uses[i].last_use == point)
2196 : {
2197 : /* This register is now live again. */
2198 0 : mark_regno_birth_or_death (NULL, delta,
2199 : uses[i].regno, true);
2200 :
2201 : /* Remove this use from the array. */
2202 0 : uses[i] = uses[num_uses - 1];
2203 0 : num_uses--;
2204 0 : num_pending_births--;
2205 : }
2206 : else
2207 0 : i++;
2208 : }
2209 :
2210 0 : if (sched_verbose >= 5)
2211 : {
2212 0 : if (!print_p)
2213 : {
2214 0 : fprintf (sched_dump, MODEL_BAR);
2215 0 : fprintf (sched_dump, ";;\t\t| New pressure for model"
2216 : " schedule\n");
2217 0 : fprintf (sched_dump, MODEL_BAR);
2218 0 : print_p = true;
2219 : }
2220 :
2221 0 : fprintf (sched_dump, ";;\t\t| %3d %4d %-30s ",
2222 0 : point, INSN_UID (insn),
2223 0 : str_pattern_slim (PATTERN (insn)));
2224 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2225 : {
2226 0 : cl = ira_pressure_classes[pci];
2227 0 : ref_pressure = MODEL_REF_PRESSURE (&model_before_pressure,
2228 : point, pci);
2229 0 : fprintf (sched_dump, " %s:[%d->%d]",
2230 : reg_class_names[ira_pressure_classes[pci]],
2231 0 : ref_pressure, ref_pressure + delta[cl]);
2232 : }
2233 0 : fprintf (sched_dump, "\n");
2234 : }
2235 : }
2236 :
2237 : /* Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1
2238 : might have changed as well. */
2239 0 : mix = num_pending_births;
2240 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2241 : {
2242 0 : cl = ira_pressure_classes[pci];
2243 0 : mix |= delta[cl];
2244 0 : mix |= model_update_pressure (&model_before_pressure,
2245 : point, pci, delta[cl]);
2246 : }
2247 : }
2248 0 : while (mix && point > model_curr_point);
2249 :
2250 0 : if (print_p)
2251 0 : fprintf (sched_dump, MODEL_BAR);
2252 0 : }
2253 :
2254 : /* After DEP, which was cancelled, has been resolved for insn NEXT,
2255 : check whether the insn's pattern needs restoring. */
2256 : static bool
2257 843078 : must_restore_pattern_p (rtx_insn *next, dep_t dep)
2258 : {
2259 843078 : if (QUEUE_INDEX (next) == QUEUE_SCHEDULED)
2260 : return false;
2261 :
2262 303439 : if (DEP_TYPE (dep) == REG_DEP_CONTROL)
2263 : {
2264 0 : gcc_assert (ORIG_PAT (next) != NULL_RTX);
2265 0 : gcc_assert (next == DEP_CON (dep));
2266 : }
2267 : else
2268 : {
2269 303439 : struct dep_replacement *desc = DEP_REPLACE (dep);
2270 303439 : if (desc->insn != next)
2271 : {
2272 240334 : gcc_assert (*desc->loc == desc->orig);
2273 : return false;
2274 : }
2275 : }
2276 : return true;
2277 : }
2278 :
2279 : /* model_spill_cost (CL, P, P') returns the cost of increasing the
2280 : pressure on CL from P to P'. We use this to calculate a "base ECC",
2281 : baseECC (CL, X), for each pressure class CL and each instruction X.
2282 : Supposing X changes the pressure on CL from P to P', and that the
2283 : maximum pressure on CL in the current model schedule is MP', then:
2284 :
2285 : * if X occurs before or at the next point of maximum pressure in
2286 : the model schedule and P' > MP', then:
2287 :
2288 : baseECC (CL, X) = model_spill_cost (CL, MP, P')
2289 :
2290 : The idea is that the pressure after scheduling a fixed set of
2291 : instructions -- in this case, the set up to and including the
2292 : next maximum pressure point -- is going to be the same regardless
2293 : of the order; we simply want to keep the intermediate pressure
2294 : under control. Thus X has a cost of zero unless scheduling it
2295 : now would exceed MP'.
2296 :
2297 : If all increases in the set are by the same amount, no zero-cost
2298 : instruction will ever cause the pressure to exceed MP'. However,
2299 : if X is instead moved past an instruction X' with pressure in the
2300 : range (MP' - (P' - P), MP'), the pressure at X' will increase
2301 : beyond MP'. Since baseECC is very much a heuristic anyway,
2302 : it doesn't seem worth the overhead of tracking cases like these.
2303 :
2304 : The cost of exceeding MP' is always based on the original maximum
2305 : pressure MP. This is so that going 2 registers over the original
2306 : limit has the same cost regardless of whether it comes from two
2307 : separate +1 deltas or from a single +2 delta.
2308 :
2309 : * if X occurs after the next point of maximum pressure in the model
2310 : schedule and P' > P, then:
2311 :
2312 : baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P))
2313 :
2314 : That is, if we move X forward across a point of maximum pressure,
2315 : and if X increases the pressure by P' - P, then we conservatively
2316 : assume that scheduling X next would increase the maximum pressure
2317 : by P' - P. Again, the cost of doing this is based on the original
2318 : maximum pressure MP, for the same reason as above.
2319 :
2320 : * if P' < P, P > MP, and X occurs at or after the next point of
2321 : maximum pressure, then:
2322 :
2323 : baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P)
2324 :
2325 : That is, if we have already exceeded the original maximum pressure MP,
2326 : and if X might reduce the maximum pressure again -- or at least push
2327 : it further back, and thus allow more scheduling freedom -- it is given
2328 : a negative cost to reflect the improvement.
2329 :
2330 : * otherwise,
2331 :
2332 : baseECC (CL, X) = 0
2333 :
2334 : In this case, X is not expected to affect the maximum pressure MP',
2335 : so it has zero cost.
2336 :
2337 : We then create a combined value baseECC (X) that is the sum of
2338 : baseECC (CL, X) for each pressure class CL.
2339 :
2340 : baseECC (X) could itself be used as the ECC value described above.
2341 : However, this is often too conservative, in the sense that it
2342 : tends to make high-priority instructions that increase pressure
2343 : wait too long in cases where introducing a spill would be better.
2344 : For this reason the final ECC is a priority-adjusted form of
2345 : baseECC (X). Specifically, we calculate:
2346 :
2347 : P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X)
2348 : baseP = MAX { P (X) | baseECC (X) <= 0 }
2349 :
2350 : Then:
2351 :
2352 : ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0)
2353 :
2354 : Thus an instruction's effect on pressure is ignored if it has a high
2355 : enough priority relative to the ones that don't increase pressure.
2356 : Negative values of baseECC (X) do not increase the priority of X
2357 : itself, but they do make it harder for other instructions to
2358 : increase the pressure further.
2359 :
2360 : This pressure cost is deliberately timid. The intention has been
2361 : to choose a heuristic that rarely interferes with the normal list
2362 : scheduler in cases where that scheduler would produce good code.
2363 : We simply want to curb some of its worst excesses. */
2364 :
2365 : /* Return the cost of increasing the pressure in class CL from FROM to TO.
2366 :
2367 : Here we use the very simplistic cost model that every register above
2368 : sched_class_regs_num[CL] has a spill cost of 1. We could use other
2369 : measures instead, such as one based on MEMORY_MOVE_COST. However:
2370 :
2371 : (1) In order for an instruction to be scheduled, the higher cost
2372 : would need to be justified in a single saving of that many stalls.
2373 : This is overly pessimistic, because the benefit of spilling is
2374 : often to avoid a sequence of several short stalls rather than
2375 : a single long one.
2376 :
2377 : (2) The cost is still arbitrary. Because we are not allocating
2378 : registers during scheduling, we have no way of knowing for
2379 : sure how many memory accesses will be required by each spill,
2380 : where the spills will be placed within the block, or even
2381 : which block(s) will contain the spills.
2382 :
2383 : So a higher cost than 1 is often too conservative in practice,
2384 : forcing blocks to contain unnecessary stalls instead of spill code.
2385 : The simple cost below seems to be the best compromise. It reduces
2386 : the interference with the normal list scheduler, which helps make
2387 : it more suitable for a default-on option. */
2388 :
2389 : static int
2390 0 : model_spill_cost (int cl, int from, int to)
2391 : {
2392 0 : from = MAX (from, sched_class_regs_num[cl]);
2393 0 : return MAX (to, from) - from;
2394 : }
2395 :
2396 : /* Return baseECC (ira_pressure_classes[PCI], POINT), given that
2397 : P = curr_reg_pressure[ira_pressure_classes[PCI]] and that
2398 : P' = P + DELTA. */
2399 :
2400 : static int
2401 0 : model_excess_group_cost (struct model_pressure_group *group,
2402 : int point, int pci, int delta)
2403 : {
2404 0 : int pressure, cl;
2405 :
2406 0 : cl = ira_pressure_classes[pci];
2407 0 : if (delta < 0)
2408 : {
2409 0 : if (point >= group->limits[pci].point)
2410 : {
2411 0 : pressure = MAX (group->limits[pci].orig_pressure,
2412 : curr_reg_pressure[cl] + delta);
2413 0 : return -model_spill_cost (cl, pressure, curr_reg_pressure[cl]);
2414 : }
2415 : /* if target prefers fewer spills, return the -ve delta indicating
2416 : pressure reduction. */
2417 0 : else if (!param_cycle_accurate_model)
2418 : return delta;
2419 : }
2420 :
2421 0 : if (delta > 0)
2422 : {
2423 0 : if (point > group->limits[pci].point)
2424 0 : pressure = group->limits[pci].pressure + delta;
2425 : else
2426 0 : pressure = curr_reg_pressure[cl] + delta;
2427 :
2428 0 : if (pressure > group->limits[pci].pressure)
2429 0 : return model_spill_cost (cl, group->limits[pci].orig_pressure,
2430 : pressure);
2431 : }
2432 :
2433 : return 0;
2434 : }
2435 :
2436 : /* Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump
2437 : if PRINT_P. */
2438 :
2439 : static int
2440 0 : model_excess_cost (rtx_insn *insn, bool print_p)
2441 : {
2442 0 : int point, pci, cl, cost, this_cost, delta;
2443 0 : struct reg_pressure_data *insn_reg_pressure;
2444 0 : int insn_death[N_REG_CLASSES];
2445 :
2446 0 : calculate_reg_deaths (insn, insn_death);
2447 0 : point = model_index (insn);
2448 0 : insn_reg_pressure = INSN_REG_PRESSURE (insn);
2449 0 : cost = 0;
2450 :
2451 0 : if (print_p)
2452 0 : fprintf (sched_dump, ";;\t\t| %3d %4d | %4d %+3d |", point,
2453 0 : INSN_UID (insn), INSN_PRIORITY (insn), insn_delay (insn));
2454 :
2455 : /* Sum up the individual costs for each register class. */
2456 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2457 : {
2458 0 : cl = ira_pressure_classes[pci];
2459 0 : delta = insn_reg_pressure[pci].set_increase - insn_death[cl];
2460 0 : this_cost = model_excess_group_cost (&model_before_pressure,
2461 : point, pci, delta);
2462 0 : cost += this_cost;
2463 0 : if (print_p)
2464 0 : fprintf (sched_dump, " %s:[%d base cost %d]",
2465 : reg_class_names[cl], delta, this_cost);
2466 : }
2467 :
2468 0 : if (print_p)
2469 0 : fprintf (sched_dump, " ECC %d\n", cost);
2470 :
2471 0 : return cost;
2472 : }
2473 :
2474 : /* Dump the next points of maximum pressure for GROUP. */
2475 :
2476 : static void
2477 0 : model_dump_pressure_points (struct model_pressure_group *group)
2478 : {
2479 0 : int pci, cl;
2480 :
2481 0 : fprintf (sched_dump, ";;\t\t| pressure points");
2482 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2483 : {
2484 0 : cl = ira_pressure_classes[pci];
2485 0 : fprintf (sched_dump, " %s:[%d->%d at ", reg_class_names[cl],
2486 : curr_reg_pressure[cl], group->limits[pci].pressure);
2487 0 : if (group->limits[pci].point < model_num_insns)
2488 0 : fprintf (sched_dump, "%d:%d]", group->limits[pci].point,
2489 0 : INSN_UID (MODEL_INSN (group->limits[pci].point)));
2490 : else
2491 0 : fprintf (sched_dump, "end]");
2492 : }
2493 0 : fprintf (sched_dump, "\n");
2494 0 : }
2495 :
2496 : /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */
2497 :
2498 : static void
2499 0 : model_set_excess_costs (rtx_insn **insns, int count)
2500 : {
2501 0 : int i, cost, priority_base, priority;
2502 0 : bool print_p;
2503 :
2504 : /* Record the baseECC value for each instruction in the model schedule,
2505 : except that for targets which prefer wider schedules (more spills)
2506 : negative costs are converted to zero ones now rather than later.
2507 : Do not assign a cost to debug instructions, since they must
2508 : not change code-generation decisions. Experiments suggest we also
2509 : get better results by not assigning a cost to instructions from
2510 : a different block.
2511 :
2512 : Set PRIORITY_BASE to baseP in the block comment above. This is the
2513 : maximum priority of the "cheap" instructions, which should always
2514 : include the next model instruction. */
2515 0 : priority_base = 0;
2516 0 : print_p = false;
2517 0 : for (i = 0; i < count; i++)
2518 0 : if (INSN_MODEL_INDEX (insns[i]))
2519 : {
2520 0 : if (sched_verbose >= 6 && !print_p)
2521 : {
2522 0 : fprintf (sched_dump, MODEL_BAR);
2523 0 : fprintf (sched_dump, ";;\t\t| Pressure costs for ready queue\n");
2524 0 : model_dump_pressure_points (&model_before_pressure);
2525 0 : fprintf (sched_dump, MODEL_BAR);
2526 0 : print_p = true;
2527 : }
2528 0 : cost = model_excess_cost (insns[i], print_p);
2529 0 : if (param_cycle_accurate_model && cost <= 0)
2530 : {
2531 0 : priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]) - cost;
2532 0 : priority_base = MAX (priority_base, priority);
2533 : cost = 0;
2534 : }
2535 0 : INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = cost;
2536 : }
2537 0 : if (print_p)
2538 0 : fprintf (sched_dump, MODEL_BAR);
2539 :
2540 : /* Typically in-order cores have a good pipeline scheduling model and the
2541 : algorithm would try to use that to minimize bubbles, favoring spills.
2542 : MAX (baseECC, 0) below changes negative baseECC (pressure reduction)
2543 : to 0 (pressure neutral) thus tending to more spills.
2544 : Otherwise return. */
2545 0 : if (!param_cycle_accurate_model)
2546 : return;
2547 :
2548 : /* Use MAX (baseECC, 0) and baseP to calculcate ECC for each
2549 : instruction. */
2550 0 : for (i = 0; i < count; i++)
2551 : {
2552 0 : cost = INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]);
2553 0 : priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]);
2554 0 : if (cost > 0 && priority > priority_base)
2555 : {
2556 0 : cost += priority_base - priority;
2557 0 : INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = MAX (cost, 0);
2558 : }
2559 : }
2560 : }
2561 :
2562 :
2563 : /* Enum of rank_for_schedule heuristic decisions. */
2564 : enum rfs_decision {
2565 : RFS_LIVE_RANGE_SHRINK1, RFS_LIVE_RANGE_SHRINK2,
2566 : RFS_SCHED_GROUP, RFS_PRESSURE_DELAY, RFS_PRESSURE_TICK,
2567 : RFS_FEEDS_BACKTRACK_INSN, RFS_PRIORITY, RFS_AUTOPREF, RFS_SPECULATION,
2568 : RFS_SCHED_RANK, RFS_LAST_INSN, RFS_PRESSURE_INDEX,
2569 : RFS_DEP_COUNT, RFS_TIE, RFS_FUSION, RFS_COST, RFS_N };
2570 :
2571 : /* Corresponding strings for print outs. */
2572 : static const char *rfs_str[RFS_N] = {
2573 : "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
2574 : "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
2575 : "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_AUTOPREF", "RFS_SPECULATION",
2576 : "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
2577 : "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION", "RFS_COST" };
2578 :
2579 : /* Statistical breakdown of rank_for_schedule decisions. */
2580 : struct rank_for_schedule_stats_t { unsigned stats[RFS_N]; };
2581 : static rank_for_schedule_stats_t rank_for_schedule_stats;
2582 :
2583 : /* Return the result of comparing insns TMP and TMP2 and update
2584 : Rank_For_Schedule statistics. */
2585 : static int
2586 332525761 : rfs_result (enum rfs_decision decision, int result, rtx tmp, rtx tmp2)
2587 : {
2588 332525761 : ++rank_for_schedule_stats.stats[decision];
2589 332525761 : if (result < 0)
2590 182517124 : INSN_LAST_RFS_WIN (tmp) = decision;
2591 150008637 : else if (result > 0)
2592 150008637 : INSN_LAST_RFS_WIN (tmp2) = decision;
2593 : else
2594 0 : gcc_unreachable ();
2595 332525761 : return result;
2596 : }
2597 :
2598 : /* Sorting predicate to move DEBUG_INSNs to the top of ready list, while
2599 : keeping normal insns in original order. */
2600 :
2601 : static int
2602 387387286 : rank_for_schedule_debug (const void *x, const void *y)
2603 : {
2604 387387286 : rtx_insn *tmp = *(rtx_insn * const *) y;
2605 387387286 : rtx_insn *tmp2 = *(rtx_insn * const *) x;
2606 :
2607 : /* Schedule debug insns as early as possible. */
2608 387387286 : if (DEBUG_INSN_P (tmp) && !DEBUG_INSN_P (tmp2))
2609 : return -1;
2610 242533067 : else if (!DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
2611 : return 1;
2612 180262378 : else if (DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
2613 0 : return INSN_LUID (tmp) - INSN_LUID (tmp2);
2614 : else
2615 180262378 : return INSN_RFS_DEBUG_ORIG_ORDER (tmp2) - INSN_RFS_DEBUG_ORIG_ORDER (tmp);
2616 : }
2617 :
2618 : /* Returns a positive value if x is preferred; returns a negative value if
2619 : y is preferred. Should never return 0, since that will make the sort
2620 : unstable. */
2621 :
2622 : static int
2623 332525761 : rank_for_schedule (const void *x, const void *y)
2624 : {
2625 332525761 : rtx_insn *tmp = *(rtx_insn * const *) y;
2626 332525761 : rtx_insn *tmp2 = *(rtx_insn * const *) x;
2627 332525761 : int tmp_class, tmp2_class;
2628 332525761 : int val, priority_val, info_val, diff;
2629 :
2630 332525761 : if (live_range_shrinkage_p)
2631 : {
2632 : /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse
2633 : code. */
2634 257525 : gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED);
2635 257525 : if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) < 0
2636 252167 : || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2) < 0)
2637 257525 : && (diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp)
2638 9318 : - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2))) != 0)
2639 9210 : return rfs_result (RFS_LIVE_RANGE_SHRINK1, diff, tmp, tmp2);
2640 : /* Sort by INSN_LUID (original insn order), so that we make the
2641 : sort stable. This minimizes instruction movement, thus
2642 : minimizing sched's effect on debugging and cross-jumping. */
2643 744945 : return rfs_result (RFS_LIVE_RANGE_SHRINK2,
2644 248315 : INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2645 : }
2646 :
2647 : /* The insn in a schedule group should be issued the first. */
2648 332268236 : if (flag_sched_group_heuristic &&
2649 332268236 : SCHED_GROUP_P (tmp) != SCHED_GROUP_P (tmp2))
2650 225 : return rfs_result (RFS_SCHED_GROUP, SCHED_GROUP_P (tmp2) ? 1 : -1,
2651 158 : tmp, tmp2);
2652 :
2653 : /* Make sure that priority of TMP and TMP2 are initialized. */
2654 332268078 : gcc_assert (INSN_PRIORITY_KNOWN (tmp) && INSN_PRIORITY_KNOWN (tmp2));
2655 :
2656 332268078 : if (sched_fusion)
2657 : {
2658 : /* The instruction that has the same fusion priority as the last
2659 : instruction is the instruction we picked next. If that is not
2660 : the case, we sort ready list firstly by fusion priority, then
2661 : by priority, and at last by INSN_LUID. */
2662 0 : int a = INSN_FUSION_PRIORITY (tmp);
2663 0 : int b = INSN_FUSION_PRIORITY (tmp2);
2664 0 : int last = -1;
2665 :
2666 0 : if (last_nondebug_scheduled_insn
2667 0 : && !NOTE_P (last_nondebug_scheduled_insn)
2668 0 : && BLOCK_FOR_INSN (tmp)
2669 0 : == BLOCK_FOR_INSN (last_nondebug_scheduled_insn))
2670 0 : last = INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn);
2671 :
2672 0 : if (a != last && b != last)
2673 : {
2674 0 : if (a == b)
2675 : {
2676 0 : a = INSN_PRIORITY (tmp);
2677 0 : b = INSN_PRIORITY (tmp2);
2678 : }
2679 0 : if (a != b)
2680 0 : return rfs_result (RFS_FUSION, b - a, tmp, tmp2);
2681 : else
2682 0 : return rfs_result (RFS_FUSION,
2683 0 : INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2684 : }
2685 0 : else if (a == b)
2686 : {
2687 0 : gcc_assert (last_nondebug_scheduled_insn
2688 : && !NOTE_P (last_nondebug_scheduled_insn));
2689 0 : last = INSN_PRIORITY (last_nondebug_scheduled_insn);
2690 :
2691 0 : a = abs (INSN_PRIORITY (tmp) - last);
2692 0 : b = abs (INSN_PRIORITY (tmp2) - last);
2693 0 : if (a != b)
2694 0 : return rfs_result (RFS_FUSION, a - b, tmp, tmp2);
2695 : else
2696 0 : return rfs_result (RFS_FUSION,
2697 0 : INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2698 : }
2699 0 : else if (a == last)
2700 0 : return rfs_result (RFS_FUSION, -1, tmp, tmp2);
2701 : else
2702 0 : return rfs_result (RFS_FUSION, 1, tmp, tmp2);
2703 : }
2704 :
2705 332268078 : if (sched_pressure != SCHED_PRESSURE_NONE)
2706 : {
2707 : /* Prefer insn whose scheduling results in the smallest register
2708 : pressure excess. */
2709 13661 : if ((diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp)
2710 13661 : + insn_delay (tmp)
2711 13661 : - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2)
2712 13661 : - insn_delay (tmp2))))
2713 5021 : return rfs_result (RFS_PRESSURE_DELAY, diff, tmp, tmp2);
2714 : }
2715 :
2716 332263057 : if (sched_pressure != SCHED_PRESSURE_NONE
2717 8640 : && (INSN_TICK (tmp2) > clock_var || INSN_TICK (tmp) > clock_var)
2718 332264002 : && INSN_TICK (tmp2) != INSN_TICK (tmp))
2719 : {
2720 0 : diff = INSN_TICK (tmp) - INSN_TICK (tmp2);
2721 0 : return rfs_result (RFS_PRESSURE_TICK, diff, tmp, tmp2);
2722 : }
2723 :
2724 : /* If we are doing backtracking in this schedule, prefer insns that
2725 : have forward dependencies with negative cost against an insn that
2726 : was already scheduled. */
2727 332263057 : if (current_sched_info->flags & DO_BACKTRACKING)
2728 : {
2729 0 : priority_val = FEEDS_BACKTRACK_INSN (tmp2) - FEEDS_BACKTRACK_INSN (tmp);
2730 0 : if (priority_val)
2731 0 : return rfs_result (RFS_FEEDS_BACKTRACK_INSN, priority_val, tmp, tmp2);
2732 : }
2733 :
2734 : /* Prefer insn with higher priority. */
2735 332263057 : priority_val = INSN_PRIORITY (tmp2) - INSN_PRIORITY (tmp);
2736 :
2737 332263057 : if (flag_sched_critical_path_heuristic && priority_val)
2738 88269008 : return rfs_result (RFS_PRIORITY, priority_val, tmp, tmp2);
2739 :
2740 243994049 : if (param_sched_autopref_queue_depth >= 0)
2741 : {
2742 12 : int autopref = autopref_rank_for_schedule (tmp, tmp2);
2743 12 : if (autopref != 0)
2744 12 : return rfs_result (RFS_AUTOPREF, autopref, tmp, tmp2);
2745 : }
2746 :
2747 : /* Prefer speculative insn with greater dependencies weakness. */
2748 243994037 : if (flag_sched_spec_insn_heuristic && spec_info)
2749 : {
2750 0 : ds_t ds1, ds2;
2751 0 : dw_t dw1, dw2;
2752 0 : int dw;
2753 :
2754 0 : ds1 = TODO_SPEC (tmp) & SPECULATIVE;
2755 0 : if (ds1)
2756 0 : dw1 = ds_weak (ds1);
2757 : else
2758 : dw1 = NO_DEP_WEAK;
2759 :
2760 0 : ds2 = TODO_SPEC (tmp2) & SPECULATIVE;
2761 0 : if (ds2)
2762 0 : dw2 = ds_weak (ds2);
2763 : else
2764 : dw2 = NO_DEP_WEAK;
2765 :
2766 0 : dw = dw2 - dw1;
2767 0 : if (dw > (NO_DEP_WEAK / 8) || dw < -(NO_DEP_WEAK / 8))
2768 0 : return rfs_result (RFS_SPECULATION, dw, tmp, tmp2);
2769 : }
2770 :
2771 243994037 : info_val = (*current_sched_info->rank) (tmp, tmp2);
2772 243994037 : if (flag_sched_rank_heuristic && info_val)
2773 0 : return rfs_result (RFS_SCHED_RANK, info_val, tmp, tmp2);
2774 :
2775 : /* Compare insns based on their relation to the last scheduled
2776 : non-debug insn. */
2777 243994037 : if (flag_sched_last_insn_heuristic && last_nondebug_scheduled_insn)
2778 : {
2779 217421101 : dep_t dep1;
2780 217421101 : dep_t dep2;
2781 217421101 : rtx_insn *last = last_nondebug_scheduled_insn;
2782 :
2783 : /* Classify the instructions into three classes:
2784 : 1) Data dependent on last schedule insn.
2785 : 2) Anti/Output dependent on last scheduled insn.
2786 : 3) Independent of last scheduled insn, or has latency of one.
2787 : Choose the insn from the highest numbered class if different. */
2788 217421101 : dep1 = sd_find_dep_between (last, tmp, true);
2789 :
2790 217421101 : if (dep1 == NULL || dep_cost (dep1) == 1)
2791 : tmp_class = 3;
2792 16284216 : else if (/* Data dependence. */
2793 16284216 : DEP_TYPE (dep1) == REG_DEP_TRUE)
2794 : tmp_class = 1;
2795 : else
2796 14769906 : tmp_class = 2;
2797 :
2798 217421101 : dep2 = sd_find_dep_between (last, tmp2, true);
2799 :
2800 217421101 : if (dep2 == NULL || dep_cost (dep2) == 1)
2801 : tmp2_class = 3;
2802 16636433 : else if (/* Data dependence. */
2803 16636433 : DEP_TYPE (dep2) == REG_DEP_TRUE)
2804 : tmp2_class = 1;
2805 : else
2806 14905129 : tmp2_class = 2;
2807 :
2808 217421101 : if ((val = tmp2_class - tmp_class))
2809 3329685 : return rfs_result (RFS_LAST_INSN, val, tmp, tmp2);
2810 : }
2811 :
2812 : /* Prefer instructions that occur earlier in the model schedule. */
2813 240664352 : if (sched_pressure == SCHED_PRESSURE_MODEL)
2814 : {
2815 0 : diff = model_index (tmp) - model_index (tmp2);
2816 0 : if (diff != 0)
2817 0 : return rfs_result (RFS_PRESSURE_INDEX, diff, tmp, tmp2);
2818 : }
2819 :
2820 : /* Prefer the insn which has more later insns that depend on it.
2821 : This gives the scheduler more freedom when scheduling later
2822 : instructions at the expense of added register pressure. */
2823 :
2824 240664352 : val = (dep_list_size (tmp2, SD_LIST_FORW)
2825 240664352 : - dep_list_size (tmp, SD_LIST_FORW));
2826 :
2827 240664352 : if (flag_sched_dep_count_heuristic && val != 0)
2828 28655100 : return rfs_result (RFS_DEP_COUNT, val, tmp, tmp2);
2829 :
2830 : /* Sort by INSN_COST rather than INSN_LUID. This means that instructions
2831 : which take longer to execute are prioritised and it leads to more
2832 : dual-issue opportunities on in-order cores which have this feature. */
2833 :
2834 212009252 : if (INSN_COST (tmp) != INSN_COST (tmp2))
2835 13078880 : return rfs_result (RFS_COST, INSN_COST (tmp2) - INSN_COST (tmp),
2836 13078880 : tmp, tmp2);
2837 :
2838 : /* If insns are equally good, sort by INSN_LUID (original insn order),
2839 : so that we make the sort stable. This minimizes instruction movement,
2840 : thus minimizing sched's effect on debugging and cross-jumping. */
2841 198930372 : return rfs_result (RFS_TIE, INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2842 : }
2843 :
2844 : /* Resort the array A in which only element at index N may be out of order. */
2845 :
2846 : HAIFA_INLINE static void
2847 7903426 : swap_sort (rtx_insn **a, int n)
2848 : {
2849 7903426 : rtx_insn *insn = a[n - 1];
2850 7903426 : int i = n - 2;
2851 :
2852 10622981 : while (i >= 0 && rank_for_schedule (a + i, &insn) >= 0)
2853 : {
2854 2719555 : a[i + 1] = a[i];
2855 2719555 : i -= 1;
2856 : }
2857 7903426 : a[i + 1] = insn;
2858 7903426 : }
2859 :
2860 : /* Add INSN to the insn queue so that it can be executed at least
2861 : N_CYCLES after the currently executing insn. Preserve insns
2862 : chain for debugging purposes. REASON will be printed in debugging
2863 : output. */
2864 :
2865 : HAIFA_INLINE static void
2866 38783621 : queue_insn (rtx_insn *insn, int n_cycles, const char *reason)
2867 : {
2868 38783621 : int next_q = NEXT_Q_AFTER (q_ptr, n_cycles);
2869 38783621 : rtx_insn_list *link = alloc_INSN_LIST (insn, insn_queue[next_q]);
2870 38783621 : int new_tick;
2871 :
2872 38783621 : gcc_assert (n_cycles <= max_insn_queue_index);
2873 38783621 : gcc_assert (!DEBUG_INSN_P (insn));
2874 :
2875 38783621 : insn_queue[next_q] = link;
2876 38783621 : q_size += 1;
2877 :
2878 38783621 : if (sched_verbose >= 2)
2879 : {
2880 0 : fprintf (sched_dump, ";;\t\tReady-->Q: insn %s: ",
2881 0 : (*current_sched_info->print_insn) (insn, 0));
2882 :
2883 0 : fprintf (sched_dump, "queued for %d cycles (%s).\n", n_cycles, reason);
2884 : }
2885 :
2886 38783621 : QUEUE_INDEX (insn) = next_q;
2887 :
2888 38783621 : if (current_sched_info->flags & DO_BACKTRACKING)
2889 : {
2890 0 : new_tick = clock_var + n_cycles;
2891 0 : if (INSN_TICK (insn) == INVALID_TICK || INSN_TICK (insn) < new_tick)
2892 0 : INSN_TICK (insn) = new_tick;
2893 :
2894 0 : if (INSN_EXACT_TICK (insn) != INVALID_TICK
2895 0 : && INSN_EXACT_TICK (insn) < clock_var + n_cycles)
2896 : {
2897 0 : must_backtrack = true;
2898 0 : if (sched_verbose >= 2)
2899 0 : fprintf (sched_dump, ";;\t\tcausing a backtrack.\n");
2900 : }
2901 : }
2902 38783621 : }
2903 :
2904 : /* Remove INSN from queue. */
2905 : static void
2906 138267 : queue_remove (rtx_insn *insn)
2907 : {
2908 138267 : gcc_assert (QUEUE_INDEX (insn) >= 0);
2909 138267 : remove_free_INSN_LIST_elem (insn, &insn_queue[QUEUE_INDEX (insn)]);
2910 138267 : q_size--;
2911 138267 : QUEUE_INDEX (insn) = QUEUE_NOWHERE;
2912 138267 : }
2913 :
2914 : /* Return a pointer to the bottom of the ready list, i.e. the insn
2915 : with the lowest priority. */
2916 :
2917 : rtx_insn **
2918 143633490 : ready_lastpos (struct ready_list *ready)
2919 : {
2920 143633490 : gcc_assert (ready->n_ready >= 1);
2921 143633490 : return ready->vec + ready->first - ready->n_ready + 1;
2922 : }
2923 :
2924 : /* Add an element INSN to the ready list so that it ends up with the
2925 : lowest/highest priority depending on FIRST_P. */
2926 :
2927 : HAIFA_INLINE static void
2928 128130352 : ready_add (struct ready_list *ready, rtx_insn *insn, bool first_p)
2929 : {
2930 128130352 : if (!first_p)
2931 : {
2932 128128114 : if (ready->first == ready->n_ready)
2933 : {
2934 122 : memmove (ready->vec + ready->veclen - ready->n_ready,
2935 61 : ready_lastpos (ready),
2936 61 : ready->n_ready * sizeof (rtx));
2937 61 : ready->first = ready->veclen - 1;
2938 : }
2939 128128114 : ready->vec[ready->first - ready->n_ready] = insn;
2940 : }
2941 : else
2942 : {
2943 2238 : if (ready->first == ready->veclen - 1)
2944 : {
2945 950 : if (ready->n_ready)
2946 : /* ready_lastpos() fails when called with (ready->n_ready == 0). */
2947 0 : memmove (ready->vec + ready->veclen - ready->n_ready - 1,
2948 0 : ready_lastpos (ready),
2949 0 : ready->n_ready * sizeof (rtx));
2950 950 : ready->first = ready->veclen - 2;
2951 : }
2952 2238 : ready->vec[++(ready->first)] = insn;
2953 : }
2954 :
2955 128130352 : ready->n_ready++;
2956 128130352 : if (DEBUG_INSN_P (insn))
2957 48493730 : ready->n_debug++;
2958 :
2959 128130352 : gcc_assert (QUEUE_INDEX (insn) != QUEUE_READY);
2960 128130352 : QUEUE_INDEX (insn) = QUEUE_READY;
2961 :
2962 128130352 : if (INSN_EXACT_TICK (insn) != INVALID_TICK
2963 128130352 : && INSN_EXACT_TICK (insn) < clock_var)
2964 : {
2965 0 : must_backtrack = true;
2966 : }
2967 128130352 : }
2968 :
2969 : /* Remove the element with the highest priority from the ready list and
2970 : return it. */
2971 :
2972 : HAIFA_INLINE static rtx_insn *
2973 125930798 : ready_remove_first (struct ready_list *ready)
2974 : {
2975 125930798 : rtx_insn *t;
2976 :
2977 125930798 : gcc_assert (ready->n_ready);
2978 125930798 : t = ready->vec[ready->first--];
2979 125930798 : ready->n_ready--;
2980 125930798 : if (DEBUG_INSN_P (t))
2981 48493730 : ready->n_debug--;
2982 : /* If the queue becomes empty, reset it. */
2983 125930798 : if (ready->n_ready == 0)
2984 66060212 : ready->first = ready->veclen - 1;
2985 :
2986 125930798 : gcc_assert (QUEUE_INDEX (t) == QUEUE_READY);
2987 125930798 : QUEUE_INDEX (t) = QUEUE_NOWHERE;
2988 :
2989 125930798 : return t;
2990 : }
2991 :
2992 : /* The following code implements multi-pass scheduling for the first
2993 : cycle. In other words, we will try to choose ready insn which
2994 : permits to start maximum number of insns on the same cycle. */
2995 :
2996 : /* Return a pointer to the element INDEX from the ready. INDEX for
2997 : insn with the highest priority is 0, and the lowest priority has
2998 : N_READY - 1. */
2999 :
3000 : rtx_insn *
3001 1103979428 : ready_element (struct ready_list *ready, int index)
3002 : {
3003 1103979428 : gcc_assert (ready->n_ready && index < ready->n_ready);
3004 :
3005 1103979428 : return ready->vec[ready->first - index];
3006 : }
3007 :
3008 : /* Remove the element INDEX from the ready list and return it. INDEX
3009 : for insn with the highest priority is 0, and the lowest priority
3010 : has N_READY - 1. */
3011 :
3012 : HAIFA_INLINE static rtx_insn *
3013 72437967 : ready_remove (struct ready_list *ready, int index)
3014 : {
3015 72437967 : rtx_insn *t;
3016 72437967 : int i;
3017 :
3018 72437967 : if (index == 0)
3019 70238425 : return ready_remove_first (ready);
3020 2199542 : gcc_assert (ready->n_ready && index < ready->n_ready);
3021 2199542 : t = ready->vec[ready->first - index];
3022 2199542 : ready->n_ready--;
3023 2199542 : if (DEBUG_INSN_P (t))
3024 0 : ready->n_debug--;
3025 7271703 : for (i = index; i < ready->n_ready; i++)
3026 5072161 : ready->vec[ready->first - i] = ready->vec[ready->first - i - 1];
3027 2199542 : QUEUE_INDEX (t) = QUEUE_NOWHERE;
3028 2199542 : return t;
3029 : }
3030 :
3031 : /* Remove INSN from the ready list. */
3032 : static void
3033 0 : ready_remove_insn (rtx_insn *insn)
3034 : {
3035 0 : int i;
3036 :
3037 0 : for (i = 0; i < readyp->n_ready; i++)
3038 0 : if (ready_element (readyp, i) == insn)
3039 : {
3040 0 : ready_remove (readyp, i);
3041 0 : return;
3042 : }
3043 0 : gcc_unreachable ();
3044 : }
3045 :
3046 : /* Calculate difference of two statistics set WAS and NOW.
3047 : Result returned in WAS. */
3048 : static void
3049 0 : rank_for_schedule_stats_diff (rank_for_schedule_stats_t *was,
3050 : const rank_for_schedule_stats_t *now)
3051 : {
3052 0 : for (int i = 0; i < RFS_N; ++i)
3053 0 : was->stats[i] = now->stats[i] - was->stats[i];
3054 0 : }
3055 :
3056 : /* Print rank_for_schedule statistics. */
3057 : static void
3058 0 : print_rank_for_schedule_stats (const char *prefix,
3059 : const rank_for_schedule_stats_t *stats,
3060 : struct ready_list *ready)
3061 : {
3062 0 : for (int i = 0; i < RFS_N; ++i)
3063 0 : if (stats->stats[i])
3064 : {
3065 0 : fprintf (sched_dump, "%s%20s: %u", prefix, rfs_str[i], stats->stats[i]);
3066 :
3067 0 : if (ready != NULL)
3068 : /* Print out insns that won due to RFS_<I>. */
3069 : {
3070 0 : rtx_insn **p = ready_lastpos (ready);
3071 :
3072 0 : fprintf (sched_dump, ":");
3073 : /* Start with 1 since least-priority insn didn't have any wins. */
3074 0 : for (int j = 1; j < ready->n_ready; ++j)
3075 0 : if (INSN_LAST_RFS_WIN (p[j]) == i)
3076 0 : fprintf (sched_dump, " %s",
3077 0 : (*current_sched_info->print_insn) (p[j], 0));
3078 : }
3079 0 : fprintf (sched_dump, "\n");
3080 : }
3081 0 : }
3082 :
3083 : /* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end
3084 : of array. */
3085 : static void
3086 48493734 : ready_sort_debug (struct ready_list *ready)
3087 : {
3088 48493734 : int i;
3089 48493734 : rtx_insn **first = ready_lastpos (ready);
3090 :
3091 204758389 : for (i = 0; i < ready->n_ready; ++i)
3092 107770921 : if (!DEBUG_INSN_P (first[i]))
3093 59277190 : INSN_RFS_DEBUG_ORIG_ORDER (first[i]) = i;
3094 :
3095 48493734 : qsort (first, ready->n_ready, sizeof (rtx), rank_for_schedule_debug);
3096 48493734 : }
3097 :
3098 : /* Sort non-debug insns in the ready list READY by ascending priority.
3099 : Assumes that all debug insns are separated from the real insns. */
3100 : static void
3101 60206721 : ready_sort_real (struct ready_list *ready)
3102 : {
3103 60206721 : int i;
3104 60206721 : rtx_insn **first = ready_lastpos (ready);
3105 60206721 : int n_ready_real = ready->n_ready - ready->n_debug;
3106 :
3107 60206721 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
3108 28787 : for (i = 0; i < n_ready_real; ++i)
3109 23095 : setup_insn_reg_pressure_info (first[i]);
3110 60201029 : else if (sched_pressure == SCHED_PRESSURE_MODEL
3111 0 : && model_curr_point < model_num_insns)
3112 0 : model_set_excess_costs (first, n_ready_real);
3113 :
3114 60206721 : rank_for_schedule_stats_t stats1;
3115 60206721 : if (sched_verbose >= 4)
3116 0 : stats1 = rank_for_schedule_stats;
3117 :
3118 60206721 : if (n_ready_real == 2)
3119 7903426 : swap_sort (first, n_ready_real);
3120 52303295 : else if (n_ready_real > 2)
3121 7471245 : qsort (first, n_ready_real, sizeof (rtx), rank_for_schedule);
3122 :
3123 60206721 : if (sched_verbose >= 4)
3124 : {
3125 0 : rank_for_schedule_stats_diff (&stats1, &rank_for_schedule_stats);
3126 0 : print_rank_for_schedule_stats (";;\t\t", &stats1, ready);
3127 : }
3128 60206721 : }
3129 :
3130 : /* Sort the ready list READY by ascending priority. */
3131 : static void
3132 108700447 : ready_sort (struct ready_list *ready)
3133 : {
3134 108700447 : if (ready->n_debug > 0)
3135 48493730 : ready_sort_debug (ready);
3136 : else
3137 60206717 : ready_sort_real (ready);
3138 108700447 : }
3139 :
3140 : /* PREV is an insn that is ready to execute. Adjust its priority if that
3141 : will help shorten or lengthen register lifetimes as appropriate. Also
3142 : provide a hook for the target to tweak itself. */
3143 :
3144 : HAIFA_INLINE static void
3145 108621144 : adjust_priority (rtx_insn *prev)
3146 : {
3147 : /* ??? There used to be code here to try and estimate how an insn
3148 : affected register lifetimes, but it did it by looking at REG_DEAD
3149 : notes, which we removed in schedule_region. Nor did it try to
3150 : take into account register pressure or anything useful like that.
3151 :
3152 : Revisit when we have a machine model to work with and not before. */
3153 :
3154 108621144 : if (targetm.sched.adjust_priority)
3155 108621144 : INSN_PRIORITY (prev) =
3156 108621144 : targetm.sched.adjust_priority (prev, INSN_PRIORITY (prev));
3157 108621144 : }
3158 :
3159 : /* Advance DFA state STATE on one cycle. */
3160 : void
3161 45949227 : advance_state (state_t state)
3162 : {
3163 45949227 : if (targetm.sched.dfa_pre_advance_cycle)
3164 0 : targetm.sched.dfa_pre_advance_cycle ();
3165 :
3166 45949227 : if (targetm.sched.dfa_pre_cycle_insn)
3167 0 : state_transition (state,
3168 : targetm.sched.dfa_pre_cycle_insn ());
3169 :
3170 45949227 : state_transition (state, NULL);
3171 :
3172 45949227 : if (targetm.sched.dfa_post_cycle_insn)
3173 0 : state_transition (state,
3174 0 : targetm.sched.dfa_post_cycle_insn ());
3175 :
3176 45949227 : if (targetm.sched.dfa_post_advance_cycle)
3177 45774424 : targetm.sched.dfa_post_advance_cycle ();
3178 45949227 : }
3179 :
3180 : /* Advance time on one cycle. */
3181 : HAIFA_INLINE static void
3182 45944067 : advance_one_cycle (void)
3183 : {
3184 45944067 : int i;
3185 :
3186 45944067 : advance_state (curr_state);
3187 91888134 : for (i = 4; i <= sched_verbose; ++i)
3188 0 : fprintf (sched_dump, ";;\tAdvance the current state: %d.\n", clock_var);
3189 45944067 : }
3190 :
3191 : /* Update register pressure after scheduling INSN. */
3192 : static void
3193 22232 : update_register_pressure (rtx_insn *insn)
3194 : {
3195 22232 : struct reg_use_data *use;
3196 22232 : struct reg_set_data *set;
3197 :
3198 22232 : gcc_checking_assert (!DEBUG_INSN_P (insn));
3199 :
3200 43762 : for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
3201 21530 : if (dying_use_p (use))
3202 18383 : mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
3203 : use->regno, false);
3204 39466 : for (set = INSN_REG_SET_LIST (insn); set != NULL; set = set->next_insn_set)
3205 17234 : mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
3206 : set->regno, true);
3207 22232 : }
3208 :
3209 : /* Set up or update (if UPDATE_P) max register pressure (see its
3210 : meaning in sched-int.h::_haifa_insn_data) for all current BB insns
3211 : after insn AFTER. */
3212 : static void
3213 2579 : setup_insn_max_reg_pressure (rtx_insn *after, bool update_p)
3214 : {
3215 2579 : int i, p;
3216 2579 : bool eq_p;
3217 2579 : rtx_insn *insn;
3218 2579 : static int max_reg_pressure[N_REG_CLASSES];
3219 :
3220 2579 : save_reg_pressure ();
3221 15663 : for (i = 0; i < ira_pressure_classes_num; i++)
3222 10505 : max_reg_pressure[ira_pressure_classes[i]]
3223 10505 : = curr_reg_pressure[ira_pressure_classes[i]];
3224 24861 : for (insn = NEXT_INSN (after);
3225 24861 : insn != NULL_RTX && ! BARRIER_P (insn)
3226 49561 : && BLOCK_FOR_INSN (insn) == BLOCK_FOR_INSN (after);
3227 22282 : insn = NEXT_INSN (insn))
3228 23812 : if (NONDEBUG_INSN_P (insn))
3229 : {
3230 : eq_p = true;
3231 109172 : for (i = 0; i < ira_pressure_classes_num; i++)
3232 : {
3233 88718 : p = max_reg_pressure[ira_pressure_classes[i]];
3234 88718 : if (INSN_MAX_REG_PRESSURE (insn)[i] != p)
3235 : {
3236 21898 : eq_p = false;
3237 21898 : INSN_MAX_REG_PRESSURE (insn)[i]
3238 21898 : = max_reg_pressure[ira_pressure_classes[i]];
3239 : }
3240 : }
3241 20454 : if (update_p && eq_p)
3242 : break;
3243 18924 : update_register_pressure (insn);
3244 120341 : for (i = 0; i < ira_pressure_classes_num; i++)
3245 82493 : if (max_reg_pressure[ira_pressure_classes[i]]
3246 82493 : < curr_reg_pressure[ira_pressure_classes[i]])
3247 2883 : max_reg_pressure[ira_pressure_classes[i]]
3248 2883 : = curr_reg_pressure[ira_pressure_classes[i]];
3249 : }
3250 2579 : restore_reg_pressure ();
3251 2579 : }
3252 :
3253 : /* Update the current register pressure after scheduling INSN. Update
3254 : also max register pressure for unscheduled insns of the current
3255 : BB. */
3256 : static void
3257 3308 : update_reg_and_insn_max_reg_pressure (rtx_insn *insn)
3258 : {
3259 3308 : int i;
3260 3308 : int before[N_REG_CLASSES];
3261 :
3262 16829 : for (i = 0; i < ira_pressure_classes_num; i++)
3263 13521 : before[i] = curr_reg_pressure[ira_pressure_classes[i]];
3264 3308 : update_register_pressure (insn);
3265 11679 : for (i = 0; i < ira_pressure_classes_num; i++)
3266 7212 : if (curr_reg_pressure[ira_pressure_classes[i]] != before[i])
3267 : break;
3268 3308 : if (i < ira_pressure_classes_num)
3269 2149 : setup_insn_max_reg_pressure (insn, true);
3270 3308 : }
3271 :
3272 : /* Set up register pressure at the beginning of basic block BB whose
3273 : insns starting after insn AFTER. Set up also max register pressure
3274 : for all insns of the basic block. */
3275 : void
3276 430 : sched_setup_bb_reg_pressure_info (basic_block bb, rtx_insn *after)
3277 : {
3278 430 : gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED);
3279 430 : initiate_bb_reg_pressure_info (bb);
3280 430 : setup_insn_max_reg_pressure (after, false);
3281 430 : }
3282 :
3283 : /* If doing predication while scheduling, verify whether INSN, which
3284 : has just been scheduled, clobbers the conditions of any
3285 : instructions that must be predicated in order to break their
3286 : dependencies. If so, remove them from the queues so that they will
3287 : only be scheduled once their control dependency is resolved. */
3288 :
3289 : static void
3290 108621132 : check_clobbered_conditions (rtx_insn *insn)
3291 : {
3292 108621132 : HARD_REG_SET t;
3293 108621132 : int i;
3294 :
3295 108621132 : if ((current_sched_info->flags & DO_PREDICATION) == 0)
3296 108621132 : return;
3297 :
3298 0 : find_all_hard_reg_sets (insn, &t, true);
3299 :
3300 0 : restart:
3301 0 : for (i = 0; i < ready.n_ready; i++)
3302 : {
3303 0 : rtx_insn *x = ready_element (&ready, i);
3304 0 : if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t))
3305 : {
3306 0 : ready_remove_insn (x);
3307 0 : goto restart;
3308 : }
3309 : }
3310 0 : for (i = 0; i <= max_insn_queue_index; i++)
3311 : {
3312 0 : rtx_insn_list *link;
3313 0 : int q = NEXT_Q_AFTER (q_ptr, i);
3314 :
3315 0 : restart_queue:
3316 0 : for (link = insn_queue[q]; link; link = link->next ())
3317 : {
3318 0 : rtx_insn *x = link->insn ();
3319 0 : if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t))
3320 : {
3321 0 : queue_remove (x);
3322 0 : goto restart_queue;
3323 : }
3324 : }
3325 : }
3326 : }
3327 :
3328 : /* Return (in order):
3329 :
3330 : - positive if INSN adversely affects the pressure on one
3331 : register class
3332 :
3333 : - negative if INSN reduces the pressure on one register class
3334 :
3335 : - 0 if INSN doesn't affect the pressure on any register class. */
3336 :
3337 : static int
3338 0 : model_classify_pressure (struct model_insn_info *insn)
3339 : {
3340 0 : struct reg_pressure_data *reg_pressure;
3341 0 : int death[N_REG_CLASSES];
3342 0 : int pci, cl, sum;
3343 :
3344 0 : calculate_reg_deaths (insn->insn, death);
3345 0 : reg_pressure = INSN_REG_PRESSURE (insn->insn);
3346 0 : sum = 0;
3347 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3348 : {
3349 0 : cl = ira_pressure_classes[pci];
3350 0 : if (death[cl] < reg_pressure[pci].set_increase)
3351 : return 1;
3352 0 : sum += reg_pressure[pci].set_increase - death[cl];
3353 : }
3354 : return sum;
3355 : }
3356 :
3357 : /* Return true if INSN1 should come before INSN2 in the model schedule. */
3358 :
3359 : static int
3360 0 : model_order_p (struct model_insn_info *insn1, struct model_insn_info *insn2)
3361 : {
3362 0 : unsigned int height1, height2;
3363 0 : unsigned int priority1, priority2;
3364 :
3365 : /* Prefer instructions with a higher model priority. */
3366 0 : if (insn1->model_priority != insn2->model_priority)
3367 0 : return insn1->model_priority > insn2->model_priority;
3368 :
3369 : /* Combine the length of the longest path of satisfied true dependencies
3370 : that leads to each instruction (depth) with the length of the longest
3371 : path of any dependencies that leads from the instruction (alap).
3372 : Prefer instructions with the greatest combined length. If the combined
3373 : lengths are equal, prefer instructions with the greatest depth.
3374 :
3375 : The idea is that, if we have a set S of "equal" instructions that each
3376 : have ALAP value X, and we pick one such instruction I, any true-dependent
3377 : successors of I that have ALAP value X - 1 should be preferred over S.
3378 : This encourages the schedule to be "narrow" rather than "wide".
3379 : However, if I is a low-priority instruction that we decided to
3380 : schedule because of its model_classify_pressure, and if there
3381 : is a set of higher-priority instructions T, the aforementioned
3382 : successors of I should not have the edge over T. */
3383 0 : height1 = insn1->depth + insn1->alap;
3384 0 : height2 = insn2->depth + insn2->alap;
3385 0 : if (height1 != height2)
3386 0 : return height1 > height2;
3387 0 : if (insn1->depth != insn2->depth)
3388 0 : return insn1->depth > insn2->depth;
3389 :
3390 : /* We have no real preference between INSN1 an INSN2 as far as attempts
3391 : to reduce pressure go. Prefer instructions with higher priorities. */
3392 0 : priority1 = INSN_PRIORITY (insn1->insn);
3393 0 : priority2 = INSN_PRIORITY (insn2->insn);
3394 0 : if (priority1 != priority2)
3395 0 : return priority1 > priority2;
3396 :
3397 : /* Use the original rtl sequence as a tie-breaker. */
3398 0 : return insn1 < insn2;
3399 : }
3400 :
3401 : /* Add INSN to the model worklist immediately after PREV. Add it to the
3402 : beginning of the list if PREV is null. */
3403 :
3404 : static void
3405 0 : model_add_to_worklist_at (struct model_insn_info *insn,
3406 : struct model_insn_info *prev)
3407 : {
3408 0 : gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_NOWHERE);
3409 0 : QUEUE_INDEX (insn->insn) = QUEUE_READY;
3410 :
3411 0 : insn->prev = prev;
3412 0 : if (prev)
3413 : {
3414 0 : insn->next = prev->next;
3415 0 : prev->next = insn;
3416 : }
3417 : else
3418 : {
3419 0 : insn->next = model_worklist;
3420 0 : model_worklist = insn;
3421 : }
3422 0 : if (insn->next)
3423 0 : insn->next->prev = insn;
3424 0 : }
3425 :
3426 : /* Remove INSN from the model worklist. */
3427 :
3428 : static void
3429 0 : model_remove_from_worklist (struct model_insn_info *insn)
3430 : {
3431 0 : gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_READY);
3432 0 : QUEUE_INDEX (insn->insn) = QUEUE_NOWHERE;
3433 :
3434 0 : if (insn->prev)
3435 0 : insn->prev->next = insn->next;
3436 : else
3437 0 : model_worklist = insn->next;
3438 0 : if (insn->next)
3439 0 : insn->next->prev = insn->prev;
3440 0 : }
3441 :
3442 : /* Add INSN to the model worklist. Start looking for a suitable position
3443 : between neighbors PREV and NEXT, testing at most param_max_sched_ready_insns
3444 : insns either side. A null PREV indicates the beginning of the list and
3445 : a null NEXT indicates the end. */
3446 :
3447 : static void
3448 0 : model_add_to_worklist (struct model_insn_info *insn,
3449 : struct model_insn_info *prev,
3450 : struct model_insn_info *next)
3451 : {
3452 0 : int count;
3453 :
3454 0 : count = param_max_sched_ready_insns;
3455 0 : if (count > 0 && prev && model_order_p (insn, prev))
3456 0 : do
3457 : {
3458 0 : count--;
3459 0 : prev = prev->prev;
3460 : }
3461 0 : while (count > 0 && prev && model_order_p (insn, prev));
3462 : else
3463 0 : while (count > 0 && next && model_order_p (next, insn))
3464 : {
3465 0 : count--;
3466 0 : prev = next;
3467 0 : next = next->next;
3468 : }
3469 0 : model_add_to_worklist_at (insn, prev);
3470 0 : }
3471 :
3472 : /* INSN may now have a higher priority (in the model_order_p sense)
3473 : than before. Move it up the worklist if necessary. */
3474 :
3475 : static void
3476 0 : model_promote_insn (struct model_insn_info *insn)
3477 : {
3478 0 : struct model_insn_info *prev;
3479 0 : int count;
3480 :
3481 0 : prev = insn->prev;
3482 0 : count = param_max_sched_ready_insns;
3483 0 : while (count > 0 && prev && model_order_p (insn, prev))
3484 : {
3485 0 : count--;
3486 0 : prev = prev->prev;
3487 : }
3488 0 : if (prev != insn->prev)
3489 : {
3490 0 : model_remove_from_worklist (insn);
3491 0 : model_add_to_worklist_at (insn, prev);
3492 : }
3493 0 : }
3494 :
3495 : /* Add INSN to the end of the model schedule. */
3496 :
3497 : static void
3498 0 : model_add_to_schedule (rtx_insn *insn)
3499 : {
3500 0 : unsigned int point;
3501 :
3502 0 : gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE);
3503 0 : QUEUE_INDEX (insn) = QUEUE_SCHEDULED;
3504 :
3505 0 : point = model_schedule.length ();
3506 0 : model_schedule.quick_push (insn);
3507 0 : INSN_MODEL_INDEX (insn) = point + 1;
3508 0 : }
3509 :
3510 : /* Analyze the instructions that are to be scheduled, setting up
3511 : MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready
3512 : instructions to model_worklist. */
3513 :
3514 : static void
3515 0 : model_analyze_insns (void)
3516 : {
3517 0 : rtx_insn *start, *end, *iter;
3518 0 : sd_iterator_def sd_it;
3519 0 : dep_t dep;
3520 0 : struct model_insn_info *insn, *con;
3521 :
3522 0 : model_num_insns = 0;
3523 0 : start = PREV_INSN (current_sched_info->next_tail);
3524 0 : end = current_sched_info->prev_head;
3525 0 : for (iter = start; iter != end; iter = PREV_INSN (iter))
3526 0 : if (NONDEBUG_INSN_P (iter))
3527 : {
3528 0 : insn = MODEL_INSN_INFO (iter);
3529 0 : insn->insn = iter;
3530 0 : FOR_EACH_DEP (iter, SD_LIST_FORW, sd_it, dep)
3531 : {
3532 0 : con = MODEL_INSN_INFO (DEP_CON (dep));
3533 0 : if (con->insn && insn->alap < con->alap + 1)
3534 0 : insn->alap = con->alap + 1;
3535 : }
3536 :
3537 0 : insn->old_queue = QUEUE_INDEX (iter);
3538 0 : QUEUE_INDEX (iter) = QUEUE_NOWHERE;
3539 :
3540 0 : insn->unscheduled_preds = dep_list_size (iter, SD_LIST_HARD_BACK);
3541 0 : if (insn->unscheduled_preds == 0)
3542 0 : model_add_to_worklist (insn, NULL, model_worklist);
3543 :
3544 0 : model_num_insns++;
3545 : }
3546 0 : }
3547 :
3548 : /* The global state describes the register pressure at the start of the
3549 : model schedule. Initialize GROUP accordingly. */
3550 :
3551 : static void
3552 0 : model_init_pressure_group (struct model_pressure_group *group)
3553 : {
3554 0 : int pci, cl;
3555 :
3556 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3557 : {
3558 0 : cl = ira_pressure_classes[pci];
3559 0 : group->limits[pci].pressure = curr_reg_pressure[cl];
3560 0 : group->limits[pci].point = 0;
3561 : }
3562 : /* Use index model_num_insns to record the state after the last
3563 : instruction in the model schedule. */
3564 0 : group->model = XNEWVEC (struct model_pressure_data,
3565 : (model_num_insns + 1) * ira_pressure_classes_num);
3566 0 : }
3567 :
3568 : /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE.
3569 : Update the maximum pressure for the whole schedule. */
3570 :
3571 : static void
3572 0 : model_record_pressure (struct model_pressure_group *group,
3573 : int point, int pci, int pressure)
3574 : {
3575 0 : MODEL_REF_PRESSURE (group, point, pci) = pressure;
3576 0 : if (group->limits[pci].pressure < pressure)
3577 : {
3578 0 : group->limits[pci].pressure = pressure;
3579 0 : group->limits[pci].point = point;
3580 : }
3581 0 : }
3582 :
3583 : /* INSN has just been added to the end of the model schedule. Record its
3584 : register-pressure information. */
3585 :
3586 : static void
3587 0 : model_record_pressures (struct model_insn_info *insn)
3588 : {
3589 0 : struct reg_pressure_data *reg_pressure;
3590 0 : int point, pci, cl, delta;
3591 0 : int death[N_REG_CLASSES];
3592 :
3593 0 : point = model_index (insn->insn);
3594 0 : if (sched_verbose >= 2)
3595 : {
3596 0 : if (point == 0)
3597 : {
3598 0 : fprintf (sched_dump, "\n;;\tModel schedule:\n;;\n");
3599 0 : fprintf (sched_dump, ";;\t| idx insn | mpri hght dpth prio |\n");
3600 : }
3601 0 : fprintf (sched_dump, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ",
3602 : point, INSN_UID (insn->insn), insn->model_priority,
3603 0 : insn->depth + insn->alap, insn->depth,
3604 0 : INSN_PRIORITY (insn->insn),
3605 0 : str_pattern_slim (PATTERN (insn->insn)));
3606 : }
3607 0 : calculate_reg_deaths (insn->insn, death);
3608 0 : reg_pressure = INSN_REG_PRESSURE (insn->insn);
3609 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3610 : {
3611 0 : cl = ira_pressure_classes[pci];
3612 0 : delta = reg_pressure[pci].set_increase - death[cl];
3613 0 : if (sched_verbose >= 2)
3614 0 : fprintf (sched_dump, " %s:[%d,%+d]", reg_class_names[cl],
3615 : curr_reg_pressure[cl], delta);
3616 0 : model_record_pressure (&model_before_pressure, point, pci,
3617 : curr_reg_pressure[cl]);
3618 : }
3619 0 : if (sched_verbose >= 2)
3620 0 : fprintf (sched_dump, "\n");
3621 0 : }
3622 :
3623 : /* All instructions have been added to the model schedule. Record the
3624 : final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. */
3625 :
3626 : static void
3627 0 : model_record_final_pressures (struct model_pressure_group *group)
3628 : {
3629 0 : int point, pci, max_pressure, ref_pressure, cl;
3630 :
3631 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3632 : {
3633 : /* Record the final pressure for this class. */
3634 0 : cl = ira_pressure_classes[pci];
3635 0 : point = model_num_insns;
3636 0 : ref_pressure = curr_reg_pressure[cl];
3637 0 : model_record_pressure (group, point, pci, ref_pressure);
3638 :
3639 : /* Record the original maximum pressure. */
3640 0 : group->limits[pci].orig_pressure = group->limits[pci].pressure;
3641 :
3642 : /* Update the MODEL_MAX_PRESSURE for every point of the schedule. */
3643 0 : max_pressure = ref_pressure;
3644 0 : MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
3645 0 : while (point > 0)
3646 : {
3647 0 : point--;
3648 0 : ref_pressure = MODEL_REF_PRESSURE (group, point, pci);
3649 0 : max_pressure = MAX (max_pressure, ref_pressure);
3650 0 : MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
3651 : }
3652 : }
3653 0 : }
3654 :
3655 : /* Update all successors of INSN, given that INSN has just been scheduled. */
3656 :
3657 : static void
3658 0 : model_add_successors_to_worklist (struct model_insn_info *insn)
3659 : {
3660 0 : sd_iterator_def sd_it;
3661 0 : struct model_insn_info *con;
3662 0 : dep_t dep;
3663 :
3664 0 : FOR_EACH_DEP (insn->insn, SD_LIST_FORW, sd_it, dep)
3665 : {
3666 0 : con = MODEL_INSN_INFO (DEP_CON (dep));
3667 : /* Ignore debug instructions, and instructions from other blocks. */
3668 0 : if (con->insn)
3669 : {
3670 0 : con->unscheduled_preds--;
3671 :
3672 : /* Update the depth field of each true-dependent successor.
3673 : Increasing the depth gives them a higher priority than
3674 : before. */
3675 0 : if (DEP_TYPE (dep) == REG_DEP_TRUE && con->depth < insn->depth + 1)
3676 : {
3677 0 : con->depth = insn->depth + 1;
3678 0 : if (QUEUE_INDEX (con->insn) == QUEUE_READY)
3679 0 : model_promote_insn (con);
3680 : }
3681 :
3682 : /* If this is a true dependency, or if there are no remaining
3683 : dependencies for CON (meaning that CON only had non-true
3684 : dependencies), make sure that CON is on the worklist.
3685 : We don't bother otherwise because it would tend to fill the
3686 : worklist with a lot of low-priority instructions that are not
3687 : yet ready to issue. */
3688 0 : if ((con->depth > 0 || con->unscheduled_preds == 0)
3689 0 : && QUEUE_INDEX (con->insn) == QUEUE_NOWHERE)
3690 0 : model_add_to_worklist (con, insn, insn->next);
3691 : }
3692 : }
3693 0 : }
3694 :
3695 : /* Give INSN a higher priority than any current instruction, then give
3696 : unscheduled predecessors of INSN a higher priority still. If any of
3697 : those predecessors are not on the model worklist, do the same for its
3698 : predecessors, and so on. */
3699 :
3700 : static void
3701 0 : model_promote_predecessors (struct model_insn_info *insn)
3702 : {
3703 0 : struct model_insn_info *pro, *first;
3704 0 : sd_iterator_def sd_it;
3705 0 : dep_t dep;
3706 :
3707 0 : if (sched_verbose >= 7)
3708 0 : fprintf (sched_dump, ";;\t+--- priority of %d = %d, priority of",
3709 0 : INSN_UID (insn->insn), model_next_priority);
3710 0 : insn->model_priority = model_next_priority++;
3711 0 : model_remove_from_worklist (insn);
3712 0 : model_add_to_worklist_at (insn, NULL);
3713 :
3714 0 : first = NULL;
3715 0 : for (;;)
3716 : {
3717 0 : FOR_EACH_DEP (insn->insn, SD_LIST_HARD_BACK, sd_it, dep)
3718 : {
3719 0 : pro = MODEL_INSN_INFO (DEP_PRO (dep));
3720 : /* The first test is to ignore debug instructions, and instructions
3721 : from other blocks. */
3722 0 : if (pro->insn
3723 0 : && pro->model_priority != model_next_priority
3724 0 : && QUEUE_INDEX (pro->insn) != QUEUE_SCHEDULED)
3725 : {
3726 0 : pro->model_priority = model_next_priority;
3727 0 : if (sched_verbose >= 7)
3728 0 : fprintf (sched_dump, " %d", INSN_UID (pro->insn));
3729 0 : if (QUEUE_INDEX (pro->insn) == QUEUE_READY)
3730 : {
3731 : /* PRO is already in the worklist, but it now has
3732 : a higher priority than before. Move it at the
3733 : appropriate place. */
3734 0 : model_remove_from_worklist (pro);
3735 0 : model_add_to_worklist (pro, NULL, model_worklist);
3736 : }
3737 : else
3738 : {
3739 : /* PRO isn't in the worklist. Recursively process
3740 : its predecessors until we find one that is. */
3741 0 : pro->next = first;
3742 0 : first = pro;
3743 : }
3744 : }
3745 : }
3746 0 : if (!first)
3747 : break;
3748 0 : insn = first;
3749 0 : first = insn->next;
3750 : }
3751 0 : if (sched_verbose >= 7)
3752 0 : fprintf (sched_dump, " = %d\n", model_next_priority);
3753 0 : model_next_priority++;
3754 0 : }
3755 :
3756 : /* Pick one instruction from model_worklist and process it. */
3757 :
3758 : static void
3759 0 : model_choose_insn (void)
3760 : {
3761 0 : struct model_insn_info *insn, *fallback;
3762 0 : int count;
3763 :
3764 0 : if (sched_verbose >= 7)
3765 : {
3766 0 : fprintf (sched_dump, ";;\t+--- worklist:\n");
3767 0 : insn = model_worklist;
3768 0 : count = param_max_sched_ready_insns;
3769 0 : while (count > 0 && insn)
3770 : {
3771 0 : fprintf (sched_dump, ";;\t+--- %d [%d, %d, %d, %d][%d]\n",
3772 : INSN_UID (insn->insn), insn->model_priority,
3773 0 : insn->depth + insn->alap, insn->depth,
3774 0 : INSN_PRIORITY (insn->insn), insn->unscheduled_preds);
3775 0 : count--;
3776 0 : insn = insn->next;
3777 : }
3778 : }
3779 :
3780 : /* Look for a ready instruction whose model_classify_priority is zero
3781 : or negative, picking the highest-priority one. Adding such an
3782 : instruction to the schedule now should do no harm, and may actually
3783 : do some good.
3784 :
3785 : Failing that, see whether there is an instruction with the highest
3786 : extant model_priority that is not yet ready, but which would reduce
3787 : pressure if it became ready. This is designed to catch cases like:
3788 :
3789 : (set (mem (reg R1)) (reg R2))
3790 :
3791 : where the instruction is the last remaining use of R1 and where the
3792 : value of R2 is not yet available (or vice versa). The death of R1
3793 : means that this instruction already reduces pressure. It is of
3794 : course possible that the computation of R2 involves other registers
3795 : that are hard to kill, but such cases are rare enough for this
3796 : heuristic to be a win in general.
3797 :
3798 : Failing that, just pick the highest-priority instruction in the
3799 : worklist. */
3800 0 : count = param_max_sched_ready_insns;
3801 0 : insn = model_worklist;
3802 0 : fallback = 0;
3803 0 : for (;;)
3804 : {
3805 0 : if (count == 0 || !insn)
3806 : {
3807 0 : insn = fallback ? fallback : model_worklist;
3808 : break;
3809 : }
3810 0 : if (insn->unscheduled_preds)
3811 : {
3812 0 : if (model_worklist->model_priority == insn->model_priority
3813 0 : && !fallback
3814 0 : && model_classify_pressure (insn) < 0)
3815 : fallback = insn;
3816 : }
3817 : else
3818 : {
3819 0 : if (model_classify_pressure (insn) <= 0)
3820 : break;
3821 : }
3822 0 : count--;
3823 0 : insn = insn->next;
3824 : }
3825 :
3826 0 : if (sched_verbose >= 7 && insn != model_worklist)
3827 : {
3828 0 : if (insn->unscheduled_preds)
3829 0 : fprintf (sched_dump, ";;\t+--- promoting insn %d, with dependencies\n",
3830 0 : INSN_UID (insn->insn));
3831 : else
3832 0 : fprintf (sched_dump, ";;\t+--- promoting insn %d, which is ready\n",
3833 0 : INSN_UID (insn->insn));
3834 : }
3835 0 : if (insn->unscheduled_preds)
3836 : /* INSN isn't yet ready to issue. Give all its predecessors the
3837 : highest priority. */
3838 0 : model_promote_predecessors (insn);
3839 : else
3840 : {
3841 : /* INSN is ready. Add it to the end of model_schedule and
3842 : process its successors. */
3843 0 : model_add_successors_to_worklist (insn);
3844 0 : model_remove_from_worklist (insn);
3845 0 : model_add_to_schedule (insn->insn);
3846 0 : model_record_pressures (insn);
3847 0 : update_register_pressure (insn->insn);
3848 : }
3849 0 : }
3850 :
3851 : /* Restore all QUEUE_INDEXs to the values that they had before
3852 : model_start_schedule was called. */
3853 :
3854 : static void
3855 0 : model_reset_queue_indices (void)
3856 : {
3857 0 : unsigned int i;
3858 0 : rtx_insn *insn;
3859 :
3860 0 : FOR_EACH_VEC_ELT (model_schedule, i, insn)
3861 0 : QUEUE_INDEX (insn) = MODEL_INSN_INFO (insn)->old_queue;
3862 0 : }
3863 :
3864 : /* We have calculated the model schedule and spill costs. Print a summary
3865 : to sched_dump. */
3866 :
3867 : static void
3868 0 : model_dump_pressure_summary (basic_block bb)
3869 : {
3870 0 : int pci, cl;
3871 :
3872 0 : fprintf (sched_dump, ";; Pressure summary (bb %d):", bb->index);
3873 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3874 : {
3875 0 : cl = ira_pressure_classes[pci];
3876 0 : fprintf (sched_dump, " %s:%d", reg_class_names[cl],
3877 : model_before_pressure.limits[pci].pressure);
3878 : }
3879 0 : fprintf (sched_dump, "\n\n");
3880 0 : }
3881 :
3882 : /* Initialize the SCHED_PRESSURE_MODEL information for the current
3883 : scheduling region. */
3884 :
3885 : static void
3886 0 : model_start_schedule (basic_block bb)
3887 : {
3888 0 : model_next_priority = 1;
3889 0 : model_schedule.create (sched_max_luid);
3890 0 : model_insns = XCNEWVEC (struct model_insn_info, sched_max_luid);
3891 :
3892 0 : gcc_assert (bb == BLOCK_FOR_INSN (NEXT_INSN (current_sched_info->prev_head)));
3893 0 : initiate_reg_pressure_info (df_get_live_in (bb));
3894 :
3895 0 : model_analyze_insns ();
3896 0 : model_init_pressure_group (&model_before_pressure);
3897 0 : while (model_worklist)
3898 0 : model_choose_insn ();
3899 0 : gcc_assert (model_num_insns == (int) model_schedule.length ());
3900 0 : if (sched_verbose >= 2)
3901 0 : fprintf (sched_dump, "\n");
3902 :
3903 0 : model_record_final_pressures (&model_before_pressure);
3904 0 : model_reset_queue_indices ();
3905 :
3906 0 : XDELETEVEC (model_insns);
3907 :
3908 0 : model_curr_point = 0;
3909 0 : initiate_reg_pressure_info (df_get_live_in (bb));
3910 0 : if (sched_verbose >= 1)
3911 0 : model_dump_pressure_summary (bb);
3912 0 : }
3913 :
3914 : /* Free the information associated with GROUP. */
3915 :
3916 : static void
3917 0 : model_finalize_pressure_group (struct model_pressure_group *group)
3918 : {
3919 0 : XDELETEVEC (group->model);
3920 0 : }
3921 :
3922 : /* Free the information created by model_start_schedule. */
3923 :
3924 : static void
3925 0 : model_end_schedule (void)
3926 : {
3927 0 : model_finalize_pressure_group (&model_before_pressure);
3928 0 : model_schedule.release ();
3929 0 : }
3930 :
3931 : /* Prepare reg pressure scheduling for basic block BB. */
3932 : static void
3933 430 : sched_pressure_start_bb (basic_block bb)
3934 : {
3935 : /* Set the number of available registers for each class taking into account
3936 : relative probability of current basic block versus function prologue and
3937 : epilogue.
3938 : * If the basic block executes much more often than the prologue/epilogue
3939 : (e.g., inside a hot loop), then cost of spill in the prologue is close to
3940 : nil, so the effective number of available registers is
3941 : (ira_class_hard_regs_num[cl] - fixed_regs_num[cl] - 0).
3942 : * If the basic block executes as often as the prologue/epilogue,
3943 : then spill in the block is as costly as in the prologue, so the effective
3944 : number of available registers is
3945 : (ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
3946 : - call_saved_regs_num[cl]).
3947 : Note that all-else-equal, we prefer to spill in the prologue, since that
3948 : allows "extra" registers for other basic blocks of the function.
3949 : * If the basic block is on the cold path of the function and executes
3950 : rarely, then we should always prefer to spill in the block, rather than
3951 : in the prologue/epilogue. The effective number of available register is
3952 : (ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
3953 : - call_saved_regs_num[cl]). */
3954 430 : {
3955 430 : int i;
3956 430 : int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun);
3957 430 : int bb_freq = bb->count.to_frequency (cfun);
3958 :
3959 430 : if (bb_freq == 0)
3960 : {
3961 8 : if (entry_freq == 0)
3962 : entry_freq = bb_freq = 1;
3963 : }
3964 430 : if (bb_freq < entry_freq)
3965 : bb_freq = entry_freq;
3966 :
3967 2154 : for (i = 0; i < ira_pressure_classes_num; ++i)
3968 : {
3969 1724 : enum reg_class cl = ira_pressure_classes[i];
3970 1724 : sched_class_regs_num[cl] = ira_class_hard_regs_num[cl]
3971 1724 : - fixed_regs_num[cl];
3972 1724 : sched_class_regs_num[cl]
3973 1724 : -= (call_saved_regs_num[cl] * entry_freq) / bb_freq;
3974 : }
3975 : }
3976 :
3977 430 : if (sched_pressure == SCHED_PRESSURE_MODEL)
3978 0 : model_start_schedule (bb);
3979 430 : }
3980 :
3981 : /* A structure that holds local state for the loop in schedule_block. */
3982 : struct sched_block_state
3983 : {
3984 : /* True if no real insns have been scheduled in the current cycle. */
3985 : bool first_cycle_insn_p;
3986 : /* True if a shadow insn has been scheduled in the current cycle, which
3987 : means that no more normal insns can be issued. */
3988 : bool shadows_only_p;
3989 : /* True if we're winding down a modulo schedule, which means that we only
3990 : issue insns with INSN_EXACT_TICK set. */
3991 : bool modulo_epilogue;
3992 : /* Initialized with the machine's issue rate every cycle, and updated
3993 : by calls to the variable_issue hook. */
3994 : int can_issue_more;
3995 : };
3996 :
3997 : /* INSN is the "currently executing insn". Launch each insn which was
3998 : waiting on INSN. READY is the ready list which contains the insns
3999 : that are ready to fire. CLOCK is the current cycle. The function
4000 : returns necessary cycle advance after issuing the insn (it is not
4001 : zero for insns in a schedule group). */
4002 :
4003 : static int
4004 108621132 : schedule_insn (rtx_insn *insn)
4005 : {
4006 108621132 : sd_iterator_def sd_it;
4007 108621132 : dep_t dep;
4008 108621132 : int i;
4009 108621132 : int advance = 0;
4010 :
4011 108621132 : if (sched_verbose >= 1)
4012 : {
4013 1004 : struct reg_pressure_data *pressure_info;
4014 1004 : fprintf (sched_dump, ";;\t%3i--> %s %-40s:",
4015 1004 : clock_var, (*current_sched_info->print_insn) (insn, 1),
4016 1004 : str_pattern_slim (PATTERN (insn)));
4017 :
4018 1004 : if (recog_memoized (insn) < 0)
4019 134 : fprintf (sched_dump, "nothing");
4020 : else
4021 870 : print_reservation (sched_dump, insn);
4022 1004 : pressure_info = INSN_REG_PRESSURE (insn);
4023 1004 : if (pressure_info != NULL)
4024 : {
4025 0 : fputc (':', sched_dump);
4026 0 : for (i = 0; i < ira_pressure_classes_num; i++)
4027 0 : fprintf (sched_dump, "%s%s%+d(%d)",
4028 0 : scheduled_insns.length () > 1
4029 0 : && INSN_LUID (insn)
4030 0 : < INSN_LUID (scheduled_insns[scheduled_insns.length () - 2]) ? "@" : "",
4031 0 : reg_class_names[ira_pressure_classes[i]],
4032 0 : pressure_info[i].set_increase, pressure_info[i].change);
4033 : }
4034 1004 : if (sched_pressure == SCHED_PRESSURE_MODEL
4035 0 : && model_curr_point < model_num_insns
4036 1004 : && model_index (insn) == model_curr_point)
4037 0 : fprintf (sched_dump, ":model %d", model_curr_point);
4038 1004 : fputc ('\n', sched_dump);
4039 : }
4040 :
4041 108621132 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED && !DEBUG_INSN_P (insn))
4042 3308 : update_reg_and_insn_max_reg_pressure (insn);
4043 :
4044 : /* Scheduling instruction should have all its dependencies resolved and
4045 : should have been removed from the ready list. */
4046 108621132 : gcc_assert (sd_lists_empty_p (insn, SD_LIST_HARD_BACK));
4047 :
4048 : /* Reset debug insns invalidated by moving this insn. */
4049 108621132 : if (MAY_HAVE_DEBUG_BIND_INSNS && !DEBUG_INSN_P (insn))
4050 41356263 : for (sd_it = sd_iterator_start (insn, SD_LIST_BACK);
4051 41857645 : sd_iterator_cond (&sd_it, &dep);)
4052 : {
4053 501382 : rtx_insn *dbg = DEP_PRO (dep);
4054 501382 : struct reg_use_data *use, *next;
4055 :
4056 501382 : if (DEP_STATUS (dep) & DEP_CANCELLED)
4057 : {
4058 381457 : sd_iterator_next (&sd_it);
4059 381457 : continue;
4060 : }
4061 :
4062 119925 : gcc_assert (DEBUG_BIND_INSN_P (dbg));
4063 :
4064 119925 : if (sched_verbose >= 6)
4065 0 : fprintf (sched_dump, ";;\t\tresetting: debug insn %d\n",
4066 0 : INSN_UID (dbg));
4067 :
4068 : /* ??? Rather than resetting the debug insn, we might be able
4069 : to emit a debug temp before the just-scheduled insn, but
4070 : this would involve checking that the expression at the
4071 : point of the debug insn is equivalent to the expression
4072 : before the just-scheduled insn. They might not be: the
4073 : expression in the debug insn may depend on other insns not
4074 : yet scheduled that set MEMs, REGs or even other debug
4075 : insns. It's not clear that attempting to preserve debug
4076 : information in these cases is worth the effort, given how
4077 : uncommon these resets are and the likelihood that the debug
4078 : temps introduced won't survive the schedule change. */
4079 119925 : INSN_VAR_LOCATION_LOC (dbg) = gen_rtx_UNKNOWN_VAR_LOC ();
4080 119925 : df_insn_rescan (dbg);
4081 :
4082 : /* Unknown location doesn't use any registers. */
4083 119925 : for (use = INSN_REG_USE_LIST (dbg); use != NULL; use = next)
4084 : {
4085 : struct reg_use_data *prev = use;
4086 :
4087 : /* Remove use from the cyclic next_regno_use chain first. */
4088 0 : while (prev->next_regno_use != use)
4089 : prev = prev->next_regno_use;
4090 0 : prev->next_regno_use = use->next_regno_use;
4091 0 : next = use->next_insn_use;
4092 0 : free (use);
4093 : }
4094 119925 : INSN_REG_USE_LIST (dbg) = NULL;
4095 :
4096 : /* We delete rather than resolve these deps, otherwise we
4097 : crash in sched_free_deps(), because forward deps are
4098 : expected to be released before backward deps. */
4099 119925 : sd_delete_dep (sd_it);
4100 : }
4101 :
4102 108621132 : gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE);
4103 108621132 : QUEUE_INDEX (insn) = QUEUE_SCHEDULED;
4104 :
4105 108621132 : if (sched_pressure == SCHED_PRESSURE_MODEL
4106 0 : && model_curr_point < model_num_insns
4107 0 : && NONDEBUG_INSN_P (insn))
4108 : {
4109 0 : if (model_index (insn) == model_curr_point)
4110 0 : do
4111 0 : model_curr_point++;
4112 : while (model_curr_point < model_num_insns
4113 0 : && (QUEUE_INDEX (MODEL_INSN (model_curr_point))
4114 : == QUEUE_SCHEDULED));
4115 : else
4116 0 : model_recompute (insn);
4117 0 : model_update_limit_points ();
4118 0 : update_register_pressure (insn);
4119 0 : if (sched_verbose >= 2)
4120 0 : print_curr_reg_pressure ();
4121 : }
4122 :
4123 108621132 : gcc_assert (INSN_TICK (insn) >= MIN_TICK);
4124 108621132 : if (INSN_TICK (insn) > clock_var)
4125 : /* INSN has been prematurely moved from the queue to the ready list.
4126 : This is possible only if following flags are set. */
4127 5 : gcc_assert (flag_sched_stalled_insns || sched_fusion);
4128 :
4129 : /* ??? Probably, if INSN is scheduled prematurely, we should leave
4130 : INSN_TICK untouched. This is a machine-dependent issue, actually. */
4131 108621132 : INSN_TICK (insn) = clock_var;
4132 :
4133 108621132 : check_clobbered_conditions (insn);
4134 :
4135 : /* Update dependent instructions. First, see if by scheduling this insn
4136 : now we broke a dependence in a way that requires us to change another
4137 : insn. */
4138 108621132 : for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
4139 109160771 : sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it))
4140 : {
4141 539639 : struct dep_replacement *desc = DEP_REPLACE (dep);
4142 539639 : rtx_insn *pro = DEP_PRO (dep);
4143 539639 : if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED
4144 539639 : && desc != NULL && desc->insn == pro)
4145 511787 : apply_replacement (dep, false);
4146 : }
4147 :
4148 : /* Go through and resolve forward dependencies. */
4149 108621132 : for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
4150 316932759 : sd_iterator_cond (&sd_it, &dep);)
4151 : {
4152 208311627 : rtx_insn *next = DEP_CON (dep);
4153 208311627 : bool cancelled = (DEP_STATUS (dep) & DEP_CANCELLED) != 0;
4154 :
4155 : /* Resolve the dependence between INSN and NEXT.
4156 : sd_resolve_dep () moves current dep to another list thus
4157 : advancing the iterator. */
4158 208311627 : sd_resolve_dep (sd_it);
4159 :
4160 208311627 : if (cancelled)
4161 : {
4162 843078 : if (must_restore_pattern_p (next, dep))
4163 63105 : restore_pattern (dep, false);
4164 843078 : continue;
4165 : }
4166 :
4167 : /* Don't bother trying to mark next as ready if insn is a debug
4168 : insn. If insn is the last hard dependency, it will have
4169 : already been discounted. */
4170 207468549 : if (DEBUG_INSN_P (insn) && !DEBUG_INSN_P (next))
4171 4780399 : continue;
4172 :
4173 202688150 : if (!IS_SPECULATION_BRANCHY_CHECK_P (insn))
4174 : {
4175 202688150 : int effective_cost;
4176 :
4177 202688150 : effective_cost = try_ready (next);
4178 :
4179 202688150 : if (effective_cost >= 0
4180 19276473 : && SCHED_GROUP_P (next)
4181 202688150 : && advance < effective_cost)
4182 208311627 : advance = effective_cost;
4183 : }
4184 : else
4185 : /* Check always has only one forward dependence (to the first insn in
4186 : the recovery block), therefore, this will be executed only once. */
4187 : {
4188 0 : gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW));
4189 0 : fix_recovery_deps (RECOVERY_BLOCK (insn));
4190 : }
4191 : }
4192 :
4193 : /* Annotate the instruction with issue information -- TImode
4194 : indicates that the instruction is expected not to be able
4195 : to issue on the same cycle as the previous insn. A machine
4196 : may use this information to decide how the instruction should
4197 : be aligned. */
4198 108621132 : if (issue_rate > 1
4199 108620269 : && GET_CODE (PATTERN (insn)) != USE
4200 107911527 : && GET_CODE (PATTERN (insn)) != CLOBBER
4201 216450927 : && !DEBUG_INSN_P (insn))
4202 : {
4203 59336290 : if (reload_completed)
4204 94274672 : PUT_MODE (insn, clock_var > last_clock_var ? TImode : VOIDmode);
4205 59336290 : last_clock_var = clock_var;
4206 : }
4207 :
4208 108621132 : if (nonscheduled_insns_begin != NULL_RTX)
4209 : /* Indicate to debug counters that INSN is scheduled. */
4210 0 : nonscheduled_insns_begin = insn;
4211 :
4212 108621132 : return advance;
4213 : }
4214 :
4215 : /* Functions for handling of notes. */
4216 :
4217 : /* Add note list that ends on FROM_END to the end of TO_ENDP. */
4218 : void
4219 62 : concat_note_lists (rtx_insn *from_end, rtx_insn **to_endp)
4220 : {
4221 62 : rtx_insn *from_start;
4222 :
4223 : /* It's easy when have nothing to concat. */
4224 62 : if (from_end == NULL)
4225 : return;
4226 :
4227 : /* It's also easy when destination is empty. */
4228 0 : if (*to_endp == NULL)
4229 : {
4230 0 : *to_endp = from_end;
4231 0 : return;
4232 : }
4233 :
4234 : from_start = from_end;
4235 0 : while (PREV_INSN (from_start) != NULL)
4236 : from_start = PREV_INSN (from_start);
4237 :
4238 0 : SET_PREV_INSN (from_start) = *to_endp;
4239 0 : SET_NEXT_INSN (*to_endp) = from_start;
4240 0 : *to_endp = from_end;
4241 : }
4242 :
4243 : /* Delete notes between HEAD and TAIL and put them in the chain
4244 : of notes ended by NOTE_LIST. */
4245 : void
4246 10314455 : remove_notes (rtx_insn *head, rtx_insn *tail)
4247 : {
4248 10314455 : rtx_insn *next_tail, *insn, *next;
4249 :
4250 10314455 : note_list = 0;
4251 10314455 : if (head == tail && !INSN_P (head))
4252 : return;
4253 :
4254 10314424 : next_tail = NEXT_INSN (tail);
4255 135080221 : for (insn = head; insn != next_tail; insn = next)
4256 : {
4257 114451373 : next = NEXT_INSN (insn);
4258 114451373 : if (!NOTE_P (insn))
4259 108625968 : continue;
4260 :
4261 5825405 : switch (NOTE_KIND (insn))
4262 : {
4263 1134 : case NOTE_INSN_BASIC_BLOCK:
4264 1134 : continue;
4265 :
4266 911031 : case NOTE_INSN_EPILOGUE_BEG:
4267 911031 : if (insn != tail)
4268 : {
4269 911031 : remove_insn (insn);
4270 : /* If an insn was split just before the EPILOGUE_BEG note and
4271 : that split created new basic blocks, we could have a
4272 : BASIC_BLOCK note here. Safely advance over it in that case
4273 : and assert that we land on a real insn. */
4274 911031 : if (NOTE_P (next)
4275 0 : && NOTE_KIND (next) == NOTE_INSN_BASIC_BLOCK
4276 0 : && next != next_tail)
4277 0 : next = NEXT_INSN (next);
4278 911031 : gcc_assert (INSN_P (next));
4279 911031 : add_reg_note (next, REG_SAVE_NOTE,
4280 : GEN_INT (NOTE_INSN_EPILOGUE_BEG));
4281 911031 : break;
4282 : }
4283 : /* FALLTHRU */
4284 :
4285 4913240 : default:
4286 4913240 : remove_insn (insn);
4287 :
4288 : /* Add the note to list that ends at NOTE_LIST. */
4289 4913240 : SET_PREV_INSN (insn) = note_list;
4290 4913240 : SET_NEXT_INSN (insn) = NULL_RTX;
4291 4913240 : if (note_list)
4292 2830983 : SET_NEXT_INSN (note_list) = insn;
4293 4913240 : note_list = insn;
4294 4913240 : break;
4295 : }
4296 :
4297 5824271 : gcc_assert ((sel_sched_p () || insn != tail) && insn != head);
4298 : }
4299 : }
4300 :
4301 : /* A structure to record enough data to allow us to backtrack the scheduler to
4302 : a previous state. */
4303 : struct haifa_saved_data
4304 : {
4305 : /* Next entry on the list. */
4306 : struct haifa_saved_data *next;
4307 :
4308 : /* Backtracking is associated with scheduling insns that have delay slots.
4309 : DELAY_PAIR points to the structure that contains the insns involved, and
4310 : the number of cycles between them. */
4311 : struct delay_pair *delay_pair;
4312 :
4313 : /* Data used by the frontend (e.g. sched-ebb or sched-rgn). */
4314 : void *fe_saved_data;
4315 : /* Data used by the backend. */
4316 : void *be_saved_data;
4317 :
4318 : /* Copies of global state. */
4319 : int clock_var, last_clock_var;
4320 : struct ready_list ready;
4321 : state_t curr_state;
4322 :
4323 : rtx_insn *last_scheduled_insn;
4324 : rtx_insn *last_nondebug_scheduled_insn;
4325 : rtx_insn *nonscheduled_insns_begin;
4326 : int cycle_issued_insns;
4327 :
4328 : /* Copies of state used in the inner loop of schedule_block. */
4329 : struct sched_block_state sched_block;
4330 :
4331 : /* We don't need to save q_ptr, as its value is arbitrary and we can set it
4332 : to 0 when restoring. */
4333 : int q_size;
4334 : rtx_insn_list **insn_queue;
4335 :
4336 : /* Describe pattern replacements that occurred since this backtrack point
4337 : was queued. */
4338 : vec<dep_t> replacement_deps;
4339 : vec<int> replace_apply;
4340 :
4341 : /* A copy of the next-cycle replacement vectors at the time of the backtrack
4342 : point. */
4343 : vec<dep_t> next_cycle_deps;
4344 : vec<int> next_cycle_apply;
4345 : };
4346 :
4347 : /* A record, in reverse order, of all scheduled insns which have delay slots
4348 : and may require backtracking. */
4349 : static struct haifa_saved_data *backtrack_queue;
4350 :
4351 : /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according
4352 : to SET_P. */
4353 : static void
4354 0 : mark_backtrack_feeds (rtx_insn *insn, int set_p)
4355 : {
4356 0 : sd_iterator_def sd_it;
4357 0 : dep_t dep;
4358 0 : FOR_EACH_DEP (insn, SD_LIST_HARD_BACK, sd_it, dep)
4359 : {
4360 0 : FEEDS_BACKTRACK_INSN (DEP_PRO (dep)) = set_p;
4361 : }
4362 0 : }
4363 :
4364 : /* Save the current scheduler state so that we can backtrack to it
4365 : later if necessary. PAIR gives the insns that make it necessary to
4366 : save this point. SCHED_BLOCK is the local state of schedule_block
4367 : that need to be saved. */
4368 : static void
4369 0 : save_backtrack_point (struct delay_pair *pair,
4370 : struct sched_block_state sched_block)
4371 : {
4372 0 : int i;
4373 0 : struct haifa_saved_data *save = XNEW (struct haifa_saved_data);
4374 :
4375 0 : save->curr_state = xmalloc (dfa_state_size);
4376 0 : memcpy (save->curr_state, curr_state, dfa_state_size);
4377 :
4378 0 : save->ready.first = ready.first;
4379 0 : save->ready.n_ready = ready.n_ready;
4380 0 : save->ready.n_debug = ready.n_debug;
4381 0 : save->ready.veclen = ready.veclen;
4382 0 : save->ready.vec = XNEWVEC (rtx_insn *, ready.veclen);
4383 0 : memcpy (save->ready.vec, ready.vec, ready.veclen * sizeof (rtx));
4384 :
4385 0 : save->insn_queue = XNEWVEC (rtx_insn_list *, max_insn_queue_index + 1);
4386 0 : save->q_size = q_size;
4387 0 : for (i = 0; i <= max_insn_queue_index; i++)
4388 : {
4389 0 : int q = NEXT_Q_AFTER (q_ptr, i);
4390 0 : save->insn_queue[i] = copy_INSN_LIST (insn_queue[q]);
4391 : }
4392 :
4393 0 : save->clock_var = clock_var;
4394 0 : save->last_clock_var = last_clock_var;
4395 0 : save->cycle_issued_insns = cycle_issued_insns;
4396 0 : save->last_scheduled_insn = last_scheduled_insn;
4397 0 : save->last_nondebug_scheduled_insn = last_nondebug_scheduled_insn;
4398 0 : save->nonscheduled_insns_begin = nonscheduled_insns_begin;
4399 :
4400 0 : save->sched_block = sched_block;
4401 :
4402 0 : save->replacement_deps.create (0);
4403 0 : save->replace_apply.create (0);
4404 0 : save->next_cycle_deps = next_cycle_replace_deps.copy ();
4405 0 : save->next_cycle_apply = next_cycle_apply.copy ();
4406 :
4407 0 : if (current_sched_info->save_state)
4408 0 : save->fe_saved_data = (*current_sched_info->save_state) ();
4409 :
4410 0 : if (targetm.sched.alloc_sched_context)
4411 : {
4412 0 : save->be_saved_data = targetm.sched.alloc_sched_context ();
4413 0 : targetm.sched.init_sched_context (save->be_saved_data, false);
4414 : }
4415 : else
4416 0 : save->be_saved_data = NULL;
4417 :
4418 0 : save->delay_pair = pair;
4419 :
4420 0 : save->next = backtrack_queue;
4421 0 : backtrack_queue = save;
4422 :
4423 0 : while (pair)
4424 : {
4425 0 : mark_backtrack_feeds (pair->i2, 1);
4426 0 : INSN_TICK (pair->i2) = INVALID_TICK;
4427 0 : INSN_EXACT_TICK (pair->i2) = clock_var + pair_delay (pair);
4428 0 : SHADOW_P (pair->i2) = pair->stages == 0;
4429 0 : pair = pair->next_same_i1;
4430 : }
4431 0 : }
4432 :
4433 : /* Walk the ready list and all queues. If any insns have unresolved backwards
4434 : dependencies, these must be cancelled deps, broken by predication. Set or
4435 : clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. */
4436 :
4437 : static void
4438 0 : toggle_cancelled_flags (bool set)
4439 : {
4440 0 : int i;
4441 0 : sd_iterator_def sd_it;
4442 0 : dep_t dep;
4443 :
4444 0 : if (ready.n_ready > 0)
4445 : {
4446 0 : rtx_insn **first = ready_lastpos (&ready);
4447 0 : for (i = 0; i < ready.n_ready; i++)
4448 0 : FOR_EACH_DEP (first[i], SD_LIST_BACK, sd_it, dep)
4449 0 : if (!DEBUG_INSN_P (DEP_PRO (dep)))
4450 : {
4451 0 : if (set)
4452 0 : DEP_STATUS (dep) |= DEP_CANCELLED;
4453 : else
4454 0 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
4455 : }
4456 : }
4457 0 : for (i = 0; i <= max_insn_queue_index; i++)
4458 : {
4459 0 : int q = NEXT_Q_AFTER (q_ptr, i);
4460 0 : rtx_insn_list *link;
4461 0 : for (link = insn_queue[q]; link; link = link->next ())
4462 : {
4463 0 : rtx_insn *insn = link->insn ();
4464 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
4465 0 : if (!DEBUG_INSN_P (DEP_PRO (dep)))
4466 : {
4467 0 : if (set)
4468 0 : DEP_STATUS (dep) |= DEP_CANCELLED;
4469 : else
4470 0 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
4471 : }
4472 : }
4473 : }
4474 0 : }
4475 :
4476 : /* Undo the replacements that have occurred after backtrack point SAVE
4477 : was placed. */
4478 : static void
4479 0 : undo_replacements_for_backtrack (struct haifa_saved_data *save)
4480 : {
4481 0 : while (!save->replacement_deps.is_empty ())
4482 : {
4483 0 : dep_t dep = save->replacement_deps.pop ();
4484 0 : int apply_p = save->replace_apply.pop ();
4485 :
4486 0 : if (apply_p)
4487 0 : restore_pattern (dep, true);
4488 : else
4489 0 : apply_replacement (dep, true);
4490 : }
4491 0 : save->replacement_deps.release ();
4492 0 : save->replace_apply.release ();
4493 0 : }
4494 :
4495 : /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN.
4496 : Restore their dependencies to an unresolved state, and mark them as
4497 : queued nowhere. */
4498 :
4499 : static void
4500 0 : unschedule_insns_until (rtx_insn *insn)
4501 : {
4502 0 : auto_vec<rtx_insn *> recompute_vec;
4503 :
4504 : /* Make two passes over the insns to be unscheduled. First, we clear out
4505 : dependencies and other trivial bookkeeping. */
4506 0 : for (;;)
4507 : {
4508 0 : rtx_insn *last;
4509 0 : sd_iterator_def sd_it;
4510 0 : dep_t dep;
4511 :
4512 0 : last = scheduled_insns.pop ();
4513 :
4514 : /* This will be changed by restore_backtrack_point if the insn is in
4515 : any queue. */
4516 0 : QUEUE_INDEX (last) = QUEUE_NOWHERE;
4517 0 : if (last != insn)
4518 0 : INSN_TICK (last) = INVALID_TICK;
4519 :
4520 0 : if (modulo_ii > 0 && INSN_UID (last) < modulo_iter0_max_uid)
4521 0 : modulo_insns_scheduled--;
4522 :
4523 0 : for (sd_it = sd_iterator_start (last, SD_LIST_RES_FORW);
4524 0 : sd_iterator_cond (&sd_it, &dep);)
4525 : {
4526 0 : rtx_insn *con = DEP_CON (dep);
4527 0 : sd_unresolve_dep (sd_it);
4528 0 : if (!MUST_RECOMPUTE_SPEC_P (con))
4529 : {
4530 0 : MUST_RECOMPUTE_SPEC_P (con) = 1;
4531 0 : recompute_vec.safe_push (con);
4532 : }
4533 : }
4534 :
4535 0 : if (last == insn)
4536 : break;
4537 0 : }
4538 :
4539 : /* A second pass, to update ready and speculation status for insns
4540 : depending on the unscheduled ones. The first pass must have
4541 : popped the scheduled_insns vector up to the point where we
4542 : restart scheduling, as recompute_todo_spec requires it to be
4543 : up-to-date. */
4544 0 : while (!recompute_vec.is_empty ())
4545 : {
4546 0 : rtx_insn *con;
4547 :
4548 0 : con = recompute_vec.pop ();
4549 0 : MUST_RECOMPUTE_SPEC_P (con) = 0;
4550 0 : if (!sd_lists_empty_p (con, SD_LIST_HARD_BACK))
4551 : {
4552 0 : TODO_SPEC (con) = HARD_DEP;
4553 0 : INSN_TICK (con) = INVALID_TICK;
4554 0 : if (PREDICATED_PAT (con) != NULL_RTX)
4555 0 : haifa_change_pattern (con, ORIG_PAT (con));
4556 : }
4557 0 : else if (QUEUE_INDEX (con) != QUEUE_SCHEDULED)
4558 0 : TODO_SPEC (con) = recompute_todo_spec (con, true);
4559 : }
4560 0 : }
4561 :
4562 : /* Restore scheduler state from the topmost entry on the backtracking queue.
4563 : PSCHED_BLOCK_P points to the local data of schedule_block that we must
4564 : overwrite with the saved data.
4565 : The caller must already have called unschedule_insns_until. */
4566 :
4567 : static void
4568 0 : restore_last_backtrack_point (struct sched_block_state *psched_block)
4569 : {
4570 0 : int i;
4571 0 : struct haifa_saved_data *save = backtrack_queue;
4572 :
4573 0 : backtrack_queue = save->next;
4574 :
4575 0 : if (current_sched_info->restore_state)
4576 0 : (*current_sched_info->restore_state) (save->fe_saved_data);
4577 :
4578 0 : if (targetm.sched.alloc_sched_context)
4579 : {
4580 0 : targetm.sched.set_sched_context (save->be_saved_data);
4581 0 : targetm.sched.free_sched_context (save->be_saved_data);
4582 : }
4583 :
4584 : /* Do this first since it clobbers INSN_TICK of the involved
4585 : instructions. */
4586 0 : undo_replacements_for_backtrack (save);
4587 :
4588 : /* Clear the QUEUE_INDEX of everything in the ready list or one
4589 : of the queues. */
4590 0 : if (ready.n_ready > 0)
4591 : {
4592 0 : rtx_insn **first = ready_lastpos (&ready);
4593 0 : for (i = 0; i < ready.n_ready; i++)
4594 : {
4595 0 : rtx_insn *insn = first[i];
4596 0 : QUEUE_INDEX (insn) = QUEUE_NOWHERE;
4597 0 : INSN_TICK (insn) = INVALID_TICK;
4598 : }
4599 : }
4600 0 : for (i = 0; i <= max_insn_queue_index; i++)
4601 : {
4602 0 : int q = NEXT_Q_AFTER (q_ptr, i);
4603 :
4604 0 : for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ())
4605 : {
4606 0 : rtx_insn *x = link->insn ();
4607 0 : QUEUE_INDEX (x) = QUEUE_NOWHERE;
4608 0 : INSN_TICK (x) = INVALID_TICK;
4609 : }
4610 0 : free_INSN_LIST_list (&insn_queue[q]);
4611 : }
4612 :
4613 0 : free (ready.vec);
4614 0 : ready = save->ready;
4615 :
4616 0 : if (ready.n_ready > 0)
4617 : {
4618 0 : rtx_insn **first = ready_lastpos (&ready);
4619 0 : for (i = 0; i < ready.n_ready; i++)
4620 : {
4621 0 : rtx_insn *insn = first[i];
4622 0 : QUEUE_INDEX (insn) = QUEUE_READY;
4623 0 : TODO_SPEC (insn) = recompute_todo_spec (insn, true);
4624 0 : INSN_TICK (insn) = save->clock_var;
4625 : }
4626 : }
4627 :
4628 0 : q_ptr = 0;
4629 0 : q_size = save->q_size;
4630 0 : for (i = 0; i <= max_insn_queue_index; i++)
4631 : {
4632 0 : int q = NEXT_Q_AFTER (q_ptr, i);
4633 :
4634 0 : insn_queue[q] = save->insn_queue[q];
4635 :
4636 0 : for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ())
4637 : {
4638 0 : rtx_insn *x = link->insn ();
4639 0 : QUEUE_INDEX (x) = i;
4640 0 : TODO_SPEC (x) = recompute_todo_spec (x, true);
4641 0 : INSN_TICK (x) = save->clock_var + i;
4642 : }
4643 : }
4644 0 : free (save->insn_queue);
4645 :
4646 0 : toggle_cancelled_flags (true);
4647 :
4648 0 : clock_var = save->clock_var;
4649 0 : last_clock_var = save->last_clock_var;
4650 0 : cycle_issued_insns = save->cycle_issued_insns;
4651 0 : last_scheduled_insn = save->last_scheduled_insn;
4652 0 : last_nondebug_scheduled_insn = save->last_nondebug_scheduled_insn;
4653 0 : nonscheduled_insns_begin = save->nonscheduled_insns_begin;
4654 :
4655 0 : *psched_block = save->sched_block;
4656 :
4657 0 : memcpy (curr_state, save->curr_state, dfa_state_size);
4658 0 : free (save->curr_state);
4659 :
4660 0 : mark_backtrack_feeds (save->delay_pair->i2, 0);
4661 :
4662 0 : gcc_assert (next_cycle_replace_deps.is_empty ());
4663 0 : next_cycle_replace_deps = save->next_cycle_deps.copy ();
4664 0 : next_cycle_apply = save->next_cycle_apply.copy ();
4665 :
4666 0 : free (save);
4667 :
4668 0 : for (save = backtrack_queue; save; save = save->next)
4669 : {
4670 0 : mark_backtrack_feeds (save->delay_pair->i2, 1);
4671 : }
4672 0 : }
4673 :
4674 : /* Discard all data associated with the topmost entry in the backtrack
4675 : queue. If RESET_TICK is false, we just want to free the data. If true,
4676 : we are doing this because we discovered a reason to backtrack. In the
4677 : latter case, also reset the INSN_TICK for the shadow insn. */
4678 : static void
4679 0 : free_topmost_backtrack_point (bool reset_tick)
4680 : {
4681 0 : struct haifa_saved_data *save = backtrack_queue;
4682 0 : int i;
4683 :
4684 0 : backtrack_queue = save->next;
4685 :
4686 0 : if (reset_tick)
4687 : {
4688 0 : struct delay_pair *pair = save->delay_pair;
4689 0 : while (pair)
4690 : {
4691 0 : INSN_TICK (pair->i2) = INVALID_TICK;
4692 0 : INSN_EXACT_TICK (pair->i2) = INVALID_TICK;
4693 0 : pair = pair->next_same_i1;
4694 : }
4695 0 : undo_replacements_for_backtrack (save);
4696 : }
4697 : else
4698 : {
4699 0 : save->replacement_deps.release ();
4700 0 : save->replace_apply.release ();
4701 : }
4702 :
4703 0 : if (targetm.sched.free_sched_context)
4704 0 : targetm.sched.free_sched_context (save->be_saved_data);
4705 0 : if (current_sched_info->restore_state)
4706 0 : free (save->fe_saved_data);
4707 0 : for (i = 0; i <= max_insn_queue_index; i++)
4708 0 : free_INSN_LIST_list (&save->insn_queue[i]);
4709 0 : free (save->insn_queue);
4710 0 : free (save->curr_state);
4711 0 : free (save->ready.vec);
4712 0 : free (save);
4713 0 : }
4714 :
4715 : /* Free the entire backtrack queue. */
4716 : static void
4717 10313337 : free_backtrack_queue (void)
4718 : {
4719 10313337 : while (backtrack_queue)
4720 0 : free_topmost_backtrack_point (false);
4721 10313337 : }
4722 :
4723 : /* Apply a replacement described by DESC. If IMMEDIATELY is false, we
4724 : may have to postpone the replacement until the start of the next cycle,
4725 : at which point we will be called again with IMMEDIATELY true. This is
4726 : only done for machines which have instruction packets with explicit
4727 : parallelism however. */
4728 : static void
4729 602744 : apply_replacement (dep_t dep, bool immediately)
4730 : {
4731 602744 : struct dep_replacement *desc = DEP_REPLACE (dep);
4732 602744 : if (!immediately && targetm.sched.exposed_pipeline && reload_completed)
4733 : {
4734 0 : next_cycle_replace_deps.safe_push (dep);
4735 0 : next_cycle_apply.safe_push (1);
4736 : }
4737 : else
4738 : {
4739 602744 : bool success;
4740 :
4741 602744 : if (QUEUE_INDEX (desc->insn) == QUEUE_SCHEDULED)
4742 : return;
4743 :
4744 602744 : if (sched_verbose >= 5)
4745 0 : fprintf (sched_dump, "applying replacement for insn %d\n",
4746 : INSN_UID (desc->insn));
4747 :
4748 602744 : success = validate_change (desc->insn, desc->loc, desc->newval, 0);
4749 602744 : gcc_assert (success);
4750 :
4751 602744 : rtx_insn *insn = DEP_PRO (dep);
4752 :
4753 : /* Recompute priority since dependent priorities may have changed. */
4754 602744 : priority (insn, true);
4755 602744 : update_insn_after_change (desc->insn);
4756 :
4757 602744 : if ((TODO_SPEC (desc->insn) & (HARD_DEP | DEP_POSTPONED)) == 0)
4758 311240 : fix_tick_ready (desc->insn);
4759 :
4760 602744 : if (backtrack_queue != NULL)
4761 : {
4762 0 : backtrack_queue->replacement_deps.safe_push (dep);
4763 0 : backtrack_queue->replace_apply.safe_push (1);
4764 : }
4765 : }
4766 : }
4767 :
4768 : /* We have determined that a pattern involved in DEP must be restored.
4769 : If IMMEDIATELY is false, we may have to postpone the replacement
4770 : until the start of the next cycle, at which point we will be called
4771 : again with IMMEDIATELY true. */
4772 : static void
4773 63105 : restore_pattern (dep_t dep, bool immediately)
4774 : {
4775 63105 : rtx_insn *next = DEP_CON (dep);
4776 63105 : int tick = INSN_TICK (next);
4777 :
4778 : /* If we already scheduled the insn, the modified version is
4779 : correct. */
4780 63105 : if (QUEUE_INDEX (next) == QUEUE_SCHEDULED)
4781 : return;
4782 :
4783 63105 : if (!immediately && targetm.sched.exposed_pipeline && reload_completed)
4784 : {
4785 0 : next_cycle_replace_deps.safe_push (dep);
4786 0 : next_cycle_apply.safe_push (0);
4787 0 : return;
4788 : }
4789 :
4790 :
4791 63105 : if (DEP_TYPE (dep) == REG_DEP_CONTROL)
4792 : {
4793 0 : if (sched_verbose >= 5)
4794 0 : fprintf (sched_dump, "restoring pattern for insn %d\n",
4795 : INSN_UID (next));
4796 0 : haifa_change_pattern (next, ORIG_PAT (next));
4797 : }
4798 : else
4799 : {
4800 63105 : struct dep_replacement *desc = DEP_REPLACE (dep);
4801 63105 : bool success;
4802 :
4803 63105 : if (sched_verbose >= 5)
4804 0 : fprintf (sched_dump, "restoring pattern for insn %d\n",
4805 0 : INSN_UID (desc->insn));
4806 63105 : tick = INSN_TICK (desc->insn);
4807 :
4808 63105 : success = validate_change (desc->insn, desc->loc, desc->orig, 0);
4809 63105 : gcc_assert (success);
4810 :
4811 63105 : rtx_insn *insn = DEP_PRO (dep);
4812 :
4813 63105 : if (QUEUE_INDEX (insn) != QUEUE_SCHEDULED)
4814 : {
4815 : /* Recompute priority since dependent priorities may have changed. */
4816 0 : priority (insn, true);
4817 : }
4818 :
4819 63105 : update_insn_after_change (desc->insn);
4820 :
4821 63105 : if (backtrack_queue != NULL)
4822 : {
4823 0 : backtrack_queue->replacement_deps.safe_push (dep);
4824 0 : backtrack_queue->replace_apply.safe_push (0);
4825 : }
4826 : }
4827 63105 : INSN_TICK (next) = tick;
4828 63105 : if (TODO_SPEC (next) == DEP_POSTPONED)
4829 : return;
4830 :
4831 63105 : if (sd_lists_empty_p (next, SD_LIST_BACK))
4832 63105 : TODO_SPEC (next) = 0;
4833 0 : else if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK))
4834 0 : TODO_SPEC (next) = HARD_DEP;
4835 : }
4836 :
4837 : /* Perform pattern replacements that were queued up until the next
4838 : cycle. */
4839 : static void
4840 45271138 : perform_replacements_new_cycle (void)
4841 : {
4842 45271138 : int i;
4843 45271138 : dep_t dep;
4844 45271138 : FOR_EACH_VEC_ELT (next_cycle_replace_deps, i, dep)
4845 : {
4846 0 : int apply_p = next_cycle_apply[i];
4847 0 : if (apply_p)
4848 0 : apply_replacement (dep, true);
4849 : else
4850 0 : restore_pattern (dep, true);
4851 : }
4852 45271138 : next_cycle_replace_deps.truncate (0);
4853 45271138 : next_cycle_apply.truncate (0);
4854 45271138 : }
4855 :
4856 : /* Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of
4857 : instructions we've previously encountered, a set bit prevents
4858 : recursion. BUDGET is a limit on how far ahead we look, it is
4859 : reduced on recursive calls. Return true if we produced a good
4860 : estimate, or false if we exceeded the budget. */
4861 : static bool
4862 0 : estimate_insn_tick (bitmap processed, rtx_insn *insn, int budget)
4863 : {
4864 0 : sd_iterator_def sd_it;
4865 0 : dep_t dep;
4866 0 : int earliest = INSN_TICK (insn);
4867 :
4868 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
4869 : {
4870 0 : rtx_insn *pro = DEP_PRO (dep);
4871 0 : int t;
4872 :
4873 0 : if (DEP_STATUS (dep) & DEP_CANCELLED)
4874 0 : continue;
4875 :
4876 0 : if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED)
4877 0 : gcc_assert (INSN_TICK (pro) + dep_cost (dep) <= INSN_TICK (insn));
4878 : else
4879 : {
4880 0 : int cost = dep_cost (dep);
4881 0 : if (cost >= budget)
4882 : return false;
4883 0 : if (!bitmap_bit_p (processed, INSN_LUID (pro)))
4884 : {
4885 0 : if (!estimate_insn_tick (processed, pro, budget - cost))
4886 : return false;
4887 : }
4888 0 : gcc_assert (INSN_TICK_ESTIMATE (pro) != INVALID_TICK);
4889 0 : t = INSN_TICK_ESTIMATE (pro) + cost;
4890 0 : if (earliest == INVALID_TICK || t > earliest)
4891 0 : earliest = t;
4892 : }
4893 : }
4894 0 : bitmap_set_bit (processed, INSN_LUID (insn));
4895 0 : INSN_TICK_ESTIMATE (insn) = earliest;
4896 0 : return true;
4897 : }
4898 :
4899 : /* Examine the pair of insns in P, and estimate (optimistically, assuming
4900 : infinite resources) the cycle in which the delayed shadow can be issued.
4901 : Return the number of cycles that must pass before the real insn can be
4902 : issued in order to meet this constraint. */
4903 : static int
4904 0 : estimate_shadow_tick (struct delay_pair *p)
4905 : {
4906 0 : auto_bitmap processed;
4907 0 : int t;
4908 0 : bool cutoff;
4909 :
4910 0 : cutoff = !estimate_insn_tick (processed, p->i2,
4911 0 : max_insn_queue_index + pair_delay (p));
4912 0 : if (cutoff)
4913 : return max_insn_queue_index;
4914 0 : t = INSN_TICK_ESTIMATE (p->i2) - (clock_var + pair_delay (p) + 1);
4915 0 : if (t > 0)
4916 : return t;
4917 : return 0;
4918 0 : }
4919 :
4920 : /* If INSN has no unresolved backwards dependencies, add it to the schedule and
4921 : recursively resolve all its forward dependencies. */
4922 : static void
4923 0 : resolve_dependencies (rtx_insn *insn)
4924 : {
4925 0 : sd_iterator_def sd_it;
4926 0 : dep_t dep;
4927 :
4928 : /* Don't use sd_lists_empty_p; it ignores debug insns. */
4929 0 : if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn)) != NULL
4930 0 : || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn)) != NULL)
4931 0 : return;
4932 :
4933 0 : if (sched_verbose >= 4)
4934 0 : fprintf (sched_dump, ";;\tquickly resolving %d\n", INSN_UID (insn));
4935 :
4936 0 : if (QUEUE_INDEX (insn) >= 0)
4937 0 : queue_remove (insn);
4938 :
4939 0 : scheduled_insns.safe_push (insn);
4940 :
4941 : /* Update dependent instructions. */
4942 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
4943 0 : sd_iterator_cond (&sd_it, &dep);)
4944 : {
4945 0 : rtx_insn *next = DEP_CON (dep);
4946 :
4947 0 : if (sched_verbose >= 4)
4948 0 : fprintf (sched_dump, ";;\t\tdep %d against %d\n", INSN_UID (insn),
4949 0 : INSN_UID (next));
4950 :
4951 : /* Resolve the dependence between INSN and NEXT.
4952 : sd_resolve_dep () moves current dep to another list thus
4953 : advancing the iterator. */
4954 0 : sd_resolve_dep (sd_it);
4955 :
4956 0 : if (!IS_SPECULATION_BRANCHY_CHECK_P (insn))
4957 : {
4958 0 : resolve_dependencies (next);
4959 : }
4960 : else
4961 : /* Check always has only one forward dependence (to the first insn in
4962 : the recovery block), therefore, this will be executed only once. */
4963 : {
4964 0 : gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW));
4965 : }
4966 : }
4967 : }
4968 :
4969 :
4970 : /* Return the head and tail pointers of ebb starting at BEG and ending
4971 : at END. */
4972 : void
4973 41326222 : get_ebb_head_tail (basic_block beg, basic_block end,
4974 : rtx_insn **headp, rtx_insn **tailp)
4975 : {
4976 41326222 : rtx_insn *beg_head = BB_HEAD (beg);
4977 41326222 : rtx_insn * beg_tail = BB_END (beg);
4978 41326222 : rtx_insn * end_head = BB_HEAD (end);
4979 41326222 : rtx_insn * end_tail = BB_END (end);
4980 :
4981 : /* Don't include any notes or labels at the beginning of the BEG
4982 : basic block, or notes at the end of the END basic blocks. */
4983 :
4984 41326222 : if (LABEL_P (beg_head))
4985 20211474 : beg_head = NEXT_INSN (beg_head);
4986 :
4987 96941276 : while (beg_head != beg_tail)
4988 93025593 : if (NOTE_P (beg_head))
4989 55615054 : beg_head = NEXT_INSN (beg_head);
4990 37410539 : else if (DEBUG_INSN_P (beg_head))
4991 : {
4992 10234674 : rtx_insn * note, *next;
4993 :
4994 10234674 : for (note = NEXT_INSN (beg_head);
4995 82667992 : note != beg_tail;
4996 : note = next)
4997 : {
4998 82365022 : next = NEXT_INSN (note);
4999 82365022 : if (NOTE_P (note))
5000 : {
5001 429788 : if (sched_verbose >= 9)
5002 0 : fprintf (sched_dump, "reorder %i\n", INSN_UID (note));
5003 :
5004 429788 : reorder_insns_nobb (note, note, PREV_INSN (beg_head));
5005 :
5006 429788 : if (BLOCK_FOR_INSN (note) != beg)
5007 0 : df_insn_change_bb (note, beg);
5008 : }
5009 81935234 : else if (!DEBUG_INSN_P (note))
5010 : break;
5011 : }
5012 :
5013 : break;
5014 : }
5015 : else
5016 : break;
5017 :
5018 41326222 : *headp = beg_head;
5019 :
5020 41326222 : if (beg == end)
5021 41326222 : end_head = beg_head;
5022 67 : else if (LABEL_P (end_head))
5023 3 : end_head = NEXT_INSN (end_head);
5024 :
5025 41485048 : while (end_head != end_tail)
5026 37554787 : if (NOTE_P (end_tail))
5027 158826 : end_tail = PREV_INSN (end_tail);
5028 37395961 : else if (DEBUG_INSN_P (end_tail))
5029 : {
5030 817144 : rtx_insn * note, *prev;
5031 :
5032 817144 : for (note = PREV_INSN (end_tail);
5033 6898983 : note != end_head;
5034 : note = prev)
5035 : {
5036 6865458 : prev = PREV_INSN (note);
5037 6865458 : if (NOTE_P (note))
5038 : {
5039 1642 : if (sched_verbose >= 9)
5040 0 : fprintf (sched_dump, "reorder %i\n", INSN_UID (note));
5041 :
5042 1642 : reorder_insns_nobb (note, note, end_tail);
5043 :
5044 1642 : if (end_tail == BB_END (end))
5045 937 : BB_END (end) = note;
5046 :
5047 1642 : if (BLOCK_FOR_INSN (note) != end)
5048 0 : df_insn_change_bb (note, end);
5049 : }
5050 6863816 : else if (!DEBUG_INSN_P (note))
5051 : break;
5052 : }
5053 :
5054 : break;
5055 : }
5056 : else
5057 : break;
5058 :
5059 41326222 : *tailp = end_tail;
5060 41326222 : }
5061 :
5062 : /* Return true if there are no real insns in the range [ HEAD, TAIL ]. */
5063 :
5064 : bool
5065 30993539 : no_real_insns_p (const rtx_insn *head, const rtx_insn *tail)
5066 : {
5067 31045356 : while (head != NEXT_INSN (tail))
5068 : {
5069 30993539 : if (!NOTE_P (head) && !LABEL_P (head))
5070 : return false;
5071 51817 : head = NEXT_INSN (head);
5072 : }
5073 : return true;
5074 : }
5075 :
5076 : /* Restore-other-notes: NOTE_LIST is the end of a chain of notes
5077 : previously found among the insns. Insert them just before HEAD. */
5078 : rtx_insn *
5079 10314393 : restore_other_notes (rtx_insn *head, basic_block head_bb)
5080 : {
5081 10314393 : if (note_list != 0)
5082 : {
5083 2082257 : rtx_insn *note_head = note_list;
5084 :
5085 2082257 : if (head)
5086 2081883 : head_bb = BLOCK_FOR_INSN (head);
5087 : else
5088 374 : head = NEXT_INSN (bb_note (head_bb));
5089 :
5090 4913240 : while (PREV_INSN (note_head))
5091 : {
5092 2830983 : set_block_for_insn (note_head, head_bb);
5093 2830983 : note_head = PREV_INSN (note_head);
5094 : }
5095 : /* In the above cycle we've missed this note. */
5096 2082257 : set_block_for_insn (note_head, head_bb);
5097 :
5098 2082257 : SET_PREV_INSN (note_head) = PREV_INSN (head);
5099 2082257 : SET_NEXT_INSN (PREV_INSN (head)) = note_head;
5100 2082257 : SET_PREV_INSN (head) = note_list;
5101 2082257 : SET_NEXT_INSN (note_list) = head;
5102 :
5103 2082257 : if (BLOCK_FOR_INSN (head) != head_bb)
5104 0 : BB_END (head_bb) = note_list;
5105 :
5106 : head = note_head;
5107 : }
5108 :
5109 10314393 : return head;
5110 : }
5111 :
5112 : /* When we know we are going to discard the schedule due to a failed attempt
5113 : at modulo scheduling, undo all replacements. */
5114 : static void
5115 0 : undo_all_replacements (void)
5116 : {
5117 0 : rtx_insn *insn;
5118 0 : int i;
5119 :
5120 0 : FOR_EACH_VEC_ELT (scheduled_insns, i, insn)
5121 : {
5122 0 : sd_iterator_def sd_it;
5123 0 : dep_t dep;
5124 :
5125 : /* See if we must undo a replacement. */
5126 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_RES_FORW);
5127 0 : sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it))
5128 : {
5129 0 : struct dep_replacement *desc = DEP_REPLACE (dep);
5130 0 : if (desc != NULL)
5131 0 : validate_change (desc->insn, desc->loc, desc->orig, 0);
5132 : }
5133 : }
5134 0 : }
5135 :
5136 : /* Return first non-scheduled insn in the current scheduling block.
5137 : This is mostly used for debug-counter purposes. */
5138 : static rtx_insn *
5139 0 : first_nonscheduled_insn (void)
5140 : {
5141 0 : rtx_insn *insn = (nonscheduled_insns_begin != NULL_RTX
5142 0 : ? nonscheduled_insns_begin
5143 0 : : current_sched_info->prev_head);
5144 :
5145 0 : do
5146 : {
5147 0 : insn = next_nonnote_nondebug_insn (insn);
5148 : }
5149 0 : while (QUEUE_INDEX (insn) == QUEUE_SCHEDULED);
5150 :
5151 0 : return insn;
5152 : }
5153 :
5154 : /* Move insns that became ready to fire from queue to ready list. */
5155 :
5156 : static void
5157 34957801 : queue_to_ready (struct ready_list *ready)
5158 : {
5159 34957801 : rtx_insn *insn;
5160 34957801 : rtx_insn_list *link;
5161 34957801 : rtx_insn *skip_insn;
5162 :
5163 34957801 : q_ptr = NEXT_Q (q_ptr);
5164 :
5165 34957801 : if (dbg_cnt (sched_insn) == false)
5166 : /* If debug counter is activated do not requeue the first
5167 : nonscheduled insn. */
5168 0 : skip_insn = first_nonscheduled_insn ();
5169 : else
5170 : skip_insn = NULL;
5171 :
5172 : /* Add all pending insns that can be scheduled without stalls to the
5173 : ready list. */
5174 67082073 : for (link = insn_queue[q_ptr]; link; link = link->next ())
5175 : {
5176 32124272 : insn = link->insn ();
5177 32124272 : q_size -= 1;
5178 :
5179 32124272 : if (sched_verbose >= 2)
5180 0 : fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ",
5181 0 : (*current_sched_info->print_insn) (insn, 0));
5182 :
5183 : /* If the ready list is full, delay the insn for 1 cycle.
5184 : See the comment in schedule_block for the rationale. */
5185 32124272 : if (!reload_completed
5186 5296 : && (ready->n_ready - ready->n_debug > param_max_sched_ready_insns
5187 5119 : || (sched_pressure == SCHED_PRESSURE_MODEL
5188 : /* Limit pressure recalculations to
5189 : param_max_sched_ready_insns instructions too. */
5190 0 : && model_index (insn) > (model_curr_point
5191 0 : + param_max_sched_ready_insns)))
5192 177 : && !(sched_pressure == SCHED_PRESSURE_MODEL
5193 0 : && model_curr_point < model_num_insns
5194 : /* Always allow the next model instruction to issue. */
5195 0 : && model_index (insn) == model_curr_point)
5196 177 : && !SCHED_GROUP_P (insn)
5197 32124449 : && insn != skip_insn)
5198 : {
5199 177 : if (sched_verbose >= 2)
5200 0 : fprintf (sched_dump, "keeping in queue, ready full\n");
5201 177 : queue_insn (insn, 1, "ready full");
5202 : }
5203 : else
5204 : {
5205 32124095 : ready_add (ready, insn, false);
5206 32124095 : if (sched_verbose >= 2)
5207 0 : fprintf (sched_dump, "moving to ready without stalls\n");
5208 : }
5209 : }
5210 34957801 : free_INSN_LIST_list (&insn_queue[q_ptr]);
5211 :
5212 : /* If there are no ready insns, stall until one is ready and add all
5213 : of the pending insns at that point to the ready list. */
5214 34957801 : if (ready->n_ready == 0)
5215 : {
5216 : int stalls;
5217 :
5218 10985388 : for (stalls = 1; stalls <= max_insn_queue_index; stalls++)
5219 : {
5220 10985388 : if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]))
5221 : {
5222 12595942 : for (; link; link = link->next ())
5223 : {
5224 6521077 : insn = link->insn ();
5225 6521077 : q_size -= 1;
5226 :
5227 6521077 : if (sched_verbose >= 2)
5228 0 : fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ",
5229 0 : (*current_sched_info->print_insn) (insn, 0));
5230 :
5231 6521077 : ready_add (ready, insn, false);
5232 6521077 : if (sched_verbose >= 2)
5233 0 : fprintf (sched_dump, "moving to ready with %d stalls\n", stalls);
5234 : }
5235 6074865 : free_INSN_LIST_list (&insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]);
5236 :
5237 6074865 : advance_one_cycle ();
5238 :
5239 6074865 : break;
5240 : }
5241 :
5242 4910523 : advance_one_cycle ();
5243 : }
5244 :
5245 6074865 : q_ptr = NEXT_Q_AFTER (q_ptr, stalls);
5246 6074865 : clock_var += stalls;
5247 6074865 : if (sched_verbose >= 2)
5248 0 : fprintf (sched_dump, ";;\tAdvancing clock by %d cycle[s] to %d\n",
5249 : stalls, clock_var);
5250 : }
5251 34957801 : }
5252 :
5253 : /* Used by early_queue_to_ready. Determines whether it is "ok" to
5254 : prematurely move INSN from the queue to the ready list. Currently,
5255 : if a target defines the hook 'is_costly_dependence', this function
5256 : uses the hook to check whether there exist any dependences which are
5257 : considered costly by the target, between INSN and other insns that
5258 : have already been scheduled. Dependences are checked up to Y cycles
5259 : back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows
5260 : controlling this value.
5261 : (Other considerations could be taken into account instead (or in
5262 : addition) depending on user flags and target hooks. */
5263 :
5264 : static bool
5265 5 : ok_for_early_queue_removal (rtx_insn *insn)
5266 : {
5267 5 : if (targetm.sched.is_costly_dependence)
5268 : {
5269 0 : int n_cycles;
5270 0 : int i = scheduled_insns.length ();
5271 0 : for (n_cycles = flag_sched_stalled_insns_dep; n_cycles; n_cycles--)
5272 : {
5273 0 : while (i-- > 0)
5274 : {
5275 0 : int cost;
5276 :
5277 0 : rtx_insn *prev_insn = scheduled_insns[i];
5278 :
5279 0 : if (!NOTE_P (prev_insn))
5280 : {
5281 0 : dep_t dep;
5282 :
5283 0 : dep = sd_find_dep_between (prev_insn, insn, true);
5284 :
5285 0 : if (dep != NULL)
5286 : {
5287 0 : cost = dep_cost (dep);
5288 :
5289 0 : if (targetm.sched.is_costly_dependence (dep, cost,
5290 0 : flag_sched_stalled_insns_dep - n_cycles))
5291 : return false;
5292 : }
5293 : }
5294 :
5295 0 : if (GET_MODE (prev_insn) == TImode) /* end of dispatch group */
5296 : break;
5297 : }
5298 :
5299 0 : if (i == 0)
5300 : break;
5301 : }
5302 : }
5303 :
5304 : return true;
5305 : }
5306 :
5307 :
5308 : /* Remove insns from the queue, before they become "ready" with respect
5309 : to FU latency considerations. */
5310 :
5311 : static int
5312 33057763 : early_queue_to_ready (state_t state, struct ready_list *ready)
5313 : {
5314 33057763 : rtx_insn *insn;
5315 33057763 : rtx_insn_list *link;
5316 33057763 : rtx_insn_list *next_link;
5317 33057763 : rtx_insn_list *prev_link;
5318 33057763 : bool move_to_ready;
5319 33057763 : int cost;
5320 33057763 : state_t temp_state = alloca (dfa_state_size);
5321 33057763 : int stalls;
5322 33057763 : int insns_removed = 0;
5323 :
5324 : /*
5325 : Flag '-fsched-stalled-insns=X' determines the aggressiveness of this
5326 : function:
5327 :
5328 : X == 0: There is no limit on how many queued insns can be removed
5329 : prematurely. (flag_sched_stalled_insns = -1).
5330 :
5331 : X >= 1: Only X queued insns can be removed prematurely in each
5332 : invocation. (flag_sched_stalled_insns = X).
5333 :
5334 : Otherwise: Early queue removal is disabled.
5335 : (flag_sched_stalled_insns = 0)
5336 : */
5337 :
5338 33057763 : if (! flag_sched_stalled_insns)
5339 : return 0;
5340 :
5341 1799 : for (stalls = 0; stalls <= max_insn_queue_index; stalls++)
5342 : {
5343 1792 : if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]))
5344 : {
5345 4 : if (sched_verbose > 6)
5346 0 : fprintf (sched_dump, ";; look at index %d + %d\n", q_ptr, stalls);
5347 :
5348 : prev_link = 0;
5349 9 : while (link)
5350 : {
5351 5 : next_link = link->next ();
5352 5 : insn = link->insn ();
5353 5 : if (insn && sched_verbose > 6)
5354 0 : print_rtl_single (sched_dump, insn);
5355 :
5356 5 : memcpy (temp_state, state, dfa_state_size);
5357 5 : if (recog_memoized (insn) < 0)
5358 : /* non-negative to indicate that it's not ready
5359 : to avoid infinite Q->R->Q->R... */
5360 : cost = 0;
5361 : else
5362 5 : cost = state_transition (temp_state, insn);
5363 :
5364 5 : if (sched_verbose >= 6)
5365 0 : fprintf (sched_dump, "transition cost = %d\n", cost);
5366 :
5367 5 : move_to_ready = false;
5368 5 : if (cost < 0)
5369 : {
5370 5 : move_to_ready = ok_for_early_queue_removal (insn);
5371 5 : if (move_to_ready == true)
5372 : {
5373 : /* move from Q to R */
5374 5 : q_size -= 1;
5375 5 : ready_add (ready, insn, false);
5376 :
5377 5 : if (prev_link)
5378 0 : XEXP (prev_link, 1) = next_link;
5379 : else
5380 5 : insn_queue[NEXT_Q_AFTER (q_ptr, stalls)] = next_link;
5381 :
5382 5 : free_INSN_LIST_node (link);
5383 :
5384 5 : if (sched_verbose >= 2)
5385 0 : fprintf (sched_dump, ";;\t\tEarly Q-->Ready: insn %s\n",
5386 0 : (*current_sched_info->print_insn) (insn, 0));
5387 :
5388 5 : insns_removed++;
5389 5 : if (insns_removed == flag_sched_stalled_insns)
5390 : /* Remove no more than flag_sched_stalled_insns insns
5391 : from Q at a time. */
5392 : return insns_removed;
5393 : }
5394 : }
5395 :
5396 : if (move_to_ready == false)
5397 : prev_link = link;
5398 :
5399 : link = next_link;
5400 : } /* while link */
5401 : } /* if link */
5402 :
5403 : } /* for stalls.. */
5404 :
5405 : return insns_removed;
5406 : }
5407 :
5408 :
5409 : /* Print the ready list for debugging purposes.
5410 : If READY_TRY is non-zero then only print insns that max_issue
5411 : will consider. */
5412 : static void
5413 190 : debug_ready_list_1 (struct ready_list *ready, signed char *ready_try)
5414 : {
5415 190 : rtx_insn **p;
5416 190 : int i;
5417 :
5418 190 : if (ready->n_ready == 0)
5419 : {
5420 190 : fprintf (sched_dump, "\n");
5421 190 : return;
5422 : }
5423 :
5424 0 : p = ready_lastpos (ready);
5425 0 : for (i = 0; i < ready->n_ready; i++)
5426 : {
5427 0 : if (ready_try != NULL && ready_try[ready->n_ready - i - 1])
5428 0 : continue;
5429 :
5430 0 : fprintf (sched_dump, " %s:%d",
5431 0 : (*current_sched_info->print_insn) (p[i], 0),
5432 0 : INSN_LUID (p[i]));
5433 0 : if (sched_pressure != SCHED_PRESSURE_NONE)
5434 0 : fprintf (sched_dump, "(cost=%d",
5435 0 : INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p[i]));
5436 0 : fprintf (sched_dump, ":prio=%d", INSN_PRIORITY (p[i]));
5437 0 : if (INSN_TICK (p[i]) > clock_var)
5438 0 : fprintf (sched_dump, ":delay=%d", INSN_TICK (p[i]) - clock_var);
5439 0 : if (sched_pressure == SCHED_PRESSURE_MODEL)
5440 0 : fprintf (sched_dump, ":idx=%d",
5441 : model_index (p[i]));
5442 0 : if (sched_pressure != SCHED_PRESSURE_NONE)
5443 0 : fprintf (sched_dump, ")");
5444 : }
5445 0 : fprintf (sched_dump, "\n");
5446 : }
5447 :
5448 : /* Print the ready list. Callable from debugger. */
5449 : static void
5450 190 : debug_ready_list (struct ready_list *ready)
5451 : {
5452 0 : debug_ready_list_1 (ready, NULL);
5453 190 : }
5454 :
5455 : /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn
5456 : NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb
5457 : replaces the epilogue note in the correct basic block. */
5458 : void
5459 60132003 : reemit_notes (rtx_insn *insn)
5460 : {
5461 60132003 : rtx note;
5462 60132003 : rtx_insn *last = insn;
5463 :
5464 119199256 : for (note = REG_NOTES (insn); note; note = XEXP (note, 1))
5465 : {
5466 59067253 : if (REG_NOTE_KIND (note) == REG_SAVE_NOTE)
5467 : {
5468 911031 : enum insn_note note_type = (enum insn_note) INTVAL (XEXP (note, 0));
5469 :
5470 911031 : last = emit_note_before (note_type, last);
5471 911031 : remove_note (insn, note);
5472 911031 : df_insn_create_insn_record (last);
5473 : }
5474 : }
5475 60132003 : }
5476 :
5477 : /* Move INSN. Reemit notes if needed. Update CFG, if needed. */
5478 : static void
5479 108621132 : move_insn (rtx_insn *insn, rtx_insn *last, rtx nt)
5480 : {
5481 108621132 : if (PREV_INSN (insn) != last)
5482 : {
5483 13122891 : basic_block bb;
5484 13122891 : rtx_insn *note;
5485 13122891 : int jump_p = 0;
5486 :
5487 13122891 : bb = BLOCK_FOR_INSN (insn);
5488 :
5489 : /* BB_HEAD is either LABEL or NOTE. */
5490 13122891 : gcc_assert (BB_HEAD (bb) != insn);
5491 :
5492 13122891 : if (BB_END (bb) == insn)
5493 : /* If this is last instruction in BB, move end marker one
5494 : instruction up. */
5495 : {
5496 : /* Jumps are always placed at the end of basic block. */
5497 236129 : jump_p = control_flow_insn_p (insn);
5498 :
5499 236129 : gcc_assert (!jump_p
5500 : || ((common_sched_info->sched_pass_id == SCHED_RGN_PASS)
5501 : && IS_SPECULATION_BRANCHY_CHECK_P (insn))
5502 : || (common_sched_info->sched_pass_id
5503 : == SCHED_EBB_PASS));
5504 :
5505 236129 : gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn)) == bb);
5506 :
5507 236129 : BB_END (bb) = PREV_INSN (insn);
5508 : }
5509 :
5510 13122891 : gcc_assert (BB_END (bb) != last);
5511 :
5512 13122891 : if (jump_p)
5513 : /* We move the block note along with jump. */
5514 : {
5515 1 : gcc_assert (nt);
5516 :
5517 1 : note = NEXT_INSN (insn);
5518 2 : while (NOTE_NOT_BB_P (note) && note != nt)
5519 0 : note = NEXT_INSN (note);
5520 :
5521 1 : if (note != nt
5522 1 : && (LABEL_P (note)
5523 1 : || BARRIER_P (note)))
5524 0 : note = NEXT_INSN (note);
5525 :
5526 1 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
5527 : }
5528 : else
5529 : note = insn;
5530 :
5531 13122891 : SET_NEXT_INSN (PREV_INSN (insn)) = NEXT_INSN (note);
5532 13122891 : SET_PREV_INSN (NEXT_INSN (note)) = PREV_INSN (insn);
5533 :
5534 13122891 : SET_NEXT_INSN (note) = NEXT_INSN (last);
5535 13122891 : SET_PREV_INSN (NEXT_INSN (last)) = note;
5536 :
5537 13122891 : SET_NEXT_INSN (last) = insn;
5538 13122891 : SET_PREV_INSN (insn) = last;
5539 :
5540 13122891 : bb = BLOCK_FOR_INSN (last);
5541 :
5542 13122891 : if (jump_p)
5543 : {
5544 1 : fix_jump_move (insn);
5545 :
5546 1 : if (BLOCK_FOR_INSN (insn) != bb)
5547 0 : move_block_after_check (insn);
5548 :
5549 1 : gcc_assert (BB_END (bb) == last);
5550 : }
5551 :
5552 13122891 : df_insn_change_bb (insn, bb);
5553 :
5554 : /* Update BB_END, if needed. */
5555 13122891 : if (BB_END (bb) == last)
5556 1 : BB_END (bb) = insn;
5557 : }
5558 :
5559 108621132 : SCHED_GROUP_P (insn) = 0;
5560 108621132 : }
5561 :
5562 : /* Return true if scheduling INSN will finish current clock cycle. */
5563 : static bool
5564 111256674 : insn_finishes_cycle_p (rtx_insn *insn)
5565 : {
5566 111256674 : if (SCHED_GROUP_P (insn))
5567 : /* After issuing INSN, rest of the sched_group will be forced to issue
5568 : in order. Don't make any plans for the rest of cycle. */
5569 : return true;
5570 :
5571 : /* Finishing the block will, apparently, finish the cycle. */
5572 111256416 : if (current_sched_info->insn_finishes_block_p
5573 111256416 : && current_sched_info->insn_finishes_block_p (insn))
5574 : return true;
5575 :
5576 : return false;
5577 : }
5578 :
5579 : /* Helper for autopref_multipass_init. Given a SET in PAT and whether
5580 : we're expecting a memory WRITE or not, check that the insn is relevant to
5581 : the autoprefetcher modelling code. Return true iff that is the case.
5582 : If it is relevant, record the base register of the memory op in BASE and
5583 : the offset in OFFSET. */
5584 :
5585 : static bool
5586 3 : analyze_set_insn_for_autopref (rtx pat, bool write, rtx *base, int *offset)
5587 : {
5588 3 : if (GET_CODE (pat) != SET)
5589 : return false;
5590 :
5591 3 : rtx mem = write ? SET_DEST (pat) : SET_SRC (pat);
5592 3 : if (!MEM_P (mem))
5593 : return false;
5594 :
5595 3 : struct address_info info;
5596 3 : decompose_mem_address (&info, mem);
5597 :
5598 : /* TODO: Currently only (base+const) addressing is supported. */
5599 3 : if (info.base == NULL || !REG_P (*info.base)
5600 3 : || (info.disp != NULL && !CONST_INT_P (*info.disp)))
5601 : return false;
5602 :
5603 3 : *base = *info.base;
5604 3 : *offset = info.disp ? INTVAL (*info.disp) : 0;
5605 3 : return true;
5606 : }
5607 :
5608 : /* Functions to model cache auto-prefetcher.
5609 :
5610 : Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
5611 : memory prefetches if it sees instructions with consequitive memory accesses
5612 : in the instruction stream. Details of such hardware units are not published,
5613 : so we can only guess what exactly is going on there.
5614 : In the scheduler, we model abstract auto-prefetcher. If there are memory
5615 : insns in the ready list (or the queue) that have same memory base, but
5616 : different offsets, then we delay the insns with larger offsets until insns
5617 : with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
5618 : is "1", then we look at the ready list; if it is N>1, then we also look
5619 : through N-1 queue entries.
5620 : If the param is N>=0, then rank_for_schedule will consider auto-prefetching
5621 : among its heuristics.
5622 : Param value of "-1" disables modelling of the auto-prefetcher. */
5623 :
5624 : /* Initialize autoprefetcher model data for INSN. */
5625 : static void
5626 3 : autopref_multipass_init (const rtx_insn *insn, int write)
5627 : {
5628 3 : autopref_multipass_data_t data = &INSN_AUTOPREF_MULTIPASS_DATA (insn)[write];
5629 :
5630 3 : gcc_assert (data->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED);
5631 3 : data->base = NULL_RTX;
5632 3 : data->offset = 0;
5633 : /* Set insn entry initialized, but not relevant for auto-prefetcher. */
5634 3 : data->status = AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
5635 :
5636 3 : rtx pat = PATTERN (insn);
5637 :
5638 : /* We have a multi-set insn like a load-multiple or store-multiple.
5639 : We care about these as long as all the memory ops inside the PARALLEL
5640 : have the same base register. We care about the minimum and maximum
5641 : offsets from that base but don't check for the order of those offsets
5642 : within the PARALLEL insn itself. */
5643 3 : if (GET_CODE (pat) == PARALLEL)
5644 : {
5645 0 : int n_elems = XVECLEN (pat, 0);
5646 :
5647 0 : int i, offset;
5648 0 : rtx base, prev_base = NULL_RTX;
5649 0 : int min_offset = INT_MAX;
5650 :
5651 0 : for (i = 0; i < n_elems; i++)
5652 : {
5653 0 : rtx set = XVECEXP (pat, 0, i);
5654 0 : if (GET_CODE (set) != SET)
5655 : return;
5656 :
5657 0 : if (!analyze_set_insn_for_autopref (set, write, &base, &offset))
5658 : return;
5659 :
5660 : /* Ensure that all memory operations in the PARALLEL use the same
5661 : base register. */
5662 0 : if (i > 0 && REGNO (base) != REGNO (prev_base))
5663 : return;
5664 0 : prev_base = base;
5665 0 : min_offset = MIN (min_offset, offset);
5666 : }
5667 :
5668 : /* If we reached here then we have a valid PARALLEL of multiple memory ops
5669 : with prev_base as the base and min_offset containing the offset. */
5670 0 : gcc_assert (prev_base);
5671 0 : data->base = prev_base;
5672 0 : data->offset = min_offset;
5673 0 : data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
5674 0 : return;
5675 : }
5676 :
5677 : /* Otherwise this is a single set memory operation. */
5678 3 : rtx set = single_set (insn);
5679 3 : if (set == NULL_RTX)
5680 : return;
5681 :
5682 3 : if (!analyze_set_insn_for_autopref (set, write, &data->base,
5683 : &data->offset))
5684 : return;
5685 :
5686 : /* This insn is relevant for the auto-prefetcher.
5687 : The base and offset fields will have been filled in the
5688 : analyze_set_insn_for_autopref call above. */
5689 3 : data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
5690 : }
5691 :
5692 : /* Helper function for rank_for_schedule sorting. */
5693 : static int
5694 12 : autopref_rank_for_schedule (const rtx_insn *insn1, const rtx_insn *insn2)
5695 : {
5696 12 : int r = 0;
5697 24 : for (int write = 0; write < 2 && !r; ++write)
5698 : {
5699 12 : autopref_multipass_data_t data1
5700 12 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5701 12 : autopref_multipass_data_t data2
5702 12 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
5703 :
5704 12 : if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5705 1 : autopref_multipass_init (insn1, write);
5706 :
5707 12 : if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5708 2 : autopref_multipass_init (insn2, write);
5709 :
5710 12 : int irrel1 = data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
5711 12 : int irrel2 = data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
5712 :
5713 12 : if (!irrel1 && !irrel2)
5714 : /* Sort memory references from lowest offset to the largest. */
5715 12 : r = (data1->offset > data2->offset) - (data1->offset < data2->offset);
5716 0 : else if (write)
5717 : /* Schedule "irrelevant" insns before memory stores to resolve
5718 : as many producer dependencies of stores as possible. */
5719 0 : r = irrel2 - irrel1;
5720 : else
5721 : /* Schedule "irrelevant" insns after memory reads to avoid breaking
5722 : memory read sequences. */
5723 0 : r = irrel1 - irrel2;
5724 : }
5725 :
5726 12 : return r;
5727 : }
5728 :
5729 : /* True if header of debug dump was printed. */
5730 : static bool autopref_multipass_dfa_lookahead_guard_started_dump_p;
5731 :
5732 : /* Helper for autopref_multipass_dfa_lookahead_guard.
5733 : Return "1" if INSN1 should be delayed in favor of INSN2. */
5734 : static int
5735 0 : autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn *insn1,
5736 : const rtx_insn *insn2, int write)
5737 : {
5738 0 : autopref_multipass_data_t data1
5739 0 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5740 0 : autopref_multipass_data_t data2
5741 0 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
5742 :
5743 0 : if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5744 0 : autopref_multipass_init (insn2, write);
5745 0 : if (data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
5746 : return 0;
5747 :
5748 0 : if (rtx_equal_p (data1->base, data2->base)
5749 0 : && data1->offset > data2->offset)
5750 : {
5751 0 : if (sched_verbose >= 2)
5752 : {
5753 0 : if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
5754 : {
5755 0 : fprintf (sched_dump,
5756 : ";;\t\tnot trying in max_issue due to autoprefetch "
5757 : "model: ");
5758 0 : autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
5759 : }
5760 :
5761 0 : fprintf (sched_dump, " %d(%d)", INSN_UID (insn1), INSN_UID (insn2));
5762 : }
5763 :
5764 0 : return 1;
5765 : }
5766 :
5767 : return 0;
5768 : }
5769 :
5770 : /* General note:
5771 :
5772 : We could have also hooked autoprefetcher model into
5773 : first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
5774 : to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
5775 : (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
5776 : unblocked). We don't bother about this yet because target of interest
5777 : (ARM Cortex-A15) can issue only 1 memory operation per cycle. */
5778 :
5779 : /* Implementation of first_cycle_multipass_dfa_lookahead_guard hook.
5780 : Return "1" if INSN1 should not be considered in max_issue due to
5781 : auto-prefetcher considerations. */
5782 : int
5783 0 : autopref_multipass_dfa_lookahead_guard (rtx_insn *insn1, int ready_index)
5784 : {
5785 0 : int r = 0;
5786 :
5787 : /* Exit early if the param forbids this or if we're not entering here through
5788 : normal haifa scheduling. This can happen if selective scheduling is
5789 : explicitly enabled. */
5790 0 : if (!insn_queue || param_sched_autopref_queue_depth <= 0)
5791 : return 0;
5792 :
5793 0 : if (sched_verbose >= 2 && ready_index == 0)
5794 0 : autopref_multipass_dfa_lookahead_guard_started_dump_p = false;
5795 :
5796 0 : for (int write = 0; write < 2; ++write)
5797 : {
5798 0 : autopref_multipass_data_t data1
5799 0 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5800 :
5801 0 : if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5802 0 : autopref_multipass_init (insn1, write);
5803 0 : if (data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
5804 0 : continue;
5805 :
5806 0 : if (ready_index == 0
5807 0 : && data1->status == AUTOPREF_MULTIPASS_DATA_DONT_DELAY)
5808 : /* We allow only a single delay on priviledged instructions.
5809 : Doing otherwise would cause infinite loop. */
5810 : {
5811 0 : if (sched_verbose >= 2)
5812 : {
5813 0 : if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
5814 : {
5815 0 : fprintf (sched_dump,
5816 : ";;\t\tnot trying in max_issue due to autoprefetch "
5817 : "model: ");
5818 0 : autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
5819 : }
5820 :
5821 0 : fprintf (sched_dump, " *%d*", INSN_UID (insn1));
5822 : }
5823 0 : continue;
5824 : }
5825 :
5826 0 : for (int i2 = 0; i2 < ready.n_ready; ++i2)
5827 : {
5828 0 : rtx_insn *insn2 = get_ready_element (i2);
5829 0 : if (insn1 == insn2)
5830 0 : continue;
5831 0 : r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2, write);
5832 0 : if (r)
5833 : {
5834 0 : if (ready_index == 0)
5835 : {
5836 0 : r = -1;
5837 0 : data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
5838 : }
5839 0 : goto finish;
5840 : }
5841 : }
5842 :
5843 0 : if (param_sched_autopref_queue_depth == 1)
5844 0 : continue;
5845 :
5846 : /* Everything from the current queue slot should have been moved to
5847 : the ready list. */
5848 0 : gcc_assert (insn_queue[NEXT_Q_AFTER (q_ptr, 0)] == NULL_RTX);
5849 :
5850 0 : int n_stalls = param_sched_autopref_queue_depth - 1;
5851 0 : if (n_stalls > max_insn_queue_index)
5852 : n_stalls = max_insn_queue_index;
5853 :
5854 0 : for (int stalls = 1; stalls <= n_stalls; ++stalls)
5855 : {
5856 0 : for (rtx_insn_list *link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)];
5857 0 : link != NULL_RTX;
5858 0 : link = link->next ())
5859 : {
5860 0 : rtx_insn *insn2 = link->insn ();
5861 0 : r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2,
5862 : write);
5863 0 : if (r)
5864 : {
5865 : /* Queue INSN1 until INSN2 can issue. */
5866 0 : r = -stalls;
5867 0 : if (ready_index == 0)
5868 0 : data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
5869 0 : goto finish;
5870 : }
5871 : }
5872 : }
5873 : }
5874 :
5875 0 : finish:
5876 0 : if (sched_verbose >= 2
5877 0 : && autopref_multipass_dfa_lookahead_guard_started_dump_p
5878 0 : && (ready_index == ready.n_ready - 1 || r < 0))
5879 : /* This does not /always/ trigger. We don't output EOL if the last
5880 : insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
5881 : called. We can live with this. */
5882 0 : fprintf (sched_dump, "\n");
5883 :
5884 : return r;
5885 : }
5886 :
5887 : /* Define type for target data used in multipass scheduling. */
5888 : #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
5889 : # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
5890 : #endif
5891 : typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t;
5892 :
5893 : /* The following structure describe an entry of the stack of choices. */
5894 : struct choice_entry
5895 : {
5896 : /* Ordinal number of the issued insn in the ready queue. */
5897 : int index;
5898 : /* The number of the rest insns whose issues we should try. */
5899 : int rest;
5900 : /* The number of issued essential insns. */
5901 : int n;
5902 : /* State after issuing the insn. */
5903 : state_t state;
5904 : /* Target-specific data. */
5905 : first_cycle_multipass_data_t target_data;
5906 : };
5907 :
5908 : /* The following array is used to implement a stack of choices used in
5909 : function max_issue. */
5910 : static struct choice_entry *choice_stack;
5911 :
5912 : /* This holds the value of the target dfa_lookahead hook. */
5913 : int dfa_lookahead;
5914 :
5915 : /* The following variable value is maximal number of tries of issuing
5916 : insns for the first cycle multipass insn scheduling. We define
5917 : this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE). We would not
5918 : need this constraint if all real insns (with non-negative codes)
5919 : had reservations because in this case the algorithm complexity is
5920 : O(DFA_LOOKAHEAD**ISSUE_RATE). Unfortunately, the dfa descriptions
5921 : might be incomplete and such insn might occur. For such
5922 : descriptions, the complexity of algorithm (without the constraint)
5923 : could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */
5924 : static int max_lookahead_tries;
5925 :
5926 : /* The following function returns maximal (or close to maximal) number
5927 : of insns which can be issued on the same cycle and one of which
5928 : insns is insns with the best rank (the first insn in READY). To
5929 : make this function tries different samples of ready insns. READY
5930 : is current queue `ready'. Global array READY_TRY reflects what
5931 : insns are already issued in this try. The function stops immediately,
5932 : if it reached the such a solution, that all instruction can be issued.
5933 : INDEX will contain index of the best insn in READY. The following
5934 : function is used only for first cycle multipass scheduling.
5935 :
5936 : PRIVILEGED_N >= 0
5937 :
5938 : This function expects recognized insns only. All USEs,
5939 : CLOBBERs, etc must be filtered elsewhere. */
5940 : int
5941 55096841 : max_issue (struct ready_list *ready, int privileged_n, state_t state,
5942 : bool first_cycle_insn_p, int *index)
5943 : {
5944 55096841 : int n, i, all, n_ready, best, delay, tries_num;
5945 55096841 : int more_issue;
5946 55096841 : struct choice_entry *top;
5947 55096841 : rtx_insn *insn;
5948 :
5949 55096841 : if (sched_fusion)
5950 : return 0;
5951 :
5952 55096841 : n_ready = ready->n_ready;
5953 55096841 : gcc_assert (dfa_lookahead >= 1 && privileged_n >= 0
5954 : && privileged_n <= n_ready);
5955 :
5956 : /* Init MAX_LOOKAHEAD_TRIES. */
5957 55096841 : if (max_lookahead_tries == 0)
5958 : {
5959 963981 : max_lookahead_tries = 100;
5960 4774953 : for (i = 0; i < issue_rate; i++)
5961 3810972 : max_lookahead_tries *= dfa_lookahead;
5962 : }
5963 :
5964 : /* Init max_points. */
5965 55096841 : more_issue = issue_rate - cycle_issued_insns;
5966 55096841 : gcc_assert (more_issue >= 0);
5967 :
5968 : /* The number of the issued insns in the best solution. */
5969 55096841 : best = 0;
5970 :
5971 55096841 : top = choice_stack;
5972 :
5973 : /* Set initial state of the search. */
5974 55096841 : memcpy (top->state, state, dfa_state_size);
5975 55096841 : top->rest = dfa_lookahead;
5976 55096841 : top->n = 0;
5977 55096841 : if (targetm.sched.first_cycle_multipass_begin)
5978 54903641 : targetm.sched.first_cycle_multipass_begin (&top->target_data,
5979 : ready_try, n_ready,
5980 : first_cycle_insn_p);
5981 :
5982 : /* Count the number of the insns to search among. */
5983 148004608 : for (all = i = 0; i < n_ready; i++)
5984 92907767 : if (!ready_try [i])
5985 85089884 : all++;
5986 :
5987 55096841 : if (sched_verbose >= 2)
5988 : {
5989 0 : fprintf (sched_dump, ";;\t\tmax_issue among %d insns:", all);
5990 0 : debug_ready_list_1 (ready, ready_try);
5991 : }
5992 :
5993 : /* I is the index of the insn to try next. */
5994 : i = 0;
5995 : tries_num = 0;
5996 1054598891 : for (;;)
5997 : {
5998 554847866 : if (/* If we've reached a dead end or searched enough of what we have
5999 : been asked... */
6000 554847866 : top->rest == 0
6001 : /* or have nothing else to try... */
6002 549451180 : || i >= n_ready
6003 : /* or should not issue more. */
6004 445539038 : || top->n >= more_issue)
6005 : {
6006 : /* ??? (... || i == n_ready). */
6007 111439916 : gcc_assert (i <= n_ready);
6008 :
6009 : /* We should not issue more than issue_rate instructions. */
6010 111439916 : gcc_assert (top->n <= more_issue);
6011 :
6012 111439916 : if (top == choice_stack)
6013 : break;
6014 :
6015 102628328 : if (best < top - choice_stack)
6016 : {
6017 53496236 : if (privileged_n)
6018 : {
6019 : n = privileged_n;
6020 : /* Try to find issued privileged insn. */
6021 106985514 : while (n && !ready_try[--n])
6022 : ;
6023 : }
6024 :
6025 53492756 : if (/* If all insns are equally good... */
6026 : privileged_n == 0
6027 : /* Or a privileged insn will be issued. */
6028 53492756 : || ready_try[n])
6029 : /* Then we have a solution. */
6030 : {
6031 53494396 : best = top - choice_stack;
6032 : /* This is the index of the insn issued first in this
6033 : solution. */
6034 53494396 : *index = choice_stack [1].index;
6035 53494396 : if (top->n == more_issue || best == all)
6036 : break;
6037 : }
6038 : }
6039 :
6040 : /* Set ready-list index to point to the last insn
6041 : ('i++' below will advance it to the next insn). */
6042 56343090 : i = top->index;
6043 :
6044 : /* Backtrack. */
6045 56343090 : ready_try [i] = 0;
6046 :
6047 56343090 : if (targetm.sched.first_cycle_multipass_backtrack)
6048 56223098 : targetm.sched.first_cycle_multipass_backtrack (&top->target_data,
6049 : ready_try, n_ready);
6050 :
6051 56343090 : top--;
6052 56343090 : memcpy (state, top->state, dfa_state_size);
6053 : }
6054 443407950 : else if (!ready_try [i])
6055 : {
6056 141998666 : tries_num++;
6057 141998666 : if (tries_num > max_lookahead_tries)
6058 : break;
6059 141998651 : insn = ready_element (ready, i);
6060 141998651 : delay = state_transition (state, insn);
6061 141998651 : if (delay < 0)
6062 : {
6063 111256674 : if (state_dead_lock_p (state)
6064 111256674 : || insn_finishes_cycle_p (insn))
6065 : /* We won't issue any more instructions in the next
6066 : choice_state. */
6067 5828813 : top->rest = 0;
6068 : else
6069 105427861 : top->rest--;
6070 :
6071 111256674 : n = top->n;
6072 111256674 : if (memcmp (top->state, state, dfa_state_size) != 0)
6073 110250074 : n++;
6074 :
6075 : /* Advance to the next choice_entry. */
6076 111256674 : top++;
6077 : /* Initialize it. */
6078 111256674 : top->rest = dfa_lookahead;
6079 111256674 : top->index = i;
6080 111256674 : top->n = n;
6081 111256674 : memcpy (top->state, state, dfa_state_size);
6082 111256674 : ready_try [i] = 1;
6083 :
6084 111256674 : if (targetm.sched.first_cycle_multipass_issue)
6085 110932326 : targetm.sched.first_cycle_multipass_issue (&top->target_data,
6086 : ready_try, n_ready,
6087 : insn,
6088 110932326 : &((top - 1)
6089 : ->target_data));
6090 :
6091 : i = -1;
6092 : }
6093 : }
6094 :
6095 : /* Increase ready-list index. */
6096 499751025 : i++;
6097 : }
6098 :
6099 55096841 : if (targetm.sched.first_cycle_multipass_end)
6100 107644939 : targetm.sched.first_cycle_multipass_end (best != 0
6101 52741298 : ? &choice_stack[1].target_data
6102 : : NULL);
6103 :
6104 : /* Restore the original state of the DFA. */
6105 55096841 : memcpy (state, choice_stack->state, dfa_state_size);
6106 :
6107 55096841 : return best;
6108 : }
6109 :
6110 : /* The following function chooses insn from READY and modifies
6111 : READY. The following function is used only for first
6112 : cycle multipass scheduling.
6113 : Return:
6114 : -1 if cycle should be advanced,
6115 : 0 if INSN_PTR is set to point to the desirable insn,
6116 : 1 if choose_ready () should be restarted without advancing the cycle. */
6117 : static int
6118 60129640 : choose_ready (struct ready_list *ready, bool first_cycle_insn_p,
6119 : rtx_insn **insn_ptr)
6120 : {
6121 60129640 : if (dbg_cnt (sched_insn) == false)
6122 : {
6123 0 : if (nonscheduled_insns_begin == NULL_RTX)
6124 0 : nonscheduled_insns_begin = current_sched_info->prev_head;
6125 :
6126 0 : rtx_insn *insn = first_nonscheduled_insn ();
6127 :
6128 0 : if (QUEUE_INDEX (insn) == QUEUE_READY)
6129 : /* INSN is in the ready_list. */
6130 : {
6131 0 : ready_remove_insn (insn);
6132 0 : *insn_ptr = insn;
6133 0 : return 0;
6134 : }
6135 :
6136 : /* INSN is in the queue. Advance cycle to move it to the ready list. */
6137 0 : gcc_assert (QUEUE_INDEX (insn) >= 0);
6138 : return -1;
6139 : }
6140 :
6141 60129640 : if (SCHED_GROUP_P (ready_element (ready, 0))
6142 60129640 : || DEBUG_INSN_P (ready_element (ready, 0)))
6143 : {
6144 4179235 : *insn_ptr = ready_remove_first (ready);
6145 4179235 : return 0;
6146 : }
6147 55950405 : else if (dfa_lookahead <= 0)
6148 : {
6149 9617 : if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
6150 16 : *insn_ptr = ready_remove_first_dispatch (ready);
6151 : else
6152 9601 : *insn_ptr = ready_remove_first (ready);
6153 :
6154 9617 : return 0;
6155 : }
6156 : else
6157 : {
6158 : /* Try to choose the best insn. */
6159 55940788 : int index = 0, i;
6160 55940788 : rtx_insn *insn;
6161 :
6162 55940788 : insn = ready_element (ready, 0);
6163 55940788 : if (INSN_CODE (insn) < 0)
6164 : {
6165 847464 : *insn_ptr = ready_remove_first (ready);
6166 847464 : return 0;
6167 : }
6168 :
6169 : /* Filter the search space. */
6170 147995383 : for (i = 0; i < ready->n_ready; i++)
6171 : {
6172 92902059 : ready_try[i] = 0;
6173 :
6174 92902059 : insn = ready_element (ready, i);
6175 :
6176 : /* If this insn is recognizable we should have already
6177 : recognized it earlier.
6178 : ??? Not very clear where this is supposed to be done.
6179 : See dep_cost_1. */
6180 92902059 : gcc_checking_assert (INSN_CODE (insn) >= 0
6181 : || recog_memoized (insn) < 0);
6182 92902059 : if (INSN_CODE (insn) < 0)
6183 : {
6184 : /* Non-recognized insns at position 0 are handled above. */
6185 244559 : gcc_assert (i > 0);
6186 244559 : ready_try[i] = 1;
6187 244559 : continue;
6188 : }
6189 :
6190 92657500 : if (targetm.sched.first_cycle_multipass_dfa_lookahead_guard)
6191 : {
6192 0 : ready_try[i]
6193 0 : = (targetm.sched.first_cycle_multipass_dfa_lookahead_guard
6194 0 : (insn, i));
6195 :
6196 0 : if (ready_try[i] < 0)
6197 : /* Queue instruction for several cycles.
6198 : We need to restart choose_ready as we have changed
6199 : the ready list. */
6200 : {
6201 0 : change_queue_index (insn, -ready_try[i]);
6202 0 : return 1;
6203 : }
6204 :
6205 : /* Make sure that we didn't end up with 0'th insn filtered out.
6206 : Don't be tempted to make life easier for backends and just
6207 : requeue 0'th insn if (ready_try[0] == 0) and restart
6208 : choose_ready. Backends should be very considerate about
6209 : requeueing instructions -- especially the highest priority
6210 : one at position 0. */
6211 0 : gcc_assert (ready_try[i] == 0 || i > 0);
6212 0 : if (ready_try[i])
6213 0 : continue;
6214 : }
6215 :
6216 92657500 : gcc_assert (ready_try[i] == 0);
6217 : /* INSN made it through the scrutiny of filters! */
6218 : }
6219 :
6220 55093324 : if (max_issue (ready, 1, curr_state, first_cycle_insn_p, &index) == 0)
6221 : {
6222 2162327 : *insn_ptr = ready_remove_first (ready);
6223 2162327 : if (sched_verbose >= 4)
6224 0 : fprintf (sched_dump, ";;\t\tChosen insn (but can't issue) : %s \n",
6225 0 : (*current_sched_info->print_insn) (*insn_ptr, 0));
6226 2162327 : return 0;
6227 : }
6228 : else
6229 : {
6230 52930997 : if (sched_verbose >= 4)
6231 0 : fprintf (sched_dump, ";;\t\tChosen insn : %s\n",
6232 0 : (*current_sched_info->print_insn)
6233 0 : (ready_element (ready, index), 0));
6234 :
6235 52930997 : *insn_ptr = ready_remove (ready, index);
6236 52930997 : return 0;
6237 : }
6238 : }
6239 : }
6240 :
6241 : /* This function is called when we have successfully scheduled a
6242 : block. It uses the schedule stored in the scheduled_insns vector
6243 : to rearrange the RTL. PREV_HEAD is used as the anchor to which we
6244 : append the scheduled insns; TAIL is the insn after the scheduled
6245 : block. TARGET_BB is the argument passed to schedule_block. */
6246 :
6247 : static void
6248 10313337 : commit_schedule (rtx_insn *prev_head, rtx_insn *tail, basic_block *target_bb)
6249 : {
6250 10313337 : unsigned int i;
6251 10313337 : rtx_insn *insn;
6252 :
6253 10313337 : last_scheduled_insn = prev_head;
6254 10313337 : for (i = 0;
6255 118934469 : scheduled_insns.iterate (i, &insn);
6256 : i++)
6257 : {
6258 108621132 : if (control_flow_insn_p (last_scheduled_insn)
6259 108621132 : || current_sched_info->advance_target_bb (*target_bb, insn))
6260 : {
6261 48 : *target_bb = current_sched_info->advance_target_bb (*target_bb, 0);
6262 :
6263 48 : if (sched_verbose)
6264 : {
6265 2 : rtx_insn *x;
6266 :
6267 2 : x = next_real_insn (last_scheduled_insn);
6268 2 : gcc_assert (x);
6269 2 : dump_new_block_header (1, *target_bb, x, tail);
6270 : }
6271 :
6272 48 : last_scheduled_insn = bb_note (*target_bb);
6273 : }
6274 :
6275 108621132 : if (current_sched_info->begin_move_insn)
6276 1257 : (*current_sched_info->begin_move_insn) (insn, last_scheduled_insn);
6277 108621132 : move_insn (insn, last_scheduled_insn,
6278 108621132 : current_sched_info->next_tail);
6279 108621132 : if (!DEBUG_INSN_P (insn))
6280 60127402 : reemit_notes (insn);
6281 108621132 : last_scheduled_insn = insn;
6282 : }
6283 :
6284 10313337 : scheduled_insns.truncate (0);
6285 10313337 : }
6286 :
6287 : /* Examine all insns on the ready list and queue those which can't be
6288 : issued in this cycle. TEMP_STATE is temporary scheduler state we
6289 : can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns
6290 : have been issued for the current cycle, which means it is valid to
6291 : issue an asm statement.
6292 :
6293 : If SHADOWS_ONLY_P is true, we eliminate all real insns and only
6294 : leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true,
6295 : we only leave insns which have an INSN_EXACT_TICK. */
6296 :
6297 : static void
6298 69179689 : prune_ready_list (state_t temp_state, bool first_cycle_insn_p,
6299 : bool shadows_only_p, bool modulo_epilogue_p)
6300 : {
6301 69179689 : int i, pass;
6302 69179689 : bool sched_group_found = false;
6303 69179689 : int min_cost_group = 0;
6304 :
6305 69179689 : if (sched_fusion)
6306 : return;
6307 :
6308 189062256 : for (i = 0; i < ready.n_ready; i++)
6309 : {
6310 124082133 : rtx_insn *insn = ready_element (&ready, i);
6311 124082133 : if (SCHED_GROUP_P (insn))
6312 : {
6313 : sched_group_found = true;
6314 : break;
6315 : }
6316 : }
6317 :
6318 : /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle
6319 : such an insn first and note its cost. If at least one SCHED_GROUP_P insn
6320 : gets queued, then all other insns get queued for one cycle later. */
6321 223589893 : for (pass = sched_group_found ? 0 : 1; pass < 2; )
6322 : {
6323 85230515 : int n = ready.n_ready;
6324 202853876 : for (i = 0; i < n; i++)
6325 : {
6326 129474621 : rtx_insn *insn = ready_element (&ready, i);
6327 129474621 : int cost = 0;
6328 129474621 : const char *reason = "resource conflict";
6329 :
6330 129474621 : if (DEBUG_INSN_P (insn))
6331 6421107 : continue;
6332 :
6333 8383555 : if (sched_group_found && !SCHED_GROUP_P (insn)
6334 123053724 : && ((pass == 0) || (min_cost_group >= 1)))
6335 : {
6336 124 : if (pass == 0)
6337 105 : continue;
6338 : cost = min_cost_group;
6339 : reason = "not in sched group";
6340 : }
6341 123053390 : else if (modulo_epilogue_p
6342 123053390 : && INSN_EXACT_TICK (insn) == INVALID_TICK)
6343 : {
6344 : cost = max_insn_queue_index;
6345 : reason = "not an epilogue insn";
6346 : }
6347 123053390 : else if (shadows_only_p && !SHADOW_P (insn))
6348 : {
6349 : cost = 1;
6350 : reason = "not a shadow";
6351 : }
6352 123053390 : else if (recog_memoized (insn) < 0)
6353 : {
6354 1166192 : if (!first_cycle_insn_p
6355 1166192 : && (GET_CODE (PATTERN (insn)) == ASM_INPUT
6356 940464 : || asm_noperands (PATTERN (insn)) >= 0))
6357 : cost = 1;
6358 : reason = "asm";
6359 : }
6360 121887198 : else if (sched_pressure != SCHED_PRESSURE_NONE)
6361 : {
6362 23336 : if (sched_pressure == SCHED_PRESSURE_MODEL
6363 23336 : && INSN_TICK (insn) <= clock_var)
6364 : {
6365 0 : memcpy (temp_state, curr_state, dfa_state_size);
6366 0 : if (state_transition (temp_state, insn) >= 0)
6367 0 : INSN_TICK (insn) = clock_var + 1;
6368 : }
6369 : cost = 0;
6370 : }
6371 : else
6372 : {
6373 121863862 : int delay_cost = 0;
6374 :
6375 121863862 : if (delay_htab)
6376 : {
6377 0 : struct delay_pair *delay_entry;
6378 0 : delay_entry
6379 0 : = delay_htab->find_with_hash (insn,
6380 : htab_hash_pointer (insn));
6381 0 : while (delay_entry && delay_cost == 0)
6382 : {
6383 0 : delay_cost = estimate_shadow_tick (delay_entry);
6384 0 : if (delay_cost > max_insn_queue_index)
6385 : delay_cost = max_insn_queue_index;
6386 0 : delay_entry = delay_entry->next_same_i1;
6387 : }
6388 : }
6389 :
6390 121863862 : memcpy (temp_state, curr_state, dfa_state_size);
6391 121863862 : cost = state_transition (temp_state, insn);
6392 121863862 : if (cost < 0)
6393 : cost = 0;
6394 19454343 : else if (cost == 0)
6395 0 : cost = 1;
6396 121863862 : if (cost < delay_cost)
6397 : {
6398 0 : cost = delay_cost;
6399 0 : reason = "shadow tick";
6400 : }
6401 : }
6402 121863881 : if (cost >= 1)
6403 : {
6404 19506944 : if (SCHED_GROUP_P (insn) && cost > min_cost_group)
6405 19506944 : min_cost_group = cost;
6406 19506944 : ready_remove (&ready, i);
6407 : /* Normally we'd want to queue INSN for COST cycles. However,
6408 : if SCHED_GROUP_P is set, then we must ensure that nothing
6409 : else comes between INSN and its predecessor. If there is
6410 : some other insn ready to fire on the next cycle, then that
6411 : invariant would be broken.
6412 :
6413 : So when SCHED_GROUP_P is set, just queue this insn for a
6414 : single cycle. */
6415 19522731 : queue_insn (insn, SCHED_GROUP_P (insn) ? 1 : cost, reason);
6416 19506944 : if (i + 1 < n)
6417 : break;
6418 : }
6419 : }
6420 85230515 : if (i == n)
6421 73379255 : pass++;
6422 : }
6423 : }
6424 :
6425 : /* Called when we detect that the schedule is impossible. We examine the
6426 : backtrack queue to find the earliest insn that caused this condition. */
6427 :
6428 : static struct haifa_saved_data *
6429 0 : verify_shadows (void)
6430 : {
6431 0 : struct haifa_saved_data *save, *earliest_fail = NULL;
6432 0 : for (save = backtrack_queue; save; save = save->next)
6433 : {
6434 0 : int t;
6435 0 : struct delay_pair *pair = save->delay_pair;
6436 0 : rtx_insn *i1 = pair->i1;
6437 :
6438 0 : for (; pair; pair = pair->next_same_i1)
6439 : {
6440 0 : rtx_insn *i2 = pair->i2;
6441 :
6442 0 : if (QUEUE_INDEX (i2) == QUEUE_SCHEDULED)
6443 0 : continue;
6444 :
6445 0 : t = INSN_TICK (i1) + pair_delay (pair);
6446 0 : if (t < clock_var)
6447 : {
6448 0 : if (sched_verbose >= 2)
6449 0 : fprintf (sched_dump,
6450 : ";;\t\tfailed delay requirements for %d/%d (%d->%d)"
6451 : ", not ready\n",
6452 : INSN_UID (pair->i1), INSN_UID (pair->i2),
6453 0 : INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2));
6454 : earliest_fail = save;
6455 : break;
6456 : }
6457 0 : if (QUEUE_INDEX (i2) >= 0)
6458 : {
6459 0 : int queued_for = INSN_TICK (i2);
6460 :
6461 0 : if (t < queued_for)
6462 : {
6463 0 : if (sched_verbose >= 2)
6464 0 : fprintf (sched_dump,
6465 : ";;\t\tfailed delay requirements for %d/%d"
6466 : " (%d->%d), queued too late\n",
6467 : INSN_UID (pair->i1), INSN_UID (pair->i2),
6468 0 : INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2));
6469 : earliest_fail = save;
6470 : break;
6471 : }
6472 : }
6473 : }
6474 : }
6475 :
6476 0 : return earliest_fail;
6477 : }
6478 :
6479 : /* Print instructions together with useful scheduling information between
6480 : HEAD and TAIL (inclusive). */
6481 : static void
6482 0 : dump_insn_stream (rtx_insn *head, rtx_insn *tail)
6483 : {
6484 0 : fprintf (sched_dump, ";;\t| insn | prio |\n");
6485 :
6486 0 : rtx_insn *next_tail = NEXT_INSN (tail);
6487 0 : for (rtx_insn *insn = head; insn != next_tail; insn = NEXT_INSN (insn))
6488 : {
6489 0 : int priority = NOTE_P (insn) ? 0 : INSN_PRIORITY (insn);
6490 0 : const char *pattern = (NOTE_P (insn)
6491 0 : ? "note"
6492 0 : : str_pattern_slim (PATTERN (insn)));
6493 :
6494 0 : fprintf (sched_dump, ";;\t| %4d | %4d | %-30s ",
6495 0 : INSN_UID (insn), priority, pattern);
6496 :
6497 0 : if (sched_verbose >= 4)
6498 : {
6499 0 : if (NOTE_P (insn) || LABEL_P (insn) || recog_memoized (insn) < 0)
6500 0 : fprintf (sched_dump, "nothing");
6501 : else
6502 0 : print_reservation (sched_dump, insn);
6503 : }
6504 0 : fprintf (sched_dump, "\n");
6505 : }
6506 0 : }
6507 :
6508 : /* Use forward list scheduling to rearrange insns of block pointed to by
6509 : TARGET_BB, possibly bringing insns from subsequent blocks in the same
6510 : region. */
6511 :
6512 : bool
6513 10313337 : schedule_block (basic_block *target_bb, state_t init_state)
6514 : {
6515 10313337 : int i;
6516 10313337 : bool success = modulo_ii == 0;
6517 10313337 : struct sched_block_state ls;
6518 10313337 : state_t temp_state = NULL; /* It is used for multipass scheduling. */
6519 10313337 : int sort_p, advance, start_clock_var;
6520 :
6521 : /* Head/tail info for this block. */
6522 10313337 : rtx_insn *prev_head = current_sched_info->prev_head;
6523 10313337 : rtx_insn *next_tail = current_sched_info->next_tail;
6524 10313337 : rtx_insn *head = NEXT_INSN (prev_head);
6525 10313337 : rtx_insn *tail = PREV_INSN (next_tail);
6526 :
6527 10313337 : if ((current_sched_info->flags & DONT_BREAK_DEPENDENCIES) == 0
6528 10313337 : && sched_pressure != SCHED_PRESSURE_MODEL && !sched_fusion)
6529 10313337 : find_modifiable_mems (head, tail);
6530 :
6531 : /* We used to have code to avoid getting parameters moved from hard
6532 : argument registers into pseudos.
6533 :
6534 : However, it was removed when it proved to be of marginal benefit
6535 : and caused problems because schedule_block and compute_forward_dependences
6536 : had different notions of what the "head" insn was. */
6537 :
6538 10313337 : gcc_assert (head != tail || INSN_P (head));
6539 :
6540 10313337 : haifa_recovery_bb_recently_added_p = false;
6541 :
6542 10313337 : backtrack_queue = NULL;
6543 :
6544 : /* Debug info. */
6545 10313337 : if (sched_verbose)
6546 : {
6547 190 : dump_new_block_header (0, *target_bb, head, tail);
6548 :
6549 190 : if (sched_verbose >= 2)
6550 : {
6551 0 : dump_insn_stream (head, tail);
6552 0 : memset (&rank_for_schedule_stats, 0,
6553 : sizeof (rank_for_schedule_stats));
6554 : }
6555 : }
6556 :
6557 10313337 : if (init_state == NULL)
6558 174 : state_reset (curr_state);
6559 : else
6560 10313163 : memcpy (curr_state, init_state, dfa_state_size);
6561 :
6562 : /* Clear the ready list. */
6563 10313337 : ready.first = ready.veclen - 1;
6564 10313337 : ready.n_ready = 0;
6565 10313337 : ready.n_debug = 0;
6566 :
6567 : /* It is used for first cycle multipass scheduling. */
6568 10313337 : temp_state = alloca (dfa_state_size);
6569 :
6570 10313337 : if (targetm.sched.init)
6571 0 : targetm.sched.init (sched_dump, sched_verbose, ready.veclen);
6572 :
6573 : /* We start inserting insns after PREV_HEAD. */
6574 10313337 : last_scheduled_insn = prev_head;
6575 10313337 : last_nondebug_scheduled_insn = NULL;
6576 10313337 : nonscheduled_insns_begin = NULL;
6577 :
6578 10313337 : gcc_assert ((NOTE_P (last_scheduled_insn)
6579 : || DEBUG_INSN_P (last_scheduled_insn))
6580 : && BLOCK_FOR_INSN (last_scheduled_insn) == *target_bb);
6581 :
6582 : /* Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the
6583 : queue. */
6584 10313337 : q_ptr = 0;
6585 10313337 : q_size = 0;
6586 :
6587 10313337 : insn_queue = XALLOCAVEC (rtx_insn_list *, max_insn_queue_index + 1);
6588 10313337 : memset (insn_queue, 0, (max_insn_queue_index + 1) * sizeof (rtx));
6589 :
6590 : /* Start just before the beginning of time. */
6591 10313337 : clock_var = -1;
6592 :
6593 : /* We need queue and ready lists and clock_var be initialized
6594 : in try_ready () (which is called through init_ready_list ()). */
6595 10313337 : (*current_sched_info->init_ready_list) ();
6596 :
6597 10313337 : if (sched_pressure)
6598 430 : sched_pressure_start_bb (*target_bb);
6599 :
6600 : /* The algorithm is O(n^2) in the number of ready insns at any given
6601 : time in the worst case. Before reload we are more likely to have
6602 : big lists so truncate them to a reasonable size. */
6603 10313337 : if (!reload_completed
6604 931 : && ready.n_ready - ready.n_debug > param_max_sched_ready_insns)
6605 : {
6606 4 : ready_sort_debug (&ready);
6607 4 : ready_sort_real (&ready);
6608 :
6609 : /* Find first free-standing insn past param_max_sched_ready_insns.
6610 : If there are debug insns, we know they're first. */
6611 4 : for (i = param_max_sched_ready_insns + ready.n_debug; i < ready.n_ready;
6612 : i++)
6613 4 : if (!SCHED_GROUP_P (ready_element (&ready, i)))
6614 : break;
6615 :
6616 4 : if (sched_verbose >= 2)
6617 : {
6618 0 : fprintf (sched_dump,
6619 : ";;\t\tReady list on entry: %d insns: ", ready.n_ready);
6620 0 : debug_ready_list (&ready);
6621 0 : fprintf (sched_dump,
6622 : ";;\t\t before reload => truncated to %d insns\n", i);
6623 : }
6624 :
6625 : /* Delay all insns past it for 1 cycle. If debug counter is
6626 : activated make an exception for the insn right after
6627 : nonscheduled_insns_begin. */
6628 4 : {
6629 4 : rtx_insn *skip_insn;
6630 :
6631 4 : if (dbg_cnt (sched_insn) == false)
6632 0 : skip_insn = first_nonscheduled_insn ();
6633 : else
6634 4 : skip_insn = NULL;
6635 :
6636 30 : while (i < ready.n_ready)
6637 : {
6638 26 : rtx_insn *insn;
6639 :
6640 26 : insn = ready_remove (&ready, i);
6641 :
6642 26 : if (insn != skip_insn)
6643 26 : queue_insn (insn, 1, "list truncated");
6644 : }
6645 4 : if (skip_insn)
6646 0 : ready_add (&ready, skip_insn, true);
6647 : }
6648 : }
6649 :
6650 : /* Now we can restore basic block notes and maintain precise cfg. */
6651 10313337 : restore_bb_notes (*target_bb);
6652 :
6653 10313337 : last_clock_var = -1;
6654 :
6655 10313337 : advance = 0;
6656 :
6657 10313337 : gcc_assert (scheduled_insns.length () == 0);
6658 10313337 : sort_p = true;
6659 10313337 : must_backtrack = false;
6660 10313337 : modulo_insns_scheduled = 0;
6661 :
6662 10313337 : ls.modulo_epilogue = false;
6663 10313337 : ls.first_cycle_insn_p = true;
6664 :
6665 : /* Loop until all the insns in BB are scheduled. */
6666 45271138 : while ((*current_sched_info->schedule_more_p) ())
6667 : {
6668 34957801 : perform_replacements_new_cycle ();
6669 34957801 : do
6670 : {
6671 34957801 : start_clock_var = clock_var;
6672 :
6673 34957801 : clock_var++;
6674 :
6675 34957801 : advance_one_cycle ();
6676 :
6677 : /* Add to the ready list all pending insns that can be issued now.
6678 : If there are no ready insns, increment clock until one
6679 : is ready and add all pending insns at that point to the ready
6680 : list. */
6681 34957801 : queue_to_ready (&ready);
6682 :
6683 34957801 : gcc_assert (ready.n_ready);
6684 :
6685 34957801 : if (sched_verbose >= 2)
6686 : {
6687 0 : fprintf (sched_dump, ";;\t\tReady list after queue_to_ready:");
6688 0 : debug_ready_list (&ready);
6689 : }
6690 34957801 : advance -= clock_var - start_clock_var;
6691 : }
6692 34957801 : while (advance > 0);
6693 :
6694 34957801 : if (ls.modulo_epilogue)
6695 : {
6696 0 : int stage = clock_var / modulo_ii;
6697 0 : if (stage > modulo_last_stage * 2 + 2)
6698 : {
6699 0 : if (sched_verbose >= 2)
6700 0 : fprintf (sched_dump,
6701 : ";;\t\tmodulo scheduled succeeded at II %d\n",
6702 : modulo_ii);
6703 0 : success = true;
6704 0 : goto end_schedule;
6705 : }
6706 : }
6707 34957801 : else if (modulo_ii > 0)
6708 : {
6709 0 : int stage = clock_var / modulo_ii;
6710 0 : if (stage > modulo_max_stages)
6711 : {
6712 0 : if (sched_verbose >= 2)
6713 0 : fprintf (sched_dump,
6714 : ";;\t\tfailing schedule due to excessive stages\n");
6715 0 : goto end_schedule;
6716 : }
6717 0 : if (modulo_n_insns == modulo_insns_scheduled
6718 0 : && stage > modulo_last_stage)
6719 : {
6720 0 : if (sched_verbose >= 2)
6721 0 : fprintf (sched_dump,
6722 : ";;\t\tfound kernel after %d stages, II %d\n",
6723 : stage, modulo_ii);
6724 0 : ls.modulo_epilogue = true;
6725 : }
6726 : }
6727 :
6728 34957801 : prune_ready_list (temp_state, true, false, ls.modulo_epilogue);
6729 34957801 : if (ready.n_ready == 0)
6730 4154 : continue;
6731 34953647 : if (must_backtrack)
6732 0 : goto do_backtrack;
6733 :
6734 34953647 : ls.shadows_only_p = false;
6735 34953647 : cycle_issued_insns = 0;
6736 34953647 : ls.can_issue_more = issue_rate;
6737 155094107 : for (;;)
6738 : {
6739 95023877 : rtx_insn *insn;
6740 95023877 : int cost;
6741 95023877 : bool asm_p;
6742 :
6743 95023877 : if (sort_p && ready.n_ready > 0)
6744 : {
6745 : /* Sort the ready list based on priority. This must be
6746 : done every iteration through the loop, as schedule_insn
6747 : may have readied additional insns that will not be
6748 : sorted correctly. */
6749 62020082 : ready_sort (&ready);
6750 :
6751 62020082 : if (sched_verbose >= 2)
6752 : {
6753 0 : fprintf (sched_dump,
6754 : ";;\t\tReady list after ready_sort: ");
6755 0 : debug_ready_list (&ready);
6756 : }
6757 : }
6758 :
6759 : /* We don't want md sched reorder to even see debug isns, so put
6760 : them out right away. */
6761 62020082 : if (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0))
6762 101442844 : && (*current_sched_info->schedule_more_p) ())
6763 : {
6764 54912697 : while (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0)))
6765 : {
6766 48493730 : rtx_insn *insn = ready_remove_first (&ready);
6767 48493730 : gcc_assert (DEBUG_INSN_P (insn));
6768 48493730 : (*current_sched_info->begin_schedule_ready) (insn);
6769 48493730 : scheduled_insns.safe_push (insn);
6770 48493730 : last_scheduled_insn = insn;
6771 48493730 : advance = schedule_insn (insn);
6772 48493730 : gcc_assert (advance == 0);
6773 48493730 : if (ready.n_ready > 0)
6774 46680361 : ready_sort (&ready);
6775 : }
6776 : }
6777 :
6778 95023877 : if (ls.first_cycle_insn_p && !ready.n_ready)
6779 : break;
6780 :
6781 95022632 : resume_after_backtrack:
6782 : /* Allow the target to reorder the list, typically for
6783 : better instruction bundling. */
6784 95022632 : if (sort_p
6785 95022632 : && (ready.n_ready == 0
6786 60206713 : || !SCHED_GROUP_P (ready_element (&ready, 0))))
6787 : {
6788 90838853 : if (ls.first_cycle_insn_p && targetm.sched.reorder)
6789 34932071 : ls.can_issue_more
6790 34932071 : = targetm.sched.reorder (sched_dump, sched_verbose,
6791 : ready_lastpos (&ready),
6792 : &ready.n_ready, clock_var);
6793 55906782 : else if (!ls.first_cycle_insn_p && targetm.sched.reorder2)
6794 0 : ls.can_issue_more
6795 0 : = targetm.sched.reorder2 (sched_dump, sched_verbose,
6796 0 : ready.n_ready
6797 0 : ? ready_lastpos (&ready) : NULL,
6798 : &ready.n_ready, clock_var);
6799 : }
6800 :
6801 95022632 : restart_choose_ready:
6802 95022632 : if (sched_verbose >= 2)
6803 : {
6804 0 : fprintf (sched_dump, ";;\tReady list (t = %3d): ",
6805 : clock_var);
6806 0 : debug_ready_list (&ready);
6807 0 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
6808 0 : print_curr_reg_pressure ();
6809 : }
6810 :
6811 95022632 : if (ready.n_ready == 0
6812 34815919 : && ls.can_issue_more
6813 33060560 : && reload_completed)
6814 : {
6815 : /* Allow scheduling insns directly from the queue in case
6816 : there's nothing better to do (ready list is empty) but
6817 : there are still vacant dispatch slots in the current cycle. */
6818 33057763 : if (sched_verbose >= 6)
6819 0 : fprintf (sched_dump,";;\t\tSecond chance\n");
6820 33057763 : memcpy (temp_state, curr_state, dfa_state_size);
6821 33057763 : if (early_queue_to_ready (temp_state, &ready))
6822 4 : ready_sort (&ready);
6823 : }
6824 :
6825 95022632 : if (ready.n_ready == 0
6826 60206717 : || !ls.can_issue_more
6827 60129640 : || state_dead_lock_p (curr_state)
6828 155152272 : || !(*current_sched_info->schedule_more_p) ())
6829 : break;
6830 :
6831 : /* Select and remove the insn from the ready list. */
6832 60129640 : if (sort_p)
6833 : {
6834 60129640 : int res;
6835 :
6836 60129640 : insn = NULL;
6837 60129640 : res = choose_ready (&ready, ls.first_cycle_insn_p, &insn);
6838 :
6839 60129640 : if (res < 0)
6840 : /* Finish cycle. */
6841 : break;
6842 60129640 : if (res > 0)
6843 0 : goto restart_choose_ready;
6844 :
6845 60129640 : gcc_assert (insn != NULL_RTX);
6846 : }
6847 : else
6848 0 : insn = ready_remove_first (&ready);
6849 :
6850 60129640 : if (sched_pressure != SCHED_PRESSURE_NONE
6851 60129640 : && INSN_TICK (insn) > clock_var)
6852 : {
6853 2238 : ready_add (&ready, insn, true);
6854 2238 : advance = 1;
6855 2238 : break;
6856 : }
6857 :
6858 60127402 : if (targetm.sched.dfa_new_cycle
6859 60127402 : && targetm.sched.dfa_new_cycle (sched_dump, sched_verbose,
6860 : insn, last_clock_var,
6861 : clock_var, &sort_p))
6862 : /* SORT_P is used by the target to override sorting
6863 : of the ready list. This is needed when the target
6864 : has modified its internal structures expecting that
6865 : the insn will be issued next. As we need the insn
6866 : to have the highest priority (so it will be returned by
6867 : the ready_remove_first call above), we invoke
6868 : ready_add (&ready, insn, true).
6869 : But, still, there is one issue: INSN can be later
6870 : discarded by scheduler's front end through
6871 : current_sched_info->can_schedule_ready_p, hence, won't
6872 : be issued next. */
6873 : {
6874 0 : ready_add (&ready, insn, true);
6875 0 : break;
6876 : }
6877 :
6878 60127402 : sort_p = true;
6879 :
6880 60127402 : if (current_sched_info->can_schedule_ready_p
6881 60127402 : && ! (*current_sched_info->can_schedule_ready_p) (insn))
6882 : /* We normally get here only if we don't want to move
6883 : insn from the split block. */
6884 : {
6885 0 : TODO_SPEC (insn) = DEP_POSTPONED;
6886 0 : goto restart_choose_ready;
6887 : }
6888 :
6889 60127402 : if (delay_htab)
6890 : {
6891 : /* If this insn is the first part of a delay-slot pair, record a
6892 : backtrack point. */
6893 0 : struct delay_pair *delay_entry;
6894 0 : delay_entry
6895 0 : = delay_htab->find_with_hash (insn, htab_hash_pointer (insn));
6896 0 : if (delay_entry)
6897 : {
6898 0 : save_backtrack_point (delay_entry, ls);
6899 0 : if (sched_verbose >= 2)
6900 0 : fprintf (sched_dump, ";;\t\tsaving backtrack point\n");
6901 : }
6902 : }
6903 :
6904 : /* DECISION is made. */
6905 :
6906 60127402 : if (modulo_ii > 0 && INSN_UID (insn) < modulo_iter0_max_uid)
6907 : {
6908 0 : modulo_insns_scheduled++;
6909 0 : modulo_last_stage = clock_var / modulo_ii;
6910 : }
6911 60127402 : if (TODO_SPEC (insn) & SPECULATIVE)
6912 0 : generate_recovery_code (insn);
6913 :
6914 60127402 : if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
6915 14 : targetm.sched.dispatch_do (insn, ADD_TO_DISPATCH_WINDOW);
6916 :
6917 : /* Update counters, etc in the scheduler's front end. */
6918 60127402 : (*current_sched_info->begin_schedule_ready) (insn);
6919 60127402 : scheduled_insns.safe_push (insn);
6920 60127402 : gcc_assert (NONDEBUG_INSN_P (insn));
6921 60127402 : last_nondebug_scheduled_insn = last_scheduled_insn = insn;
6922 :
6923 60127402 : if (recog_memoized (insn) >= 0)
6924 : {
6925 59279750 : memcpy (temp_state, curr_state, dfa_state_size);
6926 59279750 : cost = state_transition (curr_state, insn);
6927 59279750 : if (sched_pressure != SCHED_PRESSURE_WEIGHTED && !sched_fusion)
6928 59276496 : gcc_assert (cost < 0);
6929 59279750 : if (memcmp (temp_state, curr_state, dfa_state_size) != 0)
6930 58888368 : cycle_issued_insns++;
6931 : asm_p = false;
6932 : }
6933 : else
6934 847652 : asm_p = (GET_CODE (PATTERN (insn)) == ASM_INPUT
6935 847652 : || asm_noperands (PATTERN (insn)) >= 0);
6936 :
6937 60127402 : if (targetm.sched.variable_issue)
6938 0 : ls.can_issue_more =
6939 0 : targetm.sched.variable_issue (sched_dump, sched_verbose,
6940 : insn, ls.can_issue_more);
6941 : /* A naked CLOBBER or USE generates no instruction, so do
6942 : not count them against the issue rate. */
6943 60127402 : else if (GET_CODE (PATTERN (insn)) != USE
6944 60127402 : && GET_CODE (PATTERN (insn)) != CLOBBER)
6945 59336922 : ls.can_issue_more--;
6946 60127402 : advance = schedule_insn (insn);
6947 :
6948 60127402 : if (SHADOW_P (insn))
6949 0 : ls.shadows_only_p = true;
6950 :
6951 : /* After issuing an asm insn we should start a new cycle. */
6952 60127402 : if (advance == 0 && asm_p)
6953 57172 : advance = 1;
6954 :
6955 60127402 : if (must_backtrack)
6956 : break;
6957 :
6958 60127402 : if (advance != 0)
6959 : break;
6960 :
6961 60070230 : ls.first_cycle_insn_p = false;
6962 60070230 : if (ready.n_ready > 0)
6963 34221888 : prune_ready_list (temp_state, false, ls.shadows_only_p,
6964 34221888 : ls.modulo_epilogue);
6965 60070230 : }
6966 :
6967 34953647 : do_backtrack:
6968 34953647 : if (!must_backtrack)
6969 35197428 : for (i = 0; i < ready.n_ready; i++)
6970 : {
6971 243781 : rtx_insn *insn = ready_element (&ready, i);
6972 243781 : if (INSN_EXACT_TICK (insn) == clock_var)
6973 : {
6974 0 : must_backtrack = true;
6975 0 : clock_var++;
6976 0 : break;
6977 : }
6978 : }
6979 34953647 : if (must_backtrack && modulo_ii > 0)
6980 : {
6981 0 : if (modulo_backtracks_left == 0)
6982 0 : goto end_schedule;
6983 0 : modulo_backtracks_left--;
6984 : }
6985 34953647 : while (must_backtrack)
6986 : {
6987 0 : struct haifa_saved_data *failed;
6988 0 : rtx_insn *failed_insn;
6989 :
6990 0 : must_backtrack = false;
6991 0 : failed = verify_shadows ();
6992 0 : gcc_assert (failed);
6993 :
6994 0 : failed_insn = failed->delay_pair->i1;
6995 : /* Clear these queues. */
6996 0 : perform_replacements_new_cycle ();
6997 0 : toggle_cancelled_flags (false);
6998 0 : unschedule_insns_until (failed_insn);
6999 0 : while (failed != backtrack_queue)
7000 0 : free_topmost_backtrack_point (true);
7001 0 : restore_last_backtrack_point (&ls);
7002 0 : if (sched_verbose >= 2)
7003 0 : fprintf (sched_dump, ";;\t\trewind to cycle %d\n", clock_var);
7004 : /* Delay by at least a cycle. This could cause additional
7005 : backtracking. */
7006 0 : queue_insn (failed_insn, 1, "backtracked");
7007 0 : advance = 0;
7008 0 : if (must_backtrack)
7009 0 : continue;
7010 0 : if (ready.n_ready > 0)
7011 0 : goto resume_after_backtrack;
7012 : else
7013 : {
7014 0 : if (clock_var == 0 && ls.first_cycle_insn_p)
7015 0 : goto end_schedule;
7016 : advance = 1;
7017 : break;
7018 : }
7019 : }
7020 34953647 : ls.first_cycle_insn_p = true;
7021 : }
7022 10313337 : if (ls.modulo_epilogue)
7023 0 : success = true;
7024 10313337 : end_schedule:
7025 10313337 : if (!ls.first_cycle_insn_p || advance)
7026 878 : advance_one_cycle ();
7027 10313337 : perform_replacements_new_cycle ();
7028 10313337 : if (modulo_ii > 0)
7029 : {
7030 : /* Once again, debug insn suckiness: they can be on the ready list
7031 : even if they have unresolved dependencies. To make our view
7032 : of the world consistent, remove such "ready" insns. */
7033 0 : restart_debug_insn_loop:
7034 0 : for (i = ready.n_ready - 1; i >= 0; i--)
7035 : {
7036 0 : rtx_insn *x;
7037 :
7038 0 : x = ready_element (&ready, i);
7039 0 : if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x)) != NULL
7040 0 : || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x)) != NULL)
7041 : {
7042 0 : ready_remove (&ready, i);
7043 0 : goto restart_debug_insn_loop;
7044 : }
7045 : }
7046 0 : for (i = ready.n_ready - 1; i >= 0; i--)
7047 : {
7048 0 : rtx_insn *x;
7049 :
7050 0 : x = ready_element (&ready, i);
7051 0 : resolve_dependencies (x);
7052 : }
7053 0 : for (i = 0; i <= max_insn_queue_index; i++)
7054 : {
7055 : rtx_insn_list *link;
7056 0 : while ((link = insn_queue[i]) != NULL)
7057 : {
7058 0 : rtx_insn *x = link->insn ();
7059 0 : insn_queue[i] = link->next ();
7060 0 : QUEUE_INDEX (x) = QUEUE_NOWHERE;
7061 0 : free_INSN_LIST_node (link);
7062 0 : resolve_dependencies (x);
7063 : }
7064 : }
7065 : }
7066 :
7067 10313337 : if (!success)
7068 0 : undo_all_replacements ();
7069 :
7070 : /* Debug info. */
7071 10313337 : if (sched_verbose)
7072 : {
7073 190 : fprintf (sched_dump, ";;\tReady list (final): ");
7074 190 : debug_ready_list (&ready);
7075 : }
7076 :
7077 10313337 : if (modulo_ii == 0 && current_sched_info->queue_must_finish_empty)
7078 : /* Sanity check -- queue must be empty now. Meaningless if region has
7079 : multiple bbs. */
7080 10313289 : gcc_assert (!q_size && !ready.n_ready && !ready.n_debug);
7081 48 : else if (modulo_ii == 0)
7082 : {
7083 : /* We must maintain QUEUE_INDEX between blocks in region. */
7084 60 : for (i = ready.n_ready - 1; i >= 0; i--)
7085 : {
7086 12 : rtx_insn *x;
7087 :
7088 12 : x = ready_element (&ready, i);
7089 12 : QUEUE_INDEX (x) = QUEUE_NOWHERE;
7090 12 : TODO_SPEC (x) = HARD_DEP;
7091 : }
7092 :
7093 48 : if (q_size)
7094 0 : for (i = 0; i <= max_insn_queue_index; i++)
7095 : {
7096 0 : rtx_insn_list *link;
7097 0 : for (link = insn_queue[i]; link; link = link->next ())
7098 : {
7099 0 : rtx_insn *x;
7100 :
7101 0 : x = link->insn ();
7102 0 : QUEUE_INDEX (x) = QUEUE_NOWHERE;
7103 0 : TODO_SPEC (x) = HARD_DEP;
7104 : }
7105 0 : free_INSN_LIST_list (&insn_queue[i]);
7106 : }
7107 : }
7108 :
7109 10313337 : if (sched_pressure == SCHED_PRESSURE_MODEL)
7110 0 : model_end_schedule ();
7111 :
7112 10313337 : if (success)
7113 : {
7114 10313337 : commit_schedule (prev_head, tail, target_bb);
7115 10313337 : if (sched_verbose)
7116 190 : fprintf (sched_dump, ";; total time = %d\n", clock_var);
7117 : }
7118 : else
7119 0 : last_scheduled_insn = tail;
7120 :
7121 10313337 : scheduled_insns.truncate (0);
7122 :
7123 10313337 : if (!current_sched_info->queue_must_finish_empty
7124 10313289 : || haifa_recovery_bb_recently_added_p)
7125 : {
7126 : /* INSN_TICK (minimum clock tick at which the insn becomes
7127 : ready) may be not correct for the insn in the subsequent
7128 : blocks of the region. We should use a correct value of
7129 : `clock_var' or modify INSN_TICK. It is better to keep
7130 : clock_var value equal to 0 at the start of a basic block.
7131 : Therefore we modify INSN_TICK here. */
7132 48 : fix_inter_tick (NEXT_INSN (prev_head), last_scheduled_insn);
7133 : }
7134 :
7135 10313337 : if (targetm.sched.finish)
7136 : {
7137 0 : targetm.sched.finish (sched_dump, sched_verbose);
7138 : /* Target might have added some instructions to the scheduled block
7139 : in its md_finish () hook. These new insns don't have any data
7140 : initialized and to identify them we extend h_i_d so that they'll
7141 : get zero luids. */
7142 0 : sched_extend_luids ();
7143 : }
7144 :
7145 : /* Update head/tail boundaries. */
7146 10313337 : head = NEXT_INSN (prev_head);
7147 10313337 : tail = last_scheduled_insn;
7148 :
7149 10313337 : if (sched_verbose)
7150 : {
7151 190 : fprintf (sched_dump, ";; new head = %d\n;; new tail = %d\n",
7152 190 : INSN_UID (head), INSN_UID (tail));
7153 :
7154 190 : if (sched_verbose >= 2)
7155 : {
7156 0 : dump_insn_stream (head, tail);
7157 0 : print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats,
7158 : NULL);
7159 : }
7160 :
7161 190 : fprintf (sched_dump, "\n");
7162 : }
7163 :
7164 10313337 : head = restore_other_notes (head, NULL);
7165 :
7166 10313337 : current_sched_info->head = head;
7167 10313337 : current_sched_info->tail = tail;
7168 :
7169 10313337 : free_backtrack_queue ();
7170 :
7171 10313337 : return success;
7172 : }
7173 :
7174 : /* Set_priorities: compute priority of each insn in the block. */
7175 :
7176 : int
7177 10314356 : set_priorities (rtx_insn *head, rtx_insn *tail)
7178 : {
7179 10314356 : rtx_insn *insn;
7180 10314356 : int n_insn;
7181 10314356 : int sched_max_insns_priority =
7182 10314356 : current_sched_info->sched_max_insns_priority;
7183 10314356 : rtx_insn *prev_head;
7184 :
7185 10314356 : if (head == tail && ! INSN_P (head))
7186 0 : gcc_unreachable ();
7187 :
7188 10314356 : n_insn = 0;
7189 :
7190 10314356 : prev_head = PREV_INSN (head);
7191 135077853 : for (insn = tail; insn != prev_head; insn = PREV_INSN (insn))
7192 : {
7193 114449141 : if (!INSN_P (insn))
7194 5823530 : continue;
7195 :
7196 108625611 : n_insn++;
7197 108625611 : (void) priority (insn);
7198 :
7199 108625611 : gcc_assert (INSN_PRIORITY_KNOWN (insn));
7200 :
7201 108625611 : sched_max_insns_priority = MAX (sched_max_insns_priority,
7202 : INSN_PRIORITY (insn));
7203 : }
7204 :
7205 10314356 : current_sched_info->sched_max_insns_priority = sched_max_insns_priority;
7206 :
7207 10314356 : return n_insn;
7208 : }
7209 :
7210 : /* Set sched_dump and sched_verbose for the desired debugging output. */
7211 : void
7212 964480 : setup_sched_dump (void)
7213 : {
7214 964480 : sched_verbose = sched_verbose_param;
7215 964480 : sched_dump = dump_file;
7216 964480 : if (!dump_file)
7217 964443 : sched_verbose = 0;
7218 964480 : }
7219 :
7220 : /* Allocate data for register pressure sensitive scheduling. */
7221 : static void
7222 964480 : alloc_global_sched_pressure_data (void)
7223 : {
7224 964480 : if (sched_pressure != SCHED_PRESSURE_NONE)
7225 : {
7226 57 : int i, max_regno = max_reg_num ();
7227 :
7228 57 : if (sched_dump != NULL)
7229 : /* We need info about pseudos for rtl dumps about pseudo
7230 : classes and costs. */
7231 0 : regstat_init_n_sets_and_refs ();
7232 57 : ira_set_pseudo_classes (true, sched_verbose ? sched_dump : NULL);
7233 57 : sched_regno_pressure_class
7234 57 : = (enum reg_class *) xmalloc (max_regno * sizeof (enum reg_class));
7235 8652 : for (i = 0; i < max_regno; i++)
7236 17190 : sched_regno_pressure_class[i]
7237 8595 : = (i < FIRST_PSEUDO_REGISTER
7238 8595 : ? ira_pressure_class_translate[REGNO_REG_CLASS (i)]
7239 3351 : : ira_pressure_class_translate[reg_allocno_class (i)]);
7240 57 : curr_reg_live = BITMAP_ALLOC (NULL);
7241 57 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
7242 : {
7243 57 : saved_reg_live = BITMAP_ALLOC (NULL);
7244 57 : region_ref_regs = BITMAP_ALLOC (NULL);
7245 : }
7246 57 : if (sched_pressure == SCHED_PRESSURE_MODEL)
7247 0 : tmp_bitmap = BITMAP_ALLOC (NULL);
7248 :
7249 : /* Calculate number of CALL_SAVED_REGS and FIXED_REGS in register classes
7250 : that we calculate register pressure for. */
7251 287 : for (int c = 0; c < ira_pressure_classes_num; ++c)
7252 : {
7253 230 : enum reg_class cl = ira_pressure_classes[c];
7254 :
7255 230 : call_saved_regs_num[cl] = 0;
7256 230 : fixed_regs_num[cl] = 0;
7257 :
7258 2905 : for (int i = 0; i < ira_class_hard_regs_num[cl]; ++i)
7259 : {
7260 2675 : unsigned int regno = ira_class_hard_regs[cl][i];
7261 2675 : if (fixed_regs[regno])
7262 0 : ++fixed_regs_num[cl];
7263 2675 : else if (!crtl->abi->clobbers_full_reg_p (regno))
7264 332 : ++call_saved_regs_num[cl];
7265 : }
7266 : }
7267 : }
7268 964480 : }
7269 :
7270 : /* Free data for register pressure sensitive scheduling. Also called
7271 : from schedule_region when stopping sched-pressure early. */
7272 : void
7273 964480 : free_global_sched_pressure_data (void)
7274 : {
7275 964480 : if (sched_pressure != SCHED_PRESSURE_NONE)
7276 : {
7277 57 : if (regstat_n_sets_and_refs != NULL)
7278 0 : regstat_free_n_sets_and_refs ();
7279 57 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
7280 : {
7281 57 : BITMAP_FREE (region_ref_regs);
7282 57 : BITMAP_FREE (saved_reg_live);
7283 : }
7284 57 : if (sched_pressure == SCHED_PRESSURE_MODEL)
7285 0 : BITMAP_FREE (tmp_bitmap);
7286 57 : BITMAP_FREE (curr_reg_live);
7287 57 : free (sched_regno_pressure_class);
7288 : }
7289 964480 : }
7290 :
7291 : /* Initialize some global state for the scheduler. This function works
7292 : with the common data shared between all the schedulers. It is called
7293 : from the scheduler specific initialization routine. */
7294 :
7295 : void
7296 964480 : sched_init (void)
7297 : {
7298 964480 : if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
7299 3 : targetm.sched.dispatch_do (NULL, DISPATCH_INIT);
7300 :
7301 964480 : if (live_range_shrinkage_p)
7302 35 : sched_pressure = SCHED_PRESSURE_WEIGHTED;
7303 964445 : else if (flag_sched_pressure
7304 50 : && !reload_completed
7305 22 : && common_sched_info->sched_pass_id == SCHED_RGN_PASS)
7306 22 : sched_pressure = ((enum sched_pressure_algorithm)
7307 22 : param_sched_pressure_algorithm);
7308 : else
7309 964423 : sched_pressure = SCHED_PRESSURE_NONE;
7310 :
7311 964480 : if (sched_pressure != SCHED_PRESSURE_NONE)
7312 57 : ira_setup_eliminable_regset ();
7313 :
7314 : /* Initialize SPEC_INFO. */
7315 964480 : if (targetm.sched.set_sched_flags)
7316 : {
7317 0 : spec_info = &spec_info_var;
7318 0 : targetm.sched.set_sched_flags (spec_info);
7319 :
7320 0 : if (spec_info->mask != 0)
7321 : {
7322 0 : spec_info->data_weakness_cutoff
7323 0 : = (param_sched_spec_prob_cutoff * MAX_DEP_WEAK) / 100;
7324 0 : spec_info->control_weakness_cutoff
7325 0 : = (param_sched_spec_prob_cutoff * REG_BR_PROB_BASE) / 100;
7326 : }
7327 : else
7328 : /* So we won't read anything accidentally. */
7329 0 : spec_info = NULL;
7330 :
7331 : }
7332 : else
7333 : /* So we won't read anything accidentally. */
7334 964480 : spec_info = 0;
7335 :
7336 : /* Initialize issue_rate. */
7337 964480 : if (targetm.sched.issue_rate)
7338 964480 : issue_rate = targetm.sched.issue_rate ();
7339 : else
7340 0 : issue_rate = 1;
7341 :
7342 964480 : if (targetm.sched.first_cycle_multipass_dfa_lookahead
7343 : /* Don't use max_issue with reg_pressure scheduling. Multipass
7344 : scheduling and reg_pressure scheduling undo each other's decisions. */
7345 964480 : && sched_pressure == SCHED_PRESSURE_NONE)
7346 964423 : dfa_lookahead = targetm.sched.first_cycle_multipass_dfa_lookahead ();
7347 : else
7348 57 : dfa_lookahead = 0;
7349 :
7350 : /* Set to "0" so that we recalculate. */
7351 964480 : max_lookahead_tries = 0;
7352 :
7353 964480 : if (targetm.sched.init_dfa_pre_cycle_insn)
7354 0 : targetm.sched.init_dfa_pre_cycle_insn ();
7355 :
7356 964480 : if (targetm.sched.init_dfa_post_cycle_insn)
7357 0 : targetm.sched.init_dfa_post_cycle_insn ();
7358 :
7359 964480 : dfa_start ();
7360 964480 : dfa_state_size = state_size ();
7361 :
7362 964480 : init_alias_analysis ();
7363 :
7364 964480 : if (!sched_no_dce)
7365 964480 : df_set_flags (DF_LR_RUN_DCE);
7366 964480 : df_note_add_problem ();
7367 :
7368 : /* More problems needed for interloop dep calculation in SMS. */
7369 964480 : if (common_sched_info->sched_pass_id == SCHED_SMS_PASS)
7370 : {
7371 267 : df_rd_add_problem ();
7372 267 : df_chain_add_problem (DF_DU_CHAIN + DF_UD_CHAIN);
7373 : }
7374 :
7375 964480 : df_analyze ();
7376 :
7377 : /* Do not run DCE after reload, as this can kill nops inserted
7378 : by bundling. */
7379 964480 : if (reload_completed)
7380 963984 : df_clear_flags (DF_LR_RUN_DCE);
7381 :
7382 964480 : regstat_compute_calls_crossed ();
7383 :
7384 964480 : if (targetm.sched.init_global)
7385 964480 : targetm.sched.init_global (sched_dump, sched_verbose, get_max_uid () + 1);
7386 :
7387 964480 : alloc_global_sched_pressure_data ();
7388 :
7389 964480 : curr_state = xmalloc (dfa_state_size);
7390 964480 : }
7391 :
7392 : static void haifa_init_only_bb (basic_block, basic_block);
7393 :
7394 : /* Initialize data structures specific to the Haifa scheduler. */
7395 : void
7396 964349 : haifa_sched_init (void)
7397 : {
7398 964349 : setup_sched_dump ();
7399 964349 : sched_init ();
7400 :
7401 964349 : scheduled_insns.create (0);
7402 :
7403 964349 : if (spec_info != NULL)
7404 : {
7405 0 : sched_deps_info->use_deps_list = 1;
7406 0 : sched_deps_info->generate_spec_deps = 1;
7407 : }
7408 :
7409 : /* Initialize luids, dependency caches, target and h_i_d for the
7410 : whole function. */
7411 964349 : {
7412 964349 : sched_init_bbs ();
7413 :
7414 964349 : auto_vec<basic_block> bbs (n_basic_blocks_for_fn (cfun));
7415 964349 : basic_block bb;
7416 11296870 : FOR_EACH_BB_FN (bb, cfun)
7417 10332521 : bbs.quick_push (bb);
7418 964349 : sched_init_luids (bbs);
7419 964349 : sched_deps_init (true);
7420 964349 : sched_extend_target ();
7421 964349 : haifa_init_h_i_d (bbs);
7422 964349 : }
7423 :
7424 964349 : sched_init_only_bb = haifa_init_only_bb;
7425 964349 : sched_split_block = sched_split_block_1;
7426 964349 : sched_create_empty_bb = sched_create_empty_bb_1;
7427 964349 : haifa_recovery_bb_ever_added_p = false;
7428 :
7429 964349 : nr_begin_data = nr_begin_control = nr_be_in_data = nr_be_in_control = 0;
7430 964349 : before_recovery = 0;
7431 964349 : after_recovery = 0;
7432 :
7433 964349 : modulo_ii = 0;
7434 964349 : }
7435 :
7436 : /* Finish work with the data specific to the Haifa scheduler. */
7437 : void
7438 964349 : haifa_sched_finish (void)
7439 : {
7440 964349 : sched_create_empty_bb = NULL;
7441 964349 : sched_split_block = NULL;
7442 964349 : sched_init_only_bb = NULL;
7443 :
7444 964349 : if (spec_info && spec_info->dump)
7445 : {
7446 0 : char c = reload_completed ? 'a' : 'b';
7447 :
7448 0 : fprintf (spec_info->dump,
7449 : ";; %s:\n", current_function_name ());
7450 :
7451 0 : fprintf (spec_info->dump,
7452 : ";; Procedure %cr-begin-data-spec motions == %d\n",
7453 : c, nr_begin_data);
7454 0 : fprintf (spec_info->dump,
7455 : ";; Procedure %cr-be-in-data-spec motions == %d\n",
7456 : c, nr_be_in_data);
7457 0 : fprintf (spec_info->dump,
7458 : ";; Procedure %cr-begin-control-spec motions == %d\n",
7459 : c, nr_begin_control);
7460 0 : fprintf (spec_info->dump,
7461 : ";; Procedure %cr-be-in-control-spec motions == %d\n",
7462 : c, nr_be_in_control);
7463 : }
7464 :
7465 964349 : scheduled_insns.release ();
7466 :
7467 : /* Finalize h_i_d, dependency caches, and luids for the whole
7468 : function. Target will be finalized in md_global_finish (). */
7469 964349 : sched_deps_finish ();
7470 964349 : sched_finish_luids ();
7471 964349 : current_sched_info = NULL;
7472 964349 : insn_queue = NULL;
7473 964349 : sched_finish ();
7474 964349 : }
7475 :
7476 : /* Free global data used during insn scheduling. This function works with
7477 : the common data shared between the schedulers. */
7478 :
7479 : void
7480 964480 : sched_finish (void)
7481 : {
7482 964480 : haifa_finish_h_i_d ();
7483 964480 : free_global_sched_pressure_data ();
7484 964480 : free (curr_state);
7485 :
7486 964480 : if (targetm.sched.finish_global)
7487 0 : targetm.sched.finish_global (sched_dump, sched_verbose);
7488 :
7489 964480 : end_alias_analysis ();
7490 :
7491 964480 : regstat_free_calls_crossed ();
7492 :
7493 964480 : dfa_finish ();
7494 964480 : }
7495 :
7496 : /* Free all delay_pair structures that were recorded. */
7497 : void
7498 0 : free_delay_pairs (void)
7499 : {
7500 0 : if (delay_htab)
7501 : {
7502 0 : delay_htab->empty ();
7503 0 : delay_htab_i2->empty ();
7504 : }
7505 0 : }
7506 :
7507 : /* Fix INSN_TICKs of the instructions in the current block as well as
7508 : INSN_TICKs of their dependents.
7509 : HEAD and TAIL are the begin and the end of the current scheduled block. */
7510 : static void
7511 48 : fix_inter_tick (rtx_insn *head, rtx_insn *tail)
7512 : {
7513 : /* Set of instructions with corrected INSN_TICK. */
7514 48 : auto_bitmap processed;
7515 : /* ??? It is doubtful if we should assume that cycle advance happens on
7516 : basic block boundaries. Basically insns that are unconditionally ready
7517 : on the start of the block are more preferable then those which have
7518 : a one cycle dependency over insn from the previous block. */
7519 48 : int next_clock = clock_var + 1;
7520 :
7521 : /* Iterates over scheduled instructions and fix their INSN_TICKs and
7522 : INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent
7523 : across different blocks. */
7524 835 : for (tail = NEXT_INSN (tail); head != tail; head = NEXT_INSN (head))
7525 : {
7526 787 : if (INSN_P (head))
7527 : {
7528 787 : int tick;
7529 787 : sd_iterator_def sd_it;
7530 787 : dep_t dep;
7531 :
7532 787 : tick = INSN_TICK (head);
7533 787 : gcc_assert (tick >= MIN_TICK);
7534 :
7535 : /* Fix INSN_TICK of instruction from just scheduled block. */
7536 787 : if (bitmap_set_bit (processed, INSN_LUID (head)))
7537 : {
7538 60 : tick -= next_clock;
7539 :
7540 60 : if (tick < MIN_TICK)
7541 : tick = MIN_TICK;
7542 :
7543 60 : INSN_TICK (head) = tick;
7544 : }
7545 :
7546 787 : if (DEBUG_INSN_P (head))
7547 120 : continue;
7548 :
7549 2899 : FOR_EACH_DEP (head, SD_LIST_RES_FORW, sd_it, dep)
7550 : {
7551 2232 : rtx_insn *next;
7552 :
7553 2232 : next = DEP_CON (dep);
7554 2232 : tick = INSN_TICK (next);
7555 :
7556 2232 : if (tick != INVALID_TICK
7557 : /* If NEXT has its INSN_TICK calculated, fix it.
7558 : If not - it will be properly calculated from
7559 : scratch later in fix_tick_ready. */
7560 2232 : && bitmap_set_bit (processed, INSN_LUID (next)))
7561 : {
7562 739 : tick -= next_clock;
7563 :
7564 739 : if (tick < MIN_TICK)
7565 : tick = MIN_TICK;
7566 :
7567 739 : if (tick > INTER_TICK (next))
7568 739 : INTER_TICK (next) = tick;
7569 : else
7570 : tick = INTER_TICK (next);
7571 :
7572 739 : INSN_TICK (next) = tick;
7573 : }
7574 : }
7575 : }
7576 : }
7577 48 : }
7578 :
7579 : /* Check if NEXT is ready to be added to the ready or queue list.
7580 : If "yes", add it to the proper list.
7581 : Returns:
7582 : -1 - is not ready yet,
7583 : 0 - added to the ready list,
7584 : 0 < N - queued for N cycles. */
7585 : int
7586 311309911 : try_ready (rtx_insn *next)
7587 : {
7588 311309911 : ds_t old_ts, new_ts;
7589 :
7590 311309911 : old_ts = TODO_SPEC (next);
7591 :
7592 311309911 : gcc_assert (!(old_ts & ~(SPECULATIVE | HARD_DEP | DEP_CONTROL | DEP_POSTPONED))
7593 : && (old_ts == HARD_DEP
7594 : || old_ts == DEP_POSTPONED
7595 : || (old_ts & SPECULATIVE)
7596 : || old_ts == DEP_CONTROL));
7597 :
7598 311309911 : new_ts = recompute_todo_spec (next, false);
7599 :
7600 311309911 : if (new_ts & (HARD_DEP | DEP_POSTPONED))
7601 202688743 : gcc_assert (new_ts == old_ts
7602 : && QUEUE_INDEX (next) == QUEUE_NOWHERE);
7603 108621168 : else if (current_sched_info->new_ready)
7604 108619911 : new_ts = current_sched_info->new_ready (next, new_ts);
7605 :
7606 : /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might
7607 : have its original pattern or changed (speculative) one. This is due
7608 : to changing ebb in region scheduling.
7609 : * But if (old_ts & SPECULATIVE), then we are pretty sure that insn
7610 : has speculative pattern.
7611 :
7612 : We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because
7613 : control-speculative NEXT could have been discarded by sched-rgn.cc
7614 : (the same case as when discarded by can_schedule_ready_p ()). */
7615 :
7616 311309911 : if ((new_ts & SPECULATIVE)
7617 : /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't
7618 : need to change anything. */
7619 0 : && new_ts != old_ts)
7620 : {
7621 0 : int res;
7622 0 : rtx new_pat;
7623 :
7624 0 : gcc_assert ((new_ts & SPECULATIVE) && !(new_ts & ~SPECULATIVE));
7625 :
7626 0 : res = haifa_speculate_insn (next, new_ts, &new_pat);
7627 :
7628 0 : switch (res)
7629 : {
7630 : case -1:
7631 : /* It would be nice to change DEP_STATUS of all dependences,
7632 : which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP,
7633 : so we won't reanalyze anything. */
7634 : new_ts = HARD_DEP;
7635 : break;
7636 :
7637 0 : case 0:
7638 : /* We follow the rule, that every speculative insn
7639 : has non-null ORIG_PAT. */
7640 0 : if (!ORIG_PAT (next))
7641 0 : ORIG_PAT (next) = PATTERN (next);
7642 : break;
7643 :
7644 0 : case 1:
7645 0 : if (!ORIG_PAT (next))
7646 : /* If we gonna to overwrite the original pattern of insn,
7647 : save it. */
7648 0 : ORIG_PAT (next) = PATTERN (next);
7649 :
7650 0 : res = haifa_change_pattern (next, new_pat);
7651 0 : gcc_assert (res);
7652 : break;
7653 :
7654 0 : default:
7655 0 : gcc_unreachable ();
7656 : }
7657 : }
7658 :
7659 : /* We need to restore pattern only if (new_ts == 0), because otherwise it is
7660 : either correct (new_ts & SPECULATIVE),
7661 : or we simply don't care (new_ts & HARD_DEP). */
7662 :
7663 311309911 : gcc_assert (!ORIG_PAT (next)
7664 : || !IS_SPECULATION_BRANCHY_CHECK_P (next));
7665 :
7666 311309911 : TODO_SPEC (next) = new_ts;
7667 :
7668 311309911 : if (new_ts & (HARD_DEP | DEP_POSTPONED))
7669 : {
7670 : /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because
7671 : control-speculative NEXT could have been discarded by sched-rgn.cc
7672 : (the same case as when discarded by can_schedule_ready_p ()). */
7673 : /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/
7674 :
7675 202688767 : change_queue_index (next, QUEUE_NOWHERE);
7676 :
7677 202688767 : return -1;
7678 : }
7679 108621144 : else if (!(new_ts & BEGIN_SPEC)
7680 108621144 : && ORIG_PAT (next) && PREDICATED_PAT (next) == NULL_RTX
7681 108621144 : && !IS_SPECULATION_CHECK_P (next))
7682 : /* We should change pattern of every previously speculative
7683 : instruction - and we determine if NEXT was speculative by using
7684 : ORIG_PAT field. Except one case - speculation checks have ORIG_PAT
7685 : pat too, so skip them. */
7686 : {
7687 0 : bool success = haifa_change_pattern (next, ORIG_PAT (next));
7688 0 : gcc_assert (success);
7689 0 : ORIG_PAT (next) = 0;
7690 : }
7691 :
7692 108621144 : if (sched_verbose >= 2)
7693 : {
7694 0 : fprintf (sched_dump, ";;\t\tdependencies resolved: insn %s",
7695 0 : (*current_sched_info->print_insn) (next, 0));
7696 :
7697 0 : if (spec_info && spec_info->dump)
7698 : {
7699 0 : if (new_ts & BEGIN_DATA)
7700 0 : fprintf (spec_info->dump, "; data-spec;");
7701 0 : if (new_ts & BEGIN_CONTROL)
7702 0 : fprintf (spec_info->dump, "; control-spec;");
7703 0 : if (new_ts & BE_IN_CONTROL)
7704 0 : fprintf (spec_info->dump, "; in-control-spec;");
7705 : }
7706 0 : if (TODO_SPEC (next) & DEP_CONTROL)
7707 0 : fprintf (sched_dump, " predicated");
7708 0 : fprintf (sched_dump, "\n");
7709 : }
7710 :
7711 108621144 : adjust_priority (next);
7712 :
7713 108621144 : return fix_tick_ready (next);
7714 : }
7715 :
7716 : /* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */
7717 : static int
7718 108932384 : fix_tick_ready (rtx_insn *next)
7719 : {
7720 108932384 : int tick, delay;
7721 :
7722 108932384 : if (!DEBUG_INSN_P (next) && !sd_lists_empty_p (next, SD_LIST_RES_BACK))
7723 : {
7724 42084101 : int full_p;
7725 42084101 : sd_iterator_def sd_it;
7726 42084101 : dep_t dep;
7727 :
7728 42084101 : tick = INSN_TICK (next);
7729 : /* if tick is not equal to INVALID_TICK, then update
7730 : INSN_TICK of NEXT with the most recent resolved dependence
7731 : cost. Otherwise, recalculate from scratch. */
7732 42084101 : full_p = (tick == INVALID_TICK);
7733 :
7734 189783037 : FOR_EACH_DEP (next, SD_LIST_RES_BACK, sd_it, dep)
7735 : {
7736 147698948 : rtx_insn *pro = DEP_PRO (dep);
7737 147698948 : int tick1;
7738 :
7739 147698948 : gcc_assert (INSN_TICK (pro) >= MIN_TICK);
7740 :
7741 147698948 : tick1 = INSN_TICK (pro) + dep_cost (dep);
7742 147698948 : if (tick1 > tick)
7743 : tick = tick1;
7744 :
7745 147698948 : if (!full_p)
7746 : break;
7747 : }
7748 : }
7749 : else
7750 : tick = -1;
7751 :
7752 108932384 : INSN_TICK (next) = tick;
7753 :
7754 108932384 : delay = tick - clock_var;
7755 108932384 : if (delay <= 0 || sched_pressure != SCHED_PRESSURE_NONE || sched_fusion)
7756 89535717 : delay = QUEUE_READY;
7757 :
7758 108932384 : change_queue_index (next, delay);
7759 :
7760 108932384 : return delay;
7761 : }
7762 :
7763 : /* Move NEXT to the proper queue list with (DELAY >= 1),
7764 : or add it to the ready list (DELAY == QUEUE_READY),
7765 : or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */
7766 : static void
7767 311621151 : change_queue_index (rtx_insn *next, int delay)
7768 : {
7769 311621151 : int i = QUEUE_INDEX (next);
7770 :
7771 311621151 : gcc_assert (QUEUE_NOWHERE <= delay && delay <= max_insn_queue_index
7772 : && delay != 0);
7773 311621151 : gcc_assert (i != QUEUE_SCHEDULED);
7774 :
7775 311621151 : if ((delay > 0 && NEXT_Q_AFTER (q_ptr, delay) == i)
7776 311500958 : || (delay < 0 && delay == i))
7777 : /* We have nothing to do. */
7778 : return;
7779 :
7780 : /* Remove NEXT from wherever it is now. */
7781 108759411 : if (i == QUEUE_READY)
7782 0 : ready_remove_insn (next);
7783 108759411 : else if (i >= 0)
7784 138267 : queue_remove (next);
7785 :
7786 : /* Add it to the proper place. */
7787 108759411 : if (delay == QUEUE_READY)
7788 89482937 : ready_add (readyp, next, false);
7789 19276474 : else if (delay >= 1)
7790 19276474 : queue_insn (next, delay, "change queue index");
7791 :
7792 108759411 : if (sched_verbose >= 2)
7793 : {
7794 0 : fprintf (sched_dump, ";;\t\ttick updated: insn %s",
7795 0 : (*current_sched_info->print_insn) (next, 0));
7796 :
7797 0 : if (delay == QUEUE_READY)
7798 0 : fprintf (sched_dump, " into ready\n");
7799 0 : else if (delay >= 1)
7800 0 : fprintf (sched_dump, " into queue with cost=%d\n", delay);
7801 : else
7802 0 : fprintf (sched_dump, " removed from ready or queue lists\n");
7803 : }
7804 : }
7805 :
7806 : static int sched_ready_n_insns = -1;
7807 :
7808 : /* Initialize per region data structures. */
7809 : void
7810 10331445 : sched_extend_ready_list (int new_sched_ready_n_insns)
7811 : {
7812 10331445 : int i;
7813 :
7814 10331445 : if (sched_ready_n_insns == -1)
7815 : /* At the first call we need to initialize one more choice_stack
7816 : entry. */
7817 : {
7818 10331332 : i = 0;
7819 10331332 : sched_ready_n_insns = 0;
7820 10331332 : scheduled_insns.reserve (new_sched_ready_n_insns);
7821 : }
7822 : else
7823 113 : i = sched_ready_n_insns + 1;
7824 :
7825 10331445 : ready.veclen = new_sched_ready_n_insns + issue_rate;
7826 10331445 : ready.vec = XRESIZEVEC (rtx_insn *, ready.vec, ready.veclen);
7827 :
7828 10331445 : gcc_assert (new_sched_ready_n_insns >= sched_ready_n_insns);
7829 :
7830 10331445 : ready_try = (signed char *) xrecalloc (ready_try, new_sched_ready_n_insns,
7831 : sched_ready_n_insns,
7832 : sizeof (*ready_try));
7833 :
7834 : /* We allocate +1 element to save initial state in the choice_stack[0]
7835 : entry. */
7836 10331445 : choice_stack = XRESIZEVEC (struct choice_entry, choice_stack,
7837 : new_sched_ready_n_insns + 1);
7838 :
7839 129285316 : for (; i <= new_sched_ready_n_insns; i++)
7840 : {
7841 118953871 : choice_stack[i].state = xmalloc (dfa_state_size);
7842 :
7843 118953871 : if (targetm.sched.first_cycle_multipass_init)
7844 118648174 : targetm.sched.first_cycle_multipass_init (&(choice_stack[i]
7845 : .target_data));
7846 : }
7847 :
7848 10331445 : sched_ready_n_insns = new_sched_ready_n_insns;
7849 10331445 : }
7850 :
7851 : /* Free per region data structures. */
7852 : void
7853 10331332 : sched_finish_ready_list (void)
7854 : {
7855 10331332 : int i;
7856 :
7857 10331332 : free (ready.vec);
7858 10331332 : ready.vec = NULL;
7859 10331332 : ready.veclen = 0;
7860 :
7861 10331332 : free (ready_try);
7862 10331332 : ready_try = NULL;
7863 :
7864 129285203 : for (i = 0; i <= sched_ready_n_insns; i++)
7865 : {
7866 118953871 : if (targetm.sched.first_cycle_multipass_fini)
7867 118648174 : targetm.sched.first_cycle_multipass_fini (&(choice_stack[i]
7868 : .target_data));
7869 :
7870 118953871 : free (choice_stack [i].state);
7871 : }
7872 10331332 : free (choice_stack);
7873 10331332 : choice_stack = NULL;
7874 :
7875 10331332 : sched_ready_n_insns = -1;
7876 10331332 : }
7877 :
7878 : static int
7879 23702372 : haifa_luid_for_non_insn (rtx x)
7880 : {
7881 23702372 : gcc_assert (NOTE_P (x) || LABEL_P (x));
7882 :
7883 23702372 : return 0;
7884 : }
7885 :
7886 : /* Generates recovery code for INSN. */
7887 : static void
7888 0 : generate_recovery_code (rtx_insn *insn)
7889 : {
7890 0 : if (TODO_SPEC (insn) & BEGIN_SPEC)
7891 0 : begin_speculative_block (insn);
7892 :
7893 : /* Here we have insn with no dependencies to
7894 : instructions other then CHECK_SPEC ones. */
7895 :
7896 0 : if (TODO_SPEC (insn) & BE_IN_SPEC)
7897 0 : add_to_speculative_block (insn);
7898 0 : }
7899 :
7900 : /* Helper function.
7901 : Tries to add speculative dependencies of type FS between instructions
7902 : in deps_list L and TWIN. */
7903 : static void
7904 0 : process_insn_forw_deps_be_in_spec (rtx_insn *insn, rtx_insn *twin, ds_t fs)
7905 : {
7906 0 : sd_iterator_def sd_it;
7907 0 : dep_t dep;
7908 :
7909 0 : FOR_EACH_DEP (insn, SD_LIST_FORW, sd_it, dep)
7910 : {
7911 0 : ds_t ds;
7912 0 : rtx_insn *consumer;
7913 :
7914 0 : consumer = DEP_CON (dep);
7915 :
7916 0 : ds = DEP_STATUS (dep);
7917 :
7918 0 : if (/* If we want to create speculative dep. */
7919 : fs
7920 : /* And we can do that because this is a true dep. */
7921 0 : && (ds & DEP_TYPES) == DEP_TRUE)
7922 : {
7923 0 : gcc_assert (!(ds & BE_IN_SPEC));
7924 :
7925 0 : if (/* If this dep can be overcome with 'begin speculation'. */
7926 0 : ds & BEGIN_SPEC)
7927 : /* Then we have a choice: keep the dep 'begin speculative'
7928 : or transform it into 'be in speculative'. */
7929 : {
7930 0 : if (/* In try_ready we assert that if insn once became ready
7931 : it can be removed from the ready (or queue) list only
7932 : due to backend decision. Hence we can't let the
7933 : probability of the speculative dep to decrease. */
7934 0 : ds_weak (ds) <= ds_weak (fs))
7935 : {
7936 0 : ds_t new_ds;
7937 :
7938 0 : new_ds = (ds & ~BEGIN_SPEC) | fs;
7939 :
7940 0 : if (/* consumer can 'be in speculative'. */
7941 0 : sched_insn_is_legitimate_for_speculation_p (consumer,
7942 : new_ds))
7943 : /* Transform it to be in speculative. */
7944 0 : ds = new_ds;
7945 : }
7946 : }
7947 : else
7948 : /* Mark the dep as 'be in speculative'. */
7949 0 : ds |= fs;
7950 : }
7951 :
7952 0 : {
7953 0 : dep_def _new_dep, *new_dep = &_new_dep;
7954 :
7955 0 : init_dep_1 (new_dep, twin, consumer, DEP_TYPE (dep), ds);
7956 0 : sd_add_dep (new_dep, false);
7957 : }
7958 : }
7959 0 : }
7960 :
7961 : /* Generates recovery code for BEGIN speculative INSN. */
7962 : static void
7963 0 : begin_speculative_block (rtx_insn *insn)
7964 : {
7965 0 : if (TODO_SPEC (insn) & BEGIN_DATA)
7966 0 : nr_begin_data++;
7967 0 : if (TODO_SPEC (insn) & BEGIN_CONTROL)
7968 0 : nr_begin_control++;
7969 :
7970 0 : create_check_block_twin (insn, false);
7971 :
7972 0 : TODO_SPEC (insn) &= ~BEGIN_SPEC;
7973 0 : }
7974 :
7975 : static void haifa_init_insn (rtx_insn *);
7976 :
7977 : /* Generates recovery code for BE_IN speculative INSN. */
7978 : static void
7979 0 : add_to_speculative_block (rtx_insn *insn)
7980 : {
7981 0 : ds_t ts;
7982 0 : sd_iterator_def sd_it;
7983 0 : dep_t dep;
7984 0 : auto_vec<rtx_insn *, 10> twins;
7985 :
7986 0 : ts = TODO_SPEC (insn);
7987 0 : gcc_assert (!(ts & ~BE_IN_SPEC));
7988 :
7989 0 : if (ts & BE_IN_DATA)
7990 0 : nr_be_in_data++;
7991 0 : if (ts & BE_IN_CONTROL)
7992 0 : nr_be_in_control++;
7993 :
7994 0 : TODO_SPEC (insn) &= ~BE_IN_SPEC;
7995 0 : gcc_assert (!TODO_SPEC (insn));
7996 :
7997 0 : DONE_SPEC (insn) |= ts;
7998 :
7999 : /* First we convert all simple checks to branchy. */
8000 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8001 0 : sd_iterator_cond (&sd_it, &dep);)
8002 : {
8003 0 : rtx_insn *check = DEP_PRO (dep);
8004 :
8005 0 : if (IS_SPECULATION_SIMPLE_CHECK_P (check))
8006 : {
8007 0 : create_check_block_twin (check, true);
8008 :
8009 : /* Restart search. */
8010 0 : sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8011 : }
8012 : else
8013 : /* Continue search. */
8014 0 : sd_iterator_next (&sd_it);
8015 : }
8016 :
8017 0 : auto_vec<rtx_insn *> priorities_roots;
8018 0 : clear_priorities (insn, &priorities_roots);
8019 :
8020 0 : while (1)
8021 : {
8022 0 : rtx_insn *check, *twin;
8023 0 : basic_block rec;
8024 :
8025 : /* Get the first backward dependency of INSN. */
8026 0 : sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8027 0 : if (!sd_iterator_cond (&sd_it, &dep))
8028 : /* INSN has no backward dependencies left. */
8029 : break;
8030 :
8031 0 : gcc_assert ((DEP_STATUS (dep) & BEGIN_SPEC) == 0
8032 : && (DEP_STATUS (dep) & BE_IN_SPEC) != 0
8033 : && (DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE);
8034 :
8035 0 : check = DEP_PRO (dep);
8036 :
8037 0 : gcc_assert (!IS_SPECULATION_CHECK_P (check) && !ORIG_PAT (check)
8038 : && QUEUE_INDEX (check) == QUEUE_NOWHERE);
8039 :
8040 0 : rec = BLOCK_FOR_INSN (check);
8041 :
8042 0 : twin = emit_insn_before (copy_insn (PATTERN (insn)), BB_END (rec));
8043 0 : haifa_init_insn (twin);
8044 :
8045 0 : sd_copy_back_deps (twin, insn, true);
8046 :
8047 0 : if (sched_verbose && spec_info->dump)
8048 : /* INSN_BB (insn) isn't determined for twin insns yet.
8049 : So we can't use current_sched_info->print_insn. */
8050 0 : fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n",
8051 0 : INSN_UID (twin), rec->index);
8052 :
8053 0 : twins.safe_push (twin);
8054 :
8055 : /* Add dependences between TWIN and all appropriate
8056 : instructions from REC. */
8057 0 : FOR_EACH_DEP (insn, SD_LIST_SPEC_BACK, sd_it, dep)
8058 : {
8059 0 : rtx_insn *pro = DEP_PRO (dep);
8060 :
8061 0 : gcc_assert (DEP_TYPE (dep) == REG_DEP_TRUE);
8062 :
8063 : /* INSN might have dependencies from the instructions from
8064 : several recovery blocks. At this iteration we process those
8065 : producers that reside in REC. */
8066 0 : if (BLOCK_FOR_INSN (pro) == rec)
8067 : {
8068 0 : dep_def _new_dep, *new_dep = &_new_dep;
8069 :
8070 0 : init_dep (new_dep, pro, twin, REG_DEP_TRUE);
8071 0 : sd_add_dep (new_dep, false);
8072 : }
8073 : }
8074 :
8075 0 : process_insn_forw_deps_be_in_spec (insn, twin, ts);
8076 :
8077 : /* Remove all dependencies between INSN and insns in REC. */
8078 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8079 0 : sd_iterator_cond (&sd_it, &dep);)
8080 : {
8081 0 : rtx_insn *pro = DEP_PRO (dep);
8082 :
8083 0 : if (BLOCK_FOR_INSN (pro) == rec)
8084 0 : sd_delete_dep (sd_it);
8085 : else
8086 0 : sd_iterator_next (&sd_it);
8087 : }
8088 0 : }
8089 :
8090 : /* We couldn't have added the dependencies between INSN and TWINS earlier
8091 : because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */
8092 0 : unsigned int i;
8093 0 : rtx_insn *twin;
8094 0 : FOR_EACH_VEC_ELT_REVERSE (twins, i, twin)
8095 : {
8096 0 : dep_def _new_dep, *new_dep = &_new_dep;
8097 :
8098 0 : init_dep (new_dep, insn, twin, REG_DEP_OUTPUT);
8099 0 : sd_add_dep (new_dep, false);
8100 : }
8101 :
8102 0 : calc_priorities (priorities_roots);
8103 0 : }
8104 :
8105 : /* Extends and fills with zeros (only the new part) array pointed to by P. */
8106 : void *
8107 10331552 : xrecalloc (void *p, size_t new_nmemb, size_t old_nmemb, size_t size)
8108 : {
8109 10331552 : gcc_assert (new_nmemb >= old_nmemb);
8110 10331552 : p = XRESIZEVAR (void, p, new_nmemb * size);
8111 10331552 : memset (((char *) p) + old_nmemb * size, 0, (new_nmemb - old_nmemb) * size);
8112 10331552 : return p;
8113 : }
8114 :
8115 : /* Helper function.
8116 : Find fallthru edge from PRED. */
8117 : edge
8118 108 : find_fallthru_edge_from (basic_block pred)
8119 : {
8120 108 : edge e;
8121 108 : basic_block succ;
8122 :
8123 108 : succ = pred->next_bb;
8124 108 : gcc_assert (succ->prev_bb == pred);
8125 :
8126 324 : if (EDGE_COUNT (pred->succs) <= EDGE_COUNT (succ->preds))
8127 : {
8128 35 : e = find_fallthru_edge (pred->succs);
8129 :
8130 35 : if (e)
8131 : {
8132 23 : gcc_assert (e->dest == succ || e->dest->index == EXIT_BLOCK);
8133 : return e;
8134 : }
8135 : }
8136 : else
8137 : {
8138 73 : e = find_fallthru_edge (succ->preds);
8139 :
8140 73 : if (e)
8141 : {
8142 73 : gcc_assert (e->src == pred);
8143 : return e;
8144 : }
8145 : }
8146 :
8147 : return NULL;
8148 : }
8149 :
8150 : /* Extend per basic block data structures. */
8151 : static void
8152 964556 : sched_extend_bb (void)
8153 : {
8154 : /* The following is done to keep current_sched_info->next_tail non null. */
8155 964556 : rtx_insn *end = BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb);
8156 964556 : rtx_insn *insn = DEBUG_INSN_P (end) ? prev_nondebug_insn (end) : end;
8157 964556 : if (NEXT_INSN (end) == 0
8158 964556 : || (!NOTE_P (insn)
8159 964094 : && !LABEL_P (insn)
8160 : /* Don't emit a NOTE if it would end up before a BARRIER. */
8161 964075 : && !BARRIER_P (next_nondebug_insn (end))))
8162 : {
8163 552 : rtx_note *note = emit_note_after (NOTE_INSN_DELETED, end);
8164 : /* Make note appear outside BB. */
8165 552 : set_block_for_insn (note, NULL);
8166 552 : BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb) = end;
8167 : }
8168 964556 : }
8169 :
8170 : /* Init per basic block data structures. */
8171 : void
8172 964556 : sched_init_bbs (void)
8173 : {
8174 964556 : sched_extend_bb ();
8175 964556 : }
8176 :
8177 : /* Initialize BEFORE_RECOVERY variable. */
8178 : static void
8179 0 : init_before_recovery (basic_block *before_recovery_ptr)
8180 : {
8181 0 : basic_block last;
8182 0 : edge e;
8183 :
8184 0 : last = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
8185 0 : e = find_fallthru_edge_from (last);
8186 :
8187 0 : if (e)
8188 : {
8189 : /* We create two basic blocks:
8190 : 1. Single instruction block is inserted right after E->SRC
8191 : and has jump to
8192 : 2. Empty block right before EXIT_BLOCK.
8193 : Between these two blocks recovery blocks will be emitted. */
8194 :
8195 0 : basic_block single, empty;
8196 :
8197 : /* If the fallthrough edge to exit we've found is from the block we've
8198 : created before, don't do anything more. */
8199 0 : if (last == after_recovery)
8200 : return;
8201 :
8202 0 : adding_bb_to_current_region_p = false;
8203 :
8204 0 : single = sched_create_empty_bb (last);
8205 0 : empty = sched_create_empty_bb (single);
8206 :
8207 : /* Add new blocks to the root loop. */
8208 0 : if (current_loops != NULL)
8209 : {
8210 0 : add_bb_to_loop (single, (*current_loops->larray)[0]);
8211 0 : add_bb_to_loop (empty, (*current_loops->larray)[0]);
8212 : }
8213 :
8214 0 : single->count = last->count;
8215 0 : empty->count = last->count;
8216 0 : BB_COPY_PARTITION (single, last);
8217 0 : BB_COPY_PARTITION (empty, last);
8218 :
8219 0 : redirect_edge_succ (e, single);
8220 0 : make_single_succ_edge (single, empty, 0);
8221 0 : make_single_succ_edge (empty, EXIT_BLOCK_PTR_FOR_FN (cfun),
8222 : EDGE_FALLTHRU);
8223 :
8224 0 : rtx_code_label *label = block_label (empty);
8225 0 : rtx_jump_insn *x = emit_jump_insn_after (targetm.gen_jump (label),
8226 0 : BB_END (single));
8227 0 : JUMP_LABEL (x) = label;
8228 0 : LABEL_NUSES (label)++;
8229 0 : haifa_init_insn (x);
8230 :
8231 0 : emit_barrier_after (x);
8232 :
8233 0 : sched_init_only_bb (empty, NULL);
8234 0 : sched_init_only_bb (single, NULL);
8235 0 : sched_extend_bb ();
8236 :
8237 0 : adding_bb_to_current_region_p = true;
8238 0 : before_recovery = single;
8239 0 : after_recovery = empty;
8240 :
8241 0 : if (before_recovery_ptr)
8242 0 : *before_recovery_ptr = before_recovery;
8243 :
8244 0 : if (sched_verbose >= 2 && spec_info->dump)
8245 0 : fprintf (spec_info->dump,
8246 : ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n",
8247 : last->index, single->index, empty->index);
8248 : }
8249 : else
8250 0 : before_recovery = last;
8251 : }
8252 :
8253 : /* Returns new recovery block. */
8254 : basic_block
8255 0 : sched_create_recovery_block (basic_block *before_recovery_ptr)
8256 : {
8257 0 : rtx_insn *barrier;
8258 0 : basic_block rec;
8259 :
8260 0 : haifa_recovery_bb_recently_added_p = true;
8261 0 : haifa_recovery_bb_ever_added_p = true;
8262 :
8263 0 : init_before_recovery (before_recovery_ptr);
8264 :
8265 0 : barrier = get_last_bb_insn (before_recovery);
8266 0 : gcc_assert (BARRIER_P (barrier));
8267 :
8268 0 : rtx_insn *label = emit_label_after (gen_label_rtx (), barrier);
8269 :
8270 0 : rec = create_basic_block (label, label, before_recovery);
8271 :
8272 : /* A recovery block always ends with an unconditional jump. */
8273 0 : emit_barrier_after (BB_END (rec));
8274 :
8275 0 : if (BB_PARTITION (before_recovery) != BB_UNPARTITIONED)
8276 0 : BB_SET_PARTITION (rec, BB_COLD_PARTITION);
8277 :
8278 0 : if (sched_verbose && spec_info->dump)
8279 0 : fprintf (spec_info->dump, ";;\t\tGenerated recovery block rec%d\n",
8280 : rec->index);
8281 :
8282 0 : return rec;
8283 : }
8284 :
8285 : /* Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB
8286 : and emit necessary jumps. */
8287 : void
8288 0 : sched_create_recovery_edges (basic_block first_bb, basic_block rec,
8289 : basic_block second_bb)
8290 : {
8291 0 : int edge_flags;
8292 :
8293 : /* This is fixing of incoming edge. */
8294 : /* ??? Which other flags should be specified? */
8295 0 : if (BB_PARTITION (first_bb) != BB_PARTITION (rec))
8296 : /* Partition type is the same, if it is "unpartitioned". */
8297 : edge_flags = EDGE_CROSSING;
8298 : else
8299 0 : edge_flags = 0;
8300 :
8301 0 : edge e2 = single_succ_edge (first_bb);
8302 0 : edge e = make_edge (first_bb, rec, edge_flags);
8303 :
8304 : /* TODO: The actual probability can be determined and is computed as
8305 : 'todo_spec' variable in create_check_block_twin and
8306 : in sel-sched.cc `check_ds' in create_speculation_check. */
8307 0 : e->probability = profile_probability::very_unlikely ();
8308 0 : rec->count = e->count ();
8309 0 : e2->probability = e->probability.invert ();
8310 :
8311 0 : rtx_code_label *label = block_label (second_bb);
8312 0 : rtx_jump_insn *jump = emit_jump_insn_after (targetm.gen_jump (label),
8313 0 : BB_END (rec));
8314 0 : JUMP_LABEL (jump) = label;
8315 0 : LABEL_NUSES (label)++;
8316 :
8317 0 : if (BB_PARTITION (second_bb) != BB_PARTITION (rec))
8318 : /* Partition type is the same, if it is "unpartitioned". */
8319 : {
8320 : /* Rewritten from cfgrtl.cc. */
8321 0 : if (crtl->has_bb_partition && targetm_common.have_named_sections)
8322 : {
8323 : /* We don't need the same note for the check because
8324 : any_condjump_p (check) == true. */
8325 0 : CROSSING_JUMP_P (jump) = 1;
8326 : }
8327 : edge_flags = EDGE_CROSSING;
8328 : }
8329 : else
8330 : edge_flags = 0;
8331 :
8332 0 : make_single_succ_edge (rec, second_bb, edge_flags);
8333 0 : if (dom_info_available_p (CDI_DOMINATORS))
8334 0 : set_immediate_dominator (CDI_DOMINATORS, rec, first_bb);
8335 0 : }
8336 :
8337 : /* This function creates recovery code for INSN. If MUTATE_P is nonzero,
8338 : INSN is a simple check, that should be converted to branchy one. */
8339 : static void
8340 0 : create_check_block_twin (rtx_insn *insn, bool mutate_p)
8341 : {
8342 0 : basic_block rec;
8343 0 : rtx_insn *label, *check, *twin;
8344 0 : rtx check_pat;
8345 0 : ds_t fs;
8346 0 : sd_iterator_def sd_it;
8347 0 : dep_t dep;
8348 0 : dep_def _new_dep, *new_dep = &_new_dep;
8349 0 : ds_t todo_spec;
8350 :
8351 0 : gcc_assert (ORIG_PAT (insn) != NULL_RTX);
8352 :
8353 0 : if (!mutate_p)
8354 0 : todo_spec = TODO_SPEC (insn);
8355 : else
8356 : {
8357 0 : gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn)
8358 : && (TODO_SPEC (insn) & SPECULATIVE) == 0);
8359 :
8360 0 : todo_spec = CHECK_SPEC (insn);
8361 : }
8362 :
8363 0 : todo_spec &= SPECULATIVE;
8364 :
8365 : /* Create recovery block. */
8366 0 : if (mutate_p || targetm.sched.needs_block_p (todo_spec))
8367 : {
8368 0 : rec = sched_create_recovery_block (NULL);
8369 0 : label = BB_HEAD (rec);
8370 : }
8371 : else
8372 : {
8373 0 : rec = EXIT_BLOCK_PTR_FOR_FN (cfun);
8374 0 : label = NULL;
8375 : }
8376 :
8377 : /* Emit CHECK. */
8378 0 : check_pat = targetm.sched.gen_spec_check (insn, label, todo_spec);
8379 :
8380 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8381 : {
8382 : /* To have mem_reg alive at the beginning of second_bb,
8383 : we emit check BEFORE insn, so insn after splitting
8384 : insn will be at the beginning of second_bb, which will
8385 : provide us with the correct life information. */
8386 0 : check = emit_jump_insn_before (check_pat, insn);
8387 0 : JUMP_LABEL (check) = label;
8388 0 : LABEL_NUSES (label)++;
8389 : }
8390 : else
8391 0 : check = emit_insn_before (check_pat, insn);
8392 :
8393 : /* Extend data structures. */
8394 0 : haifa_init_insn (check);
8395 :
8396 : /* CHECK is being added to current region. Extend ready list. */
8397 0 : gcc_assert (sched_ready_n_insns != -1);
8398 0 : sched_extend_ready_list (sched_ready_n_insns + 1);
8399 :
8400 0 : if (current_sched_info->add_remove_insn)
8401 0 : current_sched_info->add_remove_insn (insn, 0);
8402 :
8403 0 : RECOVERY_BLOCK (check) = rec;
8404 :
8405 0 : if (sched_verbose && spec_info->dump)
8406 0 : fprintf (spec_info->dump, ";;\t\tGenerated check insn : %s\n",
8407 0 : (*current_sched_info->print_insn) (check, 0));
8408 :
8409 0 : gcc_assert (ORIG_PAT (insn));
8410 :
8411 : /* Initialize TWIN (twin is a duplicate of original instruction
8412 : in the recovery block). */
8413 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8414 : {
8415 0 : sd_iterator_def sd_it;
8416 0 : dep_t dep;
8417 :
8418 0 : FOR_EACH_DEP (insn, SD_LIST_RES_BACK, sd_it, dep)
8419 0 : if ((DEP_STATUS (dep) & DEP_OUTPUT) != 0)
8420 : {
8421 0 : struct _dep _dep2, *dep2 = &_dep2;
8422 :
8423 0 : init_dep (dep2, DEP_PRO (dep), check, REG_DEP_TRUE);
8424 :
8425 0 : sd_add_dep (dep2, true);
8426 : }
8427 :
8428 0 : twin = emit_insn_after (ORIG_PAT (insn), BB_END (rec));
8429 0 : haifa_init_insn (twin);
8430 :
8431 0 : if (sched_verbose && spec_info->dump)
8432 : /* INSN_BB (insn) isn't determined for twin insns yet.
8433 : So we can't use current_sched_info->print_insn. */
8434 0 : fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n",
8435 0 : INSN_UID (twin), rec->index);
8436 : }
8437 : else
8438 : {
8439 0 : ORIG_PAT (check) = ORIG_PAT (insn);
8440 0 : HAS_INTERNAL_DEP (check) = 1;
8441 0 : twin = check;
8442 : /* ??? We probably should change all OUTPUT dependencies to
8443 : (TRUE | OUTPUT). */
8444 : }
8445 :
8446 : /* Copy all resolved back dependencies of INSN to TWIN. This will
8447 : provide correct value for INSN_TICK (TWIN). */
8448 0 : sd_copy_back_deps (twin, insn, true);
8449 :
8450 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8451 : /* In case of branchy check, fix CFG. */
8452 : {
8453 0 : basic_block first_bb, second_bb;
8454 0 : rtx_insn *jump;
8455 :
8456 0 : first_bb = BLOCK_FOR_INSN (check);
8457 0 : second_bb = sched_split_block (first_bb, check);
8458 :
8459 0 : sched_create_recovery_edges (first_bb, rec, second_bb);
8460 :
8461 0 : sched_init_only_bb (second_bb, first_bb);
8462 0 : sched_init_only_bb (rec, EXIT_BLOCK_PTR_FOR_FN (cfun));
8463 :
8464 0 : jump = BB_END (rec);
8465 0 : haifa_init_insn (jump);
8466 : }
8467 :
8468 : /* Move backward dependences from INSN to CHECK and
8469 : move forward dependences from INSN to TWIN. */
8470 :
8471 : /* First, create dependencies between INSN's producers and CHECK & TWIN. */
8472 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
8473 : {
8474 0 : rtx_insn *pro = DEP_PRO (dep);
8475 0 : ds_t ds;
8476 :
8477 : /* If BEGIN_DATA: [insn ~~TRUE~~> producer]:
8478 : check --TRUE--> producer ??? or ANTI ???
8479 : twin --TRUE--> producer
8480 : twin --ANTI--> check
8481 :
8482 : If BEGIN_CONTROL: [insn ~~ANTI~~> producer]:
8483 : check --ANTI--> producer
8484 : twin --ANTI--> producer
8485 : twin --ANTI--> check
8486 :
8487 : If BE_IN_SPEC: [insn ~~TRUE~~> producer]:
8488 : check ~~TRUE~~> producer
8489 : twin ~~TRUE~~> producer
8490 : twin --ANTI--> check */
8491 :
8492 0 : ds = DEP_STATUS (dep);
8493 :
8494 0 : if (ds & BEGIN_SPEC)
8495 : {
8496 0 : gcc_assert (!mutate_p);
8497 0 : ds &= ~BEGIN_SPEC;
8498 : }
8499 :
8500 0 : init_dep_1 (new_dep, pro, check, DEP_TYPE (dep), ds);
8501 0 : sd_add_dep (new_dep, false);
8502 :
8503 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8504 : {
8505 0 : DEP_CON (new_dep) = twin;
8506 0 : sd_add_dep (new_dep, false);
8507 : }
8508 : }
8509 :
8510 : /* Second, remove backward dependencies of INSN. */
8511 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8512 0 : sd_iterator_cond (&sd_it, &dep);)
8513 : {
8514 0 : if ((DEP_STATUS (dep) & BEGIN_SPEC)
8515 0 : || mutate_p)
8516 : /* We can delete this dep because we overcome it with
8517 : BEGIN_SPECULATION. */
8518 0 : sd_delete_dep (sd_it);
8519 : else
8520 0 : sd_iterator_next (&sd_it);
8521 : }
8522 :
8523 : /* Future Speculations. Determine what BE_IN speculations will be like. */
8524 0 : fs = 0;
8525 :
8526 : /* Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only
8527 : here. */
8528 :
8529 0 : gcc_assert (!DONE_SPEC (insn));
8530 :
8531 0 : if (!mutate_p)
8532 : {
8533 0 : ds_t ts = TODO_SPEC (insn);
8534 :
8535 0 : DONE_SPEC (insn) = ts & BEGIN_SPEC;
8536 0 : CHECK_SPEC (check) = ts & BEGIN_SPEC;
8537 :
8538 : /* Luckiness of future speculations solely depends upon initial
8539 : BEGIN speculation. */
8540 0 : if (ts & BEGIN_DATA)
8541 0 : fs = set_dep_weak (fs, BE_IN_DATA, get_dep_weak (ts, BEGIN_DATA));
8542 0 : if (ts & BEGIN_CONTROL)
8543 0 : fs = set_dep_weak (fs, BE_IN_CONTROL,
8544 : get_dep_weak (ts, BEGIN_CONTROL));
8545 : }
8546 : else
8547 0 : CHECK_SPEC (check) = CHECK_SPEC (insn);
8548 :
8549 : /* Future speculations: call the helper. */
8550 0 : process_insn_forw_deps_be_in_spec (insn, twin, fs);
8551 :
8552 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8553 : {
8554 : /* Which types of dependencies should we use here is,
8555 : generally, machine-dependent question... But, for now,
8556 : it is not. */
8557 :
8558 0 : if (!mutate_p)
8559 : {
8560 0 : init_dep (new_dep, insn, check, REG_DEP_TRUE);
8561 0 : sd_add_dep (new_dep, false);
8562 :
8563 0 : init_dep (new_dep, insn, twin, REG_DEP_OUTPUT);
8564 0 : sd_add_dep (new_dep, false);
8565 : }
8566 : else
8567 : {
8568 0 : if (spec_info->dump)
8569 0 : fprintf (spec_info->dump, ";;\t\tRemoved simple check : %s\n",
8570 0 : (*current_sched_info->print_insn) (insn, 0));
8571 :
8572 : /* Remove all dependencies of the INSN. */
8573 0 : {
8574 0 : sd_it = sd_iterator_start (insn, (SD_LIST_FORW
8575 : | SD_LIST_BACK
8576 : | SD_LIST_RES_BACK));
8577 0 : while (sd_iterator_cond (&sd_it, &dep))
8578 0 : sd_delete_dep (sd_it);
8579 : }
8580 :
8581 : /* If former check (INSN) already was moved to the ready (or queue)
8582 : list, add new check (CHECK) there too. */
8583 0 : if (QUEUE_INDEX (insn) != QUEUE_NOWHERE)
8584 0 : try_ready (check);
8585 :
8586 : /* Remove old check from instruction stream and free its
8587 : data. */
8588 0 : sched_remove_insn (insn);
8589 : }
8590 :
8591 0 : init_dep (new_dep, check, twin, REG_DEP_ANTI);
8592 0 : sd_add_dep (new_dep, false);
8593 : }
8594 : else
8595 : {
8596 0 : init_dep_1 (new_dep, insn, check, REG_DEP_TRUE, DEP_TRUE | DEP_OUTPUT);
8597 0 : sd_add_dep (new_dep, false);
8598 : }
8599 :
8600 0 : if (!mutate_p)
8601 : /* Fix priorities. If MUTATE_P is nonzero, this is not necessary,
8602 : because it'll be done later in add_to_speculative_block. */
8603 : {
8604 0 : auto_vec<rtx_insn *> priorities_roots;
8605 :
8606 0 : clear_priorities (twin, &priorities_roots);
8607 0 : calc_priorities (priorities_roots);
8608 0 : }
8609 0 : }
8610 :
8611 : /* Removes dependency between instructions in the recovery block REC
8612 : and usual region instructions. It keeps inner dependences so it
8613 : won't be necessary to recompute them. */
8614 : static void
8615 0 : fix_recovery_deps (basic_block rec)
8616 : {
8617 0 : rtx_insn *note, *insn, *jump;
8618 0 : auto_vec<rtx_insn *, 10> ready_list;
8619 0 : auto_bitmap in_ready;
8620 :
8621 : /* NOTE - a basic block note. */
8622 0 : note = NEXT_INSN (BB_HEAD (rec));
8623 0 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8624 0 : insn = BB_END (rec);
8625 0 : gcc_assert (JUMP_P (insn));
8626 0 : insn = PREV_INSN (insn);
8627 :
8628 0 : do
8629 : {
8630 0 : sd_iterator_def sd_it;
8631 0 : dep_t dep;
8632 :
8633 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
8634 0 : sd_iterator_cond (&sd_it, &dep);)
8635 : {
8636 0 : rtx_insn *consumer = DEP_CON (dep);
8637 :
8638 0 : if (BLOCK_FOR_INSN (consumer) != rec)
8639 : {
8640 0 : sd_delete_dep (sd_it);
8641 :
8642 0 : if (bitmap_set_bit (in_ready, INSN_LUID (consumer)))
8643 0 : ready_list.safe_push (consumer);
8644 : }
8645 : else
8646 : {
8647 0 : gcc_assert ((DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE);
8648 :
8649 0 : sd_iterator_next (&sd_it);
8650 : }
8651 : }
8652 :
8653 0 : insn = PREV_INSN (insn);
8654 : }
8655 0 : while (insn != note);
8656 :
8657 : /* Try to add instructions to the ready or queue list. */
8658 0 : unsigned int i;
8659 0 : rtx_insn *temp;
8660 0 : FOR_EACH_VEC_ELT_REVERSE (ready_list, i, temp)
8661 0 : try_ready (temp);
8662 :
8663 : /* Fixing jump's dependences. */
8664 0 : insn = BB_HEAD (rec);
8665 0 : jump = BB_END (rec);
8666 :
8667 0 : gcc_assert (LABEL_P (insn));
8668 0 : insn = NEXT_INSN (insn);
8669 :
8670 0 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn));
8671 0 : add_jump_dependencies (insn, jump);
8672 0 : }
8673 :
8674 : /* Change pattern of INSN to NEW_PAT. Invalidate cached haifa
8675 : instruction data. */
8676 : static bool
8677 0 : haifa_change_pattern (rtx_insn *insn, rtx new_pat)
8678 : {
8679 0 : int t;
8680 :
8681 0 : t = validate_change (insn, &PATTERN (insn), new_pat, 0);
8682 0 : if (!t)
8683 : return false;
8684 :
8685 0 : update_insn_after_change (insn);
8686 0 : return true;
8687 : }
8688 :
8689 : /* -1 - can't speculate,
8690 : 0 - for speculation with REQUEST mode it is OK to use
8691 : current instruction pattern,
8692 : 1 - need to change pattern for *NEW_PAT to be speculative. */
8693 : int
8694 0 : sched_speculate_insn (rtx_insn *insn, ds_t request, rtx *new_pat)
8695 : {
8696 0 : gcc_assert (current_sched_info->flags & DO_SPECULATION
8697 : && (request & SPECULATIVE)
8698 : && sched_insn_is_legitimate_for_speculation_p (insn, request));
8699 :
8700 0 : if ((request & spec_info->mask) != request)
8701 : return -1;
8702 :
8703 0 : if (request & BE_IN_SPEC
8704 0 : && !(request & BEGIN_SPEC))
8705 : return 0;
8706 :
8707 0 : return targetm.sched.speculate_insn (insn, request, new_pat);
8708 : }
8709 :
8710 : static int
8711 0 : haifa_speculate_insn (rtx_insn *insn, ds_t request, rtx *new_pat)
8712 : {
8713 0 : gcc_assert (sched_deps_info->generate_spec_deps
8714 : && !IS_SPECULATION_CHECK_P (insn));
8715 :
8716 0 : if (HAS_INTERNAL_DEP (insn)
8717 0 : || SCHED_GROUP_P (insn))
8718 : return -1;
8719 :
8720 0 : return sched_speculate_insn (insn, request, new_pat);
8721 : }
8722 :
8723 : /* Print some information about block BB, which starts with HEAD and
8724 : ends with TAIL, before scheduling it.
8725 : I is zero, if scheduler is about to start with the fresh ebb. */
8726 : static void
8727 192 : dump_new_block_header (int i, basic_block bb, rtx_insn *head, rtx_insn *tail)
8728 : {
8729 192 : if (!i)
8730 190 : fprintf (sched_dump,
8731 : ";; ======================================================\n");
8732 : else
8733 2 : fprintf (sched_dump,
8734 : ";; =====================ADVANCING TO=====================\n");
8735 192 : fprintf (sched_dump,
8736 : ";; -- basic block %d from %d to %d -- %s reload\n",
8737 192 : bb->index, INSN_UID (head), INSN_UID (tail),
8738 192 : (reload_completed ? "after" : "before"));
8739 192 : fprintf (sched_dump,
8740 : ";; ======================================================\n");
8741 192 : fprintf (sched_dump, "\n");
8742 192 : }
8743 :
8744 : /* Unlink basic block notes and labels and saves them, so they
8745 : can be easily restored. We unlink basic block notes in EBB to
8746 : provide back-compatibility with the previous code, as target backends
8747 : assume, that there'll be only instructions between
8748 : current_sched_info->{head and tail}. We restore these notes as soon
8749 : as we can.
8750 : FIRST (LAST) is the first (last) basic block in the ebb.
8751 : NB: In usual case (FIRST == LAST) nothing is really done. */
8752 : void
8753 10313337 : unlink_bb_notes (basic_block first, basic_block last)
8754 : {
8755 : /* We DON'T unlink basic block notes of the first block in the ebb. */
8756 10313337 : if (first == last)
8757 : return;
8758 :
8759 13 : bb_header = XNEWVEC (rtx_insn *, last_basic_block_for_fn (cfun));
8760 :
8761 : /* Make a sentinel. */
8762 13 : if (last->next_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
8763 13 : bb_header[last->next_bb->index] = 0;
8764 :
8765 13 : first = first->next_bb;
8766 81 : do
8767 : {
8768 47 : rtx_insn *prev, *label, *note, *next;
8769 :
8770 47 : label = BB_HEAD (last);
8771 47 : if (LABEL_P (label))
8772 0 : note = NEXT_INSN (label);
8773 : else
8774 : note = label;
8775 47 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8776 :
8777 47 : prev = PREV_INSN (label);
8778 47 : next = NEXT_INSN (note);
8779 47 : gcc_assert (prev && next);
8780 :
8781 47 : SET_NEXT_INSN (prev) = next;
8782 47 : SET_PREV_INSN (next) = prev;
8783 :
8784 47 : bb_header[last->index] = label;
8785 :
8786 47 : if (last == first)
8787 : break;
8788 :
8789 34 : last = last->prev_bb;
8790 34 : }
8791 : while (1);
8792 : }
8793 :
8794 : /* Restore basic block notes.
8795 : FIRST is the first basic block in the ebb. */
8796 : static void
8797 10313337 : restore_bb_notes (basic_block first)
8798 : {
8799 10313337 : if (!bb_header)
8800 : return;
8801 :
8802 : /* We DON'T unlink basic block notes of the first block in the ebb. */
8803 13 : first = first->next_bb;
8804 : /* Remember: FIRST is actually a second basic block in the ebb. */
8805 :
8806 13 : while (first != EXIT_BLOCK_PTR_FOR_FN (cfun)
8807 60 : && bb_header[first->index])
8808 : {
8809 47 : rtx_insn *prev, *label, *note, *next;
8810 :
8811 47 : label = bb_header[first->index];
8812 47 : prev = PREV_INSN (label);
8813 47 : next = NEXT_INSN (prev);
8814 :
8815 47 : if (LABEL_P (label))
8816 0 : note = NEXT_INSN (label);
8817 : else
8818 : note = label;
8819 47 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8820 :
8821 47 : bb_header[first->index] = 0;
8822 :
8823 47 : SET_NEXT_INSN (prev) = label;
8824 47 : SET_NEXT_INSN (note) = next;
8825 47 : SET_PREV_INSN (next) = note;
8826 :
8827 47 : first = first->next_bb;
8828 : }
8829 :
8830 13 : free (bb_header);
8831 13 : bb_header = 0;
8832 : }
8833 :
8834 : /* Helper function.
8835 : Fix CFG after both in- and inter-block movement of
8836 : control_flow_insn_p JUMP. */
8837 : static void
8838 1 : fix_jump_move (rtx_insn *jump)
8839 : {
8840 1 : basic_block bb, jump_bb, jump_bb_next;
8841 :
8842 1 : bb = BLOCK_FOR_INSN (PREV_INSN (jump));
8843 1 : jump_bb = BLOCK_FOR_INSN (jump);
8844 1 : jump_bb_next = jump_bb->next_bb;
8845 :
8846 1 : gcc_assert (common_sched_info->sched_pass_id == SCHED_EBB_PASS
8847 : || IS_SPECULATION_BRANCHY_CHECK_P (jump));
8848 :
8849 1 : if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next)))
8850 : /* if jump_bb_next is not empty. */
8851 0 : BB_END (jump_bb) = BB_END (jump_bb_next);
8852 :
8853 1 : if (BB_END (bb) != PREV_INSN (jump))
8854 : /* Then there are instruction after jump that should be placed
8855 : to jump_bb_next. */
8856 1 : BB_END (jump_bb_next) = BB_END (bb);
8857 : else
8858 : /* Otherwise jump_bb_next is empty. */
8859 0 : BB_END (jump_bb_next) = NEXT_INSN (BB_HEAD (jump_bb_next));
8860 :
8861 : /* To make assertion in move_insn happy. */
8862 1 : BB_END (bb) = PREV_INSN (jump);
8863 :
8864 1 : update_bb_for_insn (jump_bb_next);
8865 1 : }
8866 :
8867 : /* Fix CFG after interblock movement of control_flow_insn_p JUMP. */
8868 : static void
8869 0 : move_block_after_check (rtx_insn *jump)
8870 : {
8871 0 : basic_block bb, jump_bb, jump_bb_next;
8872 0 : vec<edge, va_gc> *t;
8873 :
8874 0 : bb = BLOCK_FOR_INSN (PREV_INSN (jump));
8875 0 : jump_bb = BLOCK_FOR_INSN (jump);
8876 0 : jump_bb_next = jump_bb->next_bb;
8877 :
8878 0 : update_bb_for_insn (jump_bb);
8879 :
8880 0 : gcc_assert (IS_SPECULATION_CHECK_P (jump)
8881 : || IS_SPECULATION_CHECK_P (BB_END (jump_bb_next)));
8882 :
8883 0 : unlink_block (jump_bb_next);
8884 0 : link_block (jump_bb_next, bb);
8885 :
8886 0 : t = bb->succs;
8887 0 : bb->succs = 0;
8888 0 : move_succs (&(jump_bb->succs), bb);
8889 0 : move_succs (&(jump_bb_next->succs), jump_bb);
8890 0 : move_succs (&t, jump_bb_next);
8891 :
8892 0 : df_mark_solutions_dirty ();
8893 :
8894 0 : common_sched_info->fix_recovery_cfg
8895 0 : (bb->index, jump_bb->index, jump_bb_next->index);
8896 0 : }
8897 :
8898 : /* Helper function for move_block_after_check.
8899 : This functions attaches edge vector pointed to by SUCCSP to
8900 : block TO. */
8901 : static void
8902 0 : move_succs (vec<edge, va_gc> **succsp, basic_block to)
8903 : {
8904 0 : edge e;
8905 0 : edge_iterator ei;
8906 :
8907 0 : gcc_assert (to->succs == 0);
8908 :
8909 0 : to->succs = *succsp;
8910 :
8911 0 : FOR_EACH_EDGE (e, ei, to->succs)
8912 0 : e->src = to;
8913 :
8914 0 : *succsp = 0;
8915 0 : }
8916 :
8917 : /* Remove INSN from the instruction stream.
8918 : INSN should have any dependencies. */
8919 : static void
8920 0 : sched_remove_insn (rtx_insn *insn)
8921 : {
8922 0 : sd_finish_insn (insn);
8923 :
8924 0 : change_queue_index (insn, QUEUE_NOWHERE);
8925 0 : current_sched_info->add_remove_insn (insn, 1);
8926 0 : delete_insn (insn);
8927 0 : }
8928 :
8929 : /* Clear priorities of all instructions, that are forward dependent on INSN.
8930 : Store in vector pointed to by ROOTS_PTR insns on which priority () should
8931 : be invoked to initialize all cleared priorities. */
8932 : static void
8933 0 : clear_priorities (rtx_insn *insn, rtx_vec_t *roots_ptr)
8934 : {
8935 0 : sd_iterator_def sd_it;
8936 0 : dep_t dep;
8937 0 : bool insn_is_root_p = true;
8938 :
8939 0 : gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED);
8940 :
8941 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
8942 : {
8943 0 : rtx_insn *pro = DEP_PRO (dep);
8944 :
8945 0 : if (INSN_PRIORITY_STATUS (pro) >= 0
8946 0 : && QUEUE_INDEX (insn) != QUEUE_SCHEDULED)
8947 : {
8948 : /* If DEP doesn't contribute to priority then INSN itself should
8949 : be added to priority roots. */
8950 0 : if (contributes_to_priority_p (dep))
8951 0 : insn_is_root_p = false;
8952 :
8953 0 : INSN_PRIORITY_STATUS (pro) = -1;
8954 0 : clear_priorities (pro, roots_ptr);
8955 : }
8956 : }
8957 :
8958 0 : if (insn_is_root_p)
8959 0 : roots_ptr->safe_push (insn);
8960 0 : }
8961 :
8962 : /* Recompute priorities of instructions, whose priorities might have been
8963 : changed. ROOTS is a vector of instructions whose priority computation will
8964 : trigger initialization of all cleared priorities. */
8965 : static void
8966 0 : calc_priorities (const rtx_vec_t &roots)
8967 : {
8968 0 : int i;
8969 0 : rtx_insn *insn;
8970 :
8971 0 : FOR_EACH_VEC_ELT (roots, i, insn)
8972 0 : priority (insn);
8973 0 : }
8974 :
8975 :
8976 : /* Add dependences between JUMP and other instructions in the recovery
8977 : block. INSN is the first insn the recovery block. */
8978 : static void
8979 0 : add_jump_dependencies (rtx_insn *insn, rtx_insn *jump)
8980 : {
8981 0 : do
8982 : {
8983 0 : insn = NEXT_INSN (insn);
8984 0 : if (insn == jump)
8985 : break;
8986 :
8987 0 : if (dep_list_size (insn, SD_LIST_FORW) == 0)
8988 : {
8989 0 : dep_def _new_dep, *new_dep = &_new_dep;
8990 :
8991 0 : init_dep (new_dep, insn, jump, REG_DEP_ANTI);
8992 0 : sd_add_dep (new_dep, false);
8993 : }
8994 : }
8995 : while (1);
8996 :
8997 0 : gcc_assert (!sd_lists_empty_p (jump, SD_LIST_BACK));
8998 0 : }
8999 :
9000 : /* Extend data structures for logical insn UID. */
9001 : void
9002 967243 : sched_extend_luids (void)
9003 : {
9004 967243 : int new_luids_max_uid = get_max_uid () + 1;
9005 :
9006 967243 : sched_luids.safe_grow_cleared (new_luids_max_uid, true);
9007 967243 : }
9008 :
9009 : /* Initialize LUID for INSN. */
9010 : void
9011 132342689 : sched_init_insn_luid (rtx_insn *insn)
9012 : {
9013 132342689 : int i = INSN_P (insn) ? 1 : common_sched_info->luid_for_non_insn (insn);
9014 23703933 : int luid;
9015 :
9016 23703933 : if (i >= 0)
9017 : {
9018 132341128 : luid = sched_max_luid;
9019 132341128 : sched_max_luid += i;
9020 : }
9021 : else
9022 : luid = -1;
9023 :
9024 132342689 : SET_INSN_LUID (insn, luid);
9025 132342689 : }
9026 :
9027 : /* Initialize luids for BBS.
9028 : The hook common_sched_info->luid_for_non_insn () is used to determine
9029 : if notes, labels, etc. need luids. */
9030 : void
9031 965119 : sched_init_luids (const bb_vec_t &bbs)
9032 : {
9033 965119 : int i;
9034 965119 : basic_block bb;
9035 :
9036 965119 : sched_extend_luids ();
9037 11298690 : FOR_EACH_VEC_ELT (bbs, i, bb)
9038 : {
9039 10333571 : rtx_insn *insn;
9040 :
9041 142674929 : FOR_BB_INSNS (bb, insn)
9042 132341358 : sched_init_insn_luid (insn);
9043 : }
9044 965119 : }
9045 :
9046 : /* Free LUIDs. */
9047 : void
9048 965119 : sched_finish_luids (void)
9049 : {
9050 965119 : sched_luids.release ();
9051 965119 : sched_max_luid = 1;
9052 965119 : }
9053 :
9054 : /* Return logical uid of INSN. Helpful while debugging. */
9055 : int
9056 0 : insn_luid (rtx_insn *insn)
9057 : {
9058 0 : return INSN_LUID (insn);
9059 : }
9060 :
9061 : /* Extend per insn data in the target. */
9062 : void
9063 1932978 : sched_extend_target (void)
9064 : {
9065 1932978 : if (targetm.sched.h_i_d_extended)
9066 0 : targetm.sched.h_i_d_extended ();
9067 1932978 : }
9068 :
9069 : /* Extend global scheduler structures (those, that live across calls to
9070 : schedule_block) to include information about just emitted INSN. */
9071 : static void
9072 965119 : extend_h_i_d (void)
9073 : {
9074 965119 : int reserve = (get_max_uid () + 1 - h_i_d.length ());
9075 965119 : if (reserve > 0
9076 1929600 : && ! h_i_d.space (reserve))
9077 : {
9078 964481 : h_i_d.safe_grow_cleared (3U * get_max_uid () / 2, true);
9079 964481 : sched_extend_target ();
9080 : }
9081 965119 : }
9082 :
9083 : /* Initialize h_i_d entry of the INSN with default values.
9084 : Values, that are not explicitly initialized here, hold zero. */
9085 : static void
9086 132341358 : init_h_i_d (rtx_insn *insn)
9087 : {
9088 132341358 : if (INSN_LUID (insn) > 0)
9089 : {
9090 132339797 : INSN_COST (insn) = -1;
9091 132339797 : QUEUE_INDEX (insn) = QUEUE_NOWHERE;
9092 132339797 : INSN_TICK (insn) = INVALID_TICK;
9093 132339797 : INSN_EXACT_TICK (insn) = INVALID_TICK;
9094 132339797 : INTER_TICK (insn) = INVALID_TICK;
9095 132339797 : TODO_SPEC (insn) = HARD_DEP;
9096 132339797 : INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status
9097 132339797 : = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
9098 132339797 : INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status
9099 132339797 : = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
9100 : }
9101 132341358 : }
9102 :
9103 : /* Initialize haifa_insn_data for BBS. */
9104 : void
9105 965119 : haifa_init_h_i_d (const bb_vec_t &bbs)
9106 : {
9107 965119 : int i;
9108 965119 : basic_block bb;
9109 :
9110 965119 : extend_h_i_d ();
9111 11298690 : FOR_EACH_VEC_ELT (bbs, i, bb)
9112 : {
9113 10333571 : rtx_insn *insn;
9114 :
9115 142674929 : FOR_BB_INSNS (bb, insn)
9116 132341358 : init_h_i_d (insn);
9117 : }
9118 965119 : }
9119 :
9120 : /* Finalize haifa_insn_data. */
9121 : void
9122 964480 : haifa_finish_h_i_d (void)
9123 : {
9124 964480 : int i;
9125 964480 : haifa_insn_data_t data;
9126 964480 : reg_use_data *use, *next_use;
9127 964480 : reg_set_data *set, *next_set;
9128 :
9129 309052319 : FOR_EACH_VEC_ELT (h_i_d, i, data)
9130 : {
9131 308087839 : free (data->max_reg_pressure);
9132 308087839 : free (data->reg_pressure);
9133 308090201 : for (use = data->reg_use_list; use != NULL; use = next_use)
9134 : {
9135 2362 : next_use = use->next_insn_use;
9136 2362 : free (use);
9137 : }
9138 308090067 : for (set = data->reg_set_list; set != NULL; set = next_set)
9139 : {
9140 2228 : next_set = set->next_insn_set;
9141 2228 : free (set);
9142 : }
9143 :
9144 : }
9145 964480 : h_i_d.release ();
9146 964480 : }
9147 :
9148 : /* Init data for the new insn INSN. */
9149 : static void
9150 0 : haifa_init_insn (rtx_insn *insn)
9151 : {
9152 0 : gcc_assert (insn != NULL);
9153 :
9154 0 : sched_extend_luids ();
9155 0 : sched_init_insn_luid (insn);
9156 0 : sched_extend_target ();
9157 0 : sched_deps_init (false);
9158 0 : extend_h_i_d ();
9159 0 : init_h_i_d (insn);
9160 :
9161 0 : if (adding_bb_to_current_region_p)
9162 : {
9163 0 : sd_init_insn (insn);
9164 :
9165 : /* Extend dependency caches by one element. */
9166 0 : extend_dependency_caches (1, false);
9167 : }
9168 0 : if (sched_pressure != SCHED_PRESSURE_NONE)
9169 0 : init_insn_reg_pressure_info (insn);
9170 0 : }
9171 :
9172 : /* Init data for the new basic block BB which comes after AFTER. */
9173 : static void
9174 1 : haifa_init_only_bb (basic_block bb, basic_block after)
9175 : {
9176 1 : gcc_assert (bb != NULL);
9177 :
9178 1 : sched_init_bbs ();
9179 :
9180 1 : if (common_sched_info->add_block)
9181 : /* This changes only data structures of the front-end. */
9182 1 : common_sched_info->add_block (bb, after);
9183 1 : }
9184 :
9185 : /* A generic version of sched_split_block (). */
9186 : basic_block
9187 67 : sched_split_block_1 (basic_block first_bb, rtx after)
9188 : {
9189 67 : edge e;
9190 :
9191 67 : e = split_block (first_bb, after);
9192 67 : gcc_assert (e->src == first_bb);
9193 :
9194 : /* sched_split_block emits note if *check == BB_END. Probably it
9195 : is better to rip that note off. */
9196 :
9197 67 : return e->dest;
9198 : }
9199 :
9200 : /* A generic version of sched_create_empty_bb (). */
9201 : basic_block
9202 0 : sched_create_empty_bb_1 (basic_block after)
9203 : {
9204 0 : return create_empty_bb (after);
9205 : }
9206 :
9207 : /* Insert PAT as an INSN into the schedule and update the necessary data
9208 : structures to account for it. */
9209 : rtx_insn *
9210 0 : sched_emit_insn (rtx pat)
9211 : {
9212 0 : rtx_insn *insn = emit_insn_before (pat, first_nonscheduled_insn ());
9213 0 : haifa_init_insn (insn);
9214 :
9215 0 : if (current_sched_info->add_remove_insn)
9216 0 : current_sched_info->add_remove_insn (insn, 0);
9217 :
9218 0 : (*current_sched_info->begin_schedule_ready) (insn);
9219 0 : scheduled_insns.safe_push (insn);
9220 :
9221 0 : last_scheduled_insn = insn;
9222 0 : return insn;
9223 : }
9224 :
9225 : /* This function returns a candidate satisfying dispatch constraints from
9226 : the ready list. */
9227 :
9228 : static rtx_insn *
9229 16 : ready_remove_first_dispatch (struct ready_list *ready)
9230 : {
9231 16 : int i;
9232 16 : rtx_insn *insn = ready_element (ready, 0);
9233 :
9234 16 : if (ready->n_ready == 1
9235 0 : || !INSN_P (insn)
9236 0 : || INSN_CODE (insn) < 0
9237 0 : || !active_insn_p (insn)
9238 16 : || targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW))
9239 16 : return ready_remove_first (ready);
9240 :
9241 0 : for (i = 1; i < ready->n_ready; i++)
9242 : {
9243 0 : insn = ready_element (ready, i);
9244 :
9245 0 : if (!INSN_P (insn)
9246 0 : || INSN_CODE (insn) < 0
9247 0 : || !active_insn_p (insn))
9248 0 : continue;
9249 :
9250 0 : if (targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW))
9251 : {
9252 : /* Return ith element of ready. */
9253 0 : insn = ready_remove (ready, i);
9254 0 : return insn;
9255 : }
9256 : }
9257 :
9258 0 : if (targetm.sched.dispatch (NULL, DISPATCH_VIOLATION))
9259 0 : return ready_remove_first (ready);
9260 :
9261 0 : for (i = 1; i < ready->n_ready; i++)
9262 : {
9263 0 : insn = ready_element (ready, i);
9264 :
9265 0 : if (!INSN_P (insn)
9266 0 : || INSN_CODE (insn) < 0
9267 0 : || !active_insn_p (insn))
9268 0 : continue;
9269 :
9270 : /* Return i-th element of ready. */
9271 0 : if (targetm.sched.dispatch (insn, IS_CMP))
9272 0 : return ready_remove (ready, i);
9273 : }
9274 :
9275 0 : return ready_remove_first (ready);
9276 : }
9277 :
9278 : /* Get number of ready insn in the ready list. */
9279 :
9280 : int
9281 0 : number_in_ready (void)
9282 : {
9283 0 : return ready.n_ready;
9284 : }
9285 :
9286 : /* Get number of ready's in the ready list. */
9287 :
9288 : rtx_insn *
9289 267919350 : get_ready_element (int i)
9290 : {
9291 267919350 : return ready_element (&ready, i);
9292 : }
9293 :
9294 : #endif /* INSN_SCHEDULING */
|
