Line data Source code
1 : /* Instruction scheduling pass.
2 : Copyright (C) 1992-2026 Free Software Foundation, Inc.
3 : Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
4 : and currently maintained by, Jim Wilson (wilson@cygnus.com)
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : /* Instruction scheduling pass. This file, along with sched-deps.cc,
23 : contains the generic parts. The actual entry point for
24 : the normal instruction scheduling pass is found in sched-rgn.cc.
25 :
26 : We compute insn priorities based on data dependencies. Flow
27 : analysis only creates a fraction of the data-dependencies we must
28 : observe: namely, only those dependencies which the combiner can be
29 : expected to use. For this pass, we must therefore create the
30 : remaining dependencies we need to observe: register dependencies,
31 : memory dependencies, dependencies to keep function calls in order,
32 : and the dependence between a conditional branch and the setting of
33 : condition codes are all dealt with here.
34 :
35 : The scheduler first traverses the data flow graph, starting with
36 : the last instruction, and proceeding to the first, assigning values
37 : to insn_priority as it goes. This sorts the instructions
38 : topologically by data dependence.
39 :
40 : Once priorities have been established, we order the insns using
41 : list scheduling. This works as follows: starting with a list of
42 : all the ready insns, and sorted according to priority number, we
43 : schedule the insn from the end of the list by placing its
44 : predecessors in the list according to their priority order. We
45 : consider this insn scheduled by setting the pointer to the "end" of
46 : the list to point to the previous insn. When an insn has no
47 : predecessors, we either queue it until sufficient time has elapsed
48 : or add it to the ready list. As the instructions are scheduled or
49 : when stalls are introduced, the queue advances and dumps insns into
50 : the ready list. When all insns down to the lowest priority have
51 : been scheduled, the critical path of the basic block has been made
52 : as short as possible. The remaining insns are then scheduled in
53 : remaining slots.
54 :
55 : The following list shows the order in which we want to break ties
56 : among insns in the ready list:
57 :
58 : 1. choose insn with the longest path to end of bb, ties
59 : broken by
60 : 2. choose insn with least contribution to register pressure,
61 : ties broken by
62 : 3. prefer in-block upon interblock motion, ties broken by
63 : 4. prefer useful upon speculative motion, ties broken by
64 : 5. choose insn with largest control flow probability, ties
65 : broken by
66 : 6. choose insn with the least dependences upon the previously
67 : scheduled insn, or finally
68 : 7 choose the insn which has the most insns dependent on it.
69 : 8. choose insn with lowest UID.
70 :
71 : Memory references complicate matters. Only if we can be certain
72 : that memory references are not part of the data dependency graph
73 : (via true, anti, or output dependence), can we move operations past
74 : memory references. To first approximation, reads can be done
75 : independently, while writes introduce dependencies. Better
76 : approximations will yield fewer dependencies.
77 :
78 : Before reload, an extended analysis of interblock data dependences
79 : is required for interblock scheduling. This is performed in
80 : compute_block_dependences ().
81 :
82 : Dependencies set up by memory references are treated in exactly the
83 : same way as other dependencies, by using insn backward dependences
84 : INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences
85 : INSN_FORW_DEPS for the purpose of forward list scheduling.
86 :
87 : Having optimized the critical path, we may have also unduly
88 : extended the lifetimes of some registers. If an operation requires
89 : that constants be loaded into registers, it is certainly desirable
90 : to load those constants as early as necessary, but no earlier.
91 : I.e., it will not do to load up a bunch of registers at the
92 : beginning of a basic block only to use them at the end, if they
93 : could be loaded later, since this may result in excessive register
94 : utilization.
95 :
96 : Note that since branches are never in basic blocks, but only end
97 : basic blocks, this pass will not move branches. But that is ok,
98 : since we can use GNU's delayed branch scheduling pass to take care
99 : of this case.
100 :
101 : Also note that no further optimizations based on algebraic
102 : identities are performed, so this pass would be a good one to
103 : perform instruction splitting, such as breaking up a multiply
104 : instruction into shifts and adds where that is profitable.
105 :
106 : Given the memory aliasing analysis that this pass should perform,
107 : it should be possible to remove redundant stores to memory, and to
108 : load values from registers instead of hitting memory.
109 :
110 : Before reload, speculative insns are moved only if a 'proof' exists
111 : that no exception will be caused by this, and if no live registers
112 : exist that inhibit the motion (live registers constraints are not
113 : represented by data dependence edges).
114 :
115 : This pass must update information that subsequent passes expect to
116 : be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths,
117 : reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END.
118 :
119 : The information in the line number notes is carefully retained by
120 : this pass. Notes that refer to the starting and ending of
121 : exception regions are also carefully retained by this pass. All
122 : other NOTE insns are grouped in their same relative order at the
123 : beginning of basic blocks and regions that have been scheduled. */
124 :
125 : #include "config.h"
126 : #include "system.h"
127 : #include "coretypes.h"
128 : #include "backend.h"
129 : #include "target.h"
130 : #include "rtl.h"
131 : #include "cfghooks.h"
132 : #include "df.h"
133 : #include "memmodel.h"
134 : #include "tm_p.h"
135 : #include "insn-config.h"
136 : #include "regs.h"
137 : #include "ira.h"
138 : #include "recog.h"
139 : #include "insn-attr.h"
140 : #include "cfgrtl.h"
141 : #include "cfgbuild.h"
142 : #include "sched-int.h"
143 : #include "common/common-target.h"
144 : #include "dbgcnt.h"
145 : #include "cfgloop.h"
146 : #include "dumpfile.h"
147 : #include "print-rtl.h"
148 : #include "function-abi.h"
149 :
150 : #ifdef INSN_SCHEDULING
151 :
152 : /* True if we do register pressure relief through live-range
153 : shrinkage. */
154 : static bool live_range_shrinkage_p;
155 :
156 : /* Switch on live range shrinkage. */
157 : void
158 35 : initialize_live_range_shrinkage (void)
159 : {
160 35 : live_range_shrinkage_p = true;
161 35 : }
162 :
163 : /* Switch off live range shrinkage. */
164 : void
165 35 : finish_live_range_shrinkage (void)
166 : {
167 35 : live_range_shrinkage_p = false;
168 35 : }
169 :
170 : /* issue_rate is the number of insns that can be scheduled in the same
171 : machine cycle. It can be defined in the config/mach/mach.h file,
172 : otherwise we set it to 1. */
173 :
174 : int issue_rate;
175 :
176 : /* This can be set to true by a backend if the scheduler should not
177 : enable a DCE pass. */
178 : bool sched_no_dce;
179 :
180 : /* The current initiation interval used when modulo scheduling. */
181 : static int modulo_ii;
182 :
183 : /* The maximum number of stages we are prepared to handle. */
184 : static int modulo_max_stages;
185 :
186 : /* The number of insns that exist in each iteration of the loop. We use this
187 : to detect when we've scheduled all insns from the first iteration. */
188 : static int modulo_n_insns;
189 :
190 : /* The current count of insns in the first iteration of the loop that have
191 : already been scheduled. */
192 : static int modulo_insns_scheduled;
193 :
194 : /* The maximum uid of insns from the first iteration of the loop. */
195 : static int modulo_iter0_max_uid;
196 :
197 : /* The number of times we should attempt to backtrack when modulo scheduling.
198 : Decreased each time we have to backtrack. */
199 : static int modulo_backtracks_left;
200 :
201 : /* The stage in which the last insn from the original loop was
202 : scheduled. */
203 : static int modulo_last_stage;
204 :
205 : /* sched-verbose controls the amount of debugging output the
206 : scheduler prints. It is controlled by -fsched-verbose=N:
207 : N=0: no debugging output.
208 : N=1: default value.
209 : N=2: bb's probabilities, detailed ready list info, unit/insn info.
210 : N=3: rtl at abort point, control-flow, regions info.
211 : N=5: dependences info. */
212 : int sched_verbose = 0;
213 :
214 : /* Debugging file. All printouts are sent to dump. */
215 : FILE *sched_dump = 0;
216 :
217 : /* This is a placeholder for the scheduler parameters common
218 : to all schedulers. */
219 : struct common_sched_info_def *common_sched_info;
220 :
221 : #define INSN_TICK(INSN) (HID (INSN)->tick)
222 : #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick)
223 : #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate)
224 : #define INTER_TICK(INSN) (HID (INSN)->inter_tick)
225 : #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn)
226 : #define SHADOW_P(INSN) (HID (INSN)->shadow_p)
227 : #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec)
228 : /* Cached cost of the instruction. Use insn_sched_cost to get cost of the
229 : insn. -1 here means that the field is not initialized. */
230 : #define INSN_COST(INSN) (HID (INSN)->cost)
231 :
232 : /* If INSN_TICK of an instruction is equal to INVALID_TICK,
233 : then it should be recalculated from scratch. */
234 : #define INVALID_TICK (-(max_insn_queue_index + 1))
235 : /* The minimal value of the INSN_TICK of an instruction. */
236 : #define MIN_TICK (-max_insn_queue_index)
237 :
238 : /* Original order of insns in the ready list.
239 : Used to keep order of normal insns while separating DEBUG_INSNs. */
240 : #define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
241 :
242 : /* The deciding reason for INSN's place in the ready list. */
243 : #define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
244 :
245 : /* List of important notes we must keep around. This is a pointer to the
246 : last element in the list. */
247 : rtx_insn *note_list;
248 :
249 : static struct spec_info_def spec_info_var;
250 : /* Description of the speculative part of the scheduling.
251 : If NULL - no speculation. */
252 : spec_info_t spec_info = NULL;
253 :
254 : /* True, if recovery block was added during scheduling of current block.
255 : Used to determine, if we need to fix INSN_TICKs. */
256 : static bool haifa_recovery_bb_recently_added_p;
257 :
258 : /* True, if recovery block was added during this scheduling pass.
259 : Used to determine if we should have empty memory pools of dependencies
260 : after finishing current region. */
261 : bool haifa_recovery_bb_ever_added_p;
262 :
263 : /* Counters of different types of speculative instructions. */
264 : static int nr_begin_data, nr_be_in_data, nr_begin_control, nr_be_in_control;
265 :
266 : /* Array used in {unlink, restore}_bb_notes. */
267 : static rtx_insn **bb_header = 0;
268 :
269 : /* Basic block after which recovery blocks will be created. */
270 : static basic_block before_recovery;
271 :
272 : /* Basic block just before the EXIT_BLOCK and after recovery, if we have
273 : created it. */
274 : basic_block after_recovery;
275 :
276 : /* FALSE if we add bb to another region, so we don't need to initialize it. */
277 : bool adding_bb_to_current_region_p = true;
278 :
279 : /* Queues, etc. */
280 :
281 : /* An instruction is ready to be scheduled when all insns preceding it
282 : have already been scheduled. It is important to ensure that all
283 : insns which use its result will not be executed until its result
284 : has been computed. An insn is maintained in one of four structures:
285 :
286 : (P) the "Pending" set of insns which cannot be scheduled until
287 : their dependencies have been satisfied.
288 : (Q) the "Queued" set of insns that can be scheduled when sufficient
289 : time has passed.
290 : (R) the "Ready" list of unscheduled, uncommitted insns.
291 : (S) the "Scheduled" list of insns.
292 :
293 : Initially, all insns are either "Pending" or "Ready" depending on
294 : whether their dependencies are satisfied.
295 :
296 : Insns move from the "Ready" list to the "Scheduled" list as they
297 : are committed to the schedule. As this occurs, the insns in the
298 : "Pending" list have their dependencies satisfied and move to either
299 : the "Ready" list or the "Queued" set depending on whether
300 : sufficient time has passed to make them ready. As time passes,
301 : insns move from the "Queued" set to the "Ready" list.
302 :
303 : The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the
304 : unscheduled insns, i.e., those that are ready, queued, and pending.
305 : The "Queued" set (Q) is implemented by the variable `insn_queue'.
306 : The "Ready" list (R) is implemented by the variables `ready' and
307 : `n_ready'.
308 : The "Scheduled" list (S) is the new insn chain built by this pass.
309 :
310 : The transition (R->S) is implemented in the scheduling loop in
311 : `schedule_block' when the best insn to schedule is chosen.
312 : The transitions (P->R and P->Q) are implemented in `schedule_insn' as
313 : insns move from the ready list to the scheduled list.
314 : The transition (Q->R) is implemented in 'queue_to_insn' as time
315 : passes or stalls are introduced. */
316 :
317 : /* Implement a circular buffer to delay instructions until sufficient
318 : time has passed. For the new pipeline description interface,
319 : MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less
320 : than maximal time of instruction execution computed by genattr.cc on
321 : the base maximal time of functional unit reservations and getting a
322 : result. This is the longest time an insn may be queued. */
323 :
324 : static rtx_insn_list **insn_queue;
325 : static int q_ptr = 0;
326 : static int q_size = 0;
327 : #define NEXT_Q(X) (((X)+1) & max_insn_queue_index)
328 : #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index)
329 :
330 : #define QUEUE_SCHEDULED (-3)
331 : #define QUEUE_NOWHERE (-2)
332 : #define QUEUE_READY (-1)
333 : /* QUEUE_SCHEDULED - INSN is scheduled.
334 : QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in
335 : queue or ready list.
336 : QUEUE_READY - INSN is in ready list.
337 : N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. */
338 :
339 : #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index)
340 :
341 : /* The following variable value refers for all current and future
342 : reservations of the processor units. */
343 : state_t curr_state;
344 :
345 : /* The following variable value is size of memory representing all
346 : current and future reservations of the processor units. */
347 : size_t dfa_state_size;
348 :
349 : /* The following array is used to find the best insn from ready when
350 : the automaton pipeline interface is used. */
351 : signed char *ready_try = NULL;
352 :
353 : /* The ready list. */
354 : struct ready_list ready = {NULL, 0, 0, 0, 0};
355 :
356 : /* The pointer to the ready list (to be removed). */
357 : static struct ready_list *readyp = &ready;
358 :
359 : /* Scheduling clock. */
360 : static int clock_var;
361 :
362 : /* Clock at which the previous instruction was issued. */
363 : static int last_clock_var;
364 :
365 : /* Set to true if, when queuing a shadow insn, we discover that it would be
366 : scheduled too late. */
367 : static bool must_backtrack;
368 :
369 : /* The following variable value is number of essential insns issued on
370 : the current cycle. An insn is essential one if it changes the
371 : processors state. */
372 : int cycle_issued_insns;
373 :
374 : /* This records the actual schedule. It is built up during the main phase
375 : of schedule_block, and afterwards used to reorder the insns in the RTL. */
376 : static vec<rtx_insn *> scheduled_insns;
377 :
378 : static int may_trap_exp (const_rtx, int);
379 :
380 : /* Nonzero iff the address is comprised from at most 1 register. */
381 : #define CONST_BASED_ADDRESS_P(x) \
382 : (REG_P (x) \
383 : || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS \
384 : || (GET_CODE (x) == LO_SUM)) \
385 : && (CONSTANT_P (XEXP (x, 0)) \
386 : || CONSTANT_P (XEXP (x, 1)))))
387 :
388 : /* Returns a class that insn with GET_DEST(insn)=x may belong to,
389 : as found by analyzing insn's expression. */
390 :
391 :
392 : static int haifa_luid_for_non_insn (rtx x);
393 :
394 : /* Haifa version of sched_info hooks common to all headers. */
395 : const struct common_sched_info_def haifa_common_sched_info =
396 : {
397 : NULL, /* fix_recovery_cfg */
398 : NULL, /* add_block */
399 : NULL, /* estimate_number_of_insns */
400 : haifa_luid_for_non_insn, /* luid_for_non_insn */
401 : SCHED_PASS_UNKNOWN /* sched_pass_id */
402 : };
403 :
404 : /* Mapping from instruction UID to its Logical UID. */
405 : vec<int> sched_luids;
406 :
407 : /* Next LUID to assign to an instruction. */
408 : int sched_max_luid = 1;
409 :
410 : /* Haifa Instruction Data. */
411 : vec<haifa_insn_data_def> h_i_d;
412 :
413 : void (* sched_init_only_bb) (basic_block, basic_block);
414 :
415 : /* Split block function. Different schedulers might use different functions
416 : to handle their internal data consistent. */
417 : basic_block (* sched_split_block) (basic_block, rtx);
418 :
419 : /* Create empty basic block after the specified block. */
420 : basic_block (* sched_create_empty_bb) (basic_block);
421 :
422 : /* Return the number of cycles until INSN is expected to be ready.
423 : Return zero if it already is. */
424 : static int
425 27322 : insn_delay (rtx_insn *insn)
426 : {
427 27322 : return MAX (INSN_TICK (insn) - clock_var, 0);
428 : }
429 :
430 : static int
431 27399 : may_trap_exp (const_rtx x, int is_store)
432 : {
433 27399 : enum rtx_code code;
434 :
435 27399 : if (x == 0)
436 : return TRAP_FREE;
437 27399 : code = GET_CODE (x);
438 27399 : if (is_store)
439 : {
440 5623 : if (code == MEM && may_trap_p (x))
441 : return TRAP_RISKY;
442 : else
443 5369 : return TRAP_FREE;
444 : }
445 21776 : if (code == MEM)
446 : {
447 : /* The insn uses memory: a volatile load. */
448 1668 : if (MEM_VOLATILE_P (x))
449 : return IRISKY;
450 : /* An exception-free load. */
451 1646 : if (!may_trap_p (x))
452 : return IFREE;
453 : /* A load with 1 base register, to be further checked. */
454 382 : if (CONST_BASED_ADDRESS_P (XEXP (x, 0)))
455 : return PFREE_CANDIDATE;
456 : /* No info on the load, to be further checked. */
457 24 : return PRISKY_CANDIDATE;
458 : }
459 : else
460 : {
461 20108 : const char *fmt;
462 20108 : int i, insn_class = TRAP_FREE;
463 :
464 : /* Neither store nor load, check if it may cause a trap. */
465 20108 : if (may_trap_p (x))
466 : return TRAP_RISKY;
467 : /* Recursive step: walk the insn... */
468 19532 : fmt = GET_RTX_FORMAT (code);
469 44668 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
470 : {
471 25140 : if (fmt[i] == 'e')
472 : {
473 12054 : int tmp_class = may_trap_exp (XEXP (x, i), is_store);
474 12054 : insn_class = WORST_CLASS (insn_class, tmp_class);
475 : }
476 13086 : else if (fmt[i] == 'E')
477 : {
478 : int j;
479 286 : for (j = 0; j < XVECLEN (x, i); j++)
480 : {
481 248 : int tmp_class = may_trap_exp (XVECEXP (x, i, j), is_store);
482 248 : insn_class = WORST_CLASS (insn_class, tmp_class);
483 248 : if (insn_class == TRAP_RISKY || insn_class == IRISKY)
484 : break;
485 : }
486 : }
487 25140 : if (insn_class == TRAP_RISKY || insn_class == IRISKY)
488 : break;
489 : }
490 : return insn_class;
491 : }
492 : }
493 :
494 : /* Classifies rtx X of an insn for the purpose of verifying that X can be
495 : executed speculatively (and consequently the insn can be moved
496 : speculatively), by examining X, returning:
497 : TRAP_RISKY: store, or risky non-load insn (e.g. division by variable).
498 : TRAP_FREE: non-load insn.
499 : IFREE: load from a globally safe location.
500 : IRISKY: volatile load.
501 : PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for
502 : being either PFREE or PRISKY. */
503 :
504 : static int
505 6725 : haifa_classify_rtx (const_rtx x)
506 : {
507 6725 : int tmp_class = TRAP_FREE;
508 6725 : int insn_class = TRAP_FREE;
509 6725 : enum rtx_code code;
510 :
511 6725 : if (GET_CODE (x) == PARALLEL)
512 : {
513 772 : int i, len = XVECLEN (x, 0);
514 :
515 2102 : for (i = len - 1; i >= 0; i--)
516 : {
517 1491 : tmp_class = haifa_classify_rtx (XVECEXP (x, 0, i));
518 1491 : insn_class = WORST_CLASS (insn_class, tmp_class);
519 1491 : if (insn_class == TRAP_RISKY || insn_class == IRISKY)
520 : break;
521 : }
522 : }
523 : else
524 : {
525 5953 : code = GET_CODE (x);
526 5953 : switch (code)
527 : {
528 726 : case CLOBBER:
529 : /* Test if it is a 'store'. */
530 726 : tmp_class = may_trap_exp (XEXP (x, 0), 1);
531 726 : break;
532 4897 : case SET:
533 : /* Test if it is a store. */
534 4897 : tmp_class = may_trap_exp (SET_DEST (x), 1);
535 4897 : if (tmp_class == TRAP_RISKY)
536 : break;
537 : /* Test if it is a load. */
538 4737 : tmp_class =
539 4737 : WORST_CLASS (tmp_class,
540 : may_trap_exp (SET_SRC (x), 0));
541 : break;
542 0 : case COND_EXEC:
543 0 : tmp_class = haifa_classify_rtx (COND_EXEC_CODE (x));
544 0 : if (tmp_class == TRAP_RISKY)
545 : break;
546 0 : tmp_class = WORST_CLASS (tmp_class,
547 : may_trap_exp (COND_EXEC_TEST (x), 0));
548 : break;
549 : case TRAP_IF:
550 171 : tmp_class = TRAP_RISKY;
551 : break;
552 160 : default:;
553 : }
554 : insn_class = tmp_class;
555 : }
556 :
557 6725 : return insn_class;
558 : }
559 :
560 : int
561 5234 : haifa_classify_insn (const rtx_insn *insn)
562 : {
563 5234 : return haifa_classify_rtx (PATTERN (insn));
564 : }
565 :
566 : /* After the scheduler initialization function has been called, this function
567 : can be called to enable modulo scheduling. II is the initiation interval
568 : we should use, it affects the delays for delay_pairs that were recorded as
569 : separated by a given number of stages.
570 :
571 : MAX_STAGES provides us with a limit
572 : after which we give up scheduling; the caller must have unrolled at least
573 : as many copies of the loop body and recorded delay_pairs for them.
574 :
575 : INSNS is the number of real (non-debug) insns in one iteration of
576 : the loop. MAX_UID can be used to test whether an insn belongs to
577 : the first iteration of the loop; all of them have a uid lower than
578 : MAX_UID. */
579 : void
580 0 : set_modulo_params (int ii, int max_stages, int insns, int max_uid)
581 : {
582 0 : modulo_ii = ii;
583 0 : modulo_max_stages = max_stages;
584 0 : modulo_n_insns = insns;
585 0 : modulo_iter0_max_uid = max_uid;
586 0 : modulo_backtracks_left = param_max_modulo_backtrack_attempts;
587 0 : }
588 :
589 : /* A structure to record a pair of insns where the first one is a real
590 : insn that has delay slots, and the second is its delayed shadow.
591 : I1 is scheduled normally and will emit an assembly instruction,
592 : while I2 describes the side effect that takes place at the
593 : transition between cycles CYCLES and (CYCLES + 1) after I1. */
594 : struct delay_pair
595 : {
596 : struct delay_pair *next_same_i1;
597 : rtx_insn *i1, *i2;
598 : int cycles;
599 : /* When doing modulo scheduling, we a delay_pair can also be used to
600 : show that I1 and I2 are the same insn in a different stage. If that
601 : is the case, STAGES will be nonzero. */
602 : int stages;
603 : };
604 :
605 : /* Helpers for delay hashing. */
606 :
607 : struct delay_i1_hasher : nofree_ptr_hash <delay_pair>
608 : {
609 : typedef void *compare_type;
610 : static inline hashval_t hash (const delay_pair *);
611 : static inline bool equal (const delay_pair *, const void *);
612 : };
613 :
614 : /* Returns a hash value for X, based on hashing just I1. */
615 :
616 : inline hashval_t
617 0 : delay_i1_hasher::hash (const delay_pair *x)
618 : {
619 0 : return htab_hash_pointer (x->i1);
620 : }
621 :
622 : /* Return true if I1 of pair X is the same as that of pair Y. */
623 :
624 : inline bool
625 0 : delay_i1_hasher::equal (const delay_pair *x, const void *y)
626 : {
627 0 : return x->i1 == y;
628 : }
629 :
630 : struct delay_i2_hasher : free_ptr_hash <delay_pair>
631 : {
632 : typedef void *compare_type;
633 : static inline hashval_t hash (const delay_pair *);
634 : static inline bool equal (const delay_pair *, const void *);
635 : };
636 :
637 : /* Returns a hash value for X, based on hashing just I2. */
638 :
639 : inline hashval_t
640 0 : delay_i2_hasher::hash (const delay_pair *x)
641 : {
642 0 : return htab_hash_pointer (x->i2);
643 : }
644 :
645 : /* Return true if I2 of pair X is the same as that of pair Y. */
646 :
647 : inline bool
648 0 : delay_i2_hasher::equal (const delay_pair *x, const void *y)
649 : {
650 0 : return x->i2 == y;
651 : }
652 :
653 : /* Two hash tables to record delay_pairs, one indexed by I1 and the other
654 : indexed by I2. */
655 : static hash_table<delay_i1_hasher> *delay_htab;
656 : static hash_table<delay_i2_hasher> *delay_htab_i2;
657 :
658 : /* Called through htab_traverse. Walk the hashtable using I2 as
659 : index, and delete all elements involving an UID higher than
660 : that pointed to by *DATA. */
661 : int
662 0 : haifa_htab_i2_traverse (delay_pair **slot, int *data)
663 : {
664 0 : int maxuid = *data;
665 0 : struct delay_pair *p = *slot;
666 0 : if (INSN_UID (p->i2) >= maxuid || INSN_UID (p->i1) >= maxuid)
667 : {
668 0 : delay_htab_i2->clear_slot (slot);
669 : }
670 0 : return 1;
671 : }
672 :
673 : /* Called through htab_traverse. Walk the hashtable using I2 as
674 : index, and delete all elements involving an UID higher than
675 : that pointed to by *DATA. */
676 : int
677 0 : haifa_htab_i1_traverse (delay_pair **pslot, int *data)
678 : {
679 0 : int maxuid = *data;
680 0 : struct delay_pair *p, *first, **pprev;
681 :
682 0 : if (INSN_UID ((*pslot)->i1) >= maxuid)
683 : {
684 0 : delay_htab->clear_slot (pslot);
685 0 : return 1;
686 : }
687 : pprev = &first;
688 0 : for (p = *pslot; p; p = p->next_same_i1)
689 : {
690 0 : if (INSN_UID (p->i2) < maxuid)
691 : {
692 0 : *pprev = p;
693 0 : pprev = &p->next_same_i1;
694 : }
695 : }
696 0 : *pprev = NULL;
697 0 : if (first == NULL)
698 0 : delay_htab->clear_slot (pslot);
699 : else
700 0 : *pslot = first;
701 : return 1;
702 : }
703 :
704 : /* Discard all delay pairs which involve an insn with an UID higher
705 : than MAX_UID. */
706 : void
707 0 : discard_delay_pairs_above (int max_uid)
708 : {
709 0 : delay_htab->traverse <int *, haifa_htab_i1_traverse> (&max_uid);
710 0 : delay_htab_i2->traverse <int *, haifa_htab_i2_traverse> (&max_uid);
711 0 : }
712 :
713 : /* This function can be called by a port just before it starts the final
714 : scheduling pass. It records the fact that an instruction with delay
715 : slots has been split into two insns, I1 and I2. The first one will be
716 : scheduled normally and initiates the operation. The second one is a
717 : shadow which must follow a specific number of cycles after I1; its only
718 : purpose is to show the side effect that occurs at that cycle in the RTL.
719 : If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN,
720 : while I2 retains the original insn type.
721 :
722 : There are two ways in which the number of cycles can be specified,
723 : involving the CYCLES and STAGES arguments to this function. If STAGES
724 : is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor
725 : which is multiplied by MODULO_II to give the number of cycles. This is
726 : only useful if the caller also calls set_modulo_params to enable modulo
727 : scheduling. */
728 :
729 : void
730 0 : record_delay_slot_pair (rtx_insn *i1, rtx_insn *i2, int cycles, int stages)
731 : {
732 0 : struct delay_pair *p = XNEW (struct delay_pair);
733 0 : struct delay_pair **slot;
734 :
735 0 : p->i1 = i1;
736 0 : p->i2 = i2;
737 0 : p->cycles = cycles;
738 0 : p->stages = stages;
739 :
740 0 : if (!delay_htab)
741 : {
742 0 : delay_htab = new hash_table<delay_i1_hasher> (10);
743 0 : delay_htab_i2 = new hash_table<delay_i2_hasher> (10);
744 : }
745 0 : slot = delay_htab->find_slot_with_hash (i1, htab_hash_pointer (i1), INSERT);
746 0 : p->next_same_i1 = *slot;
747 0 : *slot = p;
748 0 : slot = delay_htab_i2->find_slot (p, INSERT);
749 0 : *slot = p;
750 0 : }
751 :
752 : /* Examine the delay pair hashtable to see if INSN is a shadow for another,
753 : and return the other insn if so. Return NULL otherwise. */
754 : rtx_insn *
755 0 : real_insn_for_shadow (rtx_insn *insn)
756 : {
757 0 : struct delay_pair *pair;
758 :
759 0 : if (!delay_htab)
760 : return NULL;
761 :
762 0 : pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn));
763 0 : if (!pair || pair->stages > 0)
764 : return NULL;
765 0 : return pair->i1;
766 : }
767 :
768 : /* For a pair P of insns, return the fixed distance in cycles from the first
769 : insn after which the second must be scheduled. */
770 : static int
771 0 : pair_delay (struct delay_pair *p)
772 : {
773 0 : if (p->stages == 0)
774 0 : return p->cycles;
775 : else
776 0 : return p->stages * modulo_ii;
777 : }
778 :
779 : /* Given an insn INSN, add a dependence on its delayed shadow if it
780 : has one. Also try to find situations where shadows depend on each other
781 : and add dependencies to the real insns to limit the amount of backtracking
782 : needed. */
783 : void
784 1535 : add_delay_dependencies (rtx_insn *insn)
785 : {
786 1535 : struct delay_pair *pair;
787 1535 : sd_iterator_def sd_it;
788 1535 : dep_t dep;
789 :
790 1535 : if (!delay_htab)
791 1535 : return;
792 :
793 0 : pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn));
794 0 : if (!pair)
795 : return;
796 0 : add_dependence (insn, pair->i1, REG_DEP_ANTI);
797 0 : if (pair->stages)
798 : return;
799 :
800 0 : FOR_EACH_DEP (pair->i2, SD_LIST_BACK, sd_it, dep)
801 : {
802 0 : rtx_insn *pro = DEP_PRO (dep);
803 0 : struct delay_pair *other_pair
804 0 : = delay_htab_i2->find_with_hash (pro, htab_hash_pointer (pro));
805 0 : if (!other_pair || other_pair->stages)
806 0 : continue;
807 0 : if (pair_delay (other_pair) >= pair_delay (pair))
808 : {
809 0 : if (sched_verbose >= 4)
810 : {
811 0 : fprintf (sched_dump, ";;\tadding dependence %d <- %d\n",
812 0 : INSN_UID (other_pair->i1),
813 0 : INSN_UID (pair->i1));
814 0 : fprintf (sched_dump, ";;\tpair1 %d <- %d, cost %d\n",
815 0 : INSN_UID (pair->i1),
816 0 : INSN_UID (pair->i2),
817 : pair_delay (pair));
818 0 : fprintf (sched_dump, ";;\tpair2 %d <- %d, cost %d\n",
819 0 : INSN_UID (other_pair->i1),
820 0 : INSN_UID (other_pair->i2),
821 : pair_delay (other_pair));
822 : }
823 0 : add_dependence (pair->i1, other_pair->i1, REG_DEP_ANTI);
824 : }
825 : }
826 : }
827 :
828 : /* Forward declarations. */
829 :
830 : static int priority (rtx_insn *, bool force_recompute = false);
831 : static int autopref_rank_for_schedule (const rtx_insn *, const rtx_insn *);
832 : static int rank_for_schedule (const void *, const void *);
833 : static void swap_sort (rtx_insn **, int);
834 : static void queue_insn (rtx_insn *, int, const char *);
835 : static int schedule_insn (rtx_insn *);
836 : static void adjust_priority (rtx_insn *);
837 : static void advance_one_cycle (void);
838 : static void extend_h_i_d (void);
839 :
840 :
841 : /* Notes handling mechanism:
842 : =========================
843 : Generally, NOTES are saved before scheduling and restored after scheduling.
844 : The scheduler distinguishes between two types of notes:
845 :
846 : (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes:
847 : Before scheduling a region, a pointer to the note is added to the insn
848 : that follows or precedes it. (This happens as part of the data dependence
849 : computation). After scheduling an insn, the pointer contained in it is
850 : used for regenerating the corresponding note (in reemit_notes).
851 :
852 : (2) All other notes (e.g. INSN_DELETED): Before scheduling a block,
853 : these notes are put in a list (in rm_other_notes() and
854 : unlink_other_notes ()). After scheduling the block, these notes are
855 : inserted at the beginning of the block (in schedule_block()). */
856 :
857 : static void ready_add (struct ready_list *, rtx_insn *, bool);
858 : static rtx_insn *ready_remove_first (struct ready_list *);
859 : static rtx_insn *ready_remove_first_dispatch (struct ready_list *ready);
860 :
861 : static void queue_to_ready (struct ready_list *);
862 : static int early_queue_to_ready (state_t, struct ready_list *);
863 :
864 : /* The following functions are used to implement multi-pass scheduling
865 : on the first cycle. */
866 : static rtx_insn *ready_remove (struct ready_list *, int);
867 : static void ready_remove_insn (rtx_insn *);
868 :
869 : static void fix_inter_tick (rtx_insn *, rtx_insn *);
870 : static int fix_tick_ready (rtx_insn *);
871 : static void change_queue_index (rtx_insn *, int);
872 :
873 : /* The following functions are used to implement scheduling of data/control
874 : speculative instructions. */
875 :
876 : static void extend_h_i_d (void);
877 : static void init_h_i_d (rtx_insn *);
878 : static int haifa_speculate_insn (rtx_insn *, ds_t, rtx *);
879 : static void generate_recovery_code (rtx_insn *);
880 : static void process_insn_forw_deps_be_in_spec (rtx_insn *, rtx_insn *, ds_t);
881 : static void begin_speculative_block (rtx_insn *);
882 : static void add_to_speculative_block (rtx_insn *);
883 : static void init_before_recovery (basic_block *);
884 : static void create_check_block_twin (rtx_insn *, bool);
885 : static void fix_recovery_deps (basic_block);
886 : static bool haifa_change_pattern (rtx_insn *, rtx);
887 : static void dump_new_block_header (int, basic_block, rtx_insn *, rtx_insn *);
888 : static void restore_bb_notes (basic_block);
889 : static void fix_jump_move (rtx_insn *);
890 : static void move_block_after_check (rtx_insn *);
891 : static void move_succs (vec<edge, va_gc> **, basic_block);
892 : static void sched_remove_insn (rtx_insn *);
893 : static void clear_priorities (rtx_insn *, rtx_vec_t *);
894 : static void calc_priorities (const rtx_vec_t &);
895 : static void add_jump_dependencies (rtx_insn *, rtx_insn *);
896 :
897 : #endif /* INSN_SCHEDULING */
898 :
899 : /* Point to state used for the current scheduling pass. */
900 : struct haifa_sched_info *current_sched_info;
901 :
902 : #ifndef INSN_SCHEDULING
903 : void
904 : schedule_insns (void)
905 : {
906 : }
907 : #else
908 :
909 : /* Do register pressure sensitive insn scheduling if the flag is set
910 : up. */
911 : enum sched_pressure_algorithm sched_pressure;
912 :
913 : /* Map regno -> its pressure class. The map defined only when
914 : SCHED_PRESSURE != SCHED_PRESSURE_NONE. */
915 : enum reg_class *sched_regno_pressure_class;
916 :
917 : /* The current register pressure. Only elements corresponding pressure
918 : classes are defined. */
919 : static int curr_reg_pressure[N_REG_CLASSES];
920 :
921 : /* Saved value of the previous array. */
922 : static int saved_reg_pressure[N_REG_CLASSES];
923 :
924 : /* Register living at given scheduling point. */
925 : static bitmap curr_reg_live;
926 :
927 : /* Saved value of the previous array. */
928 : static bitmap saved_reg_live;
929 :
930 : /* Registers mentioned in the current region. */
931 : static bitmap region_ref_regs;
932 :
933 : /* Temporary bitmap used for SCHED_PRESSURE_MODEL. */
934 : static bitmap tmp_bitmap;
935 :
936 : /* Effective number of available registers of a given class (see comment
937 : in sched_pressure_start_bb). */
938 : static int sched_class_regs_num[N_REG_CLASSES];
939 : /* The number of registers that the function would need to save before it
940 : uses them, and the number of fixed_regs. Helpers for calculating of
941 : sched_class_regs_num. */
942 : static int call_saved_regs_num[N_REG_CLASSES];
943 : static int fixed_regs_num[N_REG_CLASSES];
944 :
945 : /* Initiate register pressure relative info for scheduling the current
946 : region. Currently it is only clearing register mentioned in the
947 : current region. */
948 : void
949 679 : sched_init_region_reg_pressure_info (void)
950 : {
951 679 : bitmap_clear (region_ref_regs);
952 679 : }
953 :
954 : /* PRESSURE[CL] describes the pressure on register class CL. Update it
955 : for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO.
956 : LIVE tracks the set of live registers; if it is null, assume that
957 : every birth or death is genuine. */
958 : static inline void
959 49366 : mark_regno_birth_or_death (bitmap live, int *pressure, int regno, bool birth_p)
960 : {
961 49366 : enum reg_class pressure_class;
962 :
963 49366 : pressure_class = sched_regno_pressure_class[regno];
964 49366 : if (regno >= FIRST_PSEUDO_REGISTER)
965 : {
966 40914 : if (pressure_class != NO_REGS)
967 : {
968 40870 : if (birth_p)
969 : {
970 25024 : if (!live || bitmap_set_bit (live, regno))
971 24544 : pressure[pressure_class]
972 24544 : += (ira_reg_class_max_nregs
973 24544 : [pressure_class][PSEUDO_REGNO_MODE (regno)]);
974 : }
975 : else
976 : {
977 15846 : if (!live || bitmap_clear_bit (live, regno))
978 14974 : pressure[pressure_class]
979 14974 : -= (ira_reg_class_max_nregs
980 14974 : [pressure_class][PSEUDO_REGNO_MODE (regno)]);
981 : }
982 : }
983 : }
984 8452 : else if (pressure_class != NO_REGS
985 8452 : && ! TEST_HARD_REG_BIT (ira_no_alloc_regs, regno))
986 : {
987 6467 : if (birth_p)
988 : {
989 4499 : if (!live || bitmap_set_bit (live, regno))
990 4498 : pressure[pressure_class]++;
991 : }
992 : else
993 : {
994 1968 : if (!live || bitmap_clear_bit (live, regno))
995 1968 : pressure[pressure_class]--;
996 : }
997 : }
998 49366 : }
999 :
1000 : /* Initiate current register pressure related info from living
1001 : registers given by LIVE. */
1002 : static void
1003 706 : initiate_reg_pressure_info (bitmap live)
1004 : {
1005 706 : int i;
1006 706 : unsigned int j;
1007 706 : bitmap_iterator bi;
1008 :
1009 3534 : for (i = 0; i < ira_pressure_classes_num; i++)
1010 2828 : curr_reg_pressure[ira_pressure_classes[i]] = 0;
1011 706 : bitmap_clear (curr_reg_live);
1012 7460 : EXECUTE_IF_SET_IN_BITMAP (live, 0, j, bi)
1013 6754 : if (sched_pressure == SCHED_PRESSURE_MODEL
1014 6754 : || current_nr_blocks == 1
1015 7510 : || bitmap_bit_p (region_ref_regs, j))
1016 6346 : mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, j, true);
1017 706 : }
1018 :
1019 : /* Mark registers in X as mentioned in the current region. */
1020 : static void
1021 2988 : setup_ref_regs (rtx x)
1022 : {
1023 2988 : int i, j;
1024 2988 : const RTX_CODE code = GET_CODE (x);
1025 2988 : const char *fmt;
1026 :
1027 2988 : if (REG_P (x))
1028 : {
1029 1020 : bitmap_set_range (region_ref_regs, REGNO (x), REG_NREGS (x));
1030 1020 : return;
1031 : }
1032 1968 : fmt = GET_RTX_FORMAT (code);
1033 5184 : for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
1034 3216 : if (fmt[i] == 'e')
1035 2292 : setup_ref_regs (XEXP (x, i));
1036 924 : else if (fmt[i] == 'E')
1037 : {
1038 120 : for (j = 0; j < XVECLEN (x, i); j++)
1039 72 : setup_ref_regs (XVECEXP (x, i, j));
1040 : }
1041 : }
1042 :
1043 : /* Initiate current register pressure related info at the start of
1044 : basic block BB. */
1045 : static void
1046 706 : initiate_bb_reg_pressure_info (basic_block bb)
1047 : {
1048 706 : unsigned int i ATTRIBUTE_UNUSED;
1049 706 : rtx_insn *insn;
1050 :
1051 706 : if (current_nr_blocks > 1)
1052 1268 : FOR_BB_INSNS (bb, insn)
1053 1208 : if (NONDEBUG_INSN_P (insn))
1054 624 : setup_ref_regs (PATTERN (insn));
1055 706 : initiate_reg_pressure_info (df_get_live_in (bb));
1056 706 : if (bb_has_eh_pred (bb))
1057 0 : for (i = 0; ; ++i)
1058 : {
1059 0 : unsigned int regno = EH_RETURN_DATA_REGNO (i);
1060 :
1061 0 : if (regno == INVALID_REGNUM)
1062 : break;
1063 0 : if (! bitmap_bit_p (df_get_live_in (bb), regno))
1064 0 : mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
1065 : regno, true);
1066 0 : }
1067 706 : }
1068 :
1069 : /* Save current register pressure related info. */
1070 : static void
1071 2866 : save_reg_pressure (void)
1072 : {
1073 2866 : int i;
1074 :
1075 14506 : for (i = 0; i < ira_pressure_classes_num; i++)
1076 11640 : saved_reg_pressure[ira_pressure_classes[i]]
1077 11640 : = curr_reg_pressure[ira_pressure_classes[i]];
1078 2866 : bitmap_copy (saved_reg_live, curr_reg_live);
1079 2866 : }
1080 :
1081 : /* Restore saved register pressure related info. */
1082 : static void
1083 2866 : restore_reg_pressure (void)
1084 : {
1085 2866 : int i;
1086 :
1087 14506 : for (i = 0; i < ira_pressure_classes_num; i++)
1088 11640 : curr_reg_pressure[ira_pressure_classes[i]]
1089 11640 : = saved_reg_pressure[ira_pressure_classes[i]];
1090 2866 : bitmap_copy (curr_reg_live, saved_reg_live);
1091 2866 : }
1092 :
1093 : /* Return TRUE if the register is dying after its USE. */
1094 : static bool
1095 36747 : dying_use_p (struct reg_use_data *use)
1096 : {
1097 36747 : struct reg_use_data *next;
1098 :
1099 43953 : for (next = use->next_regno_use; next != use; next = next->next_regno_use)
1100 17893 : if (NONDEBUG_INSN_P (next->insn)
1101 17893 : && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED)
1102 : return false;
1103 : return true;
1104 : }
1105 :
1106 : /* Print info about the current register pressure and its excess for
1107 : each pressure class. */
1108 : static void
1109 0 : print_curr_reg_pressure (void)
1110 : {
1111 0 : int i;
1112 0 : enum reg_class cl;
1113 :
1114 0 : fprintf (sched_dump, ";;\t");
1115 0 : for (i = 0; i < ira_pressure_classes_num; i++)
1116 : {
1117 0 : cl = ira_pressure_classes[i];
1118 0 : gcc_assert (curr_reg_pressure[cl] >= 0);
1119 0 : fprintf (sched_dump, " %s:%d(%d)", reg_class_names[cl],
1120 : curr_reg_pressure[cl],
1121 0 : curr_reg_pressure[cl] - sched_class_regs_num[cl]);
1122 : }
1123 0 : fprintf (sched_dump, "\n");
1124 0 : }
1125 :
1126 : /* Determine if INSN has a condition that is clobbered if a register
1127 : in SET_REGS is modified. */
1128 : static bool
1129 0 : cond_clobbered_p (rtx_insn *insn, HARD_REG_SET set_regs)
1130 : {
1131 0 : rtx pat = PATTERN (insn);
1132 0 : gcc_assert (GET_CODE (pat) == COND_EXEC);
1133 0 : if (TEST_HARD_REG_BIT (set_regs, REGNO (XEXP (COND_EXEC_TEST (pat), 0))))
1134 : {
1135 0 : sd_iterator_def sd_it;
1136 0 : dep_t dep;
1137 0 : haifa_change_pattern (insn, ORIG_PAT (insn));
1138 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
1139 0 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
1140 0 : TODO_SPEC (insn) = HARD_DEP;
1141 0 : if (sched_verbose >= 2)
1142 0 : fprintf (sched_dump,
1143 : ";;\t\tdequeue insn %s because of clobbered condition\n",
1144 0 : (*current_sched_info->print_insn) (insn, 0));
1145 0 : return true;
1146 : }
1147 :
1148 : return false;
1149 : }
1150 :
1151 : /* This function should be called after modifying the pattern of INSN,
1152 : to update scheduler data structures as needed. */
1153 : static void
1154 669572 : update_insn_after_change (rtx_insn *insn)
1155 : {
1156 669572 : sd_iterator_def sd_it;
1157 669572 : dep_t dep;
1158 :
1159 669572 : dfa_clear_single_insn_cache (insn);
1160 :
1161 669572 : sd_it = sd_iterator_start (insn,
1162 : SD_LIST_FORW | SD_LIST_BACK | SD_LIST_RES_BACK);
1163 7298917 : while (sd_iterator_cond (&sd_it, &dep))
1164 : {
1165 6629345 : DEP_COST (dep) = UNKNOWN_DEP_COST;
1166 6629345 : sd_iterator_next (&sd_it);
1167 : }
1168 :
1169 : /* Invalidate INSN_COST, so it'll be recalculated. */
1170 669572 : INSN_COST (insn) = -1;
1171 : /* Invalidate INSN_TICK, so it'll be recalculated. */
1172 669572 : INSN_TICK (insn) = INVALID_TICK;
1173 :
1174 : /* Invalidate autoprefetch data entry. */
1175 669572 : INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status
1176 669572 : = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
1177 669572 : INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status
1178 669572 : = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
1179 669572 : }
1180 :
1181 :
1182 : /* Two VECs, one to hold dependencies for which pattern replacements
1183 : need to be applied or restored at the start of the next cycle, and
1184 : another to hold an integer that is either one, to apply the
1185 : corresponding replacement, or zero to restore it. */
1186 : static vec<dep_t> next_cycle_replace_deps;
1187 : static vec<int> next_cycle_apply;
1188 :
1189 : static void apply_replacement (dep_t, bool);
1190 : static void restore_pattern (dep_t, bool);
1191 :
1192 : /* Look at the remaining dependencies for insn NEXT, and compute and return
1193 : the TODO_SPEC value we should use for it. This is called after one of
1194 : NEXT's dependencies has been resolved.
1195 : We also perform pattern replacements for predication, and for broken
1196 : replacement dependencies. The latter is only done if FOR_BACKTRACK is
1197 : false. */
1198 :
1199 : static ds_t
1200 328933041 : recompute_todo_spec (rtx_insn *next, bool for_backtrack)
1201 : {
1202 328933041 : ds_t new_ds;
1203 328933041 : sd_iterator_def sd_it;
1204 328933041 : dep_t dep, modify_dep = NULL;
1205 328933041 : int n_spec = 0;
1206 328933041 : int n_control = 0;
1207 328933041 : int n_replace = 0;
1208 328933041 : bool first_p = true;
1209 :
1210 328933041 : if (sd_lists_empty_p (next, SD_LIST_BACK))
1211 : /* NEXT has all its dependencies resolved. */
1212 : return 0;
1213 :
1214 213225980 : if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK))
1215 : return HARD_DEP;
1216 :
1217 : /* If NEXT is intended to sit adjacent to this instruction, we don't
1218 : want to try to break any dependencies. Treat it as a HARD_DEP. */
1219 401466 : if (SCHED_GROUP_P (next))
1220 : return HARD_DEP;
1221 :
1222 : /* Now we've got NEXT with speculative deps only.
1223 : 1. Look at the deps to see what we have to do.
1224 : 2. Check if we can do 'todo'. */
1225 401466 : new_ds = 0;
1226 :
1227 1398875 : FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep)
1228 : {
1229 997409 : rtx_insn *pro = DEP_PRO (dep);
1230 997409 : ds_t ds = DEP_STATUS (dep) & SPECULATIVE;
1231 :
1232 997409 : if (DEBUG_INSN_P (pro) && !DEBUG_INSN_P (next))
1233 145520 : continue;
1234 :
1235 851889 : if (ds)
1236 : {
1237 0 : n_spec++;
1238 0 : if (first_p)
1239 : {
1240 : first_p = false;
1241 :
1242 : new_ds = ds;
1243 : }
1244 : else
1245 0 : new_ds = ds_merge (new_ds, ds);
1246 : }
1247 851889 : else if (DEP_TYPE (dep) == REG_DEP_CONTROL)
1248 : {
1249 0 : if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED)
1250 : {
1251 0 : n_control++;
1252 0 : modify_dep = dep;
1253 : }
1254 0 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
1255 : }
1256 851889 : else if (DEP_REPLACE (dep) != NULL)
1257 : {
1258 851889 : if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED)
1259 : {
1260 851889 : n_replace++;
1261 851889 : modify_dep = dep;
1262 : }
1263 851889 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
1264 : }
1265 : }
1266 :
1267 401466 : if (n_replace > 0 && n_control == 0 && n_spec == 0)
1268 : {
1269 401466 : if (!dbg_cnt (sched_breakdep))
1270 : return HARD_DEP;
1271 1398875 : FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep)
1272 : {
1273 997409 : struct dep_replacement *desc = DEP_REPLACE (dep);
1274 997409 : if (desc != NULL)
1275 : {
1276 851889 : if (desc->insn == next && !for_backtrack)
1277 : {
1278 89547 : gcc_assert (n_replace == 1);
1279 89547 : apply_replacement (dep, true);
1280 : }
1281 851889 : DEP_STATUS (dep) |= DEP_CANCELLED;
1282 : }
1283 : }
1284 : return 0;
1285 : }
1286 :
1287 0 : else if (n_control == 1 && n_replace == 0 && n_spec == 0)
1288 : {
1289 0 : rtx_insn *pro, *other;
1290 0 : rtx new_pat;
1291 0 : rtx cond = NULL_RTX;
1292 0 : bool success;
1293 0 : rtx_insn *prev = NULL;
1294 0 : int i;
1295 0 : unsigned regno;
1296 :
1297 0 : if ((current_sched_info->flags & DO_PREDICATION) == 0
1298 0 : || (ORIG_PAT (next) != NULL_RTX
1299 0 : && PREDICATED_PAT (next) == NULL_RTX))
1300 : return HARD_DEP;
1301 :
1302 0 : pro = DEP_PRO (modify_dep);
1303 0 : other = real_insn_for_shadow (pro);
1304 0 : if (other != NULL_RTX)
1305 0 : pro = other;
1306 :
1307 0 : cond = sched_get_reverse_condition_uncached (pro);
1308 0 : regno = REGNO (XEXP (cond, 0));
1309 :
1310 : /* Find the last scheduled insn that modifies the condition register.
1311 : We can stop looking once we find the insn we depend on through the
1312 : REG_DEP_CONTROL; if the condition register isn't modified after it,
1313 : we know that it still has the right value. */
1314 0 : if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED)
1315 0 : FOR_EACH_VEC_ELT_REVERSE (scheduled_insns, i, prev)
1316 : {
1317 0 : HARD_REG_SET t;
1318 :
1319 0 : find_all_hard_reg_sets (prev, &t, true);
1320 0 : if (TEST_HARD_REG_BIT (t, regno))
1321 0 : return HARD_DEP;
1322 0 : if (prev == pro)
1323 : break;
1324 : }
1325 0 : if (ORIG_PAT (next) == NULL_RTX)
1326 : {
1327 0 : ORIG_PAT (next) = PATTERN (next);
1328 :
1329 0 : new_pat = gen_rtx_COND_EXEC (VOIDmode, cond, PATTERN (next));
1330 0 : success = haifa_change_pattern (next, new_pat);
1331 0 : if (!success)
1332 : return HARD_DEP;
1333 0 : PREDICATED_PAT (next) = new_pat;
1334 : }
1335 0 : else if (PATTERN (next) != PREDICATED_PAT (next))
1336 : {
1337 0 : bool success = haifa_change_pattern (next,
1338 0 : PREDICATED_PAT (next));
1339 0 : gcc_assert (success);
1340 : }
1341 0 : DEP_STATUS (modify_dep) |= DEP_CANCELLED;
1342 0 : return DEP_CONTROL;
1343 : }
1344 :
1345 0 : if (PREDICATED_PAT (next) != NULL_RTX)
1346 : {
1347 0 : int tick = INSN_TICK (next);
1348 0 : bool success = haifa_change_pattern (next,
1349 0 : ORIG_PAT (next));
1350 0 : INSN_TICK (next) = tick;
1351 0 : gcc_assert (success);
1352 : }
1353 :
1354 : /* We can't handle the case where there are both speculative and control
1355 : dependencies, so we return HARD_DEP in such a case. Also fail if
1356 : we have speculative dependencies with not enough points, or more than
1357 : one control dependency. */
1358 0 : if ((n_spec > 0 && (n_control > 0 || n_replace > 0))
1359 0 : || (n_spec > 0
1360 : /* Too few points? */
1361 0 : && ds_weak (new_ds) < spec_info->data_weakness_cutoff)
1362 0 : || n_control > 0
1363 0 : || n_replace > 0)
1364 0 : return HARD_DEP;
1365 :
1366 : return new_ds;
1367 : }
1368 :
1369 : /* Pointer to the last instruction scheduled. */
1370 : static rtx_insn *last_scheduled_insn;
1371 :
1372 : /* Pointer to the last nondebug instruction scheduled within the
1373 : block, or the prev_head of the scheduling block. Used by
1374 : rank_for_schedule, so that insns independent of the last scheduled
1375 : insn will be preferred over dependent instructions. */
1376 : static rtx_insn *last_nondebug_scheduled_insn;
1377 :
1378 : /* Pointer that iterates through the list of unscheduled insns if we
1379 : have a dbg_cnt enabled. It always points at an insn prior to the
1380 : first unscheduled one. */
1381 : static rtx_insn *nonscheduled_insns_begin;
1382 :
1383 : /* Compute cost of executing INSN.
1384 : This is the number of cycles between instruction issue and
1385 : instruction results. */
1386 : int
1387 221249303 : insn_sched_cost (rtx_insn *insn)
1388 : {
1389 221249303 : int cost;
1390 :
1391 221249303 : if (sched_fusion)
1392 : return 0;
1393 :
1394 221249303 : if (sel_sched_p ())
1395 : {
1396 36399 : if (recog_memoized (insn) < 0)
1397 : return 0;
1398 :
1399 35784 : cost = insn_default_latency (insn);
1400 35784 : if (cost < 0)
1401 : cost = 0;
1402 :
1403 35784 : return cost;
1404 : }
1405 :
1406 221212904 : cost = INSN_COST (insn);
1407 :
1408 221212904 : if (cost < 0)
1409 : {
1410 : /* A USE insn, or something else we don't need to
1411 : understand. We can't pass these directly to
1412 : result_ready_cost or insn_default_latency because it will
1413 : trigger a fatal error for unrecognizable insns. */
1414 116696494 : if (recog_memoized (insn) < 0)
1415 : {
1416 55030273 : INSN_COST (insn) = 0;
1417 55030273 : return 0;
1418 : }
1419 : else
1420 : {
1421 61666221 : cost = insn_default_latency (insn);
1422 61666221 : if (cost < 0)
1423 : cost = 0;
1424 :
1425 61666221 : INSN_COST (insn) = cost;
1426 : }
1427 : }
1428 :
1429 : return cost;
1430 : }
1431 :
1432 : /* Compute cost of dependence LINK.
1433 : This is the number of cycles between instruction issue and
1434 : instruction results.
1435 : ??? We also use this function to call recog_memoized on all insns. */
1436 : int
1437 336596583 : dep_cost_1 (dep_t link, dw_t dw)
1438 : {
1439 336596583 : rtx_insn *insn = DEP_PRO (link);
1440 336596583 : rtx_insn *used = DEP_CON (link);
1441 336596583 : int cost;
1442 :
1443 336596583 : if (DEP_COST (link) != UNKNOWN_DEP_COST)
1444 179931492 : return DEP_COST (link);
1445 :
1446 156665091 : if (delay_htab)
1447 : {
1448 0 : struct delay_pair *delay_entry;
1449 0 : delay_entry
1450 0 : = delay_htab_i2->find_with_hash (used, htab_hash_pointer (used));
1451 0 : if (delay_entry)
1452 : {
1453 0 : if (delay_entry->i1 == insn)
1454 : {
1455 0 : DEP_COST (link) = pair_delay (delay_entry);
1456 0 : return DEP_COST (link);
1457 : }
1458 : }
1459 : }
1460 :
1461 : /* A USE insn should never require the value used to be computed.
1462 : This allows the computation of a function's result and parameter
1463 : values to overlap the return and call. We don't care about the
1464 : dependence cost when only decreasing register pressure. */
1465 156665091 : if (recog_memoized (used) < 0)
1466 : {
1467 1357749 : cost = 0;
1468 1357749 : recog_memoized (insn);
1469 : }
1470 : else
1471 : {
1472 155307342 : enum reg_note dep_type = DEP_TYPE (link);
1473 :
1474 155307342 : cost = insn_sched_cost (insn);
1475 :
1476 155307342 : if (INSN_CODE (insn) >= 0)
1477 : {
1478 149986726 : if (dep_type == REG_DEP_ANTI)
1479 : cost = 0;
1480 79668607 : else if (dep_type == REG_DEP_OUTPUT)
1481 : {
1482 24660914 : cost = (insn_default_latency (insn)
1483 24660914 : - insn_default_latency (used));
1484 24660914 : if (cost <= 0)
1485 : cost = 1;
1486 : }
1487 55007693 : else if (bypass_p (insn))
1488 1575 : cost = insn_latency (insn, used);
1489 : }
1490 :
1491 :
1492 155307342 : if (targetm.sched.adjust_cost)
1493 155307342 : cost = targetm.sched.adjust_cost (used, (int) dep_type, insn, cost,
1494 : dw);
1495 :
1496 155307342 : if (cost < 0)
1497 1357749 : cost = 0;
1498 : }
1499 :
1500 156665091 : DEP_COST (link) = cost;
1501 156665091 : return cost;
1502 : }
1503 :
1504 : /* Compute cost of dependence LINK.
1505 : This is the number of cycles between instruction issue and
1506 : instruction results. */
1507 : int
1508 336567440 : dep_cost (dep_t link)
1509 : {
1510 336567440 : return dep_cost_1 (link, 0);
1511 : }
1512 :
1513 : /* Use this sel-sched.cc friendly function in reorder2 instead of increasing
1514 : INSN_PRIORITY explicitly. */
1515 : void
1516 0 : increase_insn_priority (rtx_insn *insn, int amount)
1517 : {
1518 0 : if (!sel_sched_p ())
1519 : {
1520 : /* We're dealing with haifa-sched.cc INSN_PRIORITY. */
1521 0 : if (INSN_PRIORITY_KNOWN (insn))
1522 0 : INSN_PRIORITY (insn) += amount;
1523 : }
1524 : else
1525 : {
1526 : /* In sel-sched.cc INSN_PRIORITY is not kept up to date.
1527 : Use EXPR_PRIORITY instead. */
1528 0 : sel_add_to_insn_priority (insn, amount);
1529 : }
1530 0 : }
1531 :
1532 : /* Return 'true' if DEP should be included in priority calculations. */
1533 : static bool
1534 160528669 : contributes_to_priority_p (dep_t dep)
1535 : {
1536 160528669 : if (DEBUG_INSN_P (DEP_CON (dep))
1537 146465049 : || DEBUG_INSN_P (DEP_PRO (dep)))
1538 : return false;
1539 :
1540 : /* Critical path is meaningful in block boundaries only. */
1541 146465049 : if (!current_sched_info->contributes_to_priority (DEP_CON (dep),
1542 : DEP_PRO (dep)))
1543 : return false;
1544 :
1545 146461169 : if (DEP_REPLACE (dep) != NULL)
1546 : return false;
1547 :
1548 : /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set,
1549 : then speculative instructions will less likely be
1550 : scheduled. That is because the priority of
1551 : their producers will increase, and, thus, the
1552 : producers will more likely be scheduled, thus,
1553 : resolving the dependence. */
1554 145801198 : if (sched_deps_info->generate_spec_deps
1555 0 : && !(spec_info->flags & COUNT_SPEC_IN_CRITICAL_PATH)
1556 0 : && (DEP_STATUS (dep) & SPECULATIVE))
1557 0 : return false;
1558 :
1559 : return true;
1560 : }
1561 :
1562 : /* Compute the number of nondebug deps in list LIST for INSN. */
1563 : int
1564 677128496 : dep_list_size (rtx_insn *insn, sd_list_types_def list)
1565 : {
1566 677128496 : sd_iterator_def sd_it;
1567 677128496 : dep_t dep;
1568 677128496 : int dbgcount = 0, nodbgcount = 0;
1569 :
1570 677128496 : if (!MAY_HAVE_DEBUG_INSNS)
1571 285269866 : return sd_lists_size (insn, list);
1572 :
1573 : /* TODO: We should split normal and debug insns into separate SD_LIST_*
1574 : sub-lists, and then we'll be able to use something like
1575 : sd_lists_size(insn, list & SD_LIST_NON_DEBUG)
1576 : instead of walking dependencies below. */
1577 :
1578 1361081337 : FOR_EACH_DEP (insn, list, sd_it, dep)
1579 : {
1580 969222707 : if (DEBUG_INSN_P (DEP_CON (dep)))
1581 106396463 : dbgcount++;
1582 862826244 : else if (!DEBUG_INSN_P (DEP_PRO (dep)))
1583 854851534 : nodbgcount++;
1584 : }
1585 :
1586 391858630 : gcc_assert (dbgcount + nodbgcount == sd_lists_size (insn, list));
1587 :
1588 : return nodbgcount;
1589 : }
1590 :
1591 : bool sched_fusion;
1592 :
1593 : /* Compute the priority number for INSN. */
1594 : static int
1595 262522353 : priority (rtx_insn *insn, bool force_recompute)
1596 : {
1597 262522353 : if (! INSN_P (insn))
1598 : return 0;
1599 :
1600 : /* We should not be interested in priority of an already scheduled insn. */
1601 262522353 : gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED);
1602 :
1603 262522353 : if (force_recompute || !INSN_PRIORITY_KNOWN (insn))
1604 : {
1605 116721037 : int this_priority = -1;
1606 :
1607 116721037 : if (sched_fusion)
1608 : {
1609 0 : int this_fusion_priority;
1610 :
1611 0 : targetm.sched.fusion_priority (insn, FUSION_MAX_PRIORITY,
1612 : &this_fusion_priority, &this_priority);
1613 0 : INSN_FUSION_PRIORITY (insn) = this_fusion_priority;
1614 : }
1615 116721037 : else if (dep_list_size (insn, SD_LIST_FORW) == 0)
1616 : /* ??? We should set INSN_PRIORITY to insn_sched_cost when and insn
1617 : has some forward deps but all of them are ignored by
1618 : contributes_to_priority hook. At the moment we set priority of
1619 : such insn to 0. */
1620 65931512 : this_priority = insn_sched_cost (insn);
1621 : else
1622 : {
1623 50789525 : rtx_insn *prev_first, *twin;
1624 50789525 : basic_block rec;
1625 :
1626 : /* For recovery check instructions we calculate priority slightly
1627 : different than that of normal instructions. Instead of walking
1628 : through INSN_FORW_DEPS (check) list, we walk through
1629 : INSN_FORW_DEPS list of each instruction in the corresponding
1630 : recovery block. */
1631 :
1632 : /* Selective scheduling does not define RECOVERY_BLOCK macro. */
1633 50789525 : rec = sel_sched_p () ? NULL : RECOVERY_BLOCK (insn);
1634 50786216 : if (!rec || rec == EXIT_BLOCK_PTR_FOR_FN (cfun))
1635 : {
1636 50789525 : prev_first = PREV_INSN (insn);
1637 50789525 : twin = insn;
1638 : }
1639 : else
1640 : {
1641 0 : prev_first = NEXT_INSN (BB_HEAD (rec));
1642 0 : twin = PREV_INSN (BB_END (rec));
1643 : }
1644 :
1645 50789525 : do
1646 : {
1647 50789525 : sd_iterator_def sd_it;
1648 50789525 : dep_t dep;
1649 :
1650 211318194 : FOR_EACH_DEP (twin, SD_LIST_FORW, sd_it, dep)
1651 : {
1652 160528669 : rtx_insn *next;
1653 160528669 : int next_priority;
1654 :
1655 160528669 : next = DEP_CON (dep);
1656 :
1657 160528669 : if (BLOCK_FOR_INSN (next) != rec)
1658 : {
1659 160528669 : int cost;
1660 :
1661 160528669 : if (!contributes_to_priority_p (dep))
1662 14727471 : continue;
1663 :
1664 145801198 : if (twin == insn)
1665 145801198 : cost = dep_cost (dep);
1666 : else
1667 : {
1668 0 : struct _dep _dep1, *dep1 = &_dep1;
1669 :
1670 0 : init_dep (dep1, insn, next, REG_DEP_ANTI);
1671 :
1672 0 : cost = dep_cost (dep1);
1673 : }
1674 :
1675 145801198 : next_priority = cost + priority (next);
1676 :
1677 145801198 : if (next_priority > this_priority)
1678 104464051 : this_priority = next_priority;
1679 : }
1680 : }
1681 :
1682 50789525 : twin = PREV_INSN (twin);
1683 : }
1684 50789525 : while (twin != prev_first);
1685 : }
1686 :
1687 116721037 : if (this_priority < 0)
1688 : {
1689 10449 : gcc_assert (this_priority == -1);
1690 :
1691 10449 : this_priority = insn_sched_cost (insn);
1692 : }
1693 :
1694 116721037 : INSN_PRIORITY (insn) = this_priority;
1695 116721037 : INSN_PRIORITY_STATUS (insn) = 1;
1696 : }
1697 :
1698 262522353 : return INSN_PRIORITY (insn);
1699 : }
1700 :
1701 : /* Macros and functions for keeping the priority queue sorted, and
1702 : dealing with queuing and dequeuing of instructions. */
1703 :
1704 : /* For each pressure class CL, set DEATH[CL] to the number of registers
1705 : in that class that die in INSN. */
1706 :
1707 : static void
1708 22492 : calculate_reg_deaths (rtx_insn *insn, int *death)
1709 : {
1710 22492 : int i;
1711 22492 : struct reg_use_data *use;
1712 :
1713 122020 : for (i = 0; i < ira_pressure_classes_num; i++)
1714 99528 : death[ira_pressure_classes[i]] = 0;
1715 38133 : for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
1716 15641 : if (dying_use_p (use))
1717 8210 : mark_regno_birth_or_death (0, death, use->regno, true);
1718 22492 : }
1719 :
1720 : /* Setup info about the current register pressure impact of scheduling
1721 : INSN at the current scheduling point. */
1722 : static void
1723 22492 : setup_insn_reg_pressure_info (rtx_insn *insn)
1724 : {
1725 22492 : int i, change, before, after, hard_regno;
1726 22492 : int excess_cost_change;
1727 22492 : machine_mode mode;
1728 22492 : enum reg_class cl;
1729 22492 : struct reg_pressure_data *pressure_info;
1730 22492 : int *max_reg_pressure;
1731 22492 : static int death[N_REG_CLASSES];
1732 :
1733 22492 : gcc_checking_assert (!DEBUG_INSN_P (insn));
1734 :
1735 22492 : excess_cost_change = 0;
1736 22492 : calculate_reg_deaths (insn, death);
1737 22492 : pressure_info = INSN_REG_PRESSURE (insn);
1738 22492 : max_reg_pressure = INSN_MAX_REG_PRESSURE (insn);
1739 22492 : gcc_assert (pressure_info != NULL && max_reg_pressure != NULL);
1740 122020 : for (i = 0; i < ira_pressure_classes_num; i++)
1741 : {
1742 99528 : cl = ira_pressure_classes[i];
1743 99528 : gcc_assert (curr_reg_pressure[cl] >= 0);
1744 99528 : change = (int) pressure_info[i].set_increase - death[cl];
1745 99528 : before = MAX (0, max_reg_pressure[i] - sched_class_regs_num[cl]);
1746 99528 : after = MAX (0, max_reg_pressure[i] + change
1747 : - sched_class_regs_num[cl]);
1748 99528 : hard_regno = ira_class_hard_regs[cl][0];
1749 99528 : gcc_assert (hard_regno >= 0);
1750 99528 : mode = reg_raw_mode[hard_regno];
1751 99528 : excess_cost_change += ((after - before)
1752 99528 : * (ira_memory_move_cost[mode][cl][0]
1753 99528 : + ira_memory_move_cost[mode][cl][1]));
1754 : }
1755 22492 : INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn) = excess_cost_change;
1756 22492 : }
1757 :
1758 : /* This is the first page of code related to SCHED_PRESSURE_MODEL.
1759 : It tries to make the scheduler take register pressure into account
1760 : without introducing too many unnecessary stalls. It hooks into the
1761 : main scheduling algorithm at several points:
1762 :
1763 : - Before scheduling starts, model_start_schedule constructs a
1764 : "model schedule" for the current block. This model schedule is
1765 : chosen solely to keep register pressure down. It does not take the
1766 : target's pipeline or the original instruction order into account,
1767 : except as a tie-breaker. It also doesn't work to a particular
1768 : pressure limit.
1769 :
1770 : This model schedule gives us an idea of what pressure can be
1771 : achieved for the block and gives us an example of a schedule that
1772 : keeps to that pressure. It also makes the final schedule less
1773 : dependent on the original instruction order. This is important
1774 : because the original order can either be "wide" (many values live
1775 : at once, such as in user-scheduled code) or "narrow" (few values
1776 : live at once, such as after loop unrolling, where several
1777 : iterations are executed sequentially).
1778 :
1779 : We do not apply this model schedule to the rtx stream. We simply
1780 : record it in model_schedule. We also compute the maximum pressure,
1781 : MP, that was seen during this schedule.
1782 :
1783 : - Instructions are added to the ready queue even if they require
1784 : a stall. The length of the stall is instead computed as:
1785 :
1786 : MAX (INSN_TICK (INSN) - clock_var, 0)
1787 :
1788 : (= insn_delay). This allows rank_for_schedule to choose between
1789 : introducing a deliberate stall or increasing pressure.
1790 :
1791 : - Before sorting the ready queue, model_set_excess_costs assigns
1792 : a pressure-based cost to each ready instruction in the queue.
1793 : This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE
1794 : (ECC for short) and is effectively measured in cycles.
1795 :
1796 : - rank_for_schedule ranks instructions based on:
1797 :
1798 : ECC (insn) + insn_delay (insn)
1799 :
1800 : then as:
1801 :
1802 : insn_delay (insn)
1803 :
1804 : So, for example, an instruction X1 with an ECC of 1 that can issue
1805 : now will win over an instruction X0 with an ECC of zero that would
1806 : introduce a stall of one cycle. However, an instruction X2 with an
1807 : ECC of 2 that can issue now will lose to both X0 and X1.
1808 :
1809 : - When an instruction is scheduled, model_recompute updates the model
1810 : schedule with the new pressures (some of which might now exceed the
1811 : original maximum pressure MP). model_update_limit_points then searches
1812 : for the new point of maximum pressure, if not already known. */
1813 :
1814 : /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL
1815 : from surrounding debug information. */
1816 : #define MODEL_BAR \
1817 : ";;\t\t+------------------------------------------------------\n"
1818 :
1819 : /* Information about the pressure on a particular register class at a
1820 : particular point of the model schedule. */
1821 : struct model_pressure_data {
1822 : /* The pressure at this point of the model schedule, or -1 if the
1823 : point is associated with an instruction that has already been
1824 : scheduled. */
1825 : int ref_pressure;
1826 :
1827 : /* The maximum pressure during or after this point of the model schedule. */
1828 : int max_pressure;
1829 : };
1830 :
1831 : /* Per-instruction information that is used while building the model
1832 : schedule. Here, "schedule" refers to the model schedule rather
1833 : than the main schedule. */
1834 : struct model_insn_info {
1835 : /* The instruction itself. */
1836 : rtx_insn *insn;
1837 :
1838 : /* If this instruction is in model_worklist, these fields link to the
1839 : previous (higher-priority) and next (lower-priority) instructions
1840 : in the list. */
1841 : struct model_insn_info *prev;
1842 : struct model_insn_info *next;
1843 :
1844 : /* While constructing the schedule, QUEUE_INDEX describes whether an
1845 : instruction has already been added to the schedule (QUEUE_SCHEDULED),
1846 : is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE).
1847 : old_queue records the value that QUEUE_INDEX had before scheduling
1848 : started, so that we can restore it once the schedule is complete. */
1849 : int old_queue;
1850 :
1851 : /* The relative importance of an unscheduled instruction. Higher
1852 : values indicate greater importance. */
1853 : unsigned int model_priority;
1854 :
1855 : /* The length of the longest path of satisfied true dependencies
1856 : that leads to this instruction. */
1857 : unsigned int depth;
1858 :
1859 : /* The length of the longest path of dependencies of any kind
1860 : that leads from this instruction. */
1861 : unsigned int alap;
1862 :
1863 : /* The number of predecessor nodes that must still be scheduled. */
1864 : int unscheduled_preds;
1865 : };
1866 :
1867 : /* Information about the pressure limit for a particular register class.
1868 : This structure is used when applying a model schedule to the main
1869 : schedule. */
1870 : struct model_pressure_limit {
1871 : /* The maximum register pressure seen in the original model schedule. */
1872 : int orig_pressure;
1873 :
1874 : /* The maximum register pressure seen in the current model schedule
1875 : (which excludes instructions that have already been scheduled). */
1876 : int pressure;
1877 :
1878 : /* The point of the current model schedule at which PRESSURE is first
1879 : reached. It is set to -1 if the value needs to be recomputed. */
1880 : int point;
1881 : };
1882 :
1883 : /* Describes a particular way of measuring register pressure. */
1884 : struct model_pressure_group {
1885 : /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. */
1886 : struct model_pressure_limit limits[N_REG_CLASSES];
1887 :
1888 : /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure
1889 : on register class ira_pressure_classes[PCI] at point POINT of the
1890 : current model schedule. A POINT of model_num_insns describes the
1891 : pressure at the end of the schedule. */
1892 : struct model_pressure_data *model;
1893 : };
1894 :
1895 : /* Index POINT gives the instruction at point POINT of the model schedule.
1896 : This array doesn't change during main scheduling. */
1897 : static vec<rtx_insn *> model_schedule;
1898 :
1899 : /* The list of instructions in the model worklist, sorted in order of
1900 : decreasing priority. */
1901 : static struct model_insn_info *model_worklist;
1902 :
1903 : /* Index I describes the instruction with INSN_LUID I. */
1904 : static struct model_insn_info *model_insns;
1905 :
1906 : /* The number of instructions in the model schedule. */
1907 : static int model_num_insns;
1908 :
1909 : /* The index of the first instruction in model_schedule that hasn't yet been
1910 : added to the main schedule, or model_num_insns if all of them have. */
1911 : static int model_curr_point;
1912 :
1913 : /* Describes the pressure before each instruction in the model schedule. */
1914 : static struct model_pressure_group model_before_pressure;
1915 :
1916 : /* The first unused model_priority value (as used in model_insn_info). */
1917 : static unsigned int model_next_priority;
1918 :
1919 :
1920 : /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP
1921 : at point POINT of the model schedule. */
1922 : #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \
1923 : (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)])
1924 :
1925 : /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or
1926 : after point POINT of the model schedule. */
1927 : #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \
1928 : (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure)
1929 :
1930 : /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT
1931 : of the model schedule. */
1932 : #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \
1933 : (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure)
1934 :
1935 : /* Information about INSN that is used when creating the model schedule. */
1936 : #define MODEL_INSN_INFO(INSN) \
1937 : (&model_insns[INSN_LUID (INSN)])
1938 :
1939 : /* The instruction at point POINT of the model schedule. */
1940 : #define MODEL_INSN(POINT) \
1941 : (model_schedule[POINT])
1942 :
1943 :
1944 : /* Return INSN's index in the model schedule, or model_num_insns if it
1945 : doesn't belong to that schedule. */
1946 :
1947 : static int
1948 0 : model_index (rtx_insn *insn)
1949 : {
1950 0 : if (INSN_MODEL_INDEX (insn) == 0)
1951 0 : return model_num_insns;
1952 0 : return INSN_MODEL_INDEX (insn) - 1;
1953 : }
1954 :
1955 : /* Make sure that GROUP->limits is up-to-date for the current point
1956 : of the model schedule. */
1957 :
1958 : static void
1959 0 : model_update_limit_points_in_group (struct model_pressure_group *group)
1960 : {
1961 0 : int pci, max_pressure, point;
1962 :
1963 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
1964 : {
1965 : /* We may have passed the final point at which the pressure in
1966 : group->limits[pci].pressure was reached. Update the limit if so. */
1967 0 : max_pressure = MODEL_MAX_PRESSURE (group, model_curr_point, pci);
1968 0 : group->limits[pci].pressure = max_pressure;
1969 :
1970 : /* Find the point at which MAX_PRESSURE is first reached. We need
1971 : to search in three cases:
1972 :
1973 : - We've already moved past the previous pressure point.
1974 : In this case we search forward from model_curr_point.
1975 :
1976 : - We scheduled the previous point of maximum pressure ahead of
1977 : its position in the model schedule, but doing so didn't bring
1978 : the pressure point earlier. In this case we search forward
1979 : from that previous pressure point.
1980 :
1981 : - Scheduling an instruction early caused the maximum pressure
1982 : to decrease. In this case we will have set the pressure
1983 : point to -1, and we search forward from model_curr_point. */
1984 0 : point = MAX (group->limits[pci].point, model_curr_point);
1985 0 : while (point < model_num_insns
1986 0 : && MODEL_REF_PRESSURE (group, point, pci) < max_pressure)
1987 0 : point++;
1988 0 : group->limits[pci].point = point;
1989 :
1990 0 : gcc_assert (MODEL_REF_PRESSURE (group, point, pci) == max_pressure);
1991 0 : gcc_assert (MODEL_MAX_PRESSURE (group, point, pci) == max_pressure);
1992 : }
1993 0 : }
1994 :
1995 : /* Make sure that all register-pressure limits are up-to-date for the
1996 : current position in the model schedule. */
1997 :
1998 : static void
1999 0 : model_update_limit_points (void)
2000 : {
2001 0 : model_update_limit_points_in_group (&model_before_pressure);
2002 0 : }
2003 :
2004 : /* Return the model_index of the last unscheduled use in chain USE
2005 : outside of USE's instruction. Return -1 if there are no other uses,
2006 : or model_num_insns if the register is live at the end of the block. */
2007 :
2008 : static int
2009 0 : model_last_use_except (struct reg_use_data *use)
2010 : {
2011 0 : struct reg_use_data *next;
2012 0 : int last, index;
2013 :
2014 0 : last = -1;
2015 0 : for (next = use->next_regno_use; next != use; next = next->next_regno_use)
2016 0 : if (NONDEBUG_INSN_P (next->insn)
2017 0 : && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED)
2018 : {
2019 0 : index = model_index (next->insn);
2020 0 : if (index == model_num_insns)
2021 : return model_num_insns;
2022 0 : if (last < index)
2023 0 : last = index;
2024 : }
2025 : return last;
2026 : }
2027 :
2028 : /* An instruction with model_index POINT has just been scheduled, and it
2029 : adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1.
2030 : Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and
2031 : MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. */
2032 :
2033 : static void
2034 0 : model_start_update_pressure (struct model_pressure_group *group,
2035 : int point, int pci, int delta)
2036 : {
2037 0 : int next_max_pressure;
2038 :
2039 0 : if (point == model_num_insns)
2040 : {
2041 : /* The instruction wasn't part of the model schedule; it was moved
2042 : from a different block. Update the pressure for the end of
2043 : the model schedule. */
2044 0 : MODEL_REF_PRESSURE (group, point, pci) += delta;
2045 0 : MODEL_MAX_PRESSURE (group, point, pci) += delta;
2046 : }
2047 : else
2048 : {
2049 : /* Record that this instruction has been scheduled. Nothing now
2050 : changes between POINT and POINT + 1, so get the maximum pressure
2051 : from the latter. If the maximum pressure decreases, the new
2052 : pressure point may be before POINT. */
2053 0 : MODEL_REF_PRESSURE (group, point, pci) = -1;
2054 0 : next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci);
2055 0 : if (MODEL_MAX_PRESSURE (group, point, pci) > next_max_pressure)
2056 : {
2057 0 : MODEL_MAX_PRESSURE (group, point, pci) = next_max_pressure;
2058 0 : if (group->limits[pci].point == point)
2059 0 : group->limits[pci].point = -1;
2060 : }
2061 : }
2062 0 : }
2063 :
2064 : /* Record that scheduling a later instruction has changed the pressure
2065 : at point POINT of the model schedule by DELTA (which might be 0).
2066 : Update GROUP accordingly. Return nonzero if these changes might
2067 : trigger changes to previous points as well. */
2068 :
2069 : static int
2070 0 : model_update_pressure (struct model_pressure_group *group,
2071 : int point, int pci, int delta)
2072 : {
2073 0 : int ref_pressure, max_pressure, next_max_pressure;
2074 :
2075 : /* If POINT hasn't yet been scheduled, update its pressure. */
2076 0 : ref_pressure = MODEL_REF_PRESSURE (group, point, pci);
2077 0 : if (ref_pressure >= 0 && delta != 0)
2078 : {
2079 0 : ref_pressure += delta;
2080 0 : MODEL_REF_PRESSURE (group, point, pci) = ref_pressure;
2081 :
2082 : /* Check whether the maximum pressure in the overall schedule
2083 : has increased. (This means that the MODEL_MAX_PRESSURE of
2084 : every point <= POINT will need to increase too; see below.) */
2085 0 : if (group->limits[pci].pressure < ref_pressure)
2086 0 : group->limits[pci].pressure = ref_pressure;
2087 :
2088 : /* If we are at maximum pressure, and the maximum pressure
2089 : point was previously unknown or later than POINT,
2090 : bring it forward. */
2091 0 : if (group->limits[pci].pressure == ref_pressure
2092 0 : && !IN_RANGE (group->limits[pci].point, 0, point))
2093 0 : group->limits[pci].point = point;
2094 :
2095 : /* If POINT used to be the point of maximum pressure, but isn't
2096 : any longer, we need to recalculate it using a forward walk. */
2097 0 : if (group->limits[pci].pressure > ref_pressure
2098 0 : && group->limits[pci].point == point)
2099 0 : group->limits[pci].point = -1;
2100 : }
2101 :
2102 : /* Update the maximum pressure at POINT. Changes here might also
2103 : affect the maximum pressure at POINT - 1. */
2104 0 : next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci);
2105 0 : max_pressure = MAX (ref_pressure, next_max_pressure);
2106 0 : if (MODEL_MAX_PRESSURE (group, point, pci) != max_pressure)
2107 : {
2108 0 : MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
2109 0 : return 1;
2110 : }
2111 : return 0;
2112 : }
2113 :
2114 : /* INSN has just been scheduled. Update the model schedule accordingly. */
2115 :
2116 : static void
2117 0 : model_recompute (rtx_insn *insn)
2118 : {
2119 0 : struct {
2120 : int last_use;
2121 : int regno;
2122 : } uses[FIRST_PSEUDO_REGISTER + MAX_RECOG_OPERANDS];
2123 0 : struct reg_use_data *use;
2124 0 : struct reg_pressure_data *reg_pressure;
2125 0 : int delta[N_REG_CLASSES];
2126 0 : int pci, point, mix, new_last, cl, ref_pressure, queue;
2127 0 : unsigned int i, num_uses, num_pending_births;
2128 0 : bool print_p;
2129 :
2130 : /* The destinations of INSN were previously live from POINT onwards, but are
2131 : now live from model_curr_point onwards. Set up DELTA accordingly. */
2132 0 : point = model_index (insn);
2133 0 : reg_pressure = INSN_REG_PRESSURE (insn);
2134 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2135 : {
2136 0 : cl = ira_pressure_classes[pci];
2137 0 : delta[cl] = reg_pressure[pci].set_increase;
2138 : }
2139 :
2140 : /* Record which registers previously died at POINT, but which now die
2141 : before POINT. Adjust DELTA so that it represents the effect of
2142 : this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of
2143 : registers that will be born in the range [model_curr_point, POINT). */
2144 0 : num_uses = 0;
2145 0 : num_pending_births = 0;
2146 0 : bitmap_clear (tmp_bitmap);
2147 0 : for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
2148 : {
2149 0 : new_last = model_last_use_except (use);
2150 0 : if (new_last < point
2151 0 : && bitmap_set_bit (tmp_bitmap, use->regno)
2152 : /* df_get_live_in has not necessarily been updated to reflect the
2153 : effect of inter-block movement performed by earlier schedules.
2154 : Cope with stale live-in sets by ignoring registers that are not
2155 : currently assumed to be live. */
2156 0 : && bitmap_bit_p (curr_reg_live, use->regno))
2157 : {
2158 0 : gcc_assert (num_uses < ARRAY_SIZE (uses));
2159 0 : uses[num_uses].last_use = new_last;
2160 0 : uses[num_uses].regno = use->regno;
2161 : /* This register is no longer live after POINT - 1. */
2162 0 : mark_regno_birth_or_death (NULL, delta, use->regno, false);
2163 0 : num_uses++;
2164 0 : if (new_last >= 0)
2165 0 : num_pending_births++;
2166 : }
2167 : }
2168 :
2169 : /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT.
2170 : Also set each group pressure limit for POINT. */
2171 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2172 : {
2173 0 : cl = ira_pressure_classes[pci];
2174 0 : model_start_update_pressure (&model_before_pressure,
2175 : point, pci, delta[cl]);
2176 : }
2177 :
2178 : /* Walk the model schedule backwards, starting immediately before POINT. */
2179 0 : print_p = false;
2180 0 : if (point != model_curr_point)
2181 0 : do
2182 : {
2183 0 : point--;
2184 0 : insn = MODEL_INSN (point);
2185 0 : queue = QUEUE_INDEX (insn);
2186 :
2187 0 : if (queue != QUEUE_SCHEDULED)
2188 : {
2189 : /* DELTA describes the effect of the move on the register pressure
2190 : after POINT. Make it describe the effect on the pressure
2191 : before POINT. */
2192 : i = 0;
2193 0 : while (i < num_uses)
2194 : {
2195 0 : if (uses[i].last_use == point)
2196 : {
2197 : /* This register is now live again. */
2198 0 : mark_regno_birth_or_death (NULL, delta,
2199 : uses[i].regno, true);
2200 :
2201 : /* Remove this use from the array. */
2202 0 : uses[i] = uses[num_uses - 1];
2203 0 : num_uses--;
2204 0 : num_pending_births--;
2205 : }
2206 : else
2207 0 : i++;
2208 : }
2209 :
2210 0 : if (sched_verbose >= 5)
2211 : {
2212 0 : if (!print_p)
2213 : {
2214 0 : fprintf (sched_dump, MODEL_BAR);
2215 0 : fprintf (sched_dump, ";;\t\t| New pressure for model"
2216 : " schedule\n");
2217 0 : fprintf (sched_dump, MODEL_BAR);
2218 0 : print_p = true;
2219 : }
2220 :
2221 0 : fprintf (sched_dump, ";;\t\t| %3d %4d %-30s ",
2222 0 : point, INSN_UID (insn),
2223 0 : str_pattern_slim (PATTERN (insn)));
2224 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2225 : {
2226 0 : cl = ira_pressure_classes[pci];
2227 0 : ref_pressure = MODEL_REF_PRESSURE (&model_before_pressure,
2228 : point, pci);
2229 0 : fprintf (sched_dump, " %s:[%d->%d]",
2230 : reg_class_names[ira_pressure_classes[pci]],
2231 0 : ref_pressure, ref_pressure + delta[cl]);
2232 : }
2233 0 : fprintf (sched_dump, "\n");
2234 : }
2235 : }
2236 :
2237 : /* Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1
2238 : might have changed as well. */
2239 0 : mix = num_pending_births;
2240 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2241 : {
2242 0 : cl = ira_pressure_classes[pci];
2243 0 : mix |= delta[cl];
2244 0 : mix |= model_update_pressure (&model_before_pressure,
2245 : point, pci, delta[cl]);
2246 : }
2247 : }
2248 0 : while (mix && point > model_curr_point);
2249 :
2250 0 : if (print_p)
2251 0 : fprintf (sched_dump, MODEL_BAR);
2252 0 : }
2253 :
2254 : /* After DEP, which was cancelled, has been resolved for insn NEXT,
2255 : check whether the insn's pattern needs restoring. */
2256 : static bool
2257 851889 : must_restore_pattern_p (rtx_insn *next, dep_t dep)
2258 : {
2259 851889 : if (QUEUE_INDEX (next) == QUEUE_SCHEDULED)
2260 : return false;
2261 :
2262 304819 : if (DEP_TYPE (dep) == REG_DEP_CONTROL)
2263 : {
2264 0 : gcc_assert (ORIG_PAT (next) != NULL_RTX);
2265 0 : gcc_assert (next == DEP_CON (dep));
2266 : }
2267 : else
2268 : {
2269 304819 : struct dep_replacement *desc = DEP_REPLACE (dep);
2270 304819 : if (desc->insn != next)
2271 : {
2272 243568 : gcc_assert (*desc->loc == desc->orig);
2273 : return false;
2274 : }
2275 : }
2276 : return true;
2277 : }
2278 :
2279 : /* model_spill_cost (CL, P, P') returns the cost of increasing the
2280 : pressure on CL from P to P'. We use this to calculate a "base ECC",
2281 : baseECC (CL, X), for each pressure class CL and each instruction X.
2282 : Supposing X changes the pressure on CL from P to P', and that the
2283 : maximum pressure on CL in the current model schedule is MP', then:
2284 :
2285 : * if X occurs before or at the next point of maximum pressure in
2286 : the model schedule and P' > MP', then:
2287 :
2288 : baseECC (CL, X) = model_spill_cost (CL, MP, P')
2289 :
2290 : The idea is that the pressure after scheduling a fixed set of
2291 : instructions -- in this case, the set up to and including the
2292 : next maximum pressure point -- is going to be the same regardless
2293 : of the order; we simply want to keep the intermediate pressure
2294 : under control. Thus X has a cost of zero unless scheduling it
2295 : now would exceed MP'.
2296 :
2297 : If all increases in the set are by the same amount, no zero-cost
2298 : instruction will ever cause the pressure to exceed MP'. However,
2299 : if X is instead moved past an instruction X' with pressure in the
2300 : range (MP' - (P' - P), MP'), the pressure at X' will increase
2301 : beyond MP'. Since baseECC is very much a heuristic anyway,
2302 : it doesn't seem worth the overhead of tracking cases like these.
2303 :
2304 : The cost of exceeding MP' is always based on the original maximum
2305 : pressure MP. This is so that going 2 registers over the original
2306 : limit has the same cost regardless of whether it comes from two
2307 : separate +1 deltas or from a single +2 delta.
2308 :
2309 : * if X occurs after the next point of maximum pressure in the model
2310 : schedule and P' > P, then:
2311 :
2312 : baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P))
2313 :
2314 : That is, if we move X forward across a point of maximum pressure,
2315 : and if X increases the pressure by P' - P, then we conservatively
2316 : assume that scheduling X next would increase the maximum pressure
2317 : by P' - P. Again, the cost of doing this is based on the original
2318 : maximum pressure MP, for the same reason as above.
2319 :
2320 : * if P' < P, P > MP, and X occurs at or after the next point of
2321 : maximum pressure, then:
2322 :
2323 : baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P)
2324 :
2325 : That is, if we have already exceeded the original maximum pressure MP,
2326 : and if X might reduce the maximum pressure again -- or at least push
2327 : it further back, and thus allow more scheduling freedom -- it is given
2328 : a negative cost to reflect the improvement.
2329 :
2330 : * otherwise,
2331 :
2332 : baseECC (CL, X) = 0
2333 :
2334 : In this case, X is not expected to affect the maximum pressure MP',
2335 : so it has zero cost.
2336 :
2337 : We then create a combined value baseECC (X) that is the sum of
2338 : baseECC (CL, X) for each pressure class CL.
2339 :
2340 : baseECC (X) could itself be used as the ECC value described above.
2341 : However, this is often too conservative, in the sense that it
2342 : tends to make high-priority instructions that increase pressure
2343 : wait too long in cases where introducing a spill would be better.
2344 : For this reason the final ECC is a priority-adjusted form of
2345 : baseECC (X). Specifically, we calculate:
2346 :
2347 : P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X)
2348 : baseP = MAX { P (X) | baseECC (X) <= 0 }
2349 :
2350 : Then:
2351 :
2352 : ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0)
2353 :
2354 : Thus an instruction's effect on pressure is ignored if it has a high
2355 : enough priority relative to the ones that don't increase pressure.
2356 : Negative values of baseECC (X) do not increase the priority of X
2357 : itself, but they do make it harder for other instructions to
2358 : increase the pressure further.
2359 :
2360 : This pressure cost is deliberately timid. The intention has been
2361 : to choose a heuristic that rarely interferes with the normal list
2362 : scheduler in cases where that scheduler would produce good code.
2363 : We simply want to curb some of its worst excesses. */
2364 :
2365 : /* Return the cost of increasing the pressure in class CL from FROM to TO.
2366 :
2367 : Here we use the very simplistic cost model that every register above
2368 : sched_class_regs_num[CL] has a spill cost of 1. We could use other
2369 : measures instead, such as one based on MEMORY_MOVE_COST. However:
2370 :
2371 : (1) In order for an instruction to be scheduled, the higher cost
2372 : would need to be justified in a single saving of that many stalls.
2373 : This is overly pessimistic, because the benefit of spilling is
2374 : often to avoid a sequence of several short stalls rather than
2375 : a single long one.
2376 :
2377 : (2) The cost is still arbitrary. Because we are not allocating
2378 : registers during scheduling, we have no way of knowing for
2379 : sure how many memory accesses will be required by each spill,
2380 : where the spills will be placed within the block, or even
2381 : which block(s) will contain the spills.
2382 :
2383 : So a higher cost than 1 is often too conservative in practice,
2384 : forcing blocks to contain unnecessary stalls instead of spill code.
2385 : The simple cost below seems to be the best compromise. It reduces
2386 : the interference with the normal list scheduler, which helps make
2387 : it more suitable for a default-on option. */
2388 :
2389 : static int
2390 0 : model_spill_cost (int cl, int from, int to)
2391 : {
2392 0 : from = MAX (from, sched_class_regs_num[cl]);
2393 0 : return MAX (to, from) - from;
2394 : }
2395 :
2396 : /* Return baseECC (ira_pressure_classes[PCI], POINT), given that
2397 : P = curr_reg_pressure[ira_pressure_classes[PCI]] and that
2398 : P' = P + DELTA. */
2399 :
2400 : static int
2401 0 : model_excess_group_cost (struct model_pressure_group *group,
2402 : int point, int pci, int delta)
2403 : {
2404 0 : int pressure, cl;
2405 :
2406 0 : cl = ira_pressure_classes[pci];
2407 0 : if (delta < 0)
2408 : {
2409 0 : if (point >= group->limits[pci].point)
2410 : {
2411 0 : pressure = MAX (group->limits[pci].orig_pressure,
2412 : curr_reg_pressure[cl] + delta);
2413 0 : return -model_spill_cost (cl, pressure, curr_reg_pressure[cl]);
2414 : }
2415 : /* if target prefers fewer spills, return the -ve delta indicating
2416 : pressure reduction. */
2417 0 : else if (!param_cycle_accurate_model)
2418 : return delta;
2419 : }
2420 :
2421 0 : if (delta > 0)
2422 : {
2423 0 : if (point > group->limits[pci].point)
2424 0 : pressure = group->limits[pci].pressure + delta;
2425 : else
2426 0 : pressure = curr_reg_pressure[cl] + delta;
2427 :
2428 0 : if (pressure > group->limits[pci].pressure)
2429 0 : return model_spill_cost (cl, group->limits[pci].orig_pressure,
2430 : pressure);
2431 : }
2432 :
2433 : return 0;
2434 : }
2435 :
2436 : /* Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump
2437 : if PRINT_P. */
2438 :
2439 : static int
2440 0 : model_excess_cost (rtx_insn *insn, bool print_p)
2441 : {
2442 0 : int point, pci, cl, cost, this_cost, delta;
2443 0 : struct reg_pressure_data *insn_reg_pressure;
2444 0 : int insn_death[N_REG_CLASSES];
2445 :
2446 0 : calculate_reg_deaths (insn, insn_death);
2447 0 : point = model_index (insn);
2448 0 : insn_reg_pressure = INSN_REG_PRESSURE (insn);
2449 0 : cost = 0;
2450 :
2451 0 : if (print_p)
2452 0 : fprintf (sched_dump, ";;\t\t| %3d %4d | %4d %+3d |", point,
2453 0 : INSN_UID (insn), INSN_PRIORITY (insn), insn_delay (insn));
2454 :
2455 : /* Sum up the individual costs for each register class. */
2456 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2457 : {
2458 0 : cl = ira_pressure_classes[pci];
2459 0 : delta = insn_reg_pressure[pci].set_increase - insn_death[cl];
2460 0 : this_cost = model_excess_group_cost (&model_before_pressure,
2461 : point, pci, delta);
2462 0 : cost += this_cost;
2463 0 : if (print_p)
2464 0 : fprintf (sched_dump, " %s:[%d base cost %d]",
2465 : reg_class_names[cl], delta, this_cost);
2466 : }
2467 :
2468 0 : if (print_p)
2469 0 : fprintf (sched_dump, " ECC %d\n", cost);
2470 :
2471 0 : return cost;
2472 : }
2473 :
2474 : /* Dump the next points of maximum pressure for GROUP. */
2475 :
2476 : static void
2477 0 : model_dump_pressure_points (struct model_pressure_group *group)
2478 : {
2479 0 : int pci, cl;
2480 :
2481 0 : fprintf (sched_dump, ";;\t\t| pressure points");
2482 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
2483 : {
2484 0 : cl = ira_pressure_classes[pci];
2485 0 : fprintf (sched_dump, " %s:[%d->%d at ", reg_class_names[cl],
2486 : curr_reg_pressure[cl], group->limits[pci].pressure);
2487 0 : if (group->limits[pci].point < model_num_insns)
2488 0 : fprintf (sched_dump, "%d:%d]", group->limits[pci].point,
2489 0 : INSN_UID (MODEL_INSN (group->limits[pci].point)));
2490 : else
2491 0 : fprintf (sched_dump, "end]");
2492 : }
2493 0 : fprintf (sched_dump, "\n");
2494 0 : }
2495 :
2496 : /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */
2497 :
2498 : static void
2499 0 : model_set_excess_costs (rtx_insn **insns, int count)
2500 : {
2501 0 : int i, cost, priority_base, priority;
2502 0 : bool print_p;
2503 :
2504 : /* Record the baseECC value for each instruction in the model schedule,
2505 : except that for targets which prefer wider schedules (more spills)
2506 : negative costs are converted to zero ones now rather than later.
2507 : Do not assign a cost to debug instructions, since they must
2508 : not change code-generation decisions. Experiments suggest we also
2509 : get better results by not assigning a cost to instructions from
2510 : a different block.
2511 :
2512 : Set PRIORITY_BASE to baseP in the block comment above. This is the
2513 : maximum priority of the "cheap" instructions, which should always
2514 : include the next model instruction. */
2515 0 : priority_base = 0;
2516 0 : print_p = false;
2517 0 : for (i = 0; i < count; i++)
2518 0 : if (INSN_MODEL_INDEX (insns[i]))
2519 : {
2520 0 : if (sched_verbose >= 6 && !print_p)
2521 : {
2522 0 : fprintf (sched_dump, MODEL_BAR);
2523 0 : fprintf (sched_dump, ";;\t\t| Pressure costs for ready queue\n");
2524 0 : model_dump_pressure_points (&model_before_pressure);
2525 0 : fprintf (sched_dump, MODEL_BAR);
2526 0 : print_p = true;
2527 : }
2528 0 : cost = model_excess_cost (insns[i], print_p);
2529 0 : if (param_cycle_accurate_model && cost <= 0)
2530 : {
2531 0 : priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]) - cost;
2532 0 : priority_base = MAX (priority_base, priority);
2533 : cost = 0;
2534 : }
2535 0 : INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = cost;
2536 : }
2537 0 : if (print_p)
2538 0 : fprintf (sched_dump, MODEL_BAR);
2539 :
2540 : /* Typically in-order cores have a good pipeline scheduling model and the
2541 : algorithm would try to use that to minimize bubbles, favoring spills.
2542 : MAX (baseECC, 0) below changes negative baseECC (pressure reduction)
2543 : to 0 (pressure neutral) thus tending to more spills.
2544 : Otherwise return. */
2545 0 : if (!param_cycle_accurate_model)
2546 : return;
2547 :
2548 : /* Use MAX (baseECC, 0) and baseP to calculate ECC for each
2549 : instruction. */
2550 0 : for (i = 0; i < count; i++)
2551 : {
2552 0 : cost = INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]);
2553 0 : priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]);
2554 0 : if (cost > 0 && priority > priority_base)
2555 : {
2556 0 : cost += priority_base - priority;
2557 0 : INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = MAX (cost, 0);
2558 : }
2559 : }
2560 : }
2561 :
2562 :
2563 : /* Enum of rank_for_schedule heuristic decisions. */
2564 : enum rfs_decision {
2565 : RFS_LIVE_RANGE_SHRINK1, RFS_LIVE_RANGE_SHRINK2,
2566 : RFS_SCHED_GROUP, RFS_PRESSURE_DELAY, RFS_PRESSURE_TICK,
2567 : RFS_FEEDS_BACKTRACK_INSN, RFS_PRIORITY, RFS_AUTOPREF, RFS_SPECULATION,
2568 : RFS_SCHED_RANK, RFS_LAST_INSN, RFS_PRESSURE_INDEX,
2569 : RFS_DEP_COUNT, RFS_TIE, RFS_FUSION, RFS_COST, RFS_N };
2570 :
2571 : /* Corresponding strings for print outs. */
2572 : static const char *rfs_str[RFS_N] = {
2573 : "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
2574 : "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
2575 : "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_AUTOPREF", "RFS_SPECULATION",
2576 : "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
2577 : "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION", "RFS_COST" };
2578 :
2579 : /* Statistical breakdown of rank_for_schedule decisions. */
2580 : struct rank_for_schedule_stats_t { unsigned stats[RFS_N]; };
2581 : static rank_for_schedule_stats_t rank_for_schedule_stats;
2582 :
2583 : /* Return the result of comparing insns TMP and TMP2 and update
2584 : Rank_For_Schedule statistics. */
2585 : static int
2586 343434222 : rfs_result (enum rfs_decision decision, int result, rtx_insn *tmp,
2587 : rtx_insn *tmp2)
2588 : {
2589 343434222 : ++rank_for_schedule_stats.stats[decision];
2590 343434222 : if (result < 0)
2591 188399222 : INSN_LAST_RFS_WIN (tmp) = decision;
2592 155035000 : else if (result > 0)
2593 155035000 : INSN_LAST_RFS_WIN (tmp2) = decision;
2594 : else
2595 0 : gcc_unreachable ();
2596 343434222 : return result;
2597 : }
2598 :
2599 : /* Sorting predicate to move DEBUG_INSNs to the top of ready list, while
2600 : keeping normal insns in original order. */
2601 :
2602 : static int
2603 435675699 : rank_for_schedule_debug (const void *x, const void *y)
2604 : {
2605 435675699 : rtx_insn *tmp = *(rtx_insn * const *) y;
2606 435675699 : rtx_insn *tmp2 = *(rtx_insn * const *) x;
2607 :
2608 : /* Schedule debug insns as early as possible. */
2609 435675699 : if (DEBUG_INSN_P (tmp) && !DEBUG_INSN_P (tmp2))
2610 : return -1;
2611 272857386 : else if (!DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
2612 : return 1;
2613 202887662 : else if (DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
2614 0 : return INSN_LUID (tmp) - INSN_LUID (tmp2);
2615 : else
2616 202887662 : return INSN_RFS_DEBUG_ORIG_ORDER (tmp2) - INSN_RFS_DEBUG_ORIG_ORDER (tmp);
2617 : }
2618 :
2619 : /* Returns a positive value if x is preferred; returns a negative value if
2620 : y is preferred. Should never return 0, since that will make the sort
2621 : unstable. */
2622 :
2623 : static int
2624 343434222 : rank_for_schedule (const void *x, const void *y)
2625 : {
2626 343434222 : rtx_insn *tmp = *(rtx_insn * const *) y;
2627 343434222 : rtx_insn *tmp2 = *(rtx_insn * const *) x;
2628 343434222 : int tmp_class, tmp2_class;
2629 343434222 : int val, priority_val, info_val, diff;
2630 :
2631 343434222 : if (live_range_shrinkage_p)
2632 : {
2633 : /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse
2634 : code. */
2635 235785 : gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED);
2636 235785 : if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) < 0
2637 230522 : || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2) < 0)
2638 235785 : && (diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp)
2639 9169 : - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2))) != 0)
2640 9061 : return rfs_result (RFS_LIVE_RANGE_SHRINK1, diff, tmp, tmp2);
2641 : /* Sort by INSN_LUID (original insn order), so that we make the
2642 : sort stable. This minimizes instruction movement, thus
2643 : minimizing sched's effect on debugging and cross-jumping. */
2644 680172 : return rfs_result (RFS_LIVE_RANGE_SHRINK2,
2645 226724 : INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2646 : }
2647 :
2648 : /* The insn in a schedule group should be issued the first. */
2649 343198437 : if (flag_sched_group_heuristic &&
2650 343198437 : SCHED_GROUP_P (tmp) != SCHED_GROUP_P (tmp2))
2651 191 : return rfs_result (RFS_SCHED_GROUP, SCHED_GROUP_P (tmp2) ? 1 : -1,
2652 138 : tmp, tmp2);
2653 :
2654 : /* Make sure that priority of TMP and TMP2 are initialized. */
2655 343198299 : gcc_assert (INSN_PRIORITY_KNOWN (tmp) && INSN_PRIORITY_KNOWN (tmp2));
2656 :
2657 343198299 : if (sched_fusion)
2658 : {
2659 : /* The instruction that has the same fusion priority as the last
2660 : instruction is the instruction we picked next. If that is not
2661 : the case, we sort ready list firstly by fusion priority, then
2662 : by priority, and at last by INSN_LUID. */
2663 0 : int a = INSN_FUSION_PRIORITY (tmp);
2664 0 : int b = INSN_FUSION_PRIORITY (tmp2);
2665 0 : int last = -1;
2666 :
2667 0 : if (last_nondebug_scheduled_insn
2668 0 : && !NOTE_P (last_nondebug_scheduled_insn)
2669 0 : && BLOCK_FOR_INSN (tmp)
2670 0 : == BLOCK_FOR_INSN (last_nondebug_scheduled_insn))
2671 0 : last = INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn);
2672 :
2673 0 : if (a != last && b != last)
2674 : {
2675 0 : if (a == b)
2676 : {
2677 0 : a = INSN_PRIORITY (tmp);
2678 0 : b = INSN_PRIORITY (tmp2);
2679 : }
2680 0 : if (a != b)
2681 0 : return rfs_result (RFS_FUSION, b - a, tmp, tmp2);
2682 : else
2683 0 : return rfs_result (RFS_FUSION,
2684 0 : INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2685 : }
2686 0 : else if (a == b)
2687 : {
2688 0 : gcc_assert (last_nondebug_scheduled_insn
2689 : && !NOTE_P (last_nondebug_scheduled_insn));
2690 0 : last = INSN_PRIORITY (last_nondebug_scheduled_insn);
2691 :
2692 0 : a = abs (INSN_PRIORITY (tmp) - last);
2693 0 : b = abs (INSN_PRIORITY (tmp2) - last);
2694 0 : if (a != b)
2695 0 : return rfs_result (RFS_FUSION, a - b, tmp, tmp2);
2696 : else
2697 0 : return rfs_result (RFS_FUSION,
2698 0 : INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2699 : }
2700 0 : else if (a == last)
2701 0 : return rfs_result (RFS_FUSION, -1, tmp, tmp2);
2702 : else
2703 0 : return rfs_result (RFS_FUSION, 1, tmp, tmp2);
2704 : }
2705 :
2706 343198299 : if (sched_pressure != SCHED_PRESSURE_NONE)
2707 : {
2708 : /* Prefer insn whose scheduling results in the smallest register
2709 : pressure excess. */
2710 13661 : if ((diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp)
2711 13661 : + insn_delay (tmp)
2712 13661 : - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2)
2713 13661 : - insn_delay (tmp2))))
2714 5021 : return rfs_result (RFS_PRESSURE_DELAY, diff, tmp, tmp2);
2715 : }
2716 :
2717 343193278 : if (sched_pressure != SCHED_PRESSURE_NONE
2718 8640 : && (INSN_TICK (tmp2) > clock_var || INSN_TICK (tmp) > clock_var)
2719 343194223 : && INSN_TICK (tmp2) != INSN_TICK (tmp))
2720 : {
2721 0 : diff = INSN_TICK (tmp) - INSN_TICK (tmp2);
2722 0 : return rfs_result (RFS_PRESSURE_TICK, diff, tmp, tmp2);
2723 : }
2724 :
2725 : /* If we are doing backtracking in this schedule, prefer insns that
2726 : have forward dependencies with negative cost against an insn that
2727 : was already scheduled. */
2728 343193278 : if (current_sched_info->flags & DO_BACKTRACKING)
2729 : {
2730 0 : priority_val = FEEDS_BACKTRACK_INSN (tmp2) - FEEDS_BACKTRACK_INSN (tmp);
2731 0 : if (priority_val)
2732 0 : return rfs_result (RFS_FEEDS_BACKTRACK_INSN, priority_val, tmp, tmp2);
2733 : }
2734 :
2735 : /* Prefer insn with higher priority. */
2736 343193278 : priority_val = INSN_PRIORITY (tmp2) - INSN_PRIORITY (tmp);
2737 :
2738 343193278 : if (flag_sched_critical_path_heuristic && priority_val)
2739 93141113 : return rfs_result (RFS_PRIORITY, priority_val, tmp, tmp2);
2740 :
2741 250052165 : if (param_sched_autopref_queue_depth >= 0)
2742 : {
2743 12 : int autopref = autopref_rank_for_schedule (tmp, tmp2);
2744 12 : if (autopref != 0)
2745 12 : return rfs_result (RFS_AUTOPREF, autopref, tmp, tmp2);
2746 : }
2747 :
2748 : /* Prefer speculative insn with greater dependencies weakness. */
2749 250052153 : if (flag_sched_spec_insn_heuristic && spec_info)
2750 : {
2751 0 : ds_t ds1, ds2;
2752 0 : dw_t dw1, dw2;
2753 0 : int dw;
2754 :
2755 0 : ds1 = TODO_SPEC (tmp) & SPECULATIVE;
2756 0 : if (ds1)
2757 0 : dw1 = ds_weak (ds1);
2758 : else
2759 : dw1 = NO_DEP_WEAK;
2760 :
2761 0 : ds2 = TODO_SPEC (tmp2) & SPECULATIVE;
2762 0 : if (ds2)
2763 0 : dw2 = ds_weak (ds2);
2764 : else
2765 : dw2 = NO_DEP_WEAK;
2766 :
2767 0 : dw = dw2 - dw1;
2768 0 : if (dw > (NO_DEP_WEAK / 8) || dw < -(NO_DEP_WEAK / 8))
2769 0 : return rfs_result (RFS_SPECULATION, dw, tmp, tmp2);
2770 : }
2771 :
2772 250052153 : info_val = (*current_sched_info->rank) (tmp, tmp2);
2773 250052153 : if (flag_sched_rank_heuristic && info_val)
2774 0 : return rfs_result (RFS_SCHED_RANK, info_val, tmp, tmp2);
2775 :
2776 : /* Compare insns based on their relation to the last scheduled
2777 : non-debug insn. */
2778 250052153 : if (flag_sched_last_insn_heuristic && last_nondebug_scheduled_insn)
2779 : {
2780 222256374 : dep_t dep1;
2781 222256374 : dep_t dep2;
2782 222256374 : rtx_insn *last = last_nondebug_scheduled_insn;
2783 :
2784 : /* Classify the instructions into three classes:
2785 : 1) Data dependent on last schedule insn.
2786 : 2) Anti/Output dependent on last scheduled insn.
2787 : 3) Independent of last scheduled insn, or has latency of one.
2788 : Choose the insn from the highest numbered class if different. */
2789 222256374 : dep1 = sd_find_dep_between (last, tmp, true);
2790 :
2791 222256374 : if (dep1 == NULL || dep_cost (dep1) == 1)
2792 : tmp_class = 3;
2793 16680044 : else if (/* Data dependence. */
2794 16680044 : DEP_TYPE (dep1) == REG_DEP_TRUE)
2795 : tmp_class = 1;
2796 : else
2797 15170681 : tmp_class = 2;
2798 :
2799 222256374 : dep2 = sd_find_dep_between (last, tmp2, true);
2800 :
2801 222256374 : if (dep2 == NULL || dep_cost (dep2) == 1)
2802 : tmp2_class = 3;
2803 17035962 : else if (/* Data dependence. */
2804 17035962 : DEP_TYPE (dep2) == REG_DEP_TRUE)
2805 : tmp2_class = 1;
2806 : else
2807 15315954 : tmp2_class = 2;
2808 :
2809 222256374 : if ((val = tmp2_class - tmp_class))
2810 3393840 : return rfs_result (RFS_LAST_INSN, val, tmp, tmp2);
2811 : }
2812 :
2813 : /* Prefer instructions that occur earlier in the model schedule. */
2814 246658313 : if (sched_pressure == SCHED_PRESSURE_MODEL)
2815 : {
2816 0 : diff = model_index (tmp) - model_index (tmp2);
2817 0 : if (diff != 0)
2818 0 : return rfs_result (RFS_PRESSURE_INDEX, diff, tmp, tmp2);
2819 : }
2820 :
2821 : /* Prefer the insn which has more later insns that depend on it.
2822 : This gives the scheduler more freedom when scheduling later
2823 : instructions at the expense of added register pressure. */
2824 :
2825 246658313 : val = (dep_list_size (tmp2, SD_LIST_FORW)
2826 246658313 : - dep_list_size (tmp, SD_LIST_FORW));
2827 :
2828 246658313 : if (flag_sched_dep_count_heuristic && val != 0)
2829 29843831 : return rfs_result (RFS_DEP_COUNT, val, tmp, tmp2);
2830 :
2831 : /* Sort by INSN_COST rather than INSN_LUID. This means that instructions
2832 : which take longer to execute are prioritised and it leads to more
2833 : dual-issue opportunities on in-order cores which have this feature. */
2834 :
2835 216814482 : if (INSN_COST (tmp) != INSN_COST (tmp2))
2836 13823821 : return rfs_result (RFS_COST, INSN_COST (tmp2) - INSN_COST (tmp),
2837 13823821 : tmp, tmp2);
2838 :
2839 : /* If insns are equally good, sort by INSN_LUID (original insn order),
2840 : so that we make the sort stable. This minimizes instruction movement,
2841 : thus minimizing sched's effect on debugging and cross-jumping. */
2842 202990661 : return rfs_result (RFS_TIE, INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
2843 : }
2844 :
2845 : /* Resort the array A in which only element at index N may be out of order. */
2846 :
2847 : HAIFA_INLINE static void
2848 8220939 : swap_sort (rtx_insn **a, int n)
2849 : {
2850 8220939 : rtx_insn *insn = a[n - 1];
2851 8220939 : int i = n - 2;
2852 :
2853 11054331 : while (i >= 0 && rank_for_schedule (a + i, &insn) >= 0)
2854 : {
2855 2833392 : a[i + 1] = a[i];
2856 2833392 : i -= 1;
2857 : }
2858 8220939 : a[i + 1] = insn;
2859 8220939 : }
2860 :
2861 : /* Add INSN to the insn queue so that it can be executed at least
2862 : N_CYCLES after the currently executing insn. Preserve insns
2863 : chain for debugging purposes. REASON will be printed in debugging
2864 : output. */
2865 :
2866 : HAIFA_INLINE static void
2867 39963714 : queue_insn (rtx_insn *insn, int n_cycles, const char *reason)
2868 : {
2869 39963714 : int next_q = NEXT_Q_AFTER (q_ptr, n_cycles);
2870 39963714 : rtx_insn_list *link = alloc_INSN_LIST (insn, insn_queue[next_q]);
2871 39963714 : int new_tick;
2872 :
2873 39963714 : gcc_assert (n_cycles <= max_insn_queue_index);
2874 39963714 : gcc_assert (!DEBUG_INSN_P (insn));
2875 :
2876 39963714 : insn_queue[next_q] = link;
2877 39963714 : q_size += 1;
2878 :
2879 39963714 : if (sched_verbose >= 2)
2880 : {
2881 0 : fprintf (sched_dump, ";;\t\tReady-->Q: insn %s: ",
2882 0 : (*current_sched_info->print_insn) (insn, 0));
2883 :
2884 0 : fprintf (sched_dump, "queued for %d cycles (%s).\n", n_cycles, reason);
2885 : }
2886 :
2887 39963714 : QUEUE_INDEX (insn) = next_q;
2888 :
2889 39963714 : if (current_sched_info->flags & DO_BACKTRACKING)
2890 : {
2891 0 : new_tick = clock_var + n_cycles;
2892 0 : if (INSN_TICK (insn) == INVALID_TICK || INSN_TICK (insn) < new_tick)
2893 0 : INSN_TICK (insn) = new_tick;
2894 :
2895 0 : if (INSN_EXACT_TICK (insn) != INVALID_TICK
2896 0 : && INSN_EXACT_TICK (insn) < clock_var + n_cycles)
2897 : {
2898 0 : must_backtrack = true;
2899 0 : if (sched_verbose >= 2)
2900 0 : fprintf (sched_dump, ";;\t\tcausing a backtrack.\n");
2901 : }
2902 : }
2903 39963714 : }
2904 :
2905 : /* Remove INSN from queue. */
2906 : static void
2907 137697 : queue_remove (rtx_insn *insn)
2908 : {
2909 137697 : gcc_assert (QUEUE_INDEX (insn) >= 0);
2910 137697 : remove_free_INSN_LIST_elem (insn, &insn_queue[QUEUE_INDEX (insn)]);
2911 137697 : q_size--;
2912 137697 : QUEUE_INDEX (insn) = QUEUE_NOWHERE;
2913 137697 : }
2914 :
2915 : /* Return a pointer to the bottom of the ready list, i.e. the insn
2916 : with the lowest priority. */
2917 :
2918 : rtx_insn **
2919 152167692 : ready_lastpos (struct ready_list *ready)
2920 : {
2921 152167692 : gcc_assert (ready->n_ready >= 1);
2922 152167692 : return ready->vec + ready->first - ready->n_ready + 1;
2923 : }
2924 :
2925 : /* Add an element INSN to the ready list so that it ends up with the
2926 : lowest/highest priority depending on FIRST_P. */
2927 :
2928 : HAIFA_INLINE static void
2929 136081140 : ready_add (struct ready_list *ready, rtx_insn *insn, bool first_p)
2930 : {
2931 136081140 : if (!first_p)
2932 : {
2933 136078904 : if (ready->first == ready->n_ready)
2934 : {
2935 140 : memmove (ready->vec + ready->veclen - ready->n_ready,
2936 70 : ready_lastpos (ready),
2937 70 : ready->n_ready * sizeof (rtx));
2938 70 : ready->first = ready->veclen - 1;
2939 : }
2940 136078904 : ready->vec[ready->first - ready->n_ready] = insn;
2941 : }
2942 : else
2943 : {
2944 2236 : if (ready->first == ready->veclen - 1)
2945 : {
2946 950 : if (ready->n_ready)
2947 : /* ready_lastpos() fails when called with (ready->n_ready == 0). */
2948 0 : memmove (ready->vec + ready->veclen - ready->n_ready - 1,
2949 0 : ready_lastpos (ready),
2950 0 : ready->n_ready * sizeof (rtx));
2951 950 : ready->first = ready->veclen - 2;
2952 : }
2953 2236 : ready->vec[++(ready->first)] = insn;
2954 : }
2955 :
2956 136081140 : ready->n_ready++;
2957 136081140 : if (DEBUG_INSN_P (insn))
2958 54175777 : ready->n_debug++;
2959 :
2960 136081140 : gcc_assert (QUEUE_INDEX (insn) != QUEUE_READY);
2961 136081140 : QUEUE_INDEX (insn) = QUEUE_READY;
2962 :
2963 136081140 : if (INSN_EXACT_TICK (insn) != INVALID_TICK
2964 136081140 : && INSN_EXACT_TICK (insn) < clock_var)
2965 : {
2966 0 : must_backtrack = true;
2967 : }
2968 136081140 : }
2969 :
2970 : /* Remove the element with the highest priority from the ready list and
2971 : return it. */
2972 :
2973 : HAIFA_INLINE static rtx_insn *
2974 133786062 : ready_remove_first (struct ready_list *ready)
2975 : {
2976 133786062 : rtx_insn *t;
2977 :
2978 133786062 : gcc_assert (ready->n_ready);
2979 133786062 : t = ready->vec[ready->first--];
2980 133786062 : ready->n_ready--;
2981 133786062 : if (DEBUG_INSN_P (t))
2982 54175777 : ready->n_debug--;
2983 : /* If the queue becomes empty, reset it. */
2984 133786062 : if (ready->n_ready == 0)
2985 68967326 : ready->first = ready->veclen - 1;
2986 :
2987 133786062 : gcc_assert (QUEUE_INDEX (t) == QUEUE_READY);
2988 133786062 : QUEUE_INDEX (t) = QUEUE_NOWHERE;
2989 :
2990 133786062 : return t;
2991 : }
2992 :
2993 : /* The following code implements multi-pass scheduling for the first
2994 : cycle. In other words, we will try to choose ready insn which
2995 : permits to start maximum number of insns on the same cycle. */
2996 :
2997 : /* Return a pointer to the element INDEX from the ready. INDEX for
2998 : insn with the highest priority is 0, and the lowest priority has
2999 : N_READY - 1. */
3000 :
3001 : rtx_insn *
3002 1148677418 : ready_element (struct ready_list *ready, int index)
3003 : {
3004 1148677418 : gcc_assert (ready->n_ready && index < ready->n_ready);
3005 :
3006 1148677418 : return ready->vec[ready->first - index];
3007 : }
3008 :
3009 : /* Remove the element INDEX from the ready list and return it. INDEX
3010 : for insn with the highest priority is 0, and the lowest priority
3011 : has N_READY - 1. */
3012 :
3013 : HAIFA_INLINE static rtx_insn *
3014 74535495 : ready_remove (struct ready_list *ready, int index)
3015 : {
3016 74535495 : rtx_insn *t;
3017 74535495 : int i;
3018 :
3019 74535495 : if (index == 0)
3020 72240429 : return ready_remove_first (ready);
3021 2295066 : gcc_assert (ready->n_ready && index < ready->n_ready);
3022 2295066 : t = ready->vec[ready->first - index];
3023 2295066 : ready->n_ready--;
3024 2295066 : if (DEBUG_INSN_P (t))
3025 0 : ready->n_debug--;
3026 7503142 : for (i = index; i < ready->n_ready; i++)
3027 5208076 : ready->vec[ready->first - i] = ready->vec[ready->first - i - 1];
3028 2295066 : QUEUE_INDEX (t) = QUEUE_NOWHERE;
3029 2295066 : return t;
3030 : }
3031 :
3032 : /* Remove INSN from the ready list. */
3033 : static void
3034 0 : ready_remove_insn (rtx_insn *insn)
3035 : {
3036 0 : int i;
3037 :
3038 0 : for (i = 0; i < readyp->n_ready; i++)
3039 0 : if (ready_element (readyp, i) == insn)
3040 : {
3041 0 : ready_remove (readyp, i);
3042 0 : return;
3043 : }
3044 0 : gcc_unreachable ();
3045 : }
3046 :
3047 : /* Calculate difference of two statistics set WAS and NOW.
3048 : Result returned in WAS. */
3049 : static void
3050 0 : rank_for_schedule_stats_diff (rank_for_schedule_stats_t *was,
3051 : const rank_for_schedule_stats_t *now)
3052 : {
3053 0 : for (int i = 0; i < RFS_N; ++i)
3054 0 : was->stats[i] = now->stats[i] - was->stats[i];
3055 0 : }
3056 :
3057 : /* Print rank_for_schedule statistics. */
3058 : static void
3059 0 : print_rank_for_schedule_stats (const char *prefix,
3060 : const rank_for_schedule_stats_t *stats,
3061 : struct ready_list *ready)
3062 : {
3063 0 : for (int i = 0; i < RFS_N; ++i)
3064 0 : if (stats->stats[i])
3065 : {
3066 0 : fprintf (sched_dump, "%s%20s: %u", prefix, rfs_str[i], stats->stats[i]);
3067 :
3068 0 : if (ready != NULL)
3069 : /* Print out insns that won due to RFS_<I>. */
3070 : {
3071 0 : rtx_insn **p = ready_lastpos (ready);
3072 :
3073 0 : fprintf (sched_dump, ":");
3074 : /* Start with 1 since least-priority insn didn't have any wins. */
3075 0 : for (int j = 1; j < ready->n_ready; ++j)
3076 0 : if (INSN_LAST_RFS_WIN (p[j]) == i)
3077 0 : fprintf (sched_dump, " %s",
3078 0 : (*current_sched_info->print_insn) (p[j], 0));
3079 : }
3080 0 : fprintf (sched_dump, "\n");
3081 : }
3082 0 : }
3083 :
3084 : /* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end
3085 : of array. */
3086 : static void
3087 54175781 : ready_sort_debug (struct ready_list *ready)
3088 : {
3089 54175781 : int i;
3090 54175781 : rtx_insn **first = ready_lastpos (ready);
3091 :
3092 229227995 : for (i = 0; i < ready->n_ready; ++i)
3093 120876433 : if (!DEBUG_INSN_P (first[i]))
3094 66700655 : INSN_RFS_DEBUG_ORIG_ORDER (first[i]) = i;
3095 :
3096 54175781 : qsort (first, ready->n_ready, sizeof (rtx), rank_for_schedule_debug);
3097 54175781 : }
3098 :
3099 : /* Sort non-debug insns in the ready list READY by ascending priority.
3100 : Assumes that all debug insns are separated from the real insns. */
3101 : static void
3102 62019063 : ready_sort_real (struct ready_list *ready)
3103 : {
3104 62019063 : int i;
3105 62019063 : rtx_insn **first = ready_lastpos (ready);
3106 62019063 : int n_ready_real = ready->n_ready - ready->n_debug;
3107 :
3108 62019063 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
3109 28635 : for (i = 0; i < n_ready_real; ++i)
3110 22492 : setup_insn_reg_pressure_info (first[i]);
3111 62012920 : else if (sched_pressure == SCHED_PRESSURE_MODEL
3112 0 : && model_curr_point < model_num_insns)
3113 0 : model_set_excess_costs (first, n_ready_real);
3114 :
3115 62019063 : rank_for_schedule_stats_t stats1;
3116 62019063 : if (sched_verbose >= 4)
3117 0 : stats1 = rank_for_schedule_stats;
3118 :
3119 62019063 : if (n_ready_real == 2)
3120 8220939 : swap_sort (first, n_ready_real);
3121 53798124 : else if (n_ready_real > 2)
3122 7788902 : qsort (first, n_ready_real, sizeof (rtx), rank_for_schedule);
3123 :
3124 62019063 : if (sched_verbose >= 4)
3125 : {
3126 0 : rank_for_schedule_stats_diff (&stats1, &rank_for_schedule_stats);
3127 0 : print_rank_for_schedule_stats (";;\t\t", &stats1, ready);
3128 : }
3129 62019063 : }
3130 :
3131 : /* Sort the ready list READY by ascending priority. */
3132 : static void
3133 116194836 : ready_sort (struct ready_list *ready)
3134 : {
3135 116194836 : if (ready->n_debug > 0)
3136 54175777 : ready_sort_debug (ready);
3137 : else
3138 62019059 : ready_sort_real (ready);
3139 116194836 : }
3140 :
3141 : /* PREV is an insn that is ready to execute. Adjust its priority if that
3142 : will help shorten or lengthen register lifetimes as appropriate. Also
3143 : provide a hook for the target to tweak itself. */
3144 :
3145 : HAIFA_INLINE static void
3146 116108491 : adjust_priority (rtx_insn *prev)
3147 : {
3148 : /* ??? There used to be code here to try and estimate how an insn
3149 : affected register lifetimes, but it did it by looking at REG_DEAD
3150 : notes, which we removed in schedule_region. Nor did it try to
3151 : take into account register pressure or anything useful like that.
3152 :
3153 : Revisit when we have a machine model to work with and not before. */
3154 :
3155 116108491 : if (targetm.sched.adjust_priority)
3156 116108491 : INSN_PRIORITY (prev) =
3157 116108491 : targetm.sched.adjust_priority (prev, INSN_PRIORITY (prev));
3158 116108491 : }
3159 :
3160 : /* Advance DFA state STATE on one cycle. */
3161 : void
3162 47393923 : advance_state (state_t state)
3163 : {
3164 47393923 : if (targetm.sched.dfa_pre_advance_cycle)
3165 0 : targetm.sched.dfa_pre_advance_cycle ();
3166 :
3167 47393923 : if (targetm.sched.dfa_pre_cycle_insn)
3168 0 : state_transition (state,
3169 : targetm.sched.dfa_pre_cycle_insn ());
3170 :
3171 47393923 : state_transition (state, NULL);
3172 :
3173 47393923 : if (targetm.sched.dfa_post_cycle_insn)
3174 0 : state_transition (state,
3175 0 : targetm.sched.dfa_post_cycle_insn ());
3176 :
3177 47393923 : if (targetm.sched.dfa_post_advance_cycle)
3178 47217906 : targetm.sched.dfa_post_advance_cycle ();
3179 47393923 : }
3180 :
3181 : /* Advance time on one cycle. */
3182 : HAIFA_INLINE static void
3183 47388947 : advance_one_cycle (void)
3184 : {
3185 47388947 : int i;
3186 :
3187 47388947 : advance_state (curr_state);
3188 94777894 : for (i = 4; i <= sched_verbose; ++i)
3189 0 : fprintf (sched_dump, ";;\tAdvance the current state: %d.\n", clock_var);
3190 47388947 : }
3191 :
3192 : /* Update register pressure after scheduling INSN. */
3193 : static void
3194 22684 : update_register_pressure (rtx_insn *insn)
3195 : {
3196 22684 : struct reg_use_data *use;
3197 22684 : struct reg_set_data *set;
3198 :
3199 22684 : gcc_checking_assert (!DEBUG_INSN_P (insn));
3200 :
3201 43790 : for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
3202 21106 : if (dying_use_p (use))
3203 17850 : mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
3204 : use->regno, false);
3205 39644 : for (set = INSN_REG_SET_LIST (insn); set != NULL; set = set->next_insn_set)
3206 16960 : mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
3207 : set->regno, true);
3208 22684 : }
3209 :
3210 : /* Set up or update (if UPDATE_P) max register pressure (see its
3211 : meaning in sched-int.h::_haifa_insn_data) for all current BB insns
3212 : after insn AFTER. */
3213 : static void
3214 2866 : setup_insn_max_reg_pressure (rtx_insn *after, bool update_p)
3215 : {
3216 2866 : int i, p;
3217 2866 : bool eq_p;
3218 2866 : rtx_insn *insn;
3219 2866 : static int max_reg_pressure[N_REG_CLASSES];
3220 :
3221 2866 : save_reg_pressure ();
3222 17372 : for (i = 0; i < ira_pressure_classes_num; i++)
3223 11640 : max_reg_pressure[ira_pressure_classes[i]]
3224 11640 : = curr_reg_pressure[ira_pressure_classes[i]];
3225 25621 : for (insn = NEXT_INSN (after);
3226 25621 : insn != NULL_RTX && ! BARRIER_P (insn)
3227 51069 : && BLOCK_FOR_INSN (insn) == BLOCK_FOR_INSN (after);
3228 22755 : insn = NEXT_INSN (insn))
3229 24290 : if (NONDEBUG_INSN_P (insn))
3230 : {
3231 : eq_p = true;
3232 108692 : for (i = 0; i < ira_pressure_classes_num; i++)
3233 : {
3234 88236 : p = max_reg_pressure[ira_pressure_classes[i]];
3235 88236 : if (INSN_MAX_REG_PRESSURE (insn)[i] != p)
3236 : {
3237 22593 : eq_p = false;
3238 22593 : INSN_MAX_REG_PRESSURE (insn)[i]
3239 22593 : = max_reg_pressure[ira_pressure_classes[i]];
3240 : }
3241 : }
3242 20456 : if (update_p && eq_p)
3243 : break;
3244 18921 : update_register_pressure (insn);
3245 119840 : for (i = 0; i < ira_pressure_classes_num; i++)
3246 81998 : if (max_reg_pressure[ira_pressure_classes[i]]
3247 81998 : < curr_reg_pressure[ira_pressure_classes[i]])
3248 2706 : max_reg_pressure[ira_pressure_classes[i]]
3249 2706 : = curr_reg_pressure[ira_pressure_classes[i]];
3250 : }
3251 2866 : restore_reg_pressure ();
3252 2866 : }
3253 :
3254 : /* Update the current register pressure after scheduling INSN. Update
3255 : also max register pressure for unscheduled insns of the current
3256 : BB. */
3257 : static void
3258 3763 : update_reg_and_insn_max_reg_pressure (rtx_insn *insn)
3259 : {
3260 3763 : int i;
3261 3763 : int before[N_REG_CLASSES];
3262 :
3263 19091 : for (i = 0; i < ira_pressure_classes_num; i++)
3264 15328 : before[i] = curr_reg_pressure[ira_pressure_classes[i]];
3265 3763 : update_register_pressure (insn);
3266 14335 : for (i = 0; i < ira_pressure_classes_num; i++)
3267 8969 : if (curr_reg_pressure[ira_pressure_classes[i]] != before[i])
3268 : break;
3269 3763 : if (i < ira_pressure_classes_num)
3270 2160 : setup_insn_max_reg_pressure (insn, true);
3271 3763 : }
3272 :
3273 : /* Set up register pressure at the beginning of basic block BB whose
3274 : insns starting after insn AFTER. Set up also max register pressure
3275 : for all insns of the basic block. */
3276 : void
3277 706 : sched_setup_bb_reg_pressure_info (basic_block bb, rtx_insn *after)
3278 : {
3279 706 : gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED);
3280 706 : initiate_bb_reg_pressure_info (bb);
3281 706 : setup_insn_max_reg_pressure (after, false);
3282 706 : }
3283 :
3284 : /* If doing predication while scheduling, verify whether INSN, which
3285 : has just been scheduled, clobbers the conditions of any
3286 : instructions that must be predicated in order to break their
3287 : dependencies. If so, remove them from the queues so that they will
3288 : only be scheduled once their control dependency is resolved. */
3289 :
3290 : static void
3291 116108479 : check_clobbered_conditions (rtx_insn *insn)
3292 : {
3293 116108479 : HARD_REG_SET t;
3294 116108479 : int i;
3295 :
3296 116108479 : if ((current_sched_info->flags & DO_PREDICATION) == 0)
3297 116108479 : return;
3298 :
3299 0 : find_all_hard_reg_sets (insn, &t, true);
3300 :
3301 0 : restart:
3302 0 : for (i = 0; i < ready.n_ready; i++)
3303 : {
3304 0 : rtx_insn *x = ready_element (&ready, i);
3305 0 : if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t))
3306 : {
3307 0 : ready_remove_insn (x);
3308 0 : goto restart;
3309 : }
3310 : }
3311 0 : for (i = 0; i <= max_insn_queue_index; i++)
3312 : {
3313 0 : rtx_insn_list *link;
3314 0 : int q = NEXT_Q_AFTER (q_ptr, i);
3315 :
3316 0 : restart_queue:
3317 0 : for (link = insn_queue[q]; link; link = link->next ())
3318 : {
3319 0 : rtx_insn *x = link->insn ();
3320 0 : if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t))
3321 : {
3322 0 : queue_remove (x);
3323 0 : goto restart_queue;
3324 : }
3325 : }
3326 : }
3327 : }
3328 :
3329 : /* Return (in order):
3330 :
3331 : - positive if INSN adversely affects the pressure on one
3332 : register class
3333 :
3334 : - negative if INSN reduces the pressure on one register class
3335 :
3336 : - 0 if INSN doesn't affect the pressure on any register class. */
3337 :
3338 : static int
3339 0 : model_classify_pressure (struct model_insn_info *insn)
3340 : {
3341 0 : struct reg_pressure_data *reg_pressure;
3342 0 : int death[N_REG_CLASSES];
3343 0 : int pci, cl, sum;
3344 :
3345 0 : calculate_reg_deaths (insn->insn, death);
3346 0 : reg_pressure = INSN_REG_PRESSURE (insn->insn);
3347 0 : sum = 0;
3348 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3349 : {
3350 0 : cl = ira_pressure_classes[pci];
3351 0 : if (death[cl] < reg_pressure[pci].set_increase)
3352 : return 1;
3353 0 : sum += reg_pressure[pci].set_increase - death[cl];
3354 : }
3355 : return sum;
3356 : }
3357 :
3358 : /* Return true if INSN1 should come before INSN2 in the model schedule. */
3359 :
3360 : static int
3361 0 : model_order_p (struct model_insn_info *insn1, struct model_insn_info *insn2)
3362 : {
3363 0 : unsigned int height1, height2;
3364 0 : unsigned int priority1, priority2;
3365 :
3366 : /* Prefer instructions with a higher model priority. */
3367 0 : if (insn1->model_priority != insn2->model_priority)
3368 0 : return insn1->model_priority > insn2->model_priority;
3369 :
3370 : /* Combine the length of the longest path of satisfied true dependencies
3371 : that leads to each instruction (depth) with the length of the longest
3372 : path of any dependencies that leads from the instruction (alap).
3373 : Prefer instructions with the greatest combined length. If the combined
3374 : lengths are equal, prefer instructions with the greatest depth.
3375 :
3376 : The idea is that, if we have a set S of "equal" instructions that each
3377 : have ALAP value X, and we pick one such instruction I, any true-dependent
3378 : successors of I that have ALAP value X - 1 should be preferred over S.
3379 : This encourages the schedule to be "narrow" rather than "wide".
3380 : However, if I is a low-priority instruction that we decided to
3381 : schedule because of its model_classify_pressure, and if there
3382 : is a set of higher-priority instructions T, the aforementioned
3383 : successors of I should not have the edge over T. */
3384 0 : height1 = insn1->depth + insn1->alap;
3385 0 : height2 = insn2->depth + insn2->alap;
3386 0 : if (height1 != height2)
3387 0 : return height1 > height2;
3388 0 : if (insn1->depth != insn2->depth)
3389 0 : return insn1->depth > insn2->depth;
3390 :
3391 : /* We have no real preference between INSN1 an INSN2 as far as attempts
3392 : to reduce pressure go. Prefer instructions with higher priorities. */
3393 0 : priority1 = INSN_PRIORITY (insn1->insn);
3394 0 : priority2 = INSN_PRIORITY (insn2->insn);
3395 0 : if (priority1 != priority2)
3396 0 : return priority1 > priority2;
3397 :
3398 : /* Use the original rtl sequence as a tie-breaker. */
3399 0 : return insn1 < insn2;
3400 : }
3401 :
3402 : /* Add INSN to the model worklist immediately after PREV. Add it to the
3403 : beginning of the list if PREV is null. */
3404 :
3405 : static void
3406 0 : model_add_to_worklist_at (struct model_insn_info *insn,
3407 : struct model_insn_info *prev)
3408 : {
3409 0 : gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_NOWHERE);
3410 0 : QUEUE_INDEX (insn->insn) = QUEUE_READY;
3411 :
3412 0 : insn->prev = prev;
3413 0 : if (prev)
3414 : {
3415 0 : insn->next = prev->next;
3416 0 : prev->next = insn;
3417 : }
3418 : else
3419 : {
3420 0 : insn->next = model_worklist;
3421 0 : model_worklist = insn;
3422 : }
3423 0 : if (insn->next)
3424 0 : insn->next->prev = insn;
3425 0 : }
3426 :
3427 : /* Remove INSN from the model worklist. */
3428 :
3429 : static void
3430 0 : model_remove_from_worklist (struct model_insn_info *insn)
3431 : {
3432 0 : gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_READY);
3433 0 : QUEUE_INDEX (insn->insn) = QUEUE_NOWHERE;
3434 :
3435 0 : if (insn->prev)
3436 0 : insn->prev->next = insn->next;
3437 : else
3438 0 : model_worklist = insn->next;
3439 0 : if (insn->next)
3440 0 : insn->next->prev = insn->prev;
3441 0 : }
3442 :
3443 : /* Add INSN to the model worklist. Start looking for a suitable position
3444 : between neighbors PREV and NEXT, testing at most param_max_sched_ready_insns
3445 : insns either side. A null PREV indicates the beginning of the list and
3446 : a null NEXT indicates the end. */
3447 :
3448 : static void
3449 0 : model_add_to_worklist (struct model_insn_info *insn,
3450 : struct model_insn_info *prev,
3451 : struct model_insn_info *next)
3452 : {
3453 0 : int count;
3454 :
3455 0 : count = param_max_sched_ready_insns;
3456 0 : if (count > 0 && prev && model_order_p (insn, prev))
3457 0 : do
3458 : {
3459 0 : count--;
3460 0 : prev = prev->prev;
3461 : }
3462 0 : while (count > 0 && prev && model_order_p (insn, prev));
3463 : else
3464 0 : while (count > 0 && next && model_order_p (next, insn))
3465 : {
3466 0 : count--;
3467 0 : prev = next;
3468 0 : next = next->next;
3469 : }
3470 0 : model_add_to_worklist_at (insn, prev);
3471 0 : }
3472 :
3473 : /* INSN may now have a higher priority (in the model_order_p sense)
3474 : than before. Move it up the worklist if necessary. */
3475 :
3476 : static void
3477 0 : model_promote_insn (struct model_insn_info *insn)
3478 : {
3479 0 : struct model_insn_info *prev;
3480 0 : int count;
3481 :
3482 0 : prev = insn->prev;
3483 0 : count = param_max_sched_ready_insns;
3484 0 : while (count > 0 && prev && model_order_p (insn, prev))
3485 : {
3486 0 : count--;
3487 0 : prev = prev->prev;
3488 : }
3489 0 : if (prev != insn->prev)
3490 : {
3491 0 : model_remove_from_worklist (insn);
3492 0 : model_add_to_worklist_at (insn, prev);
3493 : }
3494 0 : }
3495 :
3496 : /* Add INSN to the end of the model schedule. */
3497 :
3498 : static void
3499 0 : model_add_to_schedule (rtx_insn *insn)
3500 : {
3501 0 : unsigned int point;
3502 :
3503 0 : gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE);
3504 0 : QUEUE_INDEX (insn) = QUEUE_SCHEDULED;
3505 :
3506 0 : point = model_schedule.length ();
3507 0 : model_schedule.quick_push (insn);
3508 0 : INSN_MODEL_INDEX (insn) = point + 1;
3509 0 : }
3510 :
3511 : /* Analyze the instructions that are to be scheduled, setting up
3512 : MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready
3513 : instructions to model_worklist. */
3514 :
3515 : static void
3516 0 : model_analyze_insns (void)
3517 : {
3518 0 : rtx_insn *start, *end, *iter;
3519 0 : sd_iterator_def sd_it;
3520 0 : dep_t dep;
3521 0 : struct model_insn_info *insn, *con;
3522 :
3523 0 : model_num_insns = 0;
3524 0 : start = PREV_INSN (current_sched_info->next_tail);
3525 0 : end = current_sched_info->prev_head;
3526 0 : for (iter = start; iter != end; iter = PREV_INSN (iter))
3527 0 : if (NONDEBUG_INSN_P (iter))
3528 : {
3529 0 : insn = MODEL_INSN_INFO (iter);
3530 0 : insn->insn = iter;
3531 0 : FOR_EACH_DEP (iter, SD_LIST_FORW, sd_it, dep)
3532 : {
3533 0 : con = MODEL_INSN_INFO (DEP_CON (dep));
3534 0 : if (con->insn && insn->alap < con->alap + 1)
3535 0 : insn->alap = con->alap + 1;
3536 : }
3537 :
3538 0 : insn->old_queue = QUEUE_INDEX (iter);
3539 0 : QUEUE_INDEX (iter) = QUEUE_NOWHERE;
3540 :
3541 0 : insn->unscheduled_preds = dep_list_size (iter, SD_LIST_HARD_BACK);
3542 0 : if (insn->unscheduled_preds == 0)
3543 0 : model_add_to_worklist (insn, NULL, model_worklist);
3544 :
3545 0 : model_num_insns++;
3546 : }
3547 0 : }
3548 :
3549 : /* The global state describes the register pressure at the start of the
3550 : model schedule. Initialize GROUP accordingly. */
3551 :
3552 : static void
3553 0 : model_init_pressure_group (struct model_pressure_group *group)
3554 : {
3555 0 : int pci, cl;
3556 :
3557 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3558 : {
3559 0 : cl = ira_pressure_classes[pci];
3560 0 : group->limits[pci].pressure = curr_reg_pressure[cl];
3561 0 : group->limits[pci].point = 0;
3562 : }
3563 : /* Use index model_num_insns to record the state after the last
3564 : instruction in the model schedule. */
3565 0 : group->model = XNEWVEC (struct model_pressure_data,
3566 : (model_num_insns + 1) * ira_pressure_classes_num);
3567 0 : }
3568 :
3569 : /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE.
3570 : Update the maximum pressure for the whole schedule. */
3571 :
3572 : static void
3573 0 : model_record_pressure (struct model_pressure_group *group,
3574 : int point, int pci, int pressure)
3575 : {
3576 0 : MODEL_REF_PRESSURE (group, point, pci) = pressure;
3577 0 : if (group->limits[pci].pressure < pressure)
3578 : {
3579 0 : group->limits[pci].pressure = pressure;
3580 0 : group->limits[pci].point = point;
3581 : }
3582 0 : }
3583 :
3584 : /* INSN has just been added to the end of the model schedule. Record its
3585 : register-pressure information. */
3586 :
3587 : static void
3588 0 : model_record_pressures (struct model_insn_info *insn)
3589 : {
3590 0 : struct reg_pressure_data *reg_pressure;
3591 0 : int point, pci, cl, delta;
3592 0 : int death[N_REG_CLASSES];
3593 :
3594 0 : point = model_index (insn->insn);
3595 0 : if (sched_verbose >= 2)
3596 : {
3597 0 : if (point == 0)
3598 : {
3599 0 : fprintf (sched_dump, "\n;;\tModel schedule:\n;;\n");
3600 0 : fprintf (sched_dump, ";;\t| idx insn | mpri hght dpth prio |\n");
3601 : }
3602 0 : fprintf (sched_dump, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ",
3603 : point, INSN_UID (insn->insn), insn->model_priority,
3604 0 : insn->depth + insn->alap, insn->depth,
3605 0 : INSN_PRIORITY (insn->insn),
3606 0 : str_pattern_slim (PATTERN (insn->insn)));
3607 : }
3608 0 : calculate_reg_deaths (insn->insn, death);
3609 0 : reg_pressure = INSN_REG_PRESSURE (insn->insn);
3610 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3611 : {
3612 0 : cl = ira_pressure_classes[pci];
3613 0 : delta = reg_pressure[pci].set_increase - death[cl];
3614 0 : if (sched_verbose >= 2)
3615 0 : fprintf (sched_dump, " %s:[%d,%+d]", reg_class_names[cl],
3616 : curr_reg_pressure[cl], delta);
3617 0 : model_record_pressure (&model_before_pressure, point, pci,
3618 : curr_reg_pressure[cl]);
3619 : }
3620 0 : if (sched_verbose >= 2)
3621 0 : fprintf (sched_dump, "\n");
3622 0 : }
3623 :
3624 : /* All instructions have been added to the model schedule. Record the
3625 : final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. */
3626 :
3627 : static void
3628 0 : model_record_final_pressures (struct model_pressure_group *group)
3629 : {
3630 0 : int point, pci, max_pressure, ref_pressure, cl;
3631 :
3632 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3633 : {
3634 : /* Record the final pressure for this class. */
3635 0 : cl = ira_pressure_classes[pci];
3636 0 : point = model_num_insns;
3637 0 : ref_pressure = curr_reg_pressure[cl];
3638 0 : model_record_pressure (group, point, pci, ref_pressure);
3639 :
3640 : /* Record the original maximum pressure. */
3641 0 : group->limits[pci].orig_pressure = group->limits[pci].pressure;
3642 :
3643 : /* Update the MODEL_MAX_PRESSURE for every point of the schedule. */
3644 0 : max_pressure = ref_pressure;
3645 0 : MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
3646 0 : while (point > 0)
3647 : {
3648 0 : point--;
3649 0 : ref_pressure = MODEL_REF_PRESSURE (group, point, pci);
3650 0 : max_pressure = MAX (max_pressure, ref_pressure);
3651 0 : MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
3652 : }
3653 : }
3654 0 : }
3655 :
3656 : /* Update all successors of INSN, given that INSN has just been scheduled. */
3657 :
3658 : static void
3659 0 : model_add_successors_to_worklist (struct model_insn_info *insn)
3660 : {
3661 0 : sd_iterator_def sd_it;
3662 0 : struct model_insn_info *con;
3663 0 : dep_t dep;
3664 :
3665 0 : FOR_EACH_DEP (insn->insn, SD_LIST_FORW, sd_it, dep)
3666 : {
3667 0 : con = MODEL_INSN_INFO (DEP_CON (dep));
3668 : /* Ignore debug instructions, and instructions from other blocks. */
3669 0 : if (con->insn)
3670 : {
3671 0 : con->unscheduled_preds--;
3672 :
3673 : /* Update the depth field of each true-dependent successor.
3674 : Increasing the depth gives them a higher priority than
3675 : before. */
3676 0 : if (DEP_TYPE (dep) == REG_DEP_TRUE && con->depth < insn->depth + 1)
3677 : {
3678 0 : con->depth = insn->depth + 1;
3679 0 : if (QUEUE_INDEX (con->insn) == QUEUE_READY)
3680 0 : model_promote_insn (con);
3681 : }
3682 :
3683 : /* If this is a true dependency, or if there are no remaining
3684 : dependencies for CON (meaning that CON only had non-true
3685 : dependencies), make sure that CON is on the worklist.
3686 : We don't bother otherwise because it would tend to fill the
3687 : worklist with a lot of low-priority instructions that are not
3688 : yet ready to issue. */
3689 0 : if ((con->depth > 0 || con->unscheduled_preds == 0)
3690 0 : && QUEUE_INDEX (con->insn) == QUEUE_NOWHERE)
3691 0 : model_add_to_worklist (con, insn, insn->next);
3692 : }
3693 : }
3694 0 : }
3695 :
3696 : /* Give INSN a higher priority than any current instruction, then give
3697 : unscheduled predecessors of INSN a higher priority still. If any of
3698 : those predecessors are not on the model worklist, do the same for its
3699 : predecessors, and so on. */
3700 :
3701 : static void
3702 0 : model_promote_predecessors (struct model_insn_info *insn)
3703 : {
3704 0 : struct model_insn_info *pro, *first;
3705 0 : sd_iterator_def sd_it;
3706 0 : dep_t dep;
3707 :
3708 0 : if (sched_verbose >= 7)
3709 0 : fprintf (sched_dump, ";;\t+--- priority of %d = %d, priority of",
3710 0 : INSN_UID (insn->insn), model_next_priority);
3711 0 : insn->model_priority = model_next_priority++;
3712 0 : model_remove_from_worklist (insn);
3713 0 : model_add_to_worklist_at (insn, NULL);
3714 :
3715 0 : first = NULL;
3716 0 : for (;;)
3717 : {
3718 0 : FOR_EACH_DEP (insn->insn, SD_LIST_HARD_BACK, sd_it, dep)
3719 : {
3720 0 : pro = MODEL_INSN_INFO (DEP_PRO (dep));
3721 : /* The first test is to ignore debug instructions, and instructions
3722 : from other blocks. */
3723 0 : if (pro->insn
3724 0 : && pro->model_priority != model_next_priority
3725 0 : && QUEUE_INDEX (pro->insn) != QUEUE_SCHEDULED)
3726 : {
3727 0 : pro->model_priority = model_next_priority;
3728 0 : if (sched_verbose >= 7)
3729 0 : fprintf (sched_dump, " %d", INSN_UID (pro->insn));
3730 0 : if (QUEUE_INDEX (pro->insn) == QUEUE_READY)
3731 : {
3732 : /* PRO is already in the worklist, but it now has
3733 : a higher priority than before. Move it at the
3734 : appropriate place. */
3735 0 : model_remove_from_worklist (pro);
3736 0 : model_add_to_worklist (pro, NULL, model_worklist);
3737 : }
3738 : else
3739 : {
3740 : /* PRO isn't in the worklist. Recursively process
3741 : its predecessors until we find one that is. */
3742 0 : pro->next = first;
3743 0 : first = pro;
3744 : }
3745 : }
3746 : }
3747 0 : if (!first)
3748 : break;
3749 0 : insn = first;
3750 0 : first = insn->next;
3751 : }
3752 0 : if (sched_verbose >= 7)
3753 0 : fprintf (sched_dump, " = %d\n", model_next_priority);
3754 0 : model_next_priority++;
3755 0 : }
3756 :
3757 : /* Pick one instruction from model_worklist and process it. */
3758 :
3759 : static void
3760 0 : model_choose_insn (void)
3761 : {
3762 0 : struct model_insn_info *insn, *fallback;
3763 0 : int count;
3764 :
3765 0 : if (sched_verbose >= 7)
3766 : {
3767 0 : fprintf (sched_dump, ";;\t+--- worklist:\n");
3768 0 : insn = model_worklist;
3769 0 : count = param_max_sched_ready_insns;
3770 0 : while (count > 0 && insn)
3771 : {
3772 0 : fprintf (sched_dump, ";;\t+--- %d [%d, %d, %d, %d][%d]\n",
3773 : INSN_UID (insn->insn), insn->model_priority,
3774 0 : insn->depth + insn->alap, insn->depth,
3775 0 : INSN_PRIORITY (insn->insn), insn->unscheduled_preds);
3776 0 : count--;
3777 0 : insn = insn->next;
3778 : }
3779 : }
3780 :
3781 : /* Look for a ready instruction whose model_classify_priority is zero
3782 : or negative, picking the highest-priority one. Adding such an
3783 : instruction to the schedule now should do no harm, and may actually
3784 : do some good.
3785 :
3786 : Failing that, see whether there is an instruction with the highest
3787 : extant model_priority that is not yet ready, but which would reduce
3788 : pressure if it became ready. This is designed to catch cases like:
3789 :
3790 : (set (mem (reg R1)) (reg R2))
3791 :
3792 : where the instruction is the last remaining use of R1 and where the
3793 : value of R2 is not yet available (or vice versa). The death of R1
3794 : means that this instruction already reduces pressure. It is of
3795 : course possible that the computation of R2 involves other registers
3796 : that are hard to kill, but such cases are rare enough for this
3797 : heuristic to be a win in general.
3798 :
3799 : Failing that, just pick the highest-priority instruction in the
3800 : worklist. */
3801 0 : count = param_max_sched_ready_insns;
3802 0 : insn = model_worklist;
3803 0 : fallback = 0;
3804 0 : for (;;)
3805 : {
3806 0 : if (count == 0 || !insn)
3807 : {
3808 0 : insn = fallback ? fallback : model_worklist;
3809 : break;
3810 : }
3811 0 : if (insn->unscheduled_preds)
3812 : {
3813 0 : if (model_worklist->model_priority == insn->model_priority
3814 0 : && !fallback
3815 0 : && model_classify_pressure (insn) < 0)
3816 : fallback = insn;
3817 : }
3818 : else
3819 : {
3820 0 : if (model_classify_pressure (insn) <= 0)
3821 : break;
3822 : }
3823 0 : count--;
3824 0 : insn = insn->next;
3825 : }
3826 :
3827 0 : if (sched_verbose >= 7 && insn != model_worklist)
3828 : {
3829 0 : if (insn->unscheduled_preds)
3830 0 : fprintf (sched_dump, ";;\t+--- promoting insn %d, with dependencies\n",
3831 0 : INSN_UID (insn->insn));
3832 : else
3833 0 : fprintf (sched_dump, ";;\t+--- promoting insn %d, which is ready\n",
3834 0 : INSN_UID (insn->insn));
3835 : }
3836 0 : if (insn->unscheduled_preds)
3837 : /* INSN isn't yet ready to issue. Give all its predecessors the
3838 : highest priority. */
3839 0 : model_promote_predecessors (insn);
3840 : else
3841 : {
3842 : /* INSN is ready. Add it to the end of model_schedule and
3843 : process its successors. */
3844 0 : model_add_successors_to_worklist (insn);
3845 0 : model_remove_from_worklist (insn);
3846 0 : model_add_to_schedule (insn->insn);
3847 0 : model_record_pressures (insn);
3848 0 : update_register_pressure (insn->insn);
3849 : }
3850 0 : }
3851 :
3852 : /* Restore all QUEUE_INDEXs to the values that they had before
3853 : model_start_schedule was called. */
3854 :
3855 : static void
3856 0 : model_reset_queue_indices (void)
3857 : {
3858 0 : unsigned int i;
3859 0 : rtx_insn *insn;
3860 :
3861 0 : FOR_EACH_VEC_ELT (model_schedule, i, insn)
3862 0 : QUEUE_INDEX (insn) = MODEL_INSN_INFO (insn)->old_queue;
3863 0 : }
3864 :
3865 : /* We have calculated the model schedule and spill costs. Print a summary
3866 : to sched_dump. */
3867 :
3868 : static void
3869 0 : model_dump_pressure_summary (basic_block bb)
3870 : {
3871 0 : int pci, cl;
3872 :
3873 0 : fprintf (sched_dump, ";; Pressure summary (bb %d):", bb->index);
3874 0 : for (pci = 0; pci < ira_pressure_classes_num; pci++)
3875 : {
3876 0 : cl = ira_pressure_classes[pci];
3877 0 : fprintf (sched_dump, " %s:%d", reg_class_names[cl],
3878 : model_before_pressure.limits[pci].pressure);
3879 : }
3880 0 : fprintf (sched_dump, "\n\n");
3881 0 : }
3882 :
3883 : /* Initialize the SCHED_PRESSURE_MODEL information for the current
3884 : scheduling region. */
3885 :
3886 : static void
3887 0 : model_start_schedule (basic_block bb)
3888 : {
3889 0 : model_next_priority = 1;
3890 0 : model_schedule.create (sched_max_luid);
3891 0 : model_insns = XCNEWVEC (struct model_insn_info, sched_max_luid);
3892 :
3893 0 : gcc_assert (bb == BLOCK_FOR_INSN (NEXT_INSN (current_sched_info->prev_head)));
3894 0 : initiate_reg_pressure_info (df_get_live_in (bb));
3895 :
3896 0 : model_analyze_insns ();
3897 0 : model_init_pressure_group (&model_before_pressure);
3898 0 : while (model_worklist)
3899 0 : model_choose_insn ();
3900 0 : gcc_assert (model_num_insns == (int) model_schedule.length ());
3901 0 : if (sched_verbose >= 2)
3902 0 : fprintf (sched_dump, "\n");
3903 :
3904 0 : model_record_final_pressures (&model_before_pressure);
3905 0 : model_reset_queue_indices ();
3906 :
3907 0 : XDELETEVEC (model_insns);
3908 :
3909 0 : model_curr_point = 0;
3910 0 : initiate_reg_pressure_info (df_get_live_in (bb));
3911 0 : if (sched_verbose >= 1)
3912 0 : model_dump_pressure_summary (bb);
3913 0 : }
3914 :
3915 : /* Free the information associated with GROUP. */
3916 :
3917 : static void
3918 0 : model_finalize_pressure_group (struct model_pressure_group *group)
3919 : {
3920 0 : XDELETEVEC (group->model);
3921 0 : }
3922 :
3923 : /* Free the information created by model_start_schedule. */
3924 :
3925 : static void
3926 0 : model_end_schedule (void)
3927 : {
3928 0 : model_finalize_pressure_group (&model_before_pressure);
3929 0 : model_schedule.release ();
3930 0 : }
3931 :
3932 : /* Prepare reg pressure scheduling for basic block BB. */
3933 : static void
3934 706 : sched_pressure_start_bb (basic_block bb)
3935 : {
3936 : /* Set the number of available registers for each class taking into account
3937 : relative probability of current basic block versus function prologue and
3938 : epilogue.
3939 : * If the basic block executes much more often than the prologue/epilogue
3940 : (e.g., inside a hot loop), then cost of spill in the prologue is close to
3941 : nil, so the effective number of available registers is
3942 : (ira_class_hard_regs_num[cl] - fixed_regs_num[cl] - 0).
3943 : * If the basic block executes as often as the prologue/epilogue,
3944 : then spill in the block is as costly as in the prologue, so the effective
3945 : number of available registers is
3946 : (ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
3947 : - call_saved_regs_num[cl]).
3948 : Note that all-else-equal, we prefer to spill in the prologue, since that
3949 : allows "extra" registers for other basic blocks of the function.
3950 : * If the basic block is on the cold path of the function and executes
3951 : rarely, then we should always prefer to spill in the block, rather than
3952 : in the prologue/epilogue. The effective number of available register is
3953 : (ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
3954 : - call_saved_regs_num[cl]). */
3955 706 : {
3956 706 : int i;
3957 706 : int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun);
3958 706 : int bb_freq = bb->count.to_frequency (cfun);
3959 :
3960 706 : if (bb_freq == 0)
3961 : {
3962 128 : if (entry_freq == 0)
3963 : entry_freq = bb_freq = 1;
3964 : }
3965 706 : if (bb_freq < entry_freq)
3966 : bb_freq = entry_freq;
3967 :
3968 3534 : for (i = 0; i < ira_pressure_classes_num; ++i)
3969 : {
3970 2828 : enum reg_class cl = ira_pressure_classes[i];
3971 2828 : sched_class_regs_num[cl] = ira_class_hard_regs_num[cl]
3972 2828 : - fixed_regs_num[cl];
3973 2828 : sched_class_regs_num[cl]
3974 2828 : -= (call_saved_regs_num[cl] * entry_freq) / bb_freq;
3975 : }
3976 : }
3977 :
3978 706 : if (sched_pressure == SCHED_PRESSURE_MODEL)
3979 0 : model_start_schedule (bb);
3980 706 : }
3981 :
3982 : /* A structure that holds local state for the loop in schedule_block. */
3983 : struct sched_block_state
3984 : {
3985 : /* True if no real insns have been scheduled in the current cycle. */
3986 : bool first_cycle_insn_p;
3987 : /* True if a shadow insn has been scheduled in the current cycle, which
3988 : means that no more normal insns can be issued. */
3989 : bool shadows_only_p;
3990 : /* True if we're winding down a modulo schedule, which means that we only
3991 : issue insns with INSN_EXACT_TICK set. */
3992 : bool modulo_epilogue;
3993 : /* Initialized with the machine's issue rate every cycle, and updated
3994 : by calls to the variable_issue hook. */
3995 : int can_issue_more;
3996 : };
3997 :
3998 : /* INSN is the "currently executing insn". Launch each insn which was
3999 : waiting on INSN. READY is the ready list which contains the insns
4000 : that are ready to fire. CLOCK is the current cycle. The function
4001 : returns necessary cycle advance after issuing the insn (it is not
4002 : zero for insns in a schedule group). */
4003 :
4004 : static int
4005 116108479 : schedule_insn (rtx_insn *insn)
4006 : {
4007 116108479 : sd_iterator_def sd_it;
4008 116108479 : dep_t dep;
4009 116108479 : int i;
4010 116108479 : int advance = 0;
4011 :
4012 116108479 : if (sched_verbose >= 1)
4013 : {
4014 1144 : struct reg_pressure_data *pressure_info;
4015 1144 : fprintf (sched_dump, ";;\t%3i--> %s %-40s:",
4016 1144 : clock_var, (*current_sched_info->print_insn) (insn, 1),
4017 1144 : str_pattern_slim (PATTERN (insn)));
4018 :
4019 1144 : if (recog_memoized (insn) < 0)
4020 134 : fprintf (sched_dump, "nothing");
4021 : else
4022 1010 : print_reservation (sched_dump, insn);
4023 1144 : pressure_info = INSN_REG_PRESSURE (insn);
4024 1144 : if (pressure_info != NULL)
4025 : {
4026 0 : fputc (':', sched_dump);
4027 0 : for (i = 0; i < ira_pressure_classes_num; i++)
4028 0 : fprintf (sched_dump, "%s%s%+d(%d)",
4029 0 : scheduled_insns.length () > 1
4030 0 : && INSN_LUID (insn)
4031 0 : < INSN_LUID (scheduled_insns[scheduled_insns.length () - 2]) ? "@" : "",
4032 0 : reg_class_names[ira_pressure_classes[i]],
4033 0 : pressure_info[i].set_increase, pressure_info[i].change);
4034 : }
4035 1144 : if (sched_pressure == SCHED_PRESSURE_MODEL
4036 0 : && model_curr_point < model_num_insns
4037 1144 : && model_index (insn) == model_curr_point)
4038 0 : fprintf (sched_dump, ":model %d", model_curr_point);
4039 1144 : fputc ('\n', sched_dump);
4040 : }
4041 :
4042 116108479 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED && !DEBUG_INSN_P (insn))
4043 3763 : update_reg_and_insn_max_reg_pressure (insn);
4044 :
4045 : /* Scheduling instruction should have all its dependencies resolved and
4046 : should have been removed from the ready list. */
4047 116108479 : gcc_assert (sd_lists_empty_p (insn, SD_LIST_HARD_BACK));
4048 :
4049 : /* Reset debug insns invalidated by moving this insn. */
4050 116108479 : if (MAY_HAVE_DEBUG_BIND_INSNS && !DEBUG_INSN_P (insn))
4051 42814705 : for (sd_it = sd_iterator_start (insn, SD_LIST_BACK);
4052 43347527 : sd_iterator_cond (&sd_it, &dep);)
4053 : {
4054 532822 : rtx_insn *dbg = DEP_PRO (dep);
4055 532822 : struct reg_use_data *use, *next;
4056 :
4057 532822 : if (DEP_STATUS (dep) & DEP_CANCELLED)
4058 : {
4059 383041 : sd_iterator_next (&sd_it);
4060 383041 : continue;
4061 : }
4062 :
4063 149781 : gcc_assert (DEBUG_BIND_INSN_P (dbg));
4064 :
4065 149781 : if (sched_verbose >= 6)
4066 0 : fprintf (sched_dump, ";;\t\tresetting: debug insn %d\n",
4067 0 : INSN_UID (dbg));
4068 :
4069 : /* ??? Rather than resetting the debug insn, we might be able
4070 : to emit a debug temp before the just-scheduled insn, but
4071 : this would involve checking that the expression at the
4072 : point of the debug insn is equivalent to the expression
4073 : before the just-scheduled insn. They might not be: the
4074 : expression in the debug insn may depend on other insns not
4075 : yet scheduled that set MEMs, REGs or even other debug
4076 : insns. It's not clear that attempting to preserve debug
4077 : information in these cases is worth the effort, given how
4078 : uncommon these resets are and the likelihood that the debug
4079 : temps introduced won't survive the schedule change. */
4080 149781 : INSN_VAR_LOCATION_LOC (dbg) = gen_rtx_UNKNOWN_VAR_LOC ();
4081 149781 : df_insn_rescan (dbg);
4082 :
4083 : /* Unknown location doesn't use any registers. */
4084 149781 : for (use = INSN_REG_USE_LIST (dbg); use != NULL; use = next)
4085 : {
4086 : struct reg_use_data *prev = use;
4087 :
4088 : /* Remove use from the cyclic next_regno_use chain first. */
4089 0 : while (prev->next_regno_use != use)
4090 : prev = prev->next_regno_use;
4091 0 : prev->next_regno_use = use->next_regno_use;
4092 0 : next = use->next_insn_use;
4093 0 : free (use);
4094 : }
4095 149781 : INSN_REG_USE_LIST (dbg) = NULL;
4096 :
4097 : /* We delete rather than resolve these deps, otherwise we
4098 : crash in sched_free_deps(), because forward deps are
4099 : expected to be released before backward deps. */
4100 149781 : sd_delete_dep (sd_it);
4101 : }
4102 :
4103 116108479 : gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE);
4104 116108479 : QUEUE_INDEX (insn) = QUEUE_SCHEDULED;
4105 :
4106 116108479 : if (sched_pressure == SCHED_PRESSURE_MODEL
4107 0 : && model_curr_point < model_num_insns
4108 0 : && NONDEBUG_INSN_P (insn))
4109 : {
4110 0 : if (model_index (insn) == model_curr_point)
4111 0 : do
4112 0 : model_curr_point++;
4113 : while (model_curr_point < model_num_insns
4114 0 : && (QUEUE_INDEX (MODEL_INSN (model_curr_point))
4115 : == QUEUE_SCHEDULED));
4116 : else
4117 0 : model_recompute (insn);
4118 0 : model_update_limit_points ();
4119 0 : update_register_pressure (insn);
4120 0 : if (sched_verbose >= 2)
4121 0 : print_curr_reg_pressure ();
4122 : }
4123 :
4124 116108479 : gcc_assert (INSN_TICK (insn) >= MIN_TICK);
4125 116108479 : if (INSN_TICK (insn) > clock_var)
4126 : /* INSN has been prematurely moved from the queue to the ready list.
4127 : This is possible only if following flags are set. */
4128 5 : gcc_assert (flag_sched_stalled_insns || sched_fusion);
4129 :
4130 : /* ??? Probably, if INSN is scheduled prematurely, we should leave
4131 : INSN_TICK untouched. This is a machine-dependent issue, actually. */
4132 116108479 : INSN_TICK (insn) = clock_var;
4133 :
4134 116108479 : check_clobbered_conditions (insn);
4135 :
4136 : /* Update dependent instructions. First, see if by scheduling this insn
4137 : now we broke a dependence in a way that requires us to change another
4138 : insn. */
4139 116108479 : for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
4140 116655549 : sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it))
4141 : {
4142 547070 : struct dep_replacement *desc = DEP_REPLACE (dep);
4143 547070 : rtx_insn *pro = DEP_PRO (dep);
4144 547070 : if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED
4145 547070 : && desc != NULL && desc->insn == pro)
4146 518774 : apply_replacement (dep, false);
4147 : }
4148 :
4149 : /* Go through and resolve forward dependencies. */
4150 116108479 : for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
4151 334999205 : sd_iterator_cond (&sd_it, &dep);)
4152 : {
4153 218890726 : rtx_insn *next = DEP_CON (dep);
4154 218890726 : bool cancelled = (DEP_STATUS (dep) & DEP_CANCELLED) != 0;
4155 :
4156 : /* Resolve the dependence between INSN and NEXT.
4157 : sd_resolve_dep () moves current dep to another list thus
4158 : advancing the iterator. */
4159 218890726 : sd_resolve_dep (sd_it);
4160 :
4161 218890726 : if (cancelled)
4162 : {
4163 851889 : if (must_restore_pattern_p (next, dep))
4164 61251 : restore_pattern (dep, false);
4165 851889 : continue;
4166 : }
4167 :
4168 : /* Don't bother trying to mark next as ready if insn is a debug
4169 : insn. If insn is the last hard dependency, it will have
4170 : already been discounted. */
4171 218038837 : if (DEBUG_INSN_P (insn) && !DEBUG_INSN_P (next))
4172 5214904 : continue;
4173 :
4174 212823933 : if (!IS_SPECULATION_BRANCHY_CHECK_P (insn))
4175 : {
4176 212823933 : int effective_cost;
4177 :
4178 212823933 : effective_cost = try_ready (next);
4179 :
4180 212823933 : if (effective_cost >= 0
4181 19993153 : && SCHED_GROUP_P (next)
4182 212823933 : && advance < effective_cost)
4183 : advance = effective_cost;
4184 : }
4185 : else
4186 : /* Check always has only one forward dependence (to the first insn in
4187 : the recovery block), therefore, this will be executed only once. */
4188 : {
4189 0 : gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW));
4190 0 : fix_recovery_deps (RECOVERY_BLOCK (insn));
4191 : }
4192 : }
4193 :
4194 : /* Annotate the instruction with issue information -- TImode
4195 : indicates that the instruction is expected not to be able
4196 : to issue on the same cycle as the previous insn. A machine
4197 : may use this information to decide how the instruction should
4198 : be aligned. */
4199 116108479 : if (issue_rate > 1
4200 116107616 : && GET_CODE (PATTERN (insn)) != USE
4201 115387872 : && GET_CODE (PATTERN (insn)) != CLOBBER
4202 231412939 : && !DEBUG_INSN_P (insn))
4203 : {
4204 61128908 : if (reload_completed)
4205 97107177 : PUT_MODE (insn, clock_var > last_clock_var ? TImode : VOIDmode);
4206 61128908 : last_clock_var = clock_var;
4207 : }
4208 :
4209 116108479 : if (nonscheduled_insns_begin != NULL_RTX)
4210 : /* Indicate to debug counters that INSN is scheduled. */
4211 0 : nonscheduled_insns_begin = insn;
4212 :
4213 116108479 : return advance;
4214 : }
4215 :
4216 : /* Functions for handling of notes. */
4217 :
4218 : /* Add note list that ends on FROM_END to the end of TO_ENDP. */
4219 : void
4220 64 : concat_note_lists (rtx_insn *from_end, rtx_insn **to_endp)
4221 : {
4222 64 : rtx_insn *from_start;
4223 :
4224 : /* It's easy when have nothing to concat. */
4225 64 : if (from_end == NULL)
4226 : return;
4227 :
4228 : /* It's also easy when destination is empty. */
4229 0 : if (*to_endp == NULL)
4230 : {
4231 0 : *to_endp = from_end;
4232 0 : return;
4233 : }
4234 :
4235 : from_start = from_end;
4236 0 : while (PREV_INSN (from_start) != NULL)
4237 : from_start = PREV_INSN (from_start);
4238 :
4239 0 : SET_PREV_INSN (from_start) = *to_endp;
4240 0 : SET_NEXT_INSN (*to_endp) = from_start;
4241 0 : *to_endp = from_end;
4242 : }
4243 :
4244 : /* Delete notes between HEAD and TAIL and put them in the chain
4245 : of notes ended by NOTE_LIST. */
4246 : void
4247 10611794 : remove_notes (rtx_insn *head, rtx_insn *tail)
4248 : {
4249 10611794 : rtx_insn *next_tail, *insn, *next;
4250 :
4251 10611794 : note_list = 0;
4252 10611794 : if (head == tail && !INSN_P (head))
4253 : return;
4254 :
4255 10611762 : next_tail = NEXT_INSN (tail);
4256 143280323 : for (insn = head; insn != next_tail; insn = next)
4257 : {
4258 122056799 : next = NEXT_INSN (insn);
4259 122056799 : if (!NOTE_P (insn))
4260 116113162 : continue;
4261 :
4262 5943637 : switch (NOTE_KIND (insn))
4263 : {
4264 1089 : case NOTE_INSN_BASIC_BLOCK:
4265 1089 : continue;
4266 :
4267 931558 : case NOTE_INSN_EPILOGUE_BEG:
4268 931558 : if (insn != tail)
4269 : {
4270 931558 : remove_insn (insn);
4271 : /* If an insn was split just before the EPILOGUE_BEG note and
4272 : that split created new basic blocks, we could have a
4273 : BASIC_BLOCK note here. Safely advance over it in that case
4274 : and assert that we land on a real insn. */
4275 931558 : if (NOTE_P (next)
4276 0 : && NOTE_KIND (next) == NOTE_INSN_BASIC_BLOCK
4277 0 : && next != next_tail)
4278 0 : next = NEXT_INSN (next);
4279 931558 : gcc_assert (INSN_P (next));
4280 931558 : add_reg_note (next, REG_SAVE_NOTE,
4281 : GEN_INT (NOTE_INSN_EPILOGUE_BEG));
4282 931558 : break;
4283 : }
4284 : /* FALLTHRU */
4285 :
4286 5010990 : default:
4287 5010990 : remove_insn (insn);
4288 :
4289 : /* Add the note to list that ends at NOTE_LIST. */
4290 5010990 : SET_PREV_INSN (insn) = note_list;
4291 5010990 : SET_NEXT_INSN (insn) = NULL_RTX;
4292 5010990 : if (note_list)
4293 2878178 : SET_NEXT_INSN (note_list) = insn;
4294 5010990 : note_list = insn;
4295 5010990 : break;
4296 : }
4297 :
4298 5942548 : gcc_assert ((sel_sched_p () || insn != tail) && insn != head);
4299 : }
4300 : }
4301 :
4302 : /* A structure to record enough data to allow us to backtrack the scheduler to
4303 : a previous state. */
4304 : struct haifa_saved_data
4305 : {
4306 : /* Next entry on the list. */
4307 : struct haifa_saved_data *next;
4308 :
4309 : /* Backtracking is associated with scheduling insns that have delay slots.
4310 : DELAY_PAIR points to the structure that contains the insns involved, and
4311 : the number of cycles between them. */
4312 : struct delay_pair *delay_pair;
4313 :
4314 : /* Data used by the frontend (e.g. sched-ebb or sched-rgn). */
4315 : void *fe_saved_data;
4316 : /* Data used by the backend. */
4317 : void *be_saved_data;
4318 :
4319 : /* Copies of global state. */
4320 : int clock_var, last_clock_var;
4321 : struct ready_list ready;
4322 : state_t curr_state;
4323 :
4324 : rtx_insn *last_scheduled_insn;
4325 : rtx_insn *last_nondebug_scheduled_insn;
4326 : rtx_insn *nonscheduled_insns_begin;
4327 : int cycle_issued_insns;
4328 :
4329 : /* Copies of state used in the inner loop of schedule_block. */
4330 : struct sched_block_state sched_block;
4331 :
4332 : /* We don't need to save q_ptr, as its value is arbitrary and we can set it
4333 : to 0 when restoring. */
4334 : int q_size;
4335 : rtx_insn_list **insn_queue;
4336 :
4337 : /* Describe pattern replacements that occurred since this backtrack point
4338 : was queued. */
4339 : vec<dep_t> replacement_deps;
4340 : vec<int> replace_apply;
4341 :
4342 : /* A copy of the next-cycle replacement vectors at the time of the backtrack
4343 : point. */
4344 : vec<dep_t> next_cycle_deps;
4345 : vec<int> next_cycle_apply;
4346 : };
4347 :
4348 : /* A record, in reverse order, of all scheduled insns which have delay slots
4349 : and may require backtracking. */
4350 : static struct haifa_saved_data *backtrack_queue;
4351 :
4352 : /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according
4353 : to SET_P. */
4354 : static void
4355 0 : mark_backtrack_feeds (rtx_insn *insn, int set_p)
4356 : {
4357 0 : sd_iterator_def sd_it;
4358 0 : dep_t dep;
4359 0 : FOR_EACH_DEP (insn, SD_LIST_HARD_BACK, sd_it, dep)
4360 : {
4361 0 : FEEDS_BACKTRACK_INSN (DEP_PRO (dep)) = set_p;
4362 : }
4363 0 : }
4364 :
4365 : /* Save the current scheduler state so that we can backtrack to it
4366 : later if necessary. PAIR gives the insns that make it necessary to
4367 : save this point. SCHED_BLOCK is the local state of schedule_block
4368 : that need to be saved. */
4369 : static void
4370 0 : save_backtrack_point (struct delay_pair *pair,
4371 : struct sched_block_state sched_block)
4372 : {
4373 0 : int i;
4374 0 : struct haifa_saved_data *save = XNEW (struct haifa_saved_data);
4375 :
4376 0 : save->curr_state = xmalloc (dfa_state_size);
4377 0 : memcpy (save->curr_state, curr_state, dfa_state_size);
4378 :
4379 0 : save->ready.first = ready.first;
4380 0 : save->ready.n_ready = ready.n_ready;
4381 0 : save->ready.n_debug = ready.n_debug;
4382 0 : save->ready.veclen = ready.veclen;
4383 0 : save->ready.vec = XNEWVEC (rtx_insn *, ready.veclen);
4384 0 : memcpy (save->ready.vec, ready.vec, ready.veclen * sizeof (rtx));
4385 :
4386 0 : save->insn_queue = XNEWVEC (rtx_insn_list *, max_insn_queue_index + 1);
4387 0 : save->q_size = q_size;
4388 0 : for (i = 0; i <= max_insn_queue_index; i++)
4389 : {
4390 0 : int q = NEXT_Q_AFTER (q_ptr, i);
4391 0 : save->insn_queue[i] = copy_INSN_LIST (insn_queue[q]);
4392 : }
4393 :
4394 0 : save->clock_var = clock_var;
4395 0 : save->last_clock_var = last_clock_var;
4396 0 : save->cycle_issued_insns = cycle_issued_insns;
4397 0 : save->last_scheduled_insn = last_scheduled_insn;
4398 0 : save->last_nondebug_scheduled_insn = last_nondebug_scheduled_insn;
4399 0 : save->nonscheduled_insns_begin = nonscheduled_insns_begin;
4400 :
4401 0 : save->sched_block = sched_block;
4402 :
4403 0 : save->replacement_deps.create (0);
4404 0 : save->replace_apply.create (0);
4405 0 : save->next_cycle_deps = next_cycle_replace_deps.copy ();
4406 0 : save->next_cycle_apply = next_cycle_apply.copy ();
4407 :
4408 0 : if (current_sched_info->save_state)
4409 0 : save->fe_saved_data = (*current_sched_info->save_state) ();
4410 :
4411 0 : if (targetm.sched.alloc_sched_context)
4412 : {
4413 0 : save->be_saved_data = targetm.sched.alloc_sched_context ();
4414 0 : targetm.sched.init_sched_context (save->be_saved_data, false);
4415 : }
4416 : else
4417 0 : save->be_saved_data = NULL;
4418 :
4419 0 : save->delay_pair = pair;
4420 :
4421 0 : save->next = backtrack_queue;
4422 0 : backtrack_queue = save;
4423 :
4424 0 : while (pair)
4425 : {
4426 0 : mark_backtrack_feeds (pair->i2, 1);
4427 0 : INSN_TICK (pair->i2) = INVALID_TICK;
4428 0 : INSN_EXACT_TICK (pair->i2) = clock_var + pair_delay (pair);
4429 0 : SHADOW_P (pair->i2) = pair->stages == 0;
4430 0 : pair = pair->next_same_i1;
4431 : }
4432 0 : }
4433 :
4434 : /* Walk the ready list and all queues. If any insns have unresolved backwards
4435 : dependencies, these must be cancelled deps, broken by predication. Set or
4436 : clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. */
4437 :
4438 : static void
4439 0 : toggle_cancelled_flags (bool set)
4440 : {
4441 0 : int i;
4442 0 : sd_iterator_def sd_it;
4443 0 : dep_t dep;
4444 :
4445 0 : if (ready.n_ready > 0)
4446 : {
4447 0 : rtx_insn **first = ready_lastpos (&ready);
4448 0 : for (i = 0; i < ready.n_ready; i++)
4449 0 : FOR_EACH_DEP (first[i], SD_LIST_BACK, sd_it, dep)
4450 0 : if (!DEBUG_INSN_P (DEP_PRO (dep)))
4451 : {
4452 0 : if (set)
4453 0 : DEP_STATUS (dep) |= DEP_CANCELLED;
4454 : else
4455 0 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
4456 : }
4457 : }
4458 0 : for (i = 0; i <= max_insn_queue_index; i++)
4459 : {
4460 0 : int q = NEXT_Q_AFTER (q_ptr, i);
4461 0 : rtx_insn_list *link;
4462 0 : for (link = insn_queue[q]; link; link = link->next ())
4463 : {
4464 0 : rtx_insn *insn = link->insn ();
4465 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
4466 0 : if (!DEBUG_INSN_P (DEP_PRO (dep)))
4467 : {
4468 0 : if (set)
4469 0 : DEP_STATUS (dep) |= DEP_CANCELLED;
4470 : else
4471 0 : DEP_STATUS (dep) &= ~DEP_CANCELLED;
4472 : }
4473 : }
4474 : }
4475 0 : }
4476 :
4477 : /* Undo the replacements that have occurred after backtrack point SAVE
4478 : was placed. */
4479 : static void
4480 0 : undo_replacements_for_backtrack (struct haifa_saved_data *save)
4481 : {
4482 0 : while (!save->replacement_deps.is_empty ())
4483 : {
4484 0 : dep_t dep = save->replacement_deps.pop ();
4485 0 : int apply_p = save->replace_apply.pop ();
4486 :
4487 0 : if (apply_p)
4488 0 : restore_pattern (dep, true);
4489 : else
4490 0 : apply_replacement (dep, true);
4491 : }
4492 0 : save->replacement_deps.release ();
4493 0 : save->replace_apply.release ();
4494 0 : }
4495 :
4496 : /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN.
4497 : Restore their dependencies to an unresolved state, and mark them as
4498 : queued nowhere. */
4499 :
4500 : static void
4501 0 : unschedule_insns_until (rtx_insn *insn)
4502 : {
4503 0 : auto_vec<rtx_insn *> recompute_vec;
4504 :
4505 : /* Make two passes over the insns to be unscheduled. First, we clear out
4506 : dependencies and other trivial bookkeeping. */
4507 0 : for (;;)
4508 : {
4509 0 : rtx_insn *last;
4510 0 : sd_iterator_def sd_it;
4511 0 : dep_t dep;
4512 :
4513 0 : last = scheduled_insns.pop ();
4514 :
4515 : /* This will be changed by restore_backtrack_point if the insn is in
4516 : any queue. */
4517 0 : QUEUE_INDEX (last) = QUEUE_NOWHERE;
4518 0 : if (last != insn)
4519 0 : INSN_TICK (last) = INVALID_TICK;
4520 :
4521 0 : if (modulo_ii > 0 && INSN_UID (last) < modulo_iter0_max_uid)
4522 0 : modulo_insns_scheduled--;
4523 :
4524 0 : for (sd_it = sd_iterator_start (last, SD_LIST_RES_FORW);
4525 0 : sd_iterator_cond (&sd_it, &dep);)
4526 : {
4527 0 : rtx_insn *con = DEP_CON (dep);
4528 0 : sd_unresolve_dep (sd_it);
4529 0 : if (!MUST_RECOMPUTE_SPEC_P (con))
4530 : {
4531 0 : MUST_RECOMPUTE_SPEC_P (con) = 1;
4532 0 : recompute_vec.safe_push (con);
4533 : }
4534 : }
4535 :
4536 0 : if (last == insn)
4537 : break;
4538 0 : }
4539 :
4540 : /* A second pass, to update ready and speculation status for insns
4541 : depending on the unscheduled ones. The first pass must have
4542 : popped the scheduled_insns vector up to the point where we
4543 : restart scheduling, as recompute_todo_spec requires it to be
4544 : up-to-date. */
4545 0 : while (!recompute_vec.is_empty ())
4546 : {
4547 0 : rtx_insn *con;
4548 :
4549 0 : con = recompute_vec.pop ();
4550 0 : MUST_RECOMPUTE_SPEC_P (con) = 0;
4551 0 : if (!sd_lists_empty_p (con, SD_LIST_HARD_BACK))
4552 : {
4553 0 : TODO_SPEC (con) = HARD_DEP;
4554 0 : INSN_TICK (con) = INVALID_TICK;
4555 0 : if (PREDICATED_PAT (con) != NULL_RTX)
4556 0 : haifa_change_pattern (con, ORIG_PAT (con));
4557 : }
4558 0 : else if (QUEUE_INDEX (con) != QUEUE_SCHEDULED)
4559 0 : TODO_SPEC (con) = recompute_todo_spec (con, true);
4560 : }
4561 0 : }
4562 :
4563 : /* Restore scheduler state from the topmost entry on the backtracking queue.
4564 : PSCHED_BLOCK_P points to the local data of schedule_block that we must
4565 : overwrite with the saved data.
4566 : The caller must already have called unschedule_insns_until. */
4567 :
4568 : static void
4569 0 : restore_last_backtrack_point (struct sched_block_state *psched_block)
4570 : {
4571 0 : int i;
4572 0 : struct haifa_saved_data *save = backtrack_queue;
4573 :
4574 0 : backtrack_queue = save->next;
4575 :
4576 0 : if (current_sched_info->restore_state)
4577 0 : (*current_sched_info->restore_state) (save->fe_saved_data);
4578 :
4579 0 : if (targetm.sched.alloc_sched_context)
4580 : {
4581 0 : targetm.sched.set_sched_context (save->be_saved_data);
4582 0 : targetm.sched.free_sched_context (save->be_saved_data);
4583 : }
4584 :
4585 : /* Do this first since it clobbers INSN_TICK of the involved
4586 : instructions. */
4587 0 : undo_replacements_for_backtrack (save);
4588 :
4589 : /* Clear the QUEUE_INDEX of everything in the ready list or one
4590 : of the queues. */
4591 0 : if (ready.n_ready > 0)
4592 : {
4593 0 : rtx_insn **first = ready_lastpos (&ready);
4594 0 : for (i = 0; i < ready.n_ready; i++)
4595 : {
4596 0 : rtx_insn *insn = first[i];
4597 0 : QUEUE_INDEX (insn) = QUEUE_NOWHERE;
4598 0 : INSN_TICK (insn) = INVALID_TICK;
4599 : }
4600 : }
4601 0 : for (i = 0; i <= max_insn_queue_index; i++)
4602 : {
4603 0 : int q = NEXT_Q_AFTER (q_ptr, i);
4604 :
4605 0 : for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ())
4606 : {
4607 0 : rtx_insn *x = link->insn ();
4608 0 : QUEUE_INDEX (x) = QUEUE_NOWHERE;
4609 0 : INSN_TICK (x) = INVALID_TICK;
4610 : }
4611 0 : free_INSN_LIST_list (&insn_queue[q]);
4612 : }
4613 :
4614 0 : free (ready.vec);
4615 0 : ready = save->ready;
4616 :
4617 0 : if (ready.n_ready > 0)
4618 : {
4619 0 : rtx_insn **first = ready_lastpos (&ready);
4620 0 : for (i = 0; i < ready.n_ready; i++)
4621 : {
4622 0 : rtx_insn *insn = first[i];
4623 0 : QUEUE_INDEX (insn) = QUEUE_READY;
4624 0 : TODO_SPEC (insn) = recompute_todo_spec (insn, true);
4625 0 : INSN_TICK (insn) = save->clock_var;
4626 : }
4627 : }
4628 :
4629 0 : q_ptr = 0;
4630 0 : q_size = save->q_size;
4631 0 : for (i = 0; i <= max_insn_queue_index; i++)
4632 : {
4633 0 : int q = NEXT_Q_AFTER (q_ptr, i);
4634 :
4635 0 : insn_queue[q] = save->insn_queue[q];
4636 :
4637 0 : for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ())
4638 : {
4639 0 : rtx_insn *x = link->insn ();
4640 0 : QUEUE_INDEX (x) = i;
4641 0 : TODO_SPEC (x) = recompute_todo_spec (x, true);
4642 0 : INSN_TICK (x) = save->clock_var + i;
4643 : }
4644 : }
4645 0 : free (save->insn_queue);
4646 :
4647 0 : toggle_cancelled_flags (true);
4648 :
4649 0 : clock_var = save->clock_var;
4650 0 : last_clock_var = save->last_clock_var;
4651 0 : cycle_issued_insns = save->cycle_issued_insns;
4652 0 : last_scheduled_insn = save->last_scheduled_insn;
4653 0 : last_nondebug_scheduled_insn = save->last_nondebug_scheduled_insn;
4654 0 : nonscheduled_insns_begin = save->nonscheduled_insns_begin;
4655 :
4656 0 : *psched_block = save->sched_block;
4657 :
4658 0 : memcpy (curr_state, save->curr_state, dfa_state_size);
4659 0 : free (save->curr_state);
4660 :
4661 0 : mark_backtrack_feeds (save->delay_pair->i2, 0);
4662 :
4663 0 : gcc_assert (next_cycle_replace_deps.is_empty ());
4664 0 : next_cycle_replace_deps = save->next_cycle_deps.copy ();
4665 0 : next_cycle_apply = save->next_cycle_apply.copy ();
4666 :
4667 0 : free (save);
4668 :
4669 0 : for (save = backtrack_queue; save; save = save->next)
4670 : {
4671 0 : mark_backtrack_feeds (save->delay_pair->i2, 1);
4672 : }
4673 0 : }
4674 :
4675 : /* Discard all data associated with the topmost entry in the backtrack
4676 : queue. If RESET_TICK is false, we just want to free the data. If true,
4677 : we are doing this because we discovered a reason to backtrack. In the
4678 : latter case, also reset the INSN_TICK for the shadow insn. */
4679 : static void
4680 0 : free_topmost_backtrack_point (bool reset_tick)
4681 : {
4682 0 : struct haifa_saved_data *save = backtrack_queue;
4683 0 : int i;
4684 :
4685 0 : backtrack_queue = save->next;
4686 :
4687 0 : if (reset_tick)
4688 : {
4689 0 : struct delay_pair *pair = save->delay_pair;
4690 0 : while (pair)
4691 : {
4692 0 : INSN_TICK (pair->i2) = INVALID_TICK;
4693 0 : INSN_EXACT_TICK (pair->i2) = INVALID_TICK;
4694 0 : pair = pair->next_same_i1;
4695 : }
4696 0 : undo_replacements_for_backtrack (save);
4697 : }
4698 : else
4699 : {
4700 0 : save->replacement_deps.release ();
4701 0 : save->replace_apply.release ();
4702 : }
4703 :
4704 0 : if (targetm.sched.free_sched_context)
4705 0 : targetm.sched.free_sched_context (save->be_saved_data);
4706 0 : if (current_sched_info->restore_state)
4707 0 : free (save->fe_saved_data);
4708 0 : for (i = 0; i <= max_insn_queue_index; i++)
4709 0 : free_INSN_LIST_list (&save->insn_queue[i]);
4710 0 : free (save->insn_queue);
4711 0 : free (save->curr_state);
4712 0 : free (save->ready.vec);
4713 0 : free (save);
4714 0 : }
4715 :
4716 : /* Free the entire backtrack queue. */
4717 : static void
4718 10610721 : free_backtrack_queue (void)
4719 : {
4720 10610721 : while (backtrack_queue)
4721 0 : free_topmost_backtrack_point (false);
4722 10610721 : }
4723 :
4724 : /* Apply a replacement described by DESC. If IMMEDIATELY is false, we
4725 : may have to postpone the replacement until the start of the next cycle,
4726 : at which point we will be called again with IMMEDIATELY true. This is
4727 : only done for machines which have instruction packets with explicit
4728 : parallelism however. */
4729 : static void
4730 608321 : apply_replacement (dep_t dep, bool immediately)
4731 : {
4732 608321 : struct dep_replacement *desc = DEP_REPLACE (dep);
4733 608321 : if (!immediately && targetm.sched.exposed_pipeline && reload_completed)
4734 : {
4735 0 : next_cycle_replace_deps.safe_push (dep);
4736 0 : next_cycle_apply.safe_push (1);
4737 : }
4738 : else
4739 : {
4740 608321 : bool success;
4741 :
4742 608321 : if (QUEUE_INDEX (desc->insn) == QUEUE_SCHEDULED)
4743 : return;
4744 :
4745 608321 : if (sched_verbose >= 5)
4746 0 : fprintf (sched_dump, "applying replacement for insn %d\n",
4747 : INSN_UID (desc->insn));
4748 :
4749 608321 : success = validate_change (desc->insn, desc->loc, desc->newval, 0);
4750 608321 : gcc_assert (success);
4751 :
4752 608321 : rtx_insn *insn = DEP_PRO (dep);
4753 :
4754 : /* Recompute priority since dependent priorities may have changed. */
4755 608321 : priority (insn, true);
4756 608321 : update_insn_after_change (desc->insn);
4757 :
4758 608321 : if ((TODO_SPEC (desc->insn) & (HARD_DEP | DEP_POSTPONED)) == 0)
4759 316422 : fix_tick_ready (desc->insn);
4760 :
4761 608321 : if (backtrack_queue != NULL)
4762 : {
4763 0 : backtrack_queue->replacement_deps.safe_push (dep);
4764 0 : backtrack_queue->replace_apply.safe_push (1);
4765 : }
4766 : }
4767 : }
4768 :
4769 : /* We have determined that a pattern involved in DEP must be restored.
4770 : If IMMEDIATELY is false, we may have to postpone the replacement
4771 : until the start of the next cycle, at which point we will be called
4772 : again with IMMEDIATELY true. */
4773 : static void
4774 61251 : restore_pattern (dep_t dep, bool immediately)
4775 : {
4776 61251 : rtx_insn *next = DEP_CON (dep);
4777 61251 : int tick = INSN_TICK (next);
4778 :
4779 : /* If we already scheduled the insn, the modified version is
4780 : correct. */
4781 61251 : if (QUEUE_INDEX (next) == QUEUE_SCHEDULED)
4782 : return;
4783 :
4784 61251 : if (!immediately && targetm.sched.exposed_pipeline && reload_completed)
4785 : {
4786 0 : next_cycle_replace_deps.safe_push (dep);
4787 0 : next_cycle_apply.safe_push (0);
4788 0 : return;
4789 : }
4790 :
4791 :
4792 61251 : if (DEP_TYPE (dep) == REG_DEP_CONTROL)
4793 : {
4794 0 : if (sched_verbose >= 5)
4795 0 : fprintf (sched_dump, "restoring pattern for insn %d\n",
4796 : INSN_UID (next));
4797 0 : haifa_change_pattern (next, ORIG_PAT (next));
4798 : }
4799 : else
4800 : {
4801 61251 : struct dep_replacement *desc = DEP_REPLACE (dep);
4802 61251 : bool success;
4803 :
4804 61251 : if (sched_verbose >= 5)
4805 0 : fprintf (sched_dump, "restoring pattern for insn %d\n",
4806 0 : INSN_UID (desc->insn));
4807 61251 : tick = INSN_TICK (desc->insn);
4808 :
4809 61251 : success = validate_change (desc->insn, desc->loc, desc->orig, 0);
4810 61251 : gcc_assert (success);
4811 :
4812 61251 : rtx_insn *insn = DEP_PRO (dep);
4813 :
4814 61251 : if (QUEUE_INDEX (insn) != QUEUE_SCHEDULED)
4815 : {
4816 : /* Recompute priority since dependent priorities may have changed. */
4817 0 : priority (insn, true);
4818 : }
4819 :
4820 61251 : update_insn_after_change (desc->insn);
4821 :
4822 61251 : if (backtrack_queue != NULL)
4823 : {
4824 0 : backtrack_queue->replacement_deps.safe_push (dep);
4825 0 : backtrack_queue->replace_apply.safe_push (0);
4826 : }
4827 : }
4828 61251 : INSN_TICK (next) = tick;
4829 61251 : if (TODO_SPEC (next) == DEP_POSTPONED)
4830 : return;
4831 :
4832 61251 : if (sd_lists_empty_p (next, SD_LIST_BACK))
4833 61251 : TODO_SPEC (next) = 0;
4834 0 : else if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK))
4835 0 : TODO_SPEC (next) = HARD_DEP;
4836 : }
4837 :
4838 : /* Perform pattern replacements that were queued up until the next
4839 : cycle. */
4840 : static void
4841 46608351 : perform_replacements_new_cycle (void)
4842 : {
4843 46608351 : int i;
4844 46608351 : dep_t dep;
4845 46608351 : FOR_EACH_VEC_ELT (next_cycle_replace_deps, i, dep)
4846 : {
4847 0 : int apply_p = next_cycle_apply[i];
4848 0 : if (apply_p)
4849 0 : apply_replacement (dep, true);
4850 : else
4851 0 : restore_pattern (dep, true);
4852 : }
4853 46608351 : next_cycle_replace_deps.truncate (0);
4854 46608351 : next_cycle_apply.truncate (0);
4855 46608351 : }
4856 :
4857 : /* Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of
4858 : instructions we've previously encountered, a set bit prevents
4859 : recursion. BUDGET is a limit on how far ahead we look, it is
4860 : reduced on recursive calls. Return true if we produced a good
4861 : estimate, or false if we exceeded the budget. */
4862 : static bool
4863 0 : estimate_insn_tick (bitmap processed, rtx_insn *insn, int budget)
4864 : {
4865 0 : sd_iterator_def sd_it;
4866 0 : dep_t dep;
4867 0 : int earliest = INSN_TICK (insn);
4868 :
4869 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
4870 : {
4871 0 : rtx_insn *pro = DEP_PRO (dep);
4872 0 : int t;
4873 :
4874 0 : if (DEP_STATUS (dep) & DEP_CANCELLED)
4875 0 : continue;
4876 :
4877 0 : if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED)
4878 0 : gcc_assert (INSN_TICK (pro) + dep_cost (dep) <= INSN_TICK (insn));
4879 : else
4880 : {
4881 0 : int cost = dep_cost (dep);
4882 0 : if (cost >= budget)
4883 : return false;
4884 0 : if (!bitmap_bit_p (processed, INSN_LUID (pro)))
4885 : {
4886 0 : if (!estimate_insn_tick (processed, pro, budget - cost))
4887 : return false;
4888 : }
4889 0 : gcc_assert (INSN_TICK_ESTIMATE (pro) != INVALID_TICK);
4890 0 : t = INSN_TICK_ESTIMATE (pro) + cost;
4891 0 : if (earliest == INVALID_TICK || t > earliest)
4892 0 : earliest = t;
4893 : }
4894 : }
4895 0 : bitmap_set_bit (processed, INSN_LUID (insn));
4896 0 : INSN_TICK_ESTIMATE (insn) = earliest;
4897 0 : return true;
4898 : }
4899 :
4900 : /* Examine the pair of insns in P, and estimate (optimistically, assuming
4901 : infinite resources) the cycle in which the delayed shadow can be issued.
4902 : Return the number of cycles that must pass before the real insn can be
4903 : issued in order to meet this constraint. */
4904 : static int
4905 0 : estimate_shadow_tick (struct delay_pair *p)
4906 : {
4907 0 : auto_bitmap processed;
4908 0 : int t;
4909 0 : bool cutoff;
4910 :
4911 0 : cutoff = !estimate_insn_tick (processed, p->i2,
4912 0 : max_insn_queue_index + pair_delay (p));
4913 0 : if (cutoff)
4914 : return max_insn_queue_index;
4915 0 : t = INSN_TICK_ESTIMATE (p->i2) - (clock_var + pair_delay (p) + 1);
4916 0 : if (t > 0)
4917 : return t;
4918 : return 0;
4919 0 : }
4920 :
4921 : /* If INSN has no unresolved backwards dependencies, add it to the schedule and
4922 : recursively resolve all its forward dependencies. */
4923 : static void
4924 0 : resolve_dependencies (rtx_insn *insn)
4925 : {
4926 0 : sd_iterator_def sd_it;
4927 0 : dep_t dep;
4928 :
4929 : /* Don't use sd_lists_empty_p; it ignores debug insns. */
4930 0 : if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn)) != NULL
4931 0 : || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn)) != NULL)
4932 0 : return;
4933 :
4934 0 : if (sched_verbose >= 4)
4935 0 : fprintf (sched_dump, ";;\tquickly resolving %d\n", INSN_UID (insn));
4936 :
4937 0 : if (QUEUE_INDEX (insn) >= 0)
4938 0 : queue_remove (insn);
4939 :
4940 0 : scheduled_insns.safe_push (insn);
4941 :
4942 : /* Update dependent instructions. */
4943 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
4944 0 : sd_iterator_cond (&sd_it, &dep);)
4945 : {
4946 0 : rtx_insn *next = DEP_CON (dep);
4947 :
4948 0 : if (sched_verbose >= 4)
4949 0 : fprintf (sched_dump, ";;\t\tdep %d against %d\n", INSN_UID (insn),
4950 0 : INSN_UID (next));
4951 :
4952 : /* Resolve the dependence between INSN and NEXT.
4953 : sd_resolve_dep () moves current dep to another list thus
4954 : advancing the iterator. */
4955 0 : sd_resolve_dep (sd_it);
4956 :
4957 0 : if (!IS_SPECULATION_BRANCHY_CHECK_P (insn))
4958 : {
4959 0 : resolve_dependencies (next);
4960 : }
4961 : else
4962 : /* Check always has only one forward dependence (to the first insn in
4963 : the recovery block), therefore, this will be executed only once. */
4964 : {
4965 0 : gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW));
4966 : }
4967 : }
4968 : }
4969 :
4970 :
4971 : /* Return the head and tail pointers of ebb starting at BEG and ending
4972 : at END. */
4973 : void
4974 42525633 : get_ebb_head_tail (basic_block beg, basic_block end,
4975 : rtx_insn **headp, rtx_insn **tailp)
4976 : {
4977 42525633 : rtx_insn *beg_head = BB_HEAD (beg);
4978 42525633 : rtx_insn * beg_tail = BB_END (beg);
4979 42525633 : rtx_insn * end_head = BB_HEAD (end);
4980 42525633 : rtx_insn * end_tail = BB_END (end);
4981 :
4982 : /* Don't include any notes or labels at the beginning of the BEG
4983 : basic block, or notes at the end of the END basic blocks. */
4984 :
4985 42525633 : if (LABEL_P (beg_head))
4986 20764327 : beg_head = NEXT_INSN (beg_head);
4987 :
4988 99643969 : while (beg_head != beg_tail)
4989 95662683 : if (NOTE_P (beg_head))
4990 57118336 : beg_head = NEXT_INSN (beg_head);
4991 38544347 : else if (DEBUG_INSN_P (beg_head))
4992 : {
4993 10851262 : rtx_insn * note, *next;
4994 :
4995 10851262 : for (note = NEXT_INSN (beg_head);
4996 91186775 : note != beg_tail;
4997 80335513 : note = next)
4998 : {
4999 90860767 : next = NEXT_INSN (note);
5000 90860767 : if (NOTE_P (note))
5001 : {
5002 441833 : if (sched_verbose >= 9)
5003 0 : fprintf (sched_dump, "reorder %i\n", INSN_UID (note));
5004 :
5005 441833 : reorder_insns_nobb (note, note, PREV_INSN (beg_head));
5006 :
5007 441833 : if (BLOCK_FOR_INSN (note) != beg)
5008 0 : df_insn_change_bb (note, beg);
5009 : }
5010 90418934 : else if (!DEBUG_INSN_P (note))
5011 : break;
5012 : }
5013 :
5014 : break;
5015 : }
5016 : else
5017 : break;
5018 :
5019 42525633 : *headp = beg_head;
5020 :
5021 42525633 : if (beg == end)
5022 42525633 : end_head = beg_head;
5023 63 : else if (LABEL_P (end_head))
5024 2 : end_head = NEXT_INSN (end_head);
5025 :
5026 42686440 : while (end_head != end_tail)
5027 38691914 : if (NOTE_P (end_tail))
5028 160807 : end_tail = PREV_INSN (end_tail);
5029 38531107 : else if (DEBUG_INSN_P (end_tail))
5030 : {
5031 914783 : rtx_insn * note, *prev;
5032 :
5033 914783 : for (note = PREV_INSN (end_tail);
5034 7797103 : note != end_head;
5035 6882320 : note = prev)
5036 : {
5037 7761322 : prev = PREV_INSN (note);
5038 7761322 : if (NOTE_P (note))
5039 : {
5040 1741 : if (sched_verbose >= 9)
5041 0 : fprintf (sched_dump, "reorder %i\n", INSN_UID (note));
5042 :
5043 1741 : reorder_insns_nobb (note, note, end_tail);
5044 :
5045 1741 : if (end_tail == BB_END (end))
5046 1077 : BB_END (end) = note;
5047 :
5048 1741 : if (BLOCK_FOR_INSN (note) != end)
5049 0 : df_insn_change_bb (note, end);
5050 : }
5051 7759581 : else if (!DEBUG_INSN_P (note))
5052 : break;
5053 : }
5054 :
5055 : break;
5056 : }
5057 : else
5058 : break;
5059 :
5060 42525633 : *tailp = end_tail;
5061 42525633 : }
5062 :
5063 : /* Return true if there are no real insns in the range [ HEAD, TAIL ]. */
5064 :
5065 : bool
5066 31893215 : no_real_insns_p (const rtx_insn *head, const rtx_insn *tail)
5067 : {
5068 31952347 : while (head != NEXT_INSN (tail))
5069 : {
5070 31893215 : if (!NOTE_P (head) && !LABEL_P (head))
5071 : return false;
5072 59132 : head = NEXT_INSN (head);
5073 : }
5074 : return true;
5075 : }
5076 :
5077 : /* Restore-other-notes: NOTE_LIST is the end of a chain of notes
5078 : previously found among the insns. Insert them just before HEAD. */
5079 : rtx_insn *
5080 10611730 : restore_other_notes (rtx_insn *head, basic_block head_bb)
5081 : {
5082 10611730 : if (note_list != 0)
5083 : {
5084 2132812 : rtx_insn *note_head = note_list;
5085 :
5086 2132812 : if (head)
5087 2132439 : head_bb = BLOCK_FOR_INSN (head);
5088 : else
5089 373 : head = NEXT_INSN (bb_note (head_bb));
5090 :
5091 5010990 : while (PREV_INSN (note_head))
5092 : {
5093 2878178 : set_block_for_insn (note_head, head_bb);
5094 2878178 : note_head = PREV_INSN (note_head);
5095 : }
5096 : /* In the above cycle we've missed this note. */
5097 2132812 : set_block_for_insn (note_head, head_bb);
5098 :
5099 2132812 : SET_PREV_INSN (note_head) = PREV_INSN (head);
5100 2132812 : SET_NEXT_INSN (PREV_INSN (head)) = note_head;
5101 2132812 : SET_PREV_INSN (head) = note_list;
5102 2132812 : SET_NEXT_INSN (note_list) = head;
5103 :
5104 2132812 : if (BLOCK_FOR_INSN (head) != head_bb)
5105 0 : BB_END (head_bb) = note_list;
5106 :
5107 : head = note_head;
5108 : }
5109 :
5110 10611730 : return head;
5111 : }
5112 :
5113 : /* When we know we are going to discard the schedule due to a failed attempt
5114 : at modulo scheduling, undo all replacements. */
5115 : static void
5116 0 : undo_all_replacements (void)
5117 : {
5118 0 : rtx_insn *insn;
5119 0 : int i;
5120 :
5121 0 : FOR_EACH_VEC_ELT (scheduled_insns, i, insn)
5122 : {
5123 0 : sd_iterator_def sd_it;
5124 0 : dep_t dep;
5125 :
5126 : /* See if we must undo a replacement. */
5127 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_RES_FORW);
5128 0 : sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it))
5129 : {
5130 0 : struct dep_replacement *desc = DEP_REPLACE (dep);
5131 0 : if (desc != NULL)
5132 0 : validate_change (desc->insn, desc->loc, desc->orig, 0);
5133 : }
5134 : }
5135 0 : }
5136 :
5137 : /* Return first non-scheduled insn in the current scheduling block.
5138 : This is mostly used for debug-counter purposes. */
5139 : static rtx_insn *
5140 0 : first_nonscheduled_insn (void)
5141 : {
5142 0 : rtx_insn *insn = (nonscheduled_insns_begin != NULL_RTX
5143 0 : ? nonscheduled_insns_begin
5144 0 : : current_sched_info->prev_head);
5145 :
5146 0 : do
5147 : {
5148 0 : insn = next_nonnote_nondebug_insn (insn);
5149 : }
5150 0 : while (QUEUE_INDEX (insn) == QUEUE_SCHEDULED);
5151 :
5152 0 : return insn;
5153 : }
5154 :
5155 : /* Move insns that became ready to fire from queue to ready list. */
5156 :
5157 : static void
5158 35997630 : queue_to_ready (struct ready_list *ready)
5159 : {
5160 35997630 : rtx_insn *insn;
5161 35997630 : rtx_insn_list *link;
5162 35997630 : rtx_insn *skip_insn;
5163 :
5164 35997630 : q_ptr = NEXT_Q (q_ptr);
5165 :
5166 35997630 : if (dbg_cnt (sched_insn) == false)
5167 : /* If debug counter is activated do not requeue the first
5168 : nonscheduled insn. */
5169 0 : skip_insn = first_nonscheduled_insn ();
5170 : else
5171 : skip_insn = NULL;
5172 :
5173 : /* Add all pending insns that can be scheduled without stalls to the
5174 : ready list. */
5175 69176355 : for (link = insn_queue[q_ptr]; link; link = link->next ())
5176 : {
5177 33178725 : insn = link->insn ();
5178 33178725 : q_size -= 1;
5179 :
5180 33178725 : if (sched_verbose >= 2)
5181 0 : fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ",
5182 0 : (*current_sched_info->print_insn) (insn, 0));
5183 :
5184 : /* If the ready list is full, delay the insn for 1 cycle.
5185 : See the comment in schedule_block for the rationale. */
5186 33178725 : if (!reload_completed
5187 4981 : && (ready->n_ready - ready->n_debug > param_max_sched_ready_insns
5188 4834 : || (sched_pressure == SCHED_PRESSURE_MODEL
5189 : /* Limit pressure recalculations to
5190 : param_max_sched_ready_insns instructions too. */
5191 0 : && model_index (insn) > (model_curr_point
5192 0 : + param_max_sched_ready_insns)))
5193 147 : && !(sched_pressure == SCHED_PRESSURE_MODEL
5194 0 : && model_curr_point < model_num_insns
5195 : /* Always allow the next model instruction to issue. */
5196 0 : && model_index (insn) == model_curr_point)
5197 147 : && !SCHED_GROUP_P (insn)
5198 33178872 : && insn != skip_insn)
5199 : {
5200 147 : if (sched_verbose >= 2)
5201 0 : fprintf (sched_dump, "keeping in queue, ready full\n");
5202 147 : queue_insn (insn, 1, "ready full");
5203 : }
5204 : else
5205 : {
5206 33178578 : ready_add (ready, insn, false);
5207 33178578 : if (sched_verbose >= 2)
5208 0 : fprintf (sched_dump, "moving to ready without stalls\n");
5209 : }
5210 : }
5211 35997630 : free_INSN_LIST_list (&insn_queue[q_ptr]);
5212 :
5213 : /* If there are no ready insns, stall until one is ready and add all
5214 : of the pending insns at that point to the ready list. */
5215 35997630 : if (ready->n_ready == 0)
5216 : {
5217 : int stalls;
5218 :
5219 11390379 : for (stalls = 1; stalls <= max_insn_queue_index; stalls++)
5220 : {
5221 11390379 : if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]))
5222 : {
5223 12838692 : for (; link; link = link->next ())
5224 : {
5225 6647287 : insn = link->insn ();
5226 6647287 : q_size -= 1;
5227 :
5228 6647287 : if (sched_verbose >= 2)
5229 0 : fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ",
5230 0 : (*current_sched_info->print_insn) (insn, 0));
5231 :
5232 6647287 : ready_add (ready, insn, false);
5233 6647287 : if (sched_verbose >= 2)
5234 0 : fprintf (sched_dump, "moving to ready with %d stalls\n", stalls);
5235 : }
5236 6191405 : free_INSN_LIST_list (&insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]);
5237 :
5238 6191405 : advance_one_cycle ();
5239 :
5240 6191405 : break;
5241 : }
5242 :
5243 5198974 : advance_one_cycle ();
5244 : }
5245 :
5246 6191405 : q_ptr = NEXT_Q_AFTER (q_ptr, stalls);
5247 6191405 : clock_var += stalls;
5248 6191405 : if (sched_verbose >= 2)
5249 0 : fprintf (sched_dump, ";;\tAdvancing clock by %d cycle[s] to %d\n",
5250 : stalls, clock_var);
5251 : }
5252 35997630 : }
5253 :
5254 : /* Used by early_queue_to_ready. Determines whether it is "ok" to
5255 : prematurely move INSN from the queue to the ready list. Currently,
5256 : if a target defines the hook 'is_costly_dependence', this function
5257 : uses the hook to check whether there exist any dependences which are
5258 : considered costly by the target, between INSN and other insns that
5259 : have already been scheduled. Dependences are checked up to Y cycles
5260 : back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows
5261 : controlling this value.
5262 : (Other considerations could be taken into account instead (or in
5263 : addition) depending on user flags and target hooks. */
5264 :
5265 : static bool
5266 5 : ok_for_early_queue_removal (rtx_insn *insn)
5267 : {
5268 5 : if (targetm.sched.is_costly_dependence)
5269 : {
5270 0 : int n_cycles;
5271 0 : int i = scheduled_insns.length ();
5272 0 : for (n_cycles = flag_sched_stalled_insns_dep; n_cycles; n_cycles--)
5273 : {
5274 0 : while (i-- > 0)
5275 : {
5276 0 : int cost;
5277 :
5278 0 : rtx_insn *prev_insn = scheduled_insns[i];
5279 :
5280 0 : if (!NOTE_P (prev_insn))
5281 : {
5282 0 : dep_t dep;
5283 :
5284 0 : dep = sd_find_dep_between (prev_insn, insn, true);
5285 :
5286 0 : if (dep != NULL)
5287 : {
5288 0 : cost = dep_cost (dep);
5289 :
5290 0 : if (targetm.sched.is_costly_dependence (dep, cost,
5291 0 : flag_sched_stalled_insns_dep - n_cycles))
5292 : return false;
5293 : }
5294 : }
5295 :
5296 0 : if (GET_MODE (prev_insn) == TImode) /* end of dispatch group */
5297 : break;
5298 : }
5299 :
5300 0 : if (i == 0)
5301 : break;
5302 : }
5303 : }
5304 :
5305 : return true;
5306 : }
5307 :
5308 :
5309 : /* Remove insns from the queue, before they become "ready" with respect
5310 : to FU latency considerations. */
5311 :
5312 : static int
5313 34042847 : early_queue_to_ready (state_t state, struct ready_list *ready)
5314 : {
5315 34042847 : rtx_insn *insn;
5316 34042847 : rtx_insn_list *link;
5317 34042847 : rtx_insn_list *next_link;
5318 34042847 : rtx_insn_list *prev_link;
5319 34042847 : bool move_to_ready;
5320 34042847 : int cost;
5321 34042847 : state_t temp_state = alloca (dfa_state_size);
5322 34042847 : int stalls;
5323 34042847 : int insns_removed = 0;
5324 :
5325 : /*
5326 : Flag '-fsched-stalled-insns=X' determines the aggressiveness of this
5327 : function:
5328 :
5329 : X == 0: There is no limit on how many queued insns can be removed
5330 : prematurely. (flag_sched_stalled_insns = -1).
5331 :
5332 : X >= 1: Only X queued insns can be removed prematurely in each
5333 : invocation. (flag_sched_stalled_insns = X).
5334 :
5335 : Otherwise: Early queue removal is disabled.
5336 : (flag_sched_stalled_insns = 0)
5337 : */
5338 :
5339 34042847 : if (! flag_sched_stalled_insns)
5340 : return 0;
5341 :
5342 1799 : for (stalls = 0; stalls <= max_insn_queue_index; stalls++)
5343 : {
5344 1792 : if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]))
5345 : {
5346 4 : if (sched_verbose > 6)
5347 0 : fprintf (sched_dump, ";; look at index %d + %d\n", q_ptr, stalls);
5348 :
5349 4 : prev_link = 0;
5350 9 : while (link)
5351 : {
5352 5 : next_link = link->next ();
5353 5 : insn = link->insn ();
5354 5 : if (insn && sched_verbose > 6)
5355 0 : print_rtl_single (sched_dump, insn);
5356 :
5357 5 : memcpy (temp_state, state, dfa_state_size);
5358 5 : if (recog_memoized (insn) < 0)
5359 : /* non-negative to indicate that it's not ready
5360 : to avoid infinite Q->R->Q->R... */
5361 : cost = 0;
5362 : else
5363 5 : cost = state_transition (temp_state, insn);
5364 :
5365 5 : if (sched_verbose >= 6)
5366 0 : fprintf (sched_dump, "transition cost = %d\n", cost);
5367 :
5368 5 : move_to_ready = false;
5369 5 : if (cost < 0)
5370 : {
5371 5 : move_to_ready = ok_for_early_queue_removal (insn);
5372 5 : if (move_to_ready == true)
5373 : {
5374 : /* move from Q to R */
5375 5 : q_size -= 1;
5376 5 : ready_add (ready, insn, false);
5377 :
5378 5 : if (prev_link)
5379 0 : XEXP (prev_link, 1) = next_link;
5380 : else
5381 5 : insn_queue[NEXT_Q_AFTER (q_ptr, stalls)] = next_link;
5382 :
5383 5 : free_INSN_LIST_node (link);
5384 :
5385 5 : if (sched_verbose >= 2)
5386 0 : fprintf (sched_dump, ";;\t\tEarly Q-->Ready: insn %s\n",
5387 0 : (*current_sched_info->print_insn) (insn, 0));
5388 :
5389 5 : insns_removed++;
5390 5 : if (insns_removed == flag_sched_stalled_insns)
5391 : /* Remove no more than flag_sched_stalled_insns insns
5392 : from Q at a time. */
5393 : return insns_removed;
5394 : }
5395 : }
5396 :
5397 : if (move_to_ready == false)
5398 : prev_link = link;
5399 :
5400 5 : link = next_link;
5401 : } /* while link */
5402 : } /* if link */
5403 :
5404 : } /* for stalls.. */
5405 :
5406 : return insns_removed;
5407 : }
5408 :
5409 :
5410 : /* Print the ready list for debugging purposes.
5411 : If READY_TRY is non-zero then only print insns that max_issue
5412 : will consider. */
5413 : static void
5414 190 : debug_ready_list_1 (struct ready_list *ready, signed char *ready_try)
5415 : {
5416 190 : rtx_insn **p;
5417 190 : int i;
5418 :
5419 190 : if (ready->n_ready == 0)
5420 : {
5421 190 : fprintf (sched_dump, "\n");
5422 190 : return;
5423 : }
5424 :
5425 0 : p = ready_lastpos (ready);
5426 0 : for (i = 0; i < ready->n_ready; i++)
5427 : {
5428 0 : if (ready_try != NULL && ready_try[ready->n_ready - i - 1])
5429 0 : continue;
5430 :
5431 0 : fprintf (sched_dump, " %s:%d",
5432 0 : (*current_sched_info->print_insn) (p[i], 0),
5433 0 : INSN_LUID (p[i]));
5434 0 : if (sched_pressure != SCHED_PRESSURE_NONE)
5435 0 : fprintf (sched_dump, "(cost=%d",
5436 0 : INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p[i]));
5437 0 : fprintf (sched_dump, ":prio=%d", INSN_PRIORITY (p[i]));
5438 0 : if (INSN_TICK (p[i]) > clock_var)
5439 0 : fprintf (sched_dump, ":delay=%d", INSN_TICK (p[i]) - clock_var);
5440 0 : if (sched_pressure == SCHED_PRESSURE_MODEL)
5441 0 : fprintf (sched_dump, ":idx=%d",
5442 : model_index (p[i]));
5443 0 : if (sched_pressure != SCHED_PRESSURE_NONE)
5444 0 : fprintf (sched_dump, ")");
5445 : }
5446 0 : fprintf (sched_dump, "\n");
5447 : }
5448 :
5449 : /* Print the ready list. Callable from debugger. */
5450 : static void
5451 190 : debug_ready_list (struct ready_list *ready)
5452 : {
5453 0 : debug_ready_list_1 (ready, NULL);
5454 0 : }
5455 :
5456 : /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn
5457 : NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb
5458 : replaces the epilogue note in the correct basic block. */
5459 : void
5460 61937191 : reemit_notes (rtx_insn *insn)
5461 : {
5462 61937191 : rtx note;
5463 61937191 : rtx_insn *last = insn;
5464 :
5465 122958442 : for (note = REG_NOTES (insn); note; note = XEXP (note, 1))
5466 : {
5467 61021251 : if (REG_NOTE_KIND (note) == REG_SAVE_NOTE)
5468 : {
5469 931558 : enum insn_note note_type = (enum insn_note) INTVAL (XEXP (note, 0));
5470 :
5471 931558 : last = emit_note_before (note_type, last);
5472 931558 : remove_note (insn, note);
5473 931558 : df_insn_create_insn_record (last);
5474 : }
5475 : }
5476 61937191 : }
5477 :
5478 : /* Move INSN. Reemit notes if needed. Update CFG, if needed. */
5479 : static void
5480 116108479 : move_insn (rtx_insn *insn, rtx_insn *last, rtx nt)
5481 : {
5482 116108479 : if (PREV_INSN (insn) != last)
5483 : {
5484 13856662 : basic_block bb;
5485 13856662 : rtx_insn *note;
5486 13856662 : int jump_p = 0;
5487 :
5488 13856662 : bb = BLOCK_FOR_INSN (insn);
5489 :
5490 : /* BB_HEAD is either LABEL or NOTE. */
5491 13856662 : gcc_assert (BB_HEAD (bb) != insn);
5492 :
5493 13856662 : if (BB_END (bb) == insn)
5494 : /* If this is last instruction in BB, move end marker one
5495 : instruction up. */
5496 : {
5497 : /* Jumps are always placed at the end of basic block. */
5498 234367 : jump_p = control_flow_insn_p (insn);
5499 :
5500 234367 : gcc_assert (!jump_p
5501 : || ((common_sched_info->sched_pass_id == SCHED_RGN_PASS)
5502 : && IS_SPECULATION_BRANCHY_CHECK_P (insn))
5503 : || (common_sched_info->sched_pass_id
5504 : == SCHED_EBB_PASS));
5505 :
5506 234367 : gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn)) == bb);
5507 :
5508 234367 : BB_END (bb) = PREV_INSN (insn);
5509 : }
5510 :
5511 13856662 : gcc_assert (BB_END (bb) != last);
5512 :
5513 13856662 : if (jump_p)
5514 : /* We move the block note along with jump. */
5515 : {
5516 1 : gcc_assert (nt);
5517 :
5518 1 : note = NEXT_INSN (insn);
5519 2 : while (NOTE_NOT_BB_P (note) && note != nt)
5520 0 : note = NEXT_INSN (note);
5521 :
5522 1 : if (note != nt
5523 1 : && (LABEL_P (note)
5524 1 : || BARRIER_P (note)))
5525 0 : note = NEXT_INSN (note);
5526 :
5527 1 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
5528 : }
5529 : else
5530 : note = insn;
5531 :
5532 13856662 : SET_NEXT_INSN (PREV_INSN (insn)) = NEXT_INSN (note);
5533 13856662 : SET_PREV_INSN (NEXT_INSN (note)) = PREV_INSN (insn);
5534 :
5535 13856662 : SET_NEXT_INSN (note) = NEXT_INSN (last);
5536 13856662 : SET_PREV_INSN (NEXT_INSN (last)) = note;
5537 :
5538 13856662 : SET_NEXT_INSN (last) = insn;
5539 13856662 : SET_PREV_INSN (insn) = last;
5540 :
5541 13856662 : bb = BLOCK_FOR_INSN (last);
5542 :
5543 13856662 : if (jump_p)
5544 : {
5545 1 : fix_jump_move (insn);
5546 :
5547 1 : if (BLOCK_FOR_INSN (insn) != bb)
5548 0 : move_block_after_check (insn);
5549 :
5550 1 : gcc_assert (BB_END (bb) == last);
5551 : }
5552 :
5553 13856662 : df_insn_change_bb (insn, bb);
5554 :
5555 : /* Update BB_END, if needed. */
5556 13856662 : if (BB_END (bb) == last)
5557 1 : BB_END (bb) = insn;
5558 : }
5559 :
5560 116108479 : SCHED_GROUP_P (insn) = 0;
5561 116108479 : }
5562 :
5563 : /* Return true if scheduling INSN will finish current clock cycle. */
5564 : static bool
5565 115650520 : insn_finishes_cycle_p (rtx_insn *insn)
5566 : {
5567 115650520 : if (SCHED_GROUP_P (insn))
5568 : /* After issuing INSN, rest of the sched_group will be forced to issue
5569 : in order. Don't make any plans for the rest of cycle. */
5570 : return true;
5571 :
5572 : /* Finishing the block will, apparently, finish the cycle. */
5573 115650288 : if (current_sched_info->insn_finishes_block_p
5574 115650288 : && current_sched_info->insn_finishes_block_p (insn))
5575 : return true;
5576 :
5577 : return false;
5578 : }
5579 :
5580 : /* Helper for autopref_multipass_init. Given a SET in PAT and whether
5581 : we're expecting a memory WRITE or not, check that the insn is relevant to
5582 : the autoprefetcher modelling code. Return true iff that is the case.
5583 : If it is relevant, record the base register of the memory op in BASE and
5584 : the offset in OFFSET. */
5585 :
5586 : static bool
5587 3 : analyze_set_insn_for_autopref (rtx pat, bool write, rtx *base, int *offset)
5588 : {
5589 3 : if (GET_CODE (pat) != SET)
5590 : return false;
5591 :
5592 3 : rtx mem = write ? SET_DEST (pat) : SET_SRC (pat);
5593 3 : if (!MEM_P (mem))
5594 : return false;
5595 :
5596 3 : struct address_info info;
5597 3 : decompose_mem_address (&info, mem);
5598 :
5599 : /* TODO: Currently only (base+const) addressing is supported. */
5600 3 : if (info.base == NULL || !REG_P (*info.base)
5601 3 : || (info.disp != NULL && !CONST_INT_P (*info.disp)))
5602 : return false;
5603 :
5604 3 : *base = *info.base;
5605 3 : *offset = info.disp ? INTVAL (*info.disp) : 0;
5606 3 : return true;
5607 : }
5608 :
5609 : /* Functions to model cache auto-prefetcher.
5610 :
5611 : Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
5612 : memory prefetches if it sees instructions with consecutive memory accesses
5613 : in the instruction stream. Details of such hardware units are not published,
5614 : so we can only guess what exactly is going on there.
5615 : In the scheduler, we model abstract auto-prefetcher. If there are memory
5616 : insns in the ready list (or the queue) that have same memory base, but
5617 : different offsets, then we delay the insns with larger offsets until insns
5618 : with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
5619 : is "1", then we look at the ready list; if it is N>1, then we also look
5620 : through N-1 queue entries.
5621 : If the param is N>=0, then rank_for_schedule will consider auto-prefetching
5622 : among its heuristics.
5623 : Param value of "-1" disables modelling of the auto-prefetcher. */
5624 :
5625 : /* Initialize autoprefetcher model data for INSN. */
5626 : static void
5627 3 : autopref_multipass_init (const rtx_insn *insn, int write)
5628 : {
5629 3 : autopref_multipass_data_t data = &INSN_AUTOPREF_MULTIPASS_DATA (insn)[write];
5630 :
5631 3 : gcc_assert (data->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED);
5632 3 : data->base = NULL_RTX;
5633 3 : data->offset = 0;
5634 : /* Set insn entry initialized, but not relevant for auto-prefetcher. */
5635 3 : data->status = AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
5636 :
5637 3 : rtx pat = PATTERN (insn);
5638 :
5639 : /* We have a multi-set insn like a load-multiple or store-multiple.
5640 : We care about these as long as all the memory ops inside the PARALLEL
5641 : have the same base register. We care about the minimum and maximum
5642 : offsets from that base but don't check for the order of those offsets
5643 : within the PARALLEL insn itself. */
5644 3 : if (GET_CODE (pat) == PARALLEL)
5645 : {
5646 0 : int n_elems = XVECLEN (pat, 0);
5647 :
5648 0 : int i, offset;
5649 0 : rtx base, prev_base = NULL_RTX;
5650 0 : int min_offset = INT_MAX;
5651 :
5652 0 : for (i = 0; i < n_elems; i++)
5653 : {
5654 0 : rtx set = XVECEXP (pat, 0, i);
5655 0 : if (GET_CODE (set) != SET)
5656 : return;
5657 :
5658 0 : if (!analyze_set_insn_for_autopref (set, write, &base, &offset))
5659 : return;
5660 :
5661 : /* Ensure that all memory operations in the PARALLEL use the same
5662 : base register. */
5663 0 : if (i > 0 && REGNO (base) != REGNO (prev_base))
5664 : return;
5665 0 : prev_base = base;
5666 0 : min_offset = MIN (min_offset, offset);
5667 : }
5668 :
5669 : /* If we reached here then we have a valid PARALLEL of multiple memory ops
5670 : with prev_base as the base and min_offset containing the offset. */
5671 0 : gcc_assert (prev_base);
5672 0 : data->base = prev_base;
5673 0 : data->offset = min_offset;
5674 0 : data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
5675 0 : return;
5676 : }
5677 :
5678 : /* Otherwise this is a single set memory operation. */
5679 3 : rtx set = single_set (insn);
5680 3 : if (set == NULL_RTX)
5681 : return;
5682 :
5683 3 : if (!analyze_set_insn_for_autopref (set, write, &data->base,
5684 : &data->offset))
5685 : return;
5686 :
5687 : /* This insn is relevant for the auto-prefetcher.
5688 : The base and offset fields will have been filled in the
5689 : analyze_set_insn_for_autopref call above. */
5690 3 : data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
5691 : }
5692 :
5693 : /* Helper function for rank_for_schedule sorting. */
5694 : static int
5695 12 : autopref_rank_for_schedule (const rtx_insn *insn1, const rtx_insn *insn2)
5696 : {
5697 12 : int r = 0;
5698 24 : for (int write = 0; write < 2 && !r; ++write)
5699 : {
5700 12 : autopref_multipass_data_t data1
5701 12 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5702 12 : autopref_multipass_data_t data2
5703 12 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
5704 :
5705 12 : if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5706 1 : autopref_multipass_init (insn1, write);
5707 :
5708 12 : if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5709 2 : autopref_multipass_init (insn2, write);
5710 :
5711 12 : int irrel1 = data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
5712 12 : int irrel2 = data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
5713 :
5714 12 : if (!irrel1 && !irrel2)
5715 : /* Sort memory references from lowest offset to the largest. */
5716 12 : r = (data1->offset > data2->offset) - (data1->offset < data2->offset);
5717 0 : else if (write)
5718 : /* Schedule "irrelevant" insns before memory stores to resolve
5719 : as many producer dependencies of stores as possible. */
5720 0 : r = irrel2 - irrel1;
5721 : else
5722 : /* Schedule "irrelevant" insns after memory reads to avoid breaking
5723 : memory read sequences. */
5724 0 : r = irrel1 - irrel2;
5725 : }
5726 :
5727 12 : return r;
5728 : }
5729 :
5730 : /* True if header of debug dump was printed. */
5731 : static bool autopref_multipass_dfa_lookahead_guard_started_dump_p;
5732 :
5733 : /* Helper for autopref_multipass_dfa_lookahead_guard.
5734 : Return "1" if INSN1 should be delayed in favor of INSN2. */
5735 : static int
5736 0 : autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn *insn1,
5737 : const rtx_insn *insn2, int write)
5738 : {
5739 0 : autopref_multipass_data_t data1
5740 0 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5741 0 : autopref_multipass_data_t data2
5742 0 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
5743 :
5744 0 : if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5745 0 : autopref_multipass_init (insn2, write);
5746 0 : if (data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
5747 : return 0;
5748 :
5749 0 : if (rtx_equal_p (data1->base, data2->base)
5750 0 : && data1->offset > data2->offset)
5751 : {
5752 0 : if (sched_verbose >= 2)
5753 : {
5754 0 : if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
5755 : {
5756 0 : fprintf (sched_dump,
5757 : ";;\t\tnot trying in max_issue due to autoprefetch "
5758 : "model: ");
5759 0 : autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
5760 : }
5761 :
5762 0 : fprintf (sched_dump, " %d(%d)", INSN_UID (insn1), INSN_UID (insn2));
5763 : }
5764 :
5765 : return 1;
5766 : }
5767 :
5768 : return 0;
5769 : }
5770 :
5771 : /* General note:
5772 :
5773 : We could have also hooked autoprefetcher model into
5774 : first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
5775 : to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
5776 : (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
5777 : unblocked). We don't bother about this yet because target of interest
5778 : (ARM Cortex-A15) can issue only 1 memory operation per cycle. */
5779 :
5780 : /* Implementation of first_cycle_multipass_dfa_lookahead_guard hook.
5781 : Return "1" if INSN1 should not be considered in max_issue due to
5782 : auto-prefetcher considerations. */
5783 : int
5784 0 : autopref_multipass_dfa_lookahead_guard (rtx_insn *insn1, int ready_index)
5785 : {
5786 0 : int r = 0;
5787 :
5788 : /* Exit early if the param forbids this or if we're not entering here through
5789 : normal haifa scheduling. This can happen if selective scheduling is
5790 : explicitly enabled. */
5791 0 : if (!insn_queue || param_sched_autopref_queue_depth <= 0)
5792 : return 0;
5793 :
5794 0 : if (sched_verbose >= 2 && ready_index == 0)
5795 0 : autopref_multipass_dfa_lookahead_guard_started_dump_p = false;
5796 :
5797 0 : for (int write = 0; write < 2; ++write)
5798 : {
5799 0 : autopref_multipass_data_t data1
5800 0 : = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
5801 :
5802 0 : if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
5803 0 : autopref_multipass_init (insn1, write);
5804 0 : if (data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
5805 0 : continue;
5806 :
5807 0 : if (ready_index == 0
5808 0 : && data1->status == AUTOPREF_MULTIPASS_DATA_DONT_DELAY)
5809 : /* We allow only a single delay on privileged instructions.
5810 : Doing otherwise would cause infinite loop. */
5811 : {
5812 0 : if (sched_verbose >= 2)
5813 : {
5814 0 : if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
5815 : {
5816 0 : fprintf (sched_dump,
5817 : ";;\t\tnot trying in max_issue due to autoprefetch "
5818 : "model: ");
5819 0 : autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
5820 : }
5821 :
5822 0 : fprintf (sched_dump, " *%d*", INSN_UID (insn1));
5823 : }
5824 0 : continue;
5825 : }
5826 :
5827 0 : for (int i2 = 0; i2 < ready.n_ready; ++i2)
5828 : {
5829 0 : rtx_insn *insn2 = get_ready_element (i2);
5830 0 : if (insn1 == insn2)
5831 0 : continue;
5832 0 : r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2, write);
5833 0 : if (r)
5834 : {
5835 0 : if (ready_index == 0)
5836 : {
5837 0 : r = -1;
5838 0 : data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
5839 : }
5840 0 : goto finish;
5841 : }
5842 : }
5843 :
5844 0 : if (param_sched_autopref_queue_depth == 1)
5845 0 : continue;
5846 :
5847 : /* Everything from the current queue slot should have been moved to
5848 : the ready list. */
5849 0 : gcc_assert (insn_queue[NEXT_Q_AFTER (q_ptr, 0)] == NULL_RTX);
5850 :
5851 0 : int n_stalls = param_sched_autopref_queue_depth - 1;
5852 0 : if (n_stalls > max_insn_queue_index)
5853 : n_stalls = max_insn_queue_index;
5854 :
5855 0 : for (int stalls = 1; stalls <= n_stalls; ++stalls)
5856 : {
5857 0 : for (rtx_insn_list *link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)];
5858 0 : link != NULL_RTX;
5859 0 : link = link->next ())
5860 : {
5861 0 : rtx_insn *insn2 = link->insn ();
5862 0 : r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2,
5863 : write);
5864 0 : if (r)
5865 : {
5866 : /* Queue INSN1 until INSN2 can issue. */
5867 0 : r = -stalls;
5868 0 : if (ready_index == 0)
5869 0 : data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
5870 0 : goto finish;
5871 : }
5872 : }
5873 : }
5874 : }
5875 :
5876 0 : finish:
5877 0 : if (sched_verbose >= 2
5878 0 : && autopref_multipass_dfa_lookahead_guard_started_dump_p
5879 0 : && (ready_index == ready.n_ready - 1 || r < 0))
5880 : /* This does not /always/ trigger. We don't output EOL if the last
5881 : insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
5882 : called. We can live with this. */
5883 0 : fprintf (sched_dump, "\n");
5884 :
5885 : return r;
5886 : }
5887 :
5888 : /* Define type for target data used in multipass scheduling. */
5889 : #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
5890 : # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
5891 : #endif
5892 : typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t;
5893 :
5894 : /* The following structure describe an entry of the stack of choices. */
5895 : struct choice_entry
5896 : {
5897 : /* Ordinal number of the issued insn in the ready queue. */
5898 : int index;
5899 : /* The number of the rest insns whose issues we should try. */
5900 : int rest;
5901 : /* The number of issued essential insns. */
5902 : int n;
5903 : /* State after issuing the insn. */
5904 : state_t state;
5905 : /* Target-specific data. */
5906 : first_cycle_multipass_data_t target_data;
5907 : };
5908 :
5909 : /* The following array is used to implement a stack of choices used in
5910 : function max_issue. */
5911 : static struct choice_entry *choice_stack;
5912 :
5913 : /* This holds the value of the target dfa_lookahead hook. */
5914 : int dfa_lookahead;
5915 :
5916 : /* The following variable value is maximal number of tries of issuing
5917 : insns for the first cycle multipass insn scheduling. We define
5918 : this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE). We would not
5919 : need this constraint if all real insns (with non-negative codes)
5920 : had reservations because in this case the algorithm complexity is
5921 : O(DFA_LOOKAHEAD**ISSUE_RATE). Unfortunately, the dfa descriptions
5922 : might be incomplete and such insn might occur. For such
5923 : descriptions, the complexity of algorithm (without the constraint)
5924 : could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */
5925 : static int max_lookahead_tries;
5926 :
5927 : /* The following function returns maximal (or close to maximal) number
5928 : of insns which can be issued on the same cycle and one of which
5929 : insns is insns with the best rank (the first insn in READY). To
5930 : make this function tries different samples of ready insns. READY
5931 : is current queue `ready'. Global array READY_TRY reflects what
5932 : insns are already issued in this try. The function stops immediately,
5933 : if it reached the such a solution, that all instruction can be issued.
5934 : INDEX will contain index of the best insn in READY. The following
5935 : function is used only for first cycle multipass scheduling.
5936 :
5937 : PRIVILEGED_N >= 0
5938 :
5939 : This function expects recognized insns only. All USEs,
5940 : CLOBBERs, etc must be filtered elsewhere. */
5941 : int
5942 56823345 : max_issue (struct ready_list *ready, int privileged_n, state_t state,
5943 : bool first_cycle_insn_p, int *index)
5944 : {
5945 56823345 : int n, i, all, n_ready, best, delay, tries_num;
5946 56823345 : int more_issue;
5947 56823345 : struct choice_entry *top;
5948 56823345 : rtx_insn *insn;
5949 :
5950 56823345 : if (sched_fusion)
5951 : return 0;
5952 :
5953 56823345 : n_ready = ready->n_ready;
5954 56823345 : gcc_assert (dfa_lookahead >= 1 && privileged_n >= 0
5955 : && privileged_n <= n_ready);
5956 :
5957 : /* Init MAX_LOOKAHEAD_TRIES. */
5958 56823345 : if (max_lookahead_tries == 0)
5959 : {
5960 : int64_t max_tries = 100;
5961 4896461 : for (i = 0; i < issue_rate; i++)
5962 : {
5963 3907937 : max_tries *= dfa_lookahead;
5964 3907937 : if (max_tries > INT_MAX)
5965 : break;
5966 : }
5967 988524 : max_lookahead_tries = MIN (max_tries, INT_MAX);
5968 : }
5969 :
5970 : /* Init max_points. */
5971 56823345 : more_issue = issue_rate - cycle_issued_insns;
5972 56823345 : gcc_assert (more_issue >= 0);
5973 :
5974 : /* The number of the issued insns in the best solution. */
5975 56823345 : best = 0;
5976 :
5977 56823345 : top = choice_stack;
5978 :
5979 : /* Set initial state of the search. */
5980 56823345 : memcpy (top->state, state, dfa_state_size);
5981 56823345 : top->rest = dfa_lookahead;
5982 56823345 : top->n = 0;
5983 56823345 : if (targetm.sched.first_cycle_multipass_begin)
5984 56628990 : targetm.sched.first_cycle_multipass_begin (&top->target_data,
5985 : ready_try, n_ready,
5986 : first_cycle_insn_p);
5987 :
5988 : /* Count the number of the insns to search among. */
5989 153000333 : for (all = i = 0; i < n_ready; i++)
5990 96176988 : if (!ready_try [i])
5991 88018600 : all++;
5992 :
5993 56823345 : if (sched_verbose >= 2)
5994 : {
5995 0 : fprintf (sched_dump, ";;\t\tmax_issue among %d insns:", all);
5996 0 : debug_ready_list_1 (ready, ready_try);
5997 : }
5998 :
5999 : /* I is the index of the insn to try next. */
6000 56823345 : i = 0;
6001 56823345 : tries_num = 0;
6002 1102616635 : for (;;)
6003 : {
6004 579719990 : if (/* If we've reached a dead end or searched enough of what we have
6005 : been asked... */
6006 579719990 : top->rest == 0
6007 : /* or have nothing else to try... */
6008 574008997 : || i >= n_ready
6009 : /* or should not issue more. */
6010 466118656 : || top->n >= more_issue)
6011 : {
6012 : /* ??? (... || i == n_ready). */
6013 115804087 : gcc_assert (i <= n_ready);
6014 :
6015 : /* We should not issue more than issue_rate instructions. */
6016 115804087 : gcc_assert (top->n <= more_issue);
6017 :
6018 115804087 : if (top == choice_stack)
6019 : break;
6020 :
6021 106650676 : if (best < top - choice_stack)
6022 : {
6023 55161600 : if (privileged_n)
6024 : {
6025 : n = privileged_n;
6026 : /* Try to find issued privileged insn. */
6027 110316534 : while (n && !ready_try[--n])
6028 : ;
6029 : }
6030 :
6031 55158267 : if (/* If all insns are equally good... */
6032 : privileged_n == 0
6033 : /* Or a privileged insn will be issued. */
6034 55158267 : || ready_try[n])
6035 : /* Then we have a solution. */
6036 : {
6037 55159768 : best = top - choice_stack;
6038 : /* This is the index of the insn issued first in this
6039 : solution. */
6040 55159768 : *index = choice_stack [1].index;
6041 55159768 : if (top->n == more_issue || best == all)
6042 : break;
6043 : }
6044 : }
6045 :
6046 : /* Set ready-list index to point to the last insn
6047 : ('i++' below will advance it to the next insn). */
6048 58980763 : i = top->index;
6049 :
6050 : /* Backtrack. */
6051 58980763 : ready_try [i] = 0;
6052 :
6053 58980763 : if (targetm.sched.first_cycle_multipass_backtrack)
6054 58780722 : targetm.sched.first_cycle_multipass_backtrack (&top->target_data,
6055 : ready_try, n_ready);
6056 :
6057 58980763 : top--;
6058 58980763 : memcpy (state, top->state, dfa_state_size);
6059 : }
6060 463915903 : else if (!ready_try [i])
6061 : {
6062 148101644 : tries_num++;
6063 : /* max_lookahead_tries is capped at INT_MAX, use the >= comparison
6064 : to prevent tries_num from overflowing. */
6065 148101644 : if (tries_num >= max_lookahead_tries)
6066 : break;
6067 148101623 : insn = ready_element (ready, i);
6068 148101623 : delay = state_transition (state, insn);
6069 148101623 : if (delay < 0)
6070 : {
6071 115650520 : if (state_dead_lock_p (state)
6072 115650520 : || insn_finishes_cycle_p (insn))
6073 : /* We won't issue any more instructions in the next
6074 : choice_state. */
6075 6032181 : top->rest = 0;
6076 : else
6077 109618339 : top->rest--;
6078 :
6079 115650520 : n = top->n;
6080 115650520 : if (memcmp (top->state, state, dfa_state_size) != 0)
6081 114415125 : n++;
6082 :
6083 : /* Advance to the next choice_entry. */
6084 115650520 : top++;
6085 : /* Initialize it. */
6086 115650520 : top->rest = dfa_lookahead;
6087 115650520 : top->index = i;
6088 115650520 : top->n = n;
6089 115650520 : memcpy (top->state, state, dfa_state_size);
6090 115650520 : ready_try [i] = 1;
6091 :
6092 115650520 : if (targetm.sched.first_cycle_multipass_issue)
6093 115244990 : targetm.sched.first_cycle_multipass_issue (&top->target_data,
6094 : ready_try, n_ready,
6095 : insn,
6096 115244990 : &((top - 1)
6097 : ->target_data));
6098 :
6099 : i = -1;
6100 : }
6101 : }
6102 :
6103 : /* Increase ready-list index. */
6104 522896645 : i++;
6105 : }
6106 :
6107 56823345 : if (targetm.sched.first_cycle_multipass_end)
6108 111003068 : targetm.sched.first_cycle_multipass_end (best != 0
6109 54374078 : ? &choice_stack[1].target_data
6110 : : NULL);
6111 :
6112 : /* Restore the original state of the DFA. */
6113 56823345 : memcpy (state, choice_stack->state, dfa_state_size);
6114 :
6115 56823345 : return best;
6116 : }
6117 :
6118 : /* The following function chooses insn from READY and modifies
6119 : READY. The following function is used only for first
6120 : cycle multipass scheduling.
6121 : Return:
6122 : -1 if cycle should be advanced,
6123 : 0 if INSN_PTR is set to point to the desirable insn,
6124 : 1 if choose_ready () should be restarted without advancing the cycle. */
6125 : static int
6126 61934938 : choose_ready (struct ready_list *ready, bool first_cycle_insn_p,
6127 : rtx_insn **insn_ptr)
6128 : {
6129 61934938 : if (dbg_cnt (sched_insn) == false)
6130 : {
6131 0 : if (nonscheduled_insns_begin == NULL_RTX)
6132 0 : nonscheduled_insns_begin = current_sched_info->prev_head;
6133 :
6134 0 : rtx_insn *insn = first_nonscheduled_insn ();
6135 :
6136 0 : if (QUEUE_INDEX (insn) == QUEUE_READY)
6137 : /* INSN is in the ready_list. */
6138 : {
6139 0 : ready_remove_insn (insn);
6140 0 : *insn_ptr = insn;
6141 0 : return 0;
6142 : }
6143 :
6144 : /* INSN is in the queue. Advance cycle to move it to the ready list. */
6145 0 : gcc_assert (QUEUE_INDEX (insn) >= 0);
6146 : return -1;
6147 : }
6148 :
6149 61934938 : if (SCHED_GROUP_P (ready_element (ready, 0))
6150 61934938 : || DEBUG_INSN_P (ready_element (ready, 0)))
6151 : {
6152 4250527 : *insn_ptr = ready_remove_first (ready);
6153 4250527 : return 0;
6154 : }
6155 57684411 : else if (dfa_lookahead <= 0)
6156 : {
6157 9629 : if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
6158 16 : *insn_ptr = ready_remove_first_dispatch (ready);
6159 : else
6160 9613 : *insn_ptr = ready_remove_first (ready);
6161 :
6162 9629 : return 0;
6163 : }
6164 : else
6165 : {
6166 : /* Try to choose the best insn. */
6167 57674782 : int index = 0, i;
6168 57674782 : rtx_insn *insn;
6169 :
6170 57674782 : insn = ready_element (ready, 0);
6171 57674782 : if (INSN_CODE (insn) < 0)
6172 : {
6173 854798 : *insn_ptr = ready_remove_first (ready);
6174 854798 : return 0;
6175 : }
6176 :
6177 : /* Filter the search space. */
6178 152991541 : for (i = 0; i < ready->n_ready; i++)
6179 : {
6180 96171557 : ready_try[i] = 0;
6181 :
6182 96171557 : insn = ready_element (ready, i);
6183 :
6184 : /* If this insn is recognizable we should have already
6185 : recognized it earlier.
6186 : ??? Not very clear where this is supposed to be done.
6187 : See dep_cost_1. */
6188 96171557 : gcc_checking_assert (INSN_CODE (insn) >= 0
6189 : || recog_memoized (insn) < 0);
6190 96171557 : if (INSN_CODE (insn) < 0)
6191 : {
6192 : /* Non-recognized insns at position 0 are handled above. */
6193 248193 : gcc_assert (i > 0);
6194 248193 : ready_try[i] = 1;
6195 248193 : continue;
6196 : }
6197 :
6198 95923364 : if (targetm.sched.first_cycle_multipass_dfa_lookahead_guard)
6199 : {
6200 0 : ready_try[i]
6201 0 : = (targetm.sched.first_cycle_multipass_dfa_lookahead_guard
6202 0 : (insn, i));
6203 :
6204 0 : if (ready_try[i] < 0)
6205 : /* Queue instruction for several cycles.
6206 : We need to restart choose_ready as we have changed
6207 : the ready list. */
6208 : {
6209 0 : change_queue_index (insn, -ready_try[i]);
6210 0 : return 1;
6211 : }
6212 :
6213 : /* Make sure that we didn't end up with 0'th insn filtered out.
6214 : Don't be tempted to make life easier for backends and just
6215 : requeue 0'th insn if (ready_try[0] == 0) and restart
6216 : choose_ready. Backends should be very considerate about
6217 : requeueing instructions -- especially the highest priority
6218 : one at position 0. */
6219 0 : gcc_assert (ready_try[i] == 0 || i > 0);
6220 0 : if (ready_try[i])
6221 0 : continue;
6222 : }
6223 :
6224 95923364 : gcc_assert (ready_try[i] == 0);
6225 : /* INSN made it through the scrutiny of filters! */
6226 : }
6227 :
6228 56819984 : if (max_issue (ready, 1, curr_state, first_cycle_insn_p, &index) == 0)
6229 : {
6230 2254902 : *insn_ptr = ready_remove_first (ready);
6231 2254902 : if (sched_verbose >= 4)
6232 0 : fprintf (sched_dump, ";;\t\tChosen insn (but can't issue) : %s \n",
6233 0 : (*current_sched_info->print_insn) (*insn_ptr, 0));
6234 : return 0;
6235 : }
6236 : else
6237 : {
6238 54565082 : if (sched_verbose >= 4)
6239 0 : fprintf (sched_dump, ";;\t\tChosen insn : %s\n",
6240 0 : (*current_sched_info->print_insn)
6241 0 : (ready_element (ready, index), 0));
6242 :
6243 54565082 : *insn_ptr = ready_remove (ready, index);
6244 54565082 : return 0;
6245 : }
6246 : }
6247 : }
6248 :
6249 : /* This function is called when we have successfully scheduled a
6250 : block. It uses the schedule stored in the scheduled_insns vector
6251 : to rearrange the RTL. PREV_HEAD is used as the anchor to which we
6252 : append the scheduled insns; TAIL is the insn after the scheduled
6253 : block. TARGET_BB is the argument passed to schedule_block. */
6254 :
6255 : static void
6256 10610721 : commit_schedule (rtx_insn *prev_head, rtx_insn *tail, basic_block *target_bb)
6257 : {
6258 10610721 : unsigned int i;
6259 10610721 : rtx_insn *insn;
6260 :
6261 10610721 : last_scheduled_insn = prev_head;
6262 10610721 : for (i = 0;
6263 126719200 : scheduled_insns.iterate (i, &insn);
6264 : i++)
6265 : {
6266 116108479 : if (control_flow_insn_p (last_scheduled_insn)
6267 116108479 : || current_sched_info->advance_target_bb (*target_bb, insn))
6268 : {
6269 49 : *target_bb = current_sched_info->advance_target_bb (*target_bb, 0);
6270 :
6271 49 : if (sched_verbose)
6272 : {
6273 2 : rtx_insn *x;
6274 :
6275 2 : x = next_real_insn (last_scheduled_insn);
6276 2 : gcc_assert (x);
6277 2 : dump_new_block_header (1, *target_bb, x, tail);
6278 : }
6279 :
6280 49 : last_scheduled_insn = bb_note (*target_bb);
6281 : }
6282 :
6283 116108479 : if (current_sched_info->begin_move_insn)
6284 1259 : (*current_sched_info->begin_move_insn) (insn, last_scheduled_insn);
6285 116108479 : move_insn (insn, last_scheduled_insn,
6286 116108479 : current_sched_info->next_tail);
6287 116108479 : if (!DEBUG_INSN_P (insn))
6288 61932702 : reemit_notes (insn);
6289 116108479 : last_scheduled_insn = insn;
6290 : }
6291 :
6292 10610721 : scheduled_insns.truncate (0);
6293 10610721 : }
6294 :
6295 : /* Examine all insns on the ready list and queue those which can't be
6296 : issued in this cycle. TEMP_STATE is temporary scheduler state we
6297 : can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns
6298 : have been issued for the current cycle, which means it is valid to
6299 : issue an asm statement.
6300 :
6301 : If SHADOWS_ONLY_P is true, we eliminate all real insns and only
6302 : leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true,
6303 : we only leave insns which have an INSN_EXACT_TICK. */
6304 :
6305 : static void
6306 71226317 : prune_ready_list (state_t temp_state, bool first_cycle_insn_p,
6307 : bool shadows_only_p, bool modulo_epilogue_p)
6308 : {
6309 71226317 : int i, pass;
6310 71226317 : bool sched_group_found = false;
6311 71226317 : int min_cost_group = 0;
6312 :
6313 71226317 : if (sched_fusion)
6314 : return;
6315 :
6316 195259203 : for (i = 0; i < ready.n_ready; i++)
6317 : {
6318 128304157 : rtx_insn *insn = ready_element (&ready, i);
6319 128304157 : if (SCHED_GROUP_P (insn))
6320 : {
6321 : sched_group_found = true;
6322 : break;
6323 : }
6324 : }
6325 :
6326 : /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle
6327 : such an insn first and note its cost. If at least one SCHED_GROUP_P insn
6328 : gets queued, then all other insns get queued for one cycle later. */
6329 158897247 : for (pass = sched_group_found ? 0 : 1; pass < 2; )
6330 : {
6331 87670930 : int n = ready.n_ready;
6332 209311865 : for (i = 0; i < n; i++)
6333 : {
6334 133814277 : rtx_insn *insn = ready_element (&ready, i);
6335 133814277 : int cost = 0;
6336 133814277 : const char *reason = "resource conflict";
6337 :
6338 133814277 : if (DEBUG_INSN_P (insn))
6339 6815587 : continue;
6340 :
6341 8526604 : if (sched_group_found && !SCHED_GROUP_P (insn)
6342 126998883 : && ((pass == 0) || (min_cost_group >= 1)))
6343 : {
6344 106 : if (pass == 0)
6345 95 : continue;
6346 : cost = min_cost_group;
6347 : reason = "not in sched group";
6348 : }
6349 126998584 : else if (modulo_epilogue_p
6350 126998584 : && INSN_EXACT_TICK (insn) == INVALID_TICK)
6351 : {
6352 : cost = max_insn_queue_index;
6353 : reason = "not an epilogue insn";
6354 : }
6355 126998584 : else if (shadows_only_p && !SHADOW_P (insn))
6356 : {
6357 : cost = 1;
6358 : reason = "not a shadow";
6359 : }
6360 126998584 : else if (recog_memoized (insn) < 0)
6361 : {
6362 1172440 : if (!first_cycle_insn_p
6363 1172440 : && (GET_CODE (PATTERN (insn)) == ASM_INPUT
6364 948207 : || asm_noperands (PATTERN (insn)) >= 0))
6365 : cost = 1;
6366 : reason = "asm";
6367 : }
6368 125826144 : else if (sched_pressure != SCHED_PRESSURE_NONE)
6369 : {
6370 22925 : if (sched_pressure == SCHED_PRESSURE_MODEL
6371 22925 : && INSN_TICK (insn) <= clock_var)
6372 : {
6373 0 : memcpy (temp_state, curr_state, dfa_state_size);
6374 0 : if (state_transition (temp_state, insn) >= 0)
6375 0 : INSN_TICK (insn) = clock_var + 1;
6376 : }
6377 : cost = 0;
6378 : }
6379 : else
6380 : {
6381 125803219 : int delay_cost = 0;
6382 :
6383 125803219 : if (delay_htab)
6384 : {
6385 0 : struct delay_pair *delay_entry;
6386 0 : delay_entry
6387 0 : = delay_htab->find_with_hash (insn,
6388 : htab_hash_pointer (insn));
6389 0 : while (delay_entry && delay_cost == 0)
6390 : {
6391 0 : delay_cost = estimate_shadow_tick (delay_entry);
6392 0 : if (delay_cost > max_insn_queue_index)
6393 : delay_cost = max_insn_queue_index;
6394 0 : delay_entry = delay_entry->next_same_i1;
6395 : }
6396 : }
6397 :
6398 125803219 : memcpy (temp_state, curr_state, dfa_state_size);
6399 125803219 : cost = state_transition (temp_state, insn);
6400 125803219 : if (cost < 0)
6401 : cost = 0;
6402 19923447 : else if (cost == 0)
6403 0 : cost = 1;
6404 125803219 : if (cost < delay_cost)
6405 : {
6406 0 : cost = delay_cost;
6407 0 : reason = "shadow tick";
6408 : }
6409 : }
6410 125803230 : if (cost >= 1)
6411 : {
6412 19970388 : if (SCHED_GROUP_P (insn) && cost > min_cost_group)
6413 19970388 : min_cost_group = cost;
6414 19970388 : ready_remove (&ready, i);
6415 : /* Normally we'd want to queue INSN for COST cycles. However,
6416 : if SCHED_GROUP_P is set, then we must ensure that nothing
6417 : else comes between INSN and its predecessor. If there is
6418 : some other insn ready to fire on the next cycle, then that
6419 : invariant would be broken.
6420 :
6421 : So when SCHED_GROUP_P is set, just queue this insn for a
6422 : single cycle. */
6423 19986520 : queue_insn (insn, SCHED_GROUP_P (insn) ? 1 : cost, reason);
6424 19970388 : if (i + 1 < n)
6425 : break;
6426 : }
6427 : }
6428 87670930 : if (i == n)
6429 75497588 : pass++;
6430 : }
6431 : }
6432 :
6433 : /* Called when we detect that the schedule is impossible. We examine the
6434 : backtrack queue to find the earliest insn that caused this condition. */
6435 :
6436 : static struct haifa_saved_data *
6437 0 : verify_shadows (void)
6438 : {
6439 0 : struct haifa_saved_data *save, *earliest_fail = NULL;
6440 0 : for (save = backtrack_queue; save; save = save->next)
6441 : {
6442 0 : int t;
6443 0 : struct delay_pair *pair = save->delay_pair;
6444 0 : rtx_insn *i1 = pair->i1;
6445 :
6446 0 : for (; pair; pair = pair->next_same_i1)
6447 : {
6448 0 : rtx_insn *i2 = pair->i2;
6449 :
6450 0 : if (QUEUE_INDEX (i2) == QUEUE_SCHEDULED)
6451 0 : continue;
6452 :
6453 0 : t = INSN_TICK (i1) + pair_delay (pair);
6454 0 : if (t < clock_var)
6455 : {
6456 0 : if (sched_verbose >= 2)
6457 0 : fprintf (sched_dump,
6458 : ";;\t\tfailed delay requirements for %d/%d (%d->%d)"
6459 : ", not ready\n",
6460 : INSN_UID (pair->i1), INSN_UID (pair->i2),
6461 0 : INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2));
6462 : earliest_fail = save;
6463 : break;
6464 : }
6465 0 : if (QUEUE_INDEX (i2) >= 0)
6466 : {
6467 0 : int queued_for = INSN_TICK (i2);
6468 :
6469 0 : if (t < queued_for)
6470 : {
6471 0 : if (sched_verbose >= 2)
6472 0 : fprintf (sched_dump,
6473 : ";;\t\tfailed delay requirements for %d/%d"
6474 : " (%d->%d), queued too late\n",
6475 : INSN_UID (pair->i1), INSN_UID (pair->i2),
6476 0 : INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2));
6477 : earliest_fail = save;
6478 : break;
6479 : }
6480 : }
6481 : }
6482 : }
6483 :
6484 0 : return earliest_fail;
6485 : }
6486 :
6487 : /* Print instructions together with useful scheduling information between
6488 : HEAD and TAIL (inclusive). */
6489 : static void
6490 0 : dump_insn_stream (rtx_insn *head, rtx_insn *tail)
6491 : {
6492 0 : fprintf (sched_dump, ";;\t| insn | prio |\n");
6493 :
6494 0 : rtx_insn *next_tail = NEXT_INSN (tail);
6495 0 : for (rtx_insn *insn = head; insn != next_tail; insn = NEXT_INSN (insn))
6496 : {
6497 0 : int priority = NOTE_P (insn) ? 0 : INSN_PRIORITY (insn);
6498 0 : const char *pattern = (NOTE_P (insn)
6499 0 : ? "note"
6500 0 : : str_pattern_slim (PATTERN (insn)));
6501 :
6502 0 : fprintf (sched_dump, ";;\t| %4d | %4d | %-30s ",
6503 0 : INSN_UID (insn), priority, pattern);
6504 :
6505 0 : if (sched_verbose >= 4)
6506 : {
6507 0 : if (NOTE_P (insn) || LABEL_P (insn) || recog_memoized (insn) < 0)
6508 0 : fprintf (sched_dump, "nothing");
6509 : else
6510 0 : print_reservation (sched_dump, insn);
6511 : }
6512 0 : fprintf (sched_dump, "\n");
6513 : }
6514 0 : }
6515 :
6516 : /* Use forward list scheduling to rearrange insns of block pointed to by
6517 : TARGET_BB, possibly bringing insns from subsequent blocks in the same
6518 : region. */
6519 :
6520 : bool
6521 10610721 : schedule_block (basic_block *target_bb, state_t init_state)
6522 : {
6523 10610721 : int i;
6524 10610721 : bool success = modulo_ii == 0;
6525 10610721 : struct sched_block_state ls;
6526 10610721 : state_t temp_state = NULL; /* It is used for multipass scheduling. */
6527 10610721 : int sort_p, advance, start_clock_var;
6528 :
6529 : /* Head/tail info for this block. */
6530 10610721 : rtx_insn *prev_head = current_sched_info->prev_head;
6531 10610721 : rtx_insn *next_tail = current_sched_info->next_tail;
6532 10610721 : rtx_insn *head = NEXT_INSN (prev_head);
6533 10610721 : rtx_insn *tail = PREV_INSN (next_tail);
6534 :
6535 10610721 : if ((current_sched_info->flags & DONT_BREAK_DEPENDENCIES) == 0
6536 10610721 : && sched_pressure != SCHED_PRESSURE_MODEL && !sched_fusion)
6537 10610721 : find_modifiable_mems (head, tail);
6538 :
6539 : /* We used to have code to avoid getting parameters moved from hard
6540 : argument registers into pseudos.
6541 :
6542 : However, it was removed when it proved to be of marginal benefit
6543 : and caused problems because schedule_block and compute_forward_dependences
6544 : had different notions of what the "head" insn was. */
6545 :
6546 10610721 : gcc_assert (head != tail || INSN_P (head));
6547 :
6548 10610721 : haifa_recovery_bb_recently_added_p = false;
6549 :
6550 10610721 : backtrack_queue = NULL;
6551 :
6552 : /* Debug info. */
6553 10610721 : if (sched_verbose)
6554 : {
6555 190 : dump_new_block_header (0, *target_bb, head, tail);
6556 :
6557 190 : if (sched_verbose >= 2)
6558 : {
6559 0 : dump_insn_stream (head, tail);
6560 0 : memset (&rank_for_schedule_stats, 0,
6561 : sizeof (rank_for_schedule_stats));
6562 : }
6563 : }
6564 :
6565 10610721 : if (init_state == NULL)
6566 173 : state_reset (curr_state);
6567 : else
6568 10610548 : memcpy (curr_state, init_state, dfa_state_size);
6569 :
6570 : /* Clear the ready list. */
6571 10610721 : ready.first = ready.veclen - 1;
6572 10610721 : ready.n_ready = 0;
6573 10610721 : ready.n_debug = 0;
6574 :
6575 : /* It is used for first cycle multipass scheduling. */
6576 10610721 : temp_state = alloca (dfa_state_size);
6577 :
6578 10610721 : if (targetm.sched.init)
6579 0 : targetm.sched.init (sched_dump, sched_verbose, ready.veclen);
6580 :
6581 : /* We start inserting insns after PREV_HEAD. */
6582 10610721 : last_scheduled_insn = prev_head;
6583 10610721 : last_nondebug_scheduled_insn = NULL;
6584 10610721 : nonscheduled_insns_begin = NULL;
6585 :
6586 10610721 : gcc_assert ((NOTE_P (last_scheduled_insn)
6587 : || DEBUG_INSN_P (last_scheduled_insn))
6588 : && BLOCK_FOR_INSN (last_scheduled_insn) == *target_bb);
6589 :
6590 : /* Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the
6591 : queue. */
6592 10610721 : q_ptr = 0;
6593 10610721 : q_size = 0;
6594 :
6595 10610721 : insn_queue = XALLOCAVEC (rtx_insn_list *, max_insn_queue_index + 1);
6596 10610721 : memset (insn_queue, 0, (max_insn_queue_index + 1) * sizeof (rtx));
6597 :
6598 : /* Start just before the beginning of time. */
6599 10610721 : clock_var = -1;
6600 :
6601 : /* We need queue and ready lists and clock_var be initialized
6602 : in try_ready () (which is called through init_ready_list ()). */
6603 10610721 : (*current_sched_info->init_ready_list) ();
6604 :
6605 10610721 : if (sched_pressure)
6606 706 : sched_pressure_start_bb (*target_bb);
6607 :
6608 : /* The algorithm is O(n^2) in the number of ready insns at any given
6609 : time in the worst case. Before reload we are more likely to have
6610 : big lists so truncate them to a reasonable size. */
6611 10610721 : if (!reload_completed
6612 1201 : && ready.n_ready - ready.n_debug > param_max_sched_ready_insns)
6613 : {
6614 4 : ready_sort_debug (&ready);
6615 4 : ready_sort_real (&ready);
6616 :
6617 : /* Find first free-standing insn past param_max_sched_ready_insns.
6618 : If there are debug insns, we know they're first. */
6619 4 : for (i = param_max_sched_ready_insns + ready.n_debug; i < ready.n_ready;
6620 : i++)
6621 4 : if (!SCHED_GROUP_P (ready_element (&ready, i)))
6622 : break;
6623 :
6624 4 : if (sched_verbose >= 2)
6625 : {
6626 0 : fprintf (sched_dump,
6627 : ";;\t\tReady list on entry: %d insns: ", ready.n_ready);
6628 0 : debug_ready_list (&ready);
6629 0 : fprintf (sched_dump,
6630 : ";;\t\t before reload => truncated to %d insns\n", i);
6631 : }
6632 :
6633 : /* Delay all insns past it for 1 cycle. If debug counter is
6634 : activated make an exception for the insn right after
6635 : nonscheduled_insns_begin. */
6636 4 : {
6637 4 : rtx_insn *skip_insn;
6638 :
6639 4 : if (dbg_cnt (sched_insn) == false)
6640 0 : skip_insn = first_nonscheduled_insn ();
6641 : else
6642 4 : skip_insn = NULL;
6643 :
6644 29 : while (i < ready.n_ready)
6645 : {
6646 25 : rtx_insn *insn;
6647 :
6648 25 : insn = ready_remove (&ready, i);
6649 :
6650 25 : if (insn != skip_insn)
6651 25 : queue_insn (insn, 1, "list truncated");
6652 : }
6653 4 : if (skip_insn)
6654 0 : ready_add (&ready, skip_insn, true);
6655 : }
6656 : }
6657 :
6658 : /* Now we can restore basic block notes and maintain precise cfg. */
6659 10610721 : restore_bb_notes (*target_bb);
6660 :
6661 10610721 : last_clock_var = -1;
6662 :
6663 10610721 : advance = 0;
6664 :
6665 10610721 : gcc_assert (scheduled_insns.length () == 0);
6666 10610721 : sort_p = true;
6667 10610721 : must_backtrack = false;
6668 10610721 : modulo_insns_scheduled = 0;
6669 :
6670 10610721 : ls.modulo_epilogue = false;
6671 10610721 : ls.first_cycle_insn_p = true;
6672 :
6673 : /* Loop until all the insns in BB are scheduled. */
6674 46608351 : while ((*current_sched_info->schedule_more_p) ())
6675 : {
6676 35997630 : perform_replacements_new_cycle ();
6677 35997630 : do
6678 : {
6679 35997630 : start_clock_var = clock_var;
6680 :
6681 35997630 : clock_var++;
6682 :
6683 35997630 : advance_one_cycle ();
6684 :
6685 : /* Add to the ready list all pending insns that can be issued now.
6686 : If there are no ready insns, increment clock until one
6687 : is ready and add all pending insns at that point to the ready
6688 : list. */
6689 35997630 : queue_to_ready (&ready);
6690 :
6691 35997630 : gcc_assert (ready.n_ready);
6692 :
6693 35997630 : if (sched_verbose >= 2)
6694 : {
6695 0 : fprintf (sched_dump, ";;\t\tReady list after queue_to_ready:");
6696 0 : debug_ready_list (&ready);
6697 : }
6698 35997630 : advance -= clock_var - start_clock_var;
6699 : }
6700 35997630 : while (advance > 0);
6701 :
6702 35997630 : if (ls.modulo_epilogue)
6703 : {
6704 0 : int stage = clock_var / modulo_ii;
6705 0 : if (stage > modulo_last_stage * 2 + 2)
6706 : {
6707 0 : if (sched_verbose >= 2)
6708 0 : fprintf (sched_dump,
6709 : ";;\t\tmodulo scheduled succeeded at II %d\n",
6710 : modulo_ii);
6711 0 : success = true;
6712 0 : goto end_schedule;
6713 : }
6714 : }
6715 35997630 : else if (modulo_ii > 0)
6716 : {
6717 0 : int stage = clock_var / modulo_ii;
6718 0 : if (stage > modulo_max_stages)
6719 : {
6720 0 : if (sched_verbose >= 2)
6721 0 : fprintf (sched_dump,
6722 : ";;\t\tfailing schedule due to excessive stages\n");
6723 0 : goto end_schedule;
6724 : }
6725 0 : if (modulo_n_insns == modulo_insns_scheduled
6726 0 : && stage > modulo_last_stage)
6727 : {
6728 0 : if (sched_verbose >= 2)
6729 0 : fprintf (sched_dump,
6730 : ";;\t\tfound kernel after %d stages, II %d\n",
6731 : stage, modulo_ii);
6732 0 : ls.modulo_epilogue = true;
6733 : }
6734 : }
6735 :
6736 35997630 : prune_ready_list (temp_state, true, false, ls.modulo_epilogue);
6737 35997630 : if (ready.n_ready == 0)
6738 3735 : continue;
6739 35993895 : if (must_backtrack)
6740 0 : goto do_backtrack;
6741 :
6742 35993895 : ls.shadows_only_p = false;
6743 35993895 : cycle_issued_insns = 0;
6744 35993895 : ls.can_issue_more = issue_rate;
6745 159755651 : for (;;)
6746 : {
6747 97874773 : rtx_insn *insn;
6748 97874773 : int cost;
6749 97874773 : bool asm_p;
6750 :
6751 97874773 : if (sort_p && ready.n_ready > 0)
6752 : {
6753 : /* Sort the ready list based on priority. This must be
6754 : done every iteration through the loop, as schedule_insn
6755 : may have readied additional insns that will not be
6756 : sorted correctly. */
6757 63938715 : ready_sort (&ready);
6758 :
6759 63938715 : if (sched_verbose >= 2)
6760 : {
6761 0 : fprintf (sched_dump,
6762 : ";;\t\tReady list after ready_sort: ");
6763 0 : debug_ready_list (&ready);
6764 : }
6765 : }
6766 :
6767 : /* We don't want md sched reorder to even see debug isns, so put
6768 : them out right away. */
6769 63938715 : if (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0))
6770 104689215 : && (*current_sched_info->schedule_more_p) ())
6771 : {
6772 60990219 : while (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0)))
6773 : {
6774 54175777 : rtx_insn *insn = ready_remove_first (&ready);
6775 54175777 : gcc_assert (DEBUG_INSN_P (insn));
6776 54175777 : (*current_sched_info->begin_schedule_ready) (insn);
6777 54175777 : scheduled_insns.safe_push (insn);
6778 54175777 : last_scheduled_insn = insn;
6779 54175777 : advance = schedule_insn (insn);
6780 54175777 : gcc_assert (advance == 0);
6781 54175777 : if (ready.n_ready > 0)
6782 52256117 : ready_sort (&ready);
6783 : }
6784 : }
6785 :
6786 97874773 : if (ls.first_cycle_insn_p && !ready.n_ready)
6787 : break;
6788 :
6789 97873545 : resume_after_backtrack:
6790 : /* Allow the target to reorder the list, typically for
6791 : better instruction bundling. */
6792 97873545 : if (sort_p
6793 97873545 : && (ready.n_ready == 0
6794 62019055 : || !SCHED_GROUP_P (ready_element (&ready, 0))))
6795 : {
6796 93618406 : if (ls.first_cycle_insn_p && targetm.sched.reorder)
6797 35971923 : ls.can_issue_more
6798 35971923 : = targetm.sched.reorder (sched_dump, sched_verbose,
6799 : ready_lastpos (&ready),
6800 : &ready.n_ready, clock_var);
6801 57646483 : else if (!ls.first_cycle_insn_p && targetm.sched.reorder2)
6802 0 : ls.can_issue_more
6803 0 : = targetm.sched.reorder2 (sched_dump, sched_verbose,
6804 0 : ready.n_ready
6805 0 : ? ready_lastpos (&ready) : NULL,
6806 : &ready.n_ready, clock_var);
6807 : }
6808 :
6809 97873545 : restart_choose_ready:
6810 97873545 : if (sched_verbose >= 2)
6811 : {
6812 0 : fprintf (sched_dump, ";;\tReady list (t = %3d): ",
6813 : clock_var);
6814 0 : debug_ready_list (&ready);
6815 0 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
6816 0 : print_curr_reg_pressure ();
6817 : }
6818 :
6819 97873545 : if (ready.n_ready == 0
6820 35854490 : && ls.can_issue_more
6821 34045761 : && reload_completed)
6822 : {
6823 : /* Allow scheduling insns directly from the queue in case
6824 : there's nothing better to do (ready list is empty) but
6825 : there are still vacant dispatch slots in the current cycle. */
6826 34042847 : if (sched_verbose >= 6)
6827 0 : fprintf (sched_dump,";;\t\tSecond chance\n");
6828 34042847 : memcpy (temp_state, curr_state, dfa_state_size);
6829 34042847 : if (early_queue_to_ready (temp_state, &ready))
6830 4 : ready_sort (&ready);
6831 : }
6832 :
6833 97873545 : if (ready.n_ready == 0
6834 62019059 : || !ls.can_issue_more
6835 61934938 : || state_dead_lock_p (curr_state)
6836 159808483 : || !(*current_sched_info->schedule_more_p) ())
6837 : break;
6838 :
6839 : /* Select and remove the insn from the ready list. */
6840 61934938 : if (sort_p)
6841 : {
6842 61934938 : int res;
6843 :
6844 61934938 : insn = NULL;
6845 61934938 : res = choose_ready (&ready, ls.first_cycle_insn_p, &insn);
6846 :
6847 61934938 : if (res < 0)
6848 : /* Finish cycle. */
6849 : break;
6850 61934938 : if (res > 0)
6851 0 : goto restart_choose_ready;
6852 :
6853 61934938 : gcc_assert (insn != NULL_RTX);
6854 : }
6855 : else
6856 0 : insn = ready_remove_first (&ready);
6857 :
6858 61934938 : if (sched_pressure != SCHED_PRESSURE_NONE
6859 61934938 : && INSN_TICK (insn) > clock_var)
6860 : {
6861 2236 : ready_add (&ready, insn, true);
6862 2236 : advance = 1;
6863 2236 : break;
6864 : }
6865 :
6866 61932702 : if (targetm.sched.dfa_new_cycle
6867 61932702 : && targetm.sched.dfa_new_cycle (sched_dump, sched_verbose,
6868 : insn, last_clock_var,
6869 : clock_var, &sort_p))
6870 : /* SORT_P is used by the target to override sorting
6871 : of the ready list. This is needed when the target
6872 : has modified its internal structures expecting that
6873 : the insn will be issued next. As we need the insn
6874 : to have the highest priority (so it will be returned by
6875 : the ready_remove_first call above), we invoke
6876 : ready_add (&ready, insn, true).
6877 : But, still, there is one issue: INSN can be later
6878 : discarded by scheduler's front end through
6879 : current_sched_info->can_schedule_ready_p, hence, won't
6880 : be issued next. */
6881 : {
6882 0 : ready_add (&ready, insn, true);
6883 0 : break;
6884 : }
6885 :
6886 61932702 : sort_p = true;
6887 :
6888 61932702 : if (current_sched_info->can_schedule_ready_p
6889 61932702 : && ! (*current_sched_info->can_schedule_ready_p) (insn))
6890 : /* We normally get here only if we don't want to move
6891 : insn from the split block. */
6892 : {
6893 0 : TODO_SPEC (insn) = DEP_POSTPONED;
6894 0 : goto restart_choose_ready;
6895 : }
6896 :
6897 61932702 : if (delay_htab)
6898 : {
6899 : /* If this insn is the first part of a delay-slot pair, record a
6900 : backtrack point. */
6901 0 : struct delay_pair *delay_entry;
6902 0 : delay_entry
6903 0 : = delay_htab->find_with_hash (insn, htab_hash_pointer (insn));
6904 0 : if (delay_entry)
6905 : {
6906 0 : save_backtrack_point (delay_entry, ls);
6907 0 : if (sched_verbose >= 2)
6908 0 : fprintf (sched_dump, ";;\t\tsaving backtrack point\n");
6909 : }
6910 : }
6911 :
6912 : /* DECISION is made. */
6913 :
6914 61932702 : if (modulo_ii > 0 && INSN_UID (insn) < modulo_iter0_max_uid)
6915 : {
6916 0 : modulo_insns_scheduled++;
6917 0 : modulo_last_stage = clock_var / modulo_ii;
6918 : }
6919 61932702 : if (TODO_SPEC (insn) & SPECULATIVE)
6920 0 : generate_recovery_code (insn);
6921 :
6922 61932702 : if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
6923 14 : targetm.sched.dispatch_do (insn, ADD_TO_DISPATCH_WINDOW);
6924 :
6925 : /* Update counters, etc in the scheduler's front end. */
6926 61932702 : (*current_sched_info->begin_schedule_ready) (insn);
6927 61932702 : scheduled_insns.safe_push (insn);
6928 61932702 : gcc_assert (NONDEBUG_INSN_P (insn));
6929 61932702 : last_nondebug_scheduled_insn = last_scheduled_insn = insn;
6930 :
6931 61932702 : if (recog_memoized (insn) >= 0)
6932 : {
6933 61077716 : memcpy (temp_state, curr_state, dfa_state_size);
6934 61077716 : cost = state_transition (curr_state, insn);
6935 61077716 : if (sched_pressure != SCHED_PRESSURE_WEIGHTED && !sched_fusion)
6936 61074007 : gcc_assert (cost < 0);
6937 61077716 : if (memcmp (temp_state, curr_state, dfa_state_size) != 0)
6938 60661432 : cycle_issued_insns++;
6939 : asm_p = false;
6940 : }
6941 : else
6942 854986 : asm_p = (GET_CODE (PATTERN (insn)) == ASM_INPUT
6943 854986 : || asm_noperands (PATTERN (insn)) >= 0);
6944 :
6945 61932702 : if (targetm.sched.variable_issue)
6946 0 : ls.can_issue_more =
6947 0 : targetm.sched.variable_issue (sched_dump, sched_verbose,
6948 : insn, ls.can_issue_more);
6949 : /* A naked CLOBBER or USE generates no instruction, so do
6950 : not count them against the issue rate. */
6951 61932702 : else if (GET_CODE (PATTERN (insn)) != USE
6952 61932702 : && GET_CODE (PATTERN (insn)) != CLOBBER)
6953 61129540 : ls.can_issue_more--;
6954 61932702 : advance = schedule_insn (insn);
6955 :
6956 61932702 : if (SHADOW_P (insn))
6957 0 : ls.shadows_only_p = true;
6958 :
6959 : /* After issuing an asm insn we should start a new cycle. */
6960 61932702 : if (advance == 0 && asm_p)
6961 51824 : advance = 1;
6962 :
6963 61932702 : if (must_backtrack)
6964 : break;
6965 :
6966 61932702 : if (advance != 0)
6967 : break;
6968 :
6969 61880878 : ls.first_cycle_insn_p = false;
6970 61880878 : if (ready.n_ready > 0)
6971 35228687 : prune_ready_list (temp_state, false, ls.shadows_only_p,
6972 : ls.modulo_epilogue);
6973 61880878 : }
6974 :
6975 35993895 : do_backtrack:
6976 35993895 : if (!must_backtrack)
6977 36239429 : for (i = 0; i < ready.n_ready; i++)
6978 : {
6979 245534 : rtx_insn *insn = ready_element (&ready, i);
6980 245534 : if (INSN_EXACT_TICK (insn) == clock_var)
6981 : {
6982 0 : must_backtrack = true;
6983 0 : clock_var++;
6984 0 : break;
6985 : }
6986 : }
6987 35993895 : if (must_backtrack && modulo_ii > 0)
6988 : {
6989 0 : if (modulo_backtracks_left == 0)
6990 0 : goto end_schedule;
6991 0 : modulo_backtracks_left--;
6992 : }
6993 35993895 : while (must_backtrack)
6994 : {
6995 0 : struct haifa_saved_data *failed;
6996 0 : rtx_insn *failed_insn;
6997 :
6998 0 : must_backtrack = false;
6999 0 : failed = verify_shadows ();
7000 0 : gcc_assert (failed);
7001 :
7002 0 : failed_insn = failed->delay_pair->i1;
7003 : /* Clear these queues. */
7004 0 : perform_replacements_new_cycle ();
7005 0 : toggle_cancelled_flags (false);
7006 0 : unschedule_insns_until (failed_insn);
7007 0 : while (failed != backtrack_queue)
7008 0 : free_topmost_backtrack_point (true);
7009 0 : restore_last_backtrack_point (&ls);
7010 0 : if (sched_verbose >= 2)
7011 0 : fprintf (sched_dump, ";;\t\trewind to cycle %d\n", clock_var);
7012 : /* Delay by at least a cycle. This could cause additional
7013 : backtracking. */
7014 0 : queue_insn (failed_insn, 1, "backtracked");
7015 0 : advance = 0;
7016 0 : if (must_backtrack)
7017 0 : continue;
7018 0 : if (ready.n_ready > 0)
7019 0 : goto resume_after_backtrack;
7020 : else
7021 : {
7022 0 : if (clock_var == 0 && ls.first_cycle_insn_p)
7023 0 : goto end_schedule;
7024 : advance = 1;
7025 : break;
7026 : }
7027 : }
7028 35993895 : ls.first_cycle_insn_p = true;
7029 : }
7030 10610721 : if (ls.modulo_epilogue)
7031 0 : success = true;
7032 10610721 : end_schedule:
7033 10610721 : if (!ls.first_cycle_insn_p || advance)
7034 938 : advance_one_cycle ();
7035 10610721 : perform_replacements_new_cycle ();
7036 10610721 : if (modulo_ii > 0)
7037 : {
7038 : /* Once again, debug insn suckiness: they can be on the ready list
7039 : even if they have unresolved dependencies. To make our view
7040 : of the world consistent, remove such "ready" insns. */
7041 0 : restart_debug_insn_loop:
7042 0 : for (i = ready.n_ready - 1; i >= 0; i--)
7043 : {
7044 0 : rtx_insn *x;
7045 :
7046 0 : x = ready_element (&ready, i);
7047 0 : if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x)) != NULL
7048 0 : || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x)) != NULL)
7049 : {
7050 0 : ready_remove (&ready, i);
7051 0 : goto restart_debug_insn_loop;
7052 : }
7053 : }
7054 0 : for (i = ready.n_ready - 1; i >= 0; i--)
7055 : {
7056 0 : rtx_insn *x;
7057 :
7058 0 : x = ready_element (&ready, i);
7059 0 : resolve_dependencies (x);
7060 : }
7061 0 : for (i = 0; i <= max_insn_queue_index; i++)
7062 : {
7063 : rtx_insn_list *link;
7064 0 : while ((link = insn_queue[i]) != NULL)
7065 : {
7066 0 : rtx_insn *x = link->insn ();
7067 0 : insn_queue[i] = link->next ();
7068 0 : QUEUE_INDEX (x) = QUEUE_NOWHERE;
7069 0 : free_INSN_LIST_node (link);
7070 0 : resolve_dependencies (x);
7071 : }
7072 : }
7073 : }
7074 :
7075 10610721 : if (!success)
7076 0 : undo_all_replacements ();
7077 :
7078 : /* Debug info. */
7079 10610721 : if (sched_verbose)
7080 : {
7081 190 : fprintf (sched_dump, ";;\tReady list (final): ");
7082 190 : debug_ready_list (&ready);
7083 : }
7084 :
7085 10610721 : if (modulo_ii == 0 && current_sched_info->queue_must_finish_empty)
7086 : /* Sanity check -- queue must be empty now. Meaningless if region has
7087 : multiple bbs. */
7088 10610649 : gcc_assert (!q_size && !ready.n_ready && !ready.n_debug);
7089 72 : else if (modulo_ii == 0)
7090 : {
7091 : /* We must maintain QUEUE_INDEX between blocks in region. */
7092 84 : for (i = ready.n_ready - 1; i >= 0; i--)
7093 : {
7094 12 : rtx_insn *x;
7095 :
7096 12 : x = ready_element (&ready, i);
7097 12 : QUEUE_INDEX (x) = QUEUE_NOWHERE;
7098 12 : TODO_SPEC (x) = HARD_DEP;
7099 : }
7100 :
7101 72 : if (q_size)
7102 0 : for (i = 0; i <= max_insn_queue_index; i++)
7103 : {
7104 0 : rtx_insn_list *link;
7105 0 : for (link = insn_queue[i]; link; link = link->next ())
7106 : {
7107 0 : rtx_insn *x;
7108 :
7109 0 : x = link->insn ();
7110 0 : QUEUE_INDEX (x) = QUEUE_NOWHERE;
7111 0 : TODO_SPEC (x) = HARD_DEP;
7112 : }
7113 0 : free_INSN_LIST_list (&insn_queue[i]);
7114 : }
7115 : }
7116 :
7117 10610721 : if (sched_pressure == SCHED_PRESSURE_MODEL)
7118 0 : model_end_schedule ();
7119 :
7120 10610721 : if (success)
7121 : {
7122 10610721 : commit_schedule (prev_head, tail, target_bb);
7123 10610721 : if (sched_verbose)
7124 190 : fprintf (sched_dump, ";; total time = %d\n", clock_var);
7125 : }
7126 : else
7127 0 : last_scheduled_insn = tail;
7128 :
7129 10610721 : scheduled_insns.truncate (0);
7130 :
7131 10610721 : if (!current_sched_info->queue_must_finish_empty
7132 10610649 : || haifa_recovery_bb_recently_added_p)
7133 : {
7134 : /* INSN_TICK (minimum clock tick at which the insn becomes
7135 : ready) may be not correct for the insn in the subsequent
7136 : blocks of the region. We should use a correct value of
7137 : `clock_var' or modify INSN_TICK. It is better to keep
7138 : clock_var value equal to 0 at the start of a basic block.
7139 : Therefore we modify INSN_TICK here. */
7140 72 : fix_inter_tick (NEXT_INSN (prev_head), last_scheduled_insn);
7141 : }
7142 :
7143 10610721 : if (targetm.sched.finish)
7144 : {
7145 0 : targetm.sched.finish (sched_dump, sched_verbose);
7146 : /* Target might have added some instructions to the scheduled block
7147 : in its md_finish () hook. These new insns don't have any data
7148 : initialized and to identify them we extend h_i_d so that they'll
7149 : get zero luids. */
7150 0 : sched_extend_luids ();
7151 : }
7152 :
7153 : /* Update head/tail boundaries. */
7154 10610721 : head = NEXT_INSN (prev_head);
7155 10610721 : tail = last_scheduled_insn;
7156 :
7157 10610721 : if (sched_verbose)
7158 : {
7159 190 : fprintf (sched_dump, ";; new head = %d\n;; new tail = %d\n",
7160 190 : INSN_UID (head), INSN_UID (tail));
7161 :
7162 190 : if (sched_verbose >= 2)
7163 : {
7164 0 : dump_insn_stream (head, tail);
7165 0 : print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats,
7166 : NULL);
7167 : }
7168 :
7169 190 : fprintf (sched_dump, "\n");
7170 : }
7171 :
7172 10610721 : head = restore_other_notes (head, NULL);
7173 :
7174 10610721 : current_sched_info->head = head;
7175 10610721 : current_sched_info->tail = tail;
7176 :
7177 10610721 : free_backtrack_queue ();
7178 :
7179 10610721 : return success;
7180 : }
7181 :
7182 : /* Set_priorities: compute priority of each insn in the block. */
7183 :
7184 : int
7185 10611694 : set_priorities (rtx_insn *head, rtx_insn *tail)
7186 : {
7187 10611694 : rtx_insn *insn;
7188 10611694 : int n_insn;
7189 10611694 : int sched_max_insns_priority =
7190 10611694 : current_sched_info->sched_max_insns_priority;
7191 10611694 : rtx_insn *prev_head;
7192 :
7193 10611694 : if (head == tail && ! INSN_P (head))
7194 0 : gcc_unreachable ();
7195 :
7196 10611694 : n_insn = 0;
7197 :
7198 10611694 : prev_head = PREV_INSN (head);
7199 143278025 : for (insn = tail; insn != prev_head; insn = PREV_INSN (insn))
7200 : {
7201 122054637 : if (!INSN_P (insn))
7202 5941803 : continue;
7203 :
7204 116112834 : n_insn++;
7205 116112834 : (void) priority (insn);
7206 :
7207 116112834 : gcc_assert (INSN_PRIORITY_KNOWN (insn));
7208 :
7209 116112834 : sched_max_insns_priority = MAX (sched_max_insns_priority,
7210 : INSN_PRIORITY (insn));
7211 : }
7212 :
7213 10611694 : current_sched_info->sched_max_insns_priority = sched_max_insns_priority;
7214 :
7215 10611694 : return n_insn;
7216 : }
7217 :
7218 : /* Set sched_dump and sched_verbose for the desired debugging output. */
7219 : void
7220 989027 : setup_sched_dump (void)
7221 : {
7222 989027 : sched_verbose = sched_verbose_param;
7223 989027 : sched_dump = dump_file;
7224 989027 : if (!dump_file)
7225 988990 : sched_verbose = 0;
7226 989027 : }
7227 :
7228 : /* Allocate data for register pressure sensitive scheduling. */
7229 : static void
7230 989027 : alloc_global_sched_pressure_data (void)
7231 : {
7232 989027 : if (sched_pressure != SCHED_PRESSURE_NONE)
7233 : {
7234 57 : int i, max_regno = max_reg_num ();
7235 :
7236 57 : if (sched_dump != NULL)
7237 : /* We need info about pseudos for rtl dumps about pseudo
7238 : classes and costs. */
7239 0 : regstat_init_n_sets_and_refs ();
7240 57 : ira_set_pseudo_classes (true, sched_verbose ? sched_dump : NULL);
7241 57 : sched_regno_pressure_class
7242 57 : = (enum reg_class *) xmalloc (max_regno * sizeof (enum reg_class));
7243 8732 : for (i = 0; i < max_regno; i++)
7244 17350 : sched_regno_pressure_class[i]
7245 8675 : = (i < FIRST_PSEUDO_REGISTER
7246 8675 : ? ira_pressure_class_translate[REGNO_REG_CLASS (i)]
7247 3317 : : ira_pressure_class_translate[reg_allocno_class (i)]);
7248 57 : curr_reg_live = BITMAP_ALLOC (NULL);
7249 57 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
7250 : {
7251 57 : saved_reg_live = BITMAP_ALLOC (NULL);
7252 57 : region_ref_regs = BITMAP_ALLOC (NULL);
7253 : }
7254 57 : if (sched_pressure == SCHED_PRESSURE_MODEL)
7255 0 : tmp_bitmap = BITMAP_ALLOC (NULL);
7256 :
7257 : /* Calculate number of CALL_SAVED_REGS and FIXED_REGS in register classes
7258 : that we calculate register pressure for. */
7259 287 : for (int c = 0; c < ira_pressure_classes_num; ++c)
7260 : {
7261 230 : enum reg_class cl = ira_pressure_classes[c];
7262 :
7263 230 : call_saved_regs_num[cl] = 0;
7264 230 : fixed_regs_num[cl] = 0;
7265 :
7266 2905 : for (int i = 0; i < ira_class_hard_regs_num[cl]; ++i)
7267 : {
7268 2675 : unsigned int regno = ira_class_hard_regs[cl][i];
7269 2675 : if (fixed_regs[regno])
7270 0 : ++fixed_regs_num[cl];
7271 2675 : else if (!crtl->abi->clobbers_full_reg_p (regno))
7272 332 : ++call_saved_regs_num[cl];
7273 : }
7274 : }
7275 : }
7276 989027 : }
7277 :
7278 : /* Free data for register pressure sensitive scheduling. Also called
7279 : from schedule_region when stopping sched-pressure early. */
7280 : void
7281 989027 : free_global_sched_pressure_data (void)
7282 : {
7283 989027 : if (sched_pressure != SCHED_PRESSURE_NONE)
7284 : {
7285 57 : if (regstat_n_sets_and_refs != NULL)
7286 0 : regstat_free_n_sets_and_refs ();
7287 57 : if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
7288 : {
7289 57 : BITMAP_FREE (region_ref_regs);
7290 57 : BITMAP_FREE (saved_reg_live);
7291 : }
7292 57 : if (sched_pressure == SCHED_PRESSURE_MODEL)
7293 0 : BITMAP_FREE (tmp_bitmap);
7294 57 : BITMAP_FREE (curr_reg_live);
7295 57 : free (sched_regno_pressure_class);
7296 : }
7297 989027 : }
7298 :
7299 : /* Initialize some global state for the scheduler. This function works
7300 : with the common data shared between all the schedulers. It is called
7301 : from the scheduler specific initialization routine. */
7302 :
7303 : void
7304 989027 : sched_init (void)
7305 : {
7306 989027 : if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
7307 3 : targetm.sched.dispatch_do (NULL, DISPATCH_INIT);
7308 :
7309 989027 : if (live_range_shrinkage_p)
7310 35 : sched_pressure = SCHED_PRESSURE_WEIGHTED;
7311 988992 : else if (flag_sched_pressure
7312 50 : && !reload_completed
7313 22 : && common_sched_info->sched_pass_id == SCHED_RGN_PASS)
7314 22 : sched_pressure = ((enum sched_pressure_algorithm)
7315 22 : param_sched_pressure_algorithm);
7316 : else
7317 988970 : sched_pressure = SCHED_PRESSURE_NONE;
7318 :
7319 989027 : if (sched_pressure != SCHED_PRESSURE_NONE)
7320 57 : ira_setup_eliminable_regset ();
7321 :
7322 : /* Initialize SPEC_INFO. */
7323 989027 : if (targetm.sched.set_sched_flags)
7324 : {
7325 0 : spec_info = &spec_info_var;
7326 0 : targetm.sched.set_sched_flags (spec_info);
7327 :
7328 0 : if (spec_info->mask != 0)
7329 : {
7330 0 : spec_info->data_weakness_cutoff
7331 0 : = (param_sched_spec_prob_cutoff * MAX_DEP_WEAK) / 100;
7332 0 : spec_info->control_weakness_cutoff
7333 0 : = (param_sched_spec_prob_cutoff * REG_BR_PROB_BASE) / 100;
7334 : }
7335 : else
7336 : /* So we won't read anything accidentally. */
7337 0 : spec_info = NULL;
7338 :
7339 : }
7340 : else
7341 : /* So we won't read anything accidentally. */
7342 989027 : spec_info = 0;
7343 :
7344 : /* Initialize issue_rate. */
7345 989027 : if (targetm.sched.issue_rate)
7346 989027 : issue_rate = targetm.sched.issue_rate ();
7347 : else
7348 0 : issue_rate = 1;
7349 :
7350 989027 : if (targetm.sched.first_cycle_multipass_dfa_lookahead
7351 : /* Don't use max_issue with reg_pressure scheduling. Multipass
7352 : scheduling and reg_pressure scheduling undo each other's decisions. */
7353 989027 : && sched_pressure == SCHED_PRESSURE_NONE)
7354 988970 : dfa_lookahead = targetm.sched.first_cycle_multipass_dfa_lookahead ();
7355 : else
7356 57 : dfa_lookahead = 0;
7357 :
7358 : /* Set to "0" so that we recalculate. */
7359 989027 : max_lookahead_tries = 0;
7360 :
7361 989027 : if (targetm.sched.init_dfa_pre_cycle_insn)
7362 0 : targetm.sched.init_dfa_pre_cycle_insn ();
7363 :
7364 989027 : if (targetm.sched.init_dfa_post_cycle_insn)
7365 0 : targetm.sched.init_dfa_post_cycle_insn ();
7366 :
7367 989027 : dfa_start ();
7368 989027 : dfa_state_size = state_size ();
7369 :
7370 989027 : init_alias_analysis ();
7371 :
7372 989027 : if (!sched_no_dce)
7373 989027 : df_set_flags (DF_LR_RUN_DCE);
7374 989027 : df_note_add_problem ();
7375 :
7376 : /* More problems needed for interloop dep calculation in SMS. */
7377 989027 : if (common_sched_info->sched_pass_id == SCHED_SMS_PASS)
7378 : {
7379 267 : df_rd_add_problem ();
7380 267 : df_chain_add_problem (DF_DU_CHAIN + DF_UD_CHAIN);
7381 : }
7382 :
7383 989027 : df_analyze ();
7384 :
7385 : /* Do not run DCE after reload, as this can kill nops inserted
7386 : by bundling. */
7387 989027 : if (reload_completed)
7388 988526 : df_clear_flags (DF_LR_RUN_DCE);
7389 :
7390 989027 : regstat_compute_calls_crossed ();
7391 :
7392 989027 : if (targetm.sched.init_global)
7393 989027 : targetm.sched.init_global (sched_dump, sched_verbose, get_max_uid () + 1);
7394 :
7395 989027 : alloc_global_sched_pressure_data ();
7396 :
7397 989027 : curr_state = xmalloc (dfa_state_size);
7398 989027 : }
7399 :
7400 : static void haifa_init_only_bb (basic_block, basic_block);
7401 :
7402 : /* Initialize data structures specific to the Haifa scheduler. */
7403 : void
7404 988893 : haifa_sched_init (void)
7405 : {
7406 988893 : setup_sched_dump ();
7407 988893 : sched_init ();
7408 :
7409 988893 : scheduled_insns.create (0);
7410 :
7411 988893 : if (spec_info != NULL)
7412 : {
7413 0 : sched_deps_info->use_deps_list = 1;
7414 0 : sched_deps_info->generate_spec_deps = 1;
7415 : }
7416 :
7417 : /* Initialize luids, dependency caches, target and h_i_d for the
7418 : whole function. */
7419 988893 : {
7420 988893 : sched_init_bbs ();
7421 :
7422 988893 : auto_vec<basic_block> bbs (n_basic_blocks_for_fn (cfun));
7423 988893 : basic_block bb;
7424 11621223 : FOR_EACH_BB_FN (bb, cfun)
7425 10632330 : bbs.quick_push (bb);
7426 988893 : sched_init_luids (bbs);
7427 988893 : sched_deps_init (true);
7428 988893 : sched_extend_target ();
7429 988893 : haifa_init_h_i_d (bbs);
7430 988893 : }
7431 :
7432 988893 : sched_init_only_bb = haifa_init_only_bb;
7433 988893 : sched_split_block = sched_split_block_1;
7434 988893 : sched_create_empty_bb = sched_create_empty_bb_1;
7435 988893 : haifa_recovery_bb_ever_added_p = false;
7436 :
7437 988893 : nr_begin_data = nr_begin_control = nr_be_in_data = nr_be_in_control = 0;
7438 988893 : before_recovery = 0;
7439 988893 : after_recovery = 0;
7440 :
7441 988893 : modulo_ii = 0;
7442 988893 : }
7443 :
7444 : /* Finish work with the data specific to the Haifa scheduler. */
7445 : void
7446 988893 : haifa_sched_finish (void)
7447 : {
7448 988893 : sched_create_empty_bb = NULL;
7449 988893 : sched_split_block = NULL;
7450 988893 : sched_init_only_bb = NULL;
7451 :
7452 988893 : if (spec_info && spec_info->dump)
7453 : {
7454 0 : char c = reload_completed ? 'a' : 'b';
7455 :
7456 0 : fprintf (spec_info->dump,
7457 : ";; %s:\n", current_function_name ());
7458 :
7459 0 : fprintf (spec_info->dump,
7460 : ";; Procedure %cr-begin-data-spec motions == %d\n",
7461 : c, nr_begin_data);
7462 0 : fprintf (spec_info->dump,
7463 : ";; Procedure %cr-be-in-data-spec motions == %d\n",
7464 : c, nr_be_in_data);
7465 0 : fprintf (spec_info->dump,
7466 : ";; Procedure %cr-begin-control-spec motions == %d\n",
7467 : c, nr_begin_control);
7468 0 : fprintf (spec_info->dump,
7469 : ";; Procedure %cr-be-in-control-spec motions == %d\n",
7470 : c, nr_be_in_control);
7471 : }
7472 :
7473 988893 : scheduled_insns.release ();
7474 :
7475 : /* Finalize h_i_d, dependency caches, and luids for the whole
7476 : function. Target will be finalized in md_global_finish (). */
7477 988893 : sched_deps_finish ();
7478 988893 : sched_finish_luids ();
7479 988893 : current_sched_info = NULL;
7480 988893 : insn_queue = NULL;
7481 988893 : sched_finish ();
7482 988893 : }
7483 :
7484 : /* Free global data used during insn scheduling. This function works with
7485 : the common data shared between the schedulers. */
7486 :
7487 : void
7488 989027 : sched_finish (void)
7489 : {
7490 989027 : haifa_finish_h_i_d ();
7491 989027 : free_global_sched_pressure_data ();
7492 989027 : free (curr_state);
7493 :
7494 989027 : if (targetm.sched.finish_global)
7495 0 : targetm.sched.finish_global (sched_dump, sched_verbose);
7496 :
7497 989027 : end_alias_analysis ();
7498 :
7499 989027 : regstat_free_calls_crossed ();
7500 :
7501 989027 : dfa_finish ();
7502 989027 : }
7503 :
7504 : /* Free all delay_pair structures that were recorded. */
7505 : void
7506 0 : free_delay_pairs (void)
7507 : {
7508 0 : if (delay_htab)
7509 : {
7510 0 : delay_htab->empty ();
7511 0 : delay_htab_i2->empty ();
7512 : }
7513 0 : }
7514 :
7515 : /* Fix INSN_TICKs of the instructions in the current block as well as
7516 : INSN_TICKs of their dependents.
7517 : HEAD and TAIL are the begin and the end of the current scheduled block. */
7518 : static void
7519 72 : fix_inter_tick (rtx_insn *head, rtx_insn *tail)
7520 : {
7521 : /* Set of instructions with corrected INSN_TICK. */
7522 72 : auto_bitmap processed;
7523 : /* ??? It is doubtful if we should assume that cycle advance happens on
7524 : basic block boundaries. Basically insns that are unconditionally ready
7525 : on the start of the block are more preferable then those which have
7526 : a one cycle dependency over insn from the previous block. */
7527 72 : int next_clock = clock_var + 1;
7528 :
7529 : /* Iterates over scheduled instructions and fix their INSN_TICKs and
7530 : INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent
7531 : across different blocks. */
7532 1287 : for (tail = NEXT_INSN (tail); head != tail; head = NEXT_INSN (head))
7533 : {
7534 1143 : if (INSN_P (head))
7535 : {
7536 1143 : int tick;
7537 1143 : sd_iterator_def sd_it;
7538 1143 : dep_t dep;
7539 :
7540 1143 : tick = INSN_TICK (head);
7541 1143 : gcc_assert (tick >= MIN_TICK);
7542 :
7543 : /* Fix INSN_TICK of instruction from just scheduled block. */
7544 1143 : if (bitmap_set_bit (processed, INSN_LUID (head)))
7545 : {
7546 392 : tick -= next_clock;
7547 :
7548 392 : if (tick < MIN_TICK)
7549 : tick = MIN_TICK;
7550 :
7551 392 : INSN_TICK (head) = tick;
7552 : }
7553 :
7554 1143 : if (DEBUG_INSN_P (head))
7555 428 : continue;
7556 :
7557 2995 : FOR_EACH_DEP (head, SD_LIST_RES_FORW, sd_it, dep)
7558 : {
7559 2280 : rtx_insn *next;
7560 :
7561 2280 : next = DEP_CON (dep);
7562 2280 : tick = INSN_TICK (next);
7563 :
7564 2280 : if (tick != INVALID_TICK
7565 : /* If NEXT has its INSN_TICK calculated, fix it.
7566 : If not - it will be properly calculated from
7567 : scratch later in fix_tick_ready. */
7568 2280 : && bitmap_set_bit (processed, INSN_LUID (next)))
7569 : {
7570 763 : tick -= next_clock;
7571 :
7572 763 : if (tick < MIN_TICK)
7573 : tick = MIN_TICK;
7574 :
7575 763 : if (tick > INTER_TICK (next))
7576 763 : INTER_TICK (next) = tick;
7577 : else
7578 : tick = INTER_TICK (next);
7579 :
7580 763 : INSN_TICK (next) = tick;
7581 : }
7582 : }
7583 : }
7584 : }
7585 72 : }
7586 :
7587 : /* Check if NEXT is ready to be added to the ready or queue list.
7588 : If "yes", add it to the proper list.
7589 : Returns:
7590 : -1 - is not ready yet,
7591 : 0 - added to the ready list,
7592 : 0 < N - queued for N cycles. */
7593 : int
7594 328933041 : try_ready (rtx_insn *next)
7595 : {
7596 328933041 : ds_t old_ts, new_ts;
7597 :
7598 328933041 : old_ts = TODO_SPEC (next);
7599 :
7600 328933041 : gcc_assert (!(old_ts & ~(SPECULATIVE | HARD_DEP | DEP_CONTROL | DEP_POSTPONED))
7601 : && (old_ts == HARD_DEP
7602 : || old_ts == DEP_POSTPONED
7603 : || (old_ts & SPECULATIVE)
7604 : || old_ts == DEP_CONTROL));
7605 :
7606 328933041 : new_ts = recompute_todo_spec (next, false);
7607 :
7608 328933041 : if (new_ts & (HARD_DEP | DEP_POSTPONED))
7609 212824514 : gcc_assert (new_ts == old_ts
7610 : && QUEUE_INDEX (next) == QUEUE_NOWHERE);
7611 116108527 : else if (current_sched_info->new_ready)
7612 116107268 : new_ts = current_sched_info->new_ready (next, new_ts);
7613 :
7614 : /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might
7615 : have its original pattern or changed (speculative) one. This is due
7616 : to changing ebb in region scheduling.
7617 : * But if (old_ts & SPECULATIVE), then we are pretty sure that insn
7618 : has speculative pattern.
7619 :
7620 : We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because
7621 : control-speculative NEXT could have been discarded by sched-rgn.cc
7622 : (the same case as when discarded by can_schedule_ready_p ()). */
7623 :
7624 328933041 : if ((new_ts & SPECULATIVE)
7625 : /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't
7626 : need to change anything. */
7627 0 : && new_ts != old_ts)
7628 : {
7629 0 : int res;
7630 0 : rtx new_pat;
7631 :
7632 0 : gcc_assert ((new_ts & SPECULATIVE) && !(new_ts & ~SPECULATIVE));
7633 :
7634 0 : res = haifa_speculate_insn (next, new_ts, &new_pat);
7635 :
7636 0 : switch (res)
7637 : {
7638 : case -1:
7639 : /* It would be nice to change DEP_STATUS of all dependences,
7640 : which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP,
7641 : so we won't reanalyze anything. */
7642 : new_ts = HARD_DEP;
7643 : break;
7644 :
7645 0 : case 0:
7646 : /* We follow the rule, that every speculative insn
7647 : has non-null ORIG_PAT. */
7648 0 : if (!ORIG_PAT (next))
7649 0 : ORIG_PAT (next) = PATTERN (next);
7650 : break;
7651 :
7652 0 : case 1:
7653 0 : if (!ORIG_PAT (next))
7654 : /* If we gonna to overwrite the original pattern of insn,
7655 : save it. */
7656 0 : ORIG_PAT (next) = PATTERN (next);
7657 :
7658 0 : res = haifa_change_pattern (next, new_pat);
7659 0 : gcc_assert (res);
7660 : break;
7661 :
7662 0 : default:
7663 0 : gcc_unreachable ();
7664 : }
7665 : }
7666 :
7667 : /* We need to restore pattern only if (new_ts == 0), because otherwise it is
7668 : either correct (new_ts & SPECULATIVE),
7669 : or we simply don't care (new_ts & HARD_DEP). */
7670 :
7671 328933041 : gcc_assert (!ORIG_PAT (next)
7672 : || !IS_SPECULATION_BRANCHY_CHECK_P (next));
7673 :
7674 328933041 : TODO_SPEC (next) = new_ts;
7675 :
7676 328933041 : if (new_ts & (HARD_DEP | DEP_POSTPONED))
7677 : {
7678 : /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because
7679 : control-speculative NEXT could have been discarded by sched-rgn.cc
7680 : (the same case as when discarded by can_schedule_ready_p ()). */
7681 : /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/
7682 :
7683 212824550 : change_queue_index (next, QUEUE_NOWHERE);
7684 :
7685 212824550 : return -1;
7686 : }
7687 116108491 : else if (!(new_ts & BEGIN_SPEC)
7688 116108491 : && ORIG_PAT (next) && PREDICATED_PAT (next) == NULL_RTX
7689 116108491 : && !IS_SPECULATION_CHECK_P (next))
7690 : /* We should change pattern of every previously speculative
7691 : instruction - and we determine if NEXT was speculative by using
7692 : ORIG_PAT field. Except one case - speculation checks have ORIG_PAT
7693 : pat too, so skip them. */
7694 : {
7695 0 : bool success = haifa_change_pattern (next, ORIG_PAT (next));
7696 0 : gcc_assert (success);
7697 0 : ORIG_PAT (next) = 0;
7698 : }
7699 :
7700 116108491 : if (sched_verbose >= 2)
7701 : {
7702 0 : fprintf (sched_dump, ";;\t\tdependencies resolved: insn %s",
7703 0 : (*current_sched_info->print_insn) (next, 0));
7704 :
7705 0 : if (spec_info && spec_info->dump)
7706 : {
7707 0 : if (new_ts & BEGIN_DATA)
7708 0 : fprintf (spec_info->dump, "; data-spec;");
7709 0 : if (new_ts & BEGIN_CONTROL)
7710 0 : fprintf (spec_info->dump, "; control-spec;");
7711 0 : if (new_ts & BE_IN_CONTROL)
7712 0 : fprintf (spec_info->dump, "; in-control-spec;");
7713 : }
7714 0 : if (TODO_SPEC (next) & DEP_CONTROL)
7715 0 : fprintf (sched_dump, " predicated");
7716 0 : fprintf (sched_dump, "\n");
7717 : }
7718 :
7719 116108491 : adjust_priority (next);
7720 :
7721 116108491 : return fix_tick_ready (next);
7722 : }
7723 :
7724 : /* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */
7725 : static int
7726 116424913 : fix_tick_ready (rtx_insn *next)
7727 : {
7728 116424913 : int tick, delay;
7729 :
7730 116424913 : if (!DEBUG_INSN_P (next) && !sd_lists_empty_p (next, SD_LIST_RES_BACK))
7731 : {
7732 43264760 : int full_p;
7733 43264760 : sd_iterator_def sd_it;
7734 43264760 : dep_t dep;
7735 :
7736 43264760 : tick = INSN_TICK (next);
7737 : /* if tick is not equal to INVALID_TICK, then update
7738 : INSN_TICK of NEXT with the most recent resolved dependence
7739 : cost. Otherwise, recalculate from scratch. */
7740 43264760 : full_p = (tick == INVALID_TICK);
7741 :
7742 194587376 : FOR_EACH_DEP (next, SD_LIST_RES_BACK, sd_it, dep)
7743 : {
7744 151322628 : rtx_insn *pro = DEP_PRO (dep);
7745 151322628 : int tick1;
7746 :
7747 151322628 : gcc_assert (INSN_TICK (pro) >= MIN_TICK);
7748 :
7749 151322628 : tick1 = INSN_TICK (pro) + dep_cost (dep);
7750 151322628 : if (tick1 > tick)
7751 : tick = tick1;
7752 :
7753 151322628 : if (!full_p)
7754 : break;
7755 : }
7756 : }
7757 : else
7758 : tick = -1;
7759 :
7760 116424913 : INSN_TICK (next) = tick;
7761 :
7762 116424913 : delay = tick - clock_var;
7763 116424913 : if (delay <= 0 || sched_pressure != SCHED_PRESSURE_NONE || sched_fusion)
7764 96305506 : delay = QUEUE_READY;
7765 :
7766 116424913 : change_queue_index (next, delay);
7767 :
7768 116424913 : return delay;
7769 : }
7770 :
7771 : /* Move NEXT to the proper queue list with (DELAY >= 1),
7772 : or add it to the ready list (DELAY == QUEUE_READY),
7773 : or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */
7774 : static void
7775 329249463 : change_queue_index (rtx_insn *next, int delay)
7776 : {
7777 329249463 : int i = QUEUE_INDEX (next);
7778 :
7779 329249463 : gcc_assert (QUEUE_NOWHERE <= delay && delay <= max_insn_queue_index
7780 : && delay != 0);
7781 329249463 : gcc_assert (i != QUEUE_SCHEDULED);
7782 :
7783 329249463 : if ((delay > 0 && NEXT_Q_AFTER (q_ptr, delay) == i)
7784 329123210 : || (delay < 0 && delay == i))
7785 : /* We have nothing to do. */
7786 : return;
7787 :
7788 : /* Remove NEXT from wherever it is now. */
7789 116246188 : if (i == QUEUE_READY)
7790 0 : ready_remove_insn (next);
7791 116246188 : else if (i >= 0)
7792 137697 : queue_remove (next);
7793 :
7794 : /* Add it to the proper place. */
7795 116246188 : if (delay == QUEUE_READY)
7796 96253034 : ready_add (readyp, next, false);
7797 19993154 : else if (delay >= 1)
7798 19993154 : queue_insn (next, delay, "change queue index");
7799 :
7800 116246188 : if (sched_verbose >= 2)
7801 : {
7802 0 : fprintf (sched_dump, ";;\t\ttick updated: insn %s",
7803 0 : (*current_sched_info->print_insn) (next, 0));
7804 :
7805 0 : if (delay == QUEUE_READY)
7806 0 : fprintf (sched_dump, " into ready\n");
7807 0 : else if (delay >= 1)
7808 0 : fprintf (sched_dump, " into queue with cost=%d\n", delay);
7809 : else
7810 0 : fprintf (sched_dump, " removed from ready or queue lists\n");
7811 : }
7812 : }
7813 :
7814 : static int sched_ready_n_insns = -1;
7815 :
7816 : /* Initialize per region data structures. */
7817 : void
7818 10631193 : sched_extend_ready_list (int new_sched_ready_n_insns)
7819 : {
7820 10631193 : int i;
7821 :
7822 10631193 : if (sched_ready_n_insns == -1)
7823 : /* At the first call we need to initialize one more choice_stack
7824 : entry. */
7825 : {
7826 10631087 : i = 0;
7827 10631087 : sched_ready_n_insns = 0;
7828 10631087 : scheduled_insns.reserve (new_sched_ready_n_insns);
7829 : }
7830 : else
7831 106 : i = sched_ready_n_insns + 1;
7832 :
7833 10631193 : ready.veclen = new_sched_ready_n_insns + issue_rate;
7834 10631193 : ready.vec = XRESIZEVEC (rtx_insn *, ready.vec, ready.veclen);
7835 :
7836 10631193 : gcc_assert (new_sched_ready_n_insns >= sched_ready_n_insns);
7837 :
7838 10631193 : ready_try = (signed char *) xrecalloc (ready_try, new_sched_ready_n_insns,
7839 : sched_ready_n_insns,
7840 : sizeof (*ready_try));
7841 :
7842 : /* We allocate +1 element to save initial state in the choice_stack[0]
7843 : entry. */
7844 10631193 : choice_stack = XRESIZEVEC (struct choice_entry, choice_stack,
7845 : new_sched_ready_n_insns + 1);
7846 :
7847 137372072 : for (; i <= new_sched_ready_n_insns; i++)
7848 : {
7849 126740879 : choice_stack[i].state = xmalloc (dfa_state_size);
7850 :
7851 126740879 : if (targetm.sched.first_cycle_multipass_init)
7852 126430906 : targetm.sched.first_cycle_multipass_init (&(choice_stack[i]
7853 : .target_data));
7854 : }
7855 :
7856 10631193 : sched_ready_n_insns = new_sched_ready_n_insns;
7857 10631193 : }
7858 :
7859 : /* Free per region data structures. */
7860 : void
7861 10631087 : sched_finish_ready_list (void)
7862 : {
7863 10631087 : int i;
7864 :
7865 10631087 : free (ready.vec);
7866 10631087 : ready.vec = NULL;
7867 10631087 : ready.veclen = 0;
7868 :
7869 10631087 : free (ready_try);
7870 10631087 : ready_try = NULL;
7871 :
7872 137371966 : for (i = 0; i <= sched_ready_n_insns; i++)
7873 : {
7874 126740879 : if (targetm.sched.first_cycle_multipass_fini)
7875 126430906 : targetm.sched.first_cycle_multipass_fini (&(choice_stack[i]
7876 : .target_data));
7877 :
7878 126740879 : free (choice_stack [i].state);
7879 : }
7880 10631087 : free (choice_stack);
7881 10631087 : choice_stack = NULL;
7882 :
7883 10631087 : sched_ready_n_insns = -1;
7884 10631087 : }
7885 :
7886 : static int
7887 24315328 : haifa_luid_for_non_insn (rtx x)
7888 : {
7889 24315328 : gcc_assert (NOTE_P (x) || LABEL_P (x));
7890 :
7891 24315328 : return 0;
7892 : }
7893 :
7894 : /* Generates recovery code for INSN. */
7895 : static void
7896 0 : generate_recovery_code (rtx_insn *insn)
7897 : {
7898 0 : if (TODO_SPEC (insn) & BEGIN_SPEC)
7899 0 : begin_speculative_block (insn);
7900 :
7901 : /* Here we have insn with no dependencies to
7902 : instructions other then CHECK_SPEC ones. */
7903 :
7904 0 : if (TODO_SPEC (insn) & BE_IN_SPEC)
7905 0 : add_to_speculative_block (insn);
7906 0 : }
7907 :
7908 : /* Helper function.
7909 : Tries to add speculative dependencies of type FS between instructions
7910 : in deps_list L and TWIN. */
7911 : static void
7912 0 : process_insn_forw_deps_be_in_spec (rtx_insn *insn, rtx_insn *twin, ds_t fs)
7913 : {
7914 0 : sd_iterator_def sd_it;
7915 0 : dep_t dep;
7916 :
7917 0 : FOR_EACH_DEP (insn, SD_LIST_FORW, sd_it, dep)
7918 : {
7919 0 : ds_t ds;
7920 0 : rtx_insn *consumer;
7921 :
7922 0 : consumer = DEP_CON (dep);
7923 :
7924 0 : ds = DEP_STATUS (dep);
7925 :
7926 0 : if (/* If we want to create speculative dep. */
7927 : fs
7928 : /* And we can do that because this is a true dep. */
7929 0 : && (ds & DEP_TYPES) == DEP_TRUE)
7930 : {
7931 0 : gcc_assert (!(ds & BE_IN_SPEC));
7932 :
7933 0 : if (/* If this dep can be overcome with 'begin speculation'. */
7934 0 : ds & BEGIN_SPEC)
7935 : /* Then we have a choice: keep the dep 'begin speculative'
7936 : or transform it into 'be in speculative'. */
7937 : {
7938 0 : if (/* In try_ready we assert that if insn once became ready
7939 : it can be removed from the ready (or queue) list only
7940 : due to backend decision. Hence we can't let the
7941 : probability of the speculative dep to decrease. */
7942 0 : ds_weak (ds) <= ds_weak (fs))
7943 : {
7944 0 : ds_t new_ds;
7945 :
7946 0 : new_ds = (ds & ~BEGIN_SPEC) | fs;
7947 :
7948 0 : if (/* consumer can 'be in speculative'. */
7949 0 : sched_insn_is_legitimate_for_speculation_p (consumer,
7950 : new_ds))
7951 : /* Transform it to be in speculative. */
7952 0 : ds = new_ds;
7953 : }
7954 : }
7955 : else
7956 : /* Mark the dep as 'be in speculative'. */
7957 0 : ds |= fs;
7958 : }
7959 :
7960 0 : {
7961 0 : dep_def _new_dep, *new_dep = &_new_dep;
7962 :
7963 0 : init_dep_1 (new_dep, twin, consumer, DEP_TYPE (dep), ds);
7964 0 : sd_add_dep (new_dep, false);
7965 : }
7966 : }
7967 0 : }
7968 :
7969 : /* Generates recovery code for BEGIN speculative INSN. */
7970 : static void
7971 0 : begin_speculative_block (rtx_insn *insn)
7972 : {
7973 0 : if (TODO_SPEC (insn) & BEGIN_DATA)
7974 0 : nr_begin_data++;
7975 0 : if (TODO_SPEC (insn) & BEGIN_CONTROL)
7976 0 : nr_begin_control++;
7977 :
7978 0 : create_check_block_twin (insn, false);
7979 :
7980 0 : TODO_SPEC (insn) &= ~BEGIN_SPEC;
7981 0 : }
7982 :
7983 : static void haifa_init_insn (rtx_insn *);
7984 :
7985 : /* Generates recovery code for BE_IN speculative INSN. */
7986 : static void
7987 0 : add_to_speculative_block (rtx_insn *insn)
7988 : {
7989 0 : ds_t ts;
7990 0 : sd_iterator_def sd_it;
7991 0 : dep_t dep;
7992 0 : auto_vec<rtx_insn *, 10> twins;
7993 :
7994 0 : ts = TODO_SPEC (insn);
7995 0 : gcc_assert (!(ts & ~BE_IN_SPEC));
7996 :
7997 0 : if (ts & BE_IN_DATA)
7998 0 : nr_be_in_data++;
7999 0 : if (ts & BE_IN_CONTROL)
8000 0 : nr_be_in_control++;
8001 :
8002 0 : TODO_SPEC (insn) &= ~BE_IN_SPEC;
8003 0 : gcc_assert (!TODO_SPEC (insn));
8004 :
8005 0 : DONE_SPEC (insn) |= ts;
8006 :
8007 : /* First we convert all simple checks to branchy. */
8008 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8009 0 : sd_iterator_cond (&sd_it, &dep);)
8010 : {
8011 0 : rtx_insn *check = DEP_PRO (dep);
8012 :
8013 0 : if (IS_SPECULATION_SIMPLE_CHECK_P (check))
8014 : {
8015 0 : create_check_block_twin (check, true);
8016 :
8017 : /* Restart search. */
8018 0 : sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8019 : }
8020 : else
8021 : /* Continue search. */
8022 0 : sd_iterator_next (&sd_it);
8023 : }
8024 :
8025 0 : auto_vec<rtx_insn *> priorities_roots;
8026 0 : clear_priorities (insn, &priorities_roots);
8027 :
8028 0 : while (1)
8029 : {
8030 0 : rtx_insn *check, *twin;
8031 0 : basic_block rec;
8032 :
8033 : /* Get the first backward dependency of INSN. */
8034 0 : sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8035 0 : if (!sd_iterator_cond (&sd_it, &dep))
8036 : /* INSN has no backward dependencies left. */
8037 : break;
8038 :
8039 0 : gcc_assert ((DEP_STATUS (dep) & BEGIN_SPEC) == 0
8040 : && (DEP_STATUS (dep) & BE_IN_SPEC) != 0
8041 : && (DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE);
8042 :
8043 0 : check = DEP_PRO (dep);
8044 :
8045 0 : gcc_assert (!IS_SPECULATION_CHECK_P (check) && !ORIG_PAT (check)
8046 : && QUEUE_INDEX (check) == QUEUE_NOWHERE);
8047 :
8048 0 : rec = BLOCK_FOR_INSN (check);
8049 :
8050 0 : twin = emit_insn_before (copy_insn (PATTERN (insn)), BB_END (rec));
8051 0 : haifa_init_insn (twin);
8052 :
8053 0 : sd_copy_back_deps (twin, insn, true);
8054 :
8055 0 : if (sched_verbose && spec_info->dump)
8056 : /* INSN_BB (insn) isn't determined for twin insns yet.
8057 : So we can't use current_sched_info->print_insn. */
8058 0 : fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n",
8059 0 : INSN_UID (twin), rec->index);
8060 :
8061 0 : twins.safe_push (twin);
8062 :
8063 : /* Add dependences between TWIN and all appropriate
8064 : instructions from REC. */
8065 0 : FOR_EACH_DEP (insn, SD_LIST_SPEC_BACK, sd_it, dep)
8066 : {
8067 0 : rtx_insn *pro = DEP_PRO (dep);
8068 :
8069 0 : gcc_assert (DEP_TYPE (dep) == REG_DEP_TRUE);
8070 :
8071 : /* INSN might have dependencies from the instructions from
8072 : several recovery blocks. At this iteration we process those
8073 : producers that reside in REC. */
8074 0 : if (BLOCK_FOR_INSN (pro) == rec)
8075 : {
8076 0 : dep_def _new_dep, *new_dep = &_new_dep;
8077 :
8078 0 : init_dep (new_dep, pro, twin, REG_DEP_TRUE);
8079 0 : sd_add_dep (new_dep, false);
8080 : }
8081 : }
8082 :
8083 0 : process_insn_forw_deps_be_in_spec (insn, twin, ts);
8084 :
8085 : /* Remove all dependencies between INSN and insns in REC. */
8086 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8087 0 : sd_iterator_cond (&sd_it, &dep);)
8088 : {
8089 0 : rtx_insn *pro = DEP_PRO (dep);
8090 :
8091 0 : if (BLOCK_FOR_INSN (pro) == rec)
8092 0 : sd_delete_dep (sd_it);
8093 : else
8094 0 : sd_iterator_next (&sd_it);
8095 : }
8096 0 : }
8097 :
8098 : /* We couldn't have added the dependencies between INSN and TWINS earlier
8099 : because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */
8100 0 : unsigned int i;
8101 0 : rtx_insn *twin;
8102 0 : FOR_EACH_VEC_ELT_REVERSE (twins, i, twin)
8103 : {
8104 0 : dep_def _new_dep, *new_dep = &_new_dep;
8105 :
8106 0 : init_dep (new_dep, insn, twin, REG_DEP_OUTPUT);
8107 0 : sd_add_dep (new_dep, false);
8108 : }
8109 :
8110 0 : calc_priorities (priorities_roots);
8111 0 : }
8112 :
8113 : /* Extends and fills with zeros (only the new part) array pointed to by P. */
8114 : void *
8115 10631293 : xrecalloc (void *p, size_t new_nmemb, size_t old_nmemb, size_t size)
8116 : {
8117 10631293 : gcc_assert (new_nmemb >= old_nmemb);
8118 10631293 : p = XRESIZEVAR (void, p, new_nmemb * size);
8119 10631293 : memset (((char *) p) + old_nmemb * size, 0, (new_nmemb - old_nmemb) * size);
8120 10631293 : return p;
8121 : }
8122 :
8123 : /* Helper function.
8124 : Find fallthru edge from PRED. */
8125 : edge
8126 122 : find_fallthru_edge_from (basic_block pred)
8127 : {
8128 122 : edge e;
8129 122 : basic_block succ;
8130 :
8131 122 : succ = pred->next_bb;
8132 122 : gcc_assert (succ->prev_bb == pred);
8133 :
8134 366 : if (EDGE_COUNT (pred->succs) <= EDGE_COUNT (succ->preds))
8135 : {
8136 40 : e = find_fallthru_edge (pred->succs);
8137 :
8138 40 : if (e)
8139 : {
8140 24 : gcc_assert (e->dest == succ || e->dest->index == EXIT_BLOCK);
8141 : return e;
8142 : }
8143 : }
8144 : else
8145 : {
8146 82 : e = find_fallthru_edge (succ->preds);
8147 :
8148 82 : if (e)
8149 : {
8150 82 : gcc_assert (e->src == pred);
8151 : return e;
8152 : }
8153 : }
8154 :
8155 : return NULL;
8156 : }
8157 :
8158 : /* Extend per basic block data structures. */
8159 : static void
8160 989103 : sched_extend_bb (void)
8161 : {
8162 : /* The following is done to keep current_sched_info->next_tail non null. */
8163 989103 : rtx_insn *end = BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb);
8164 989103 : rtx_insn *insn = DEBUG_INSN_P (end) ? prev_nondebug_insn (end) : end;
8165 989103 : if (NEXT_INSN (end) == 0
8166 989103 : || (!NOTE_P (insn)
8167 988638 : && !LABEL_P (insn)
8168 : /* Don't emit a NOTE if it would end up before a BARRIER. */
8169 988619 : && !BARRIER_P (next_nondebug_insn (end))))
8170 : {
8171 562 : rtx_note *note = emit_note_after (NOTE_INSN_DELETED, end);
8172 : /* Make note appear outside BB. */
8173 562 : set_block_for_insn (note, NULL);
8174 562 : BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb) = end;
8175 : }
8176 989103 : }
8177 :
8178 : /* Init per basic block data structures. */
8179 : void
8180 989103 : sched_init_bbs (void)
8181 : {
8182 989103 : sched_extend_bb ();
8183 989103 : }
8184 :
8185 : /* Initialize BEFORE_RECOVERY variable. */
8186 : static void
8187 0 : init_before_recovery (basic_block *before_recovery_ptr)
8188 : {
8189 0 : basic_block last;
8190 0 : edge e;
8191 :
8192 0 : last = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
8193 0 : e = find_fallthru_edge_from (last);
8194 :
8195 0 : if (e)
8196 : {
8197 : /* We create two basic blocks:
8198 : 1. Single instruction block is inserted right after E->SRC
8199 : and has jump to
8200 : 2. Empty block right before EXIT_BLOCK.
8201 : Between these two blocks recovery blocks will be emitted. */
8202 :
8203 0 : basic_block single, empty;
8204 :
8205 : /* If the fallthrough edge to exit we've found is from the block we've
8206 : created before, don't do anything more. */
8207 0 : if (last == after_recovery)
8208 : return;
8209 :
8210 0 : adding_bb_to_current_region_p = false;
8211 :
8212 0 : single = sched_create_empty_bb (last);
8213 0 : empty = sched_create_empty_bb (single);
8214 :
8215 : /* Add new blocks to the root loop. */
8216 0 : if (current_loops != NULL)
8217 : {
8218 0 : add_bb_to_loop (single, (*current_loops->larray)[0]);
8219 0 : add_bb_to_loop (empty, (*current_loops->larray)[0]);
8220 : }
8221 :
8222 0 : single->count = last->count;
8223 0 : empty->count = last->count;
8224 0 : BB_COPY_PARTITION (single, last);
8225 0 : BB_COPY_PARTITION (empty, last);
8226 :
8227 0 : redirect_edge_succ (e, single);
8228 0 : make_single_succ_edge (single, empty, 0);
8229 0 : make_single_succ_edge (empty, EXIT_BLOCK_PTR_FOR_FN (cfun),
8230 : EDGE_FALLTHRU);
8231 :
8232 0 : rtx_code_label *label = block_label (empty);
8233 0 : rtx_jump_insn *x = emit_jump_insn_after (targetm.gen_jump (label),
8234 0 : BB_END (single));
8235 0 : JUMP_LABEL (x) = label;
8236 0 : LABEL_NUSES (label)++;
8237 0 : haifa_init_insn (x);
8238 :
8239 0 : emit_barrier_after (x);
8240 :
8241 0 : sched_init_only_bb (empty, NULL);
8242 0 : sched_init_only_bb (single, NULL);
8243 0 : sched_extend_bb ();
8244 :
8245 0 : adding_bb_to_current_region_p = true;
8246 0 : before_recovery = single;
8247 0 : after_recovery = empty;
8248 :
8249 0 : if (before_recovery_ptr)
8250 0 : *before_recovery_ptr = before_recovery;
8251 :
8252 0 : if (sched_verbose >= 2 && spec_info->dump)
8253 0 : fprintf (spec_info->dump,
8254 : ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n",
8255 : last->index, single->index, empty->index);
8256 : }
8257 : else
8258 0 : before_recovery = last;
8259 : }
8260 :
8261 : /* Returns new recovery block. */
8262 : basic_block
8263 0 : sched_create_recovery_block (basic_block *before_recovery_ptr)
8264 : {
8265 0 : rtx_insn *barrier;
8266 0 : basic_block rec;
8267 :
8268 0 : haifa_recovery_bb_recently_added_p = true;
8269 0 : haifa_recovery_bb_ever_added_p = true;
8270 :
8271 0 : init_before_recovery (before_recovery_ptr);
8272 :
8273 0 : barrier = get_last_bb_insn (before_recovery);
8274 0 : gcc_assert (BARRIER_P (barrier));
8275 :
8276 0 : rtx_insn *label = emit_label_after (gen_label_rtx (), barrier);
8277 :
8278 0 : rec = create_basic_block (label, label, before_recovery);
8279 :
8280 : /* A recovery block always ends with an unconditional jump. */
8281 0 : emit_barrier_after (BB_END (rec));
8282 :
8283 0 : if (BB_PARTITION (before_recovery) != BB_UNPARTITIONED)
8284 0 : BB_SET_PARTITION (rec, BB_COLD_PARTITION);
8285 :
8286 0 : if (sched_verbose && spec_info->dump)
8287 0 : fprintf (spec_info->dump, ";;\t\tGenerated recovery block rec%d\n",
8288 : rec->index);
8289 :
8290 0 : return rec;
8291 : }
8292 :
8293 : /* Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB
8294 : and emit necessary jumps. */
8295 : void
8296 0 : sched_create_recovery_edges (basic_block first_bb, basic_block rec,
8297 : basic_block second_bb)
8298 : {
8299 0 : int edge_flags;
8300 :
8301 : /* This is fixing of incoming edge. */
8302 : /* ??? Which other flags should be specified? */
8303 0 : if (BB_PARTITION (first_bb) != BB_PARTITION (rec))
8304 : /* Partition type is the same, if it is "unpartitioned". */
8305 : edge_flags = EDGE_CROSSING;
8306 : else
8307 0 : edge_flags = 0;
8308 :
8309 0 : edge e2 = single_succ_edge (first_bb);
8310 0 : edge e = make_edge (first_bb, rec, edge_flags);
8311 :
8312 : /* TODO: The actual probability can be determined and is computed as
8313 : 'todo_spec' variable in create_check_block_twin and
8314 : in sel-sched.cc `check_ds' in create_speculation_check. */
8315 0 : e->probability = profile_probability::very_unlikely ();
8316 0 : rec->count = e->count ();
8317 0 : e2->probability = e->probability.invert ();
8318 :
8319 0 : rtx_code_label *label = block_label (second_bb);
8320 0 : rtx_jump_insn *jump = emit_jump_insn_after (targetm.gen_jump (label),
8321 0 : BB_END (rec));
8322 0 : JUMP_LABEL (jump) = label;
8323 0 : LABEL_NUSES (label)++;
8324 :
8325 0 : if (BB_PARTITION (second_bb) != BB_PARTITION (rec))
8326 : /* Partition type is the same, if it is "unpartitioned". */
8327 : {
8328 : /* Rewritten from cfgrtl.cc. */
8329 0 : if (crtl->has_bb_partition && targetm_common.have_named_sections)
8330 : {
8331 : /* We don't need the same note for the check because
8332 : any_condjump_p (check) == true. */
8333 0 : CROSSING_JUMP_P (jump) = 1;
8334 : }
8335 : edge_flags = EDGE_CROSSING;
8336 : }
8337 : else
8338 : edge_flags = 0;
8339 :
8340 0 : make_single_succ_edge (rec, second_bb, edge_flags);
8341 0 : if (dom_info_available_p (CDI_DOMINATORS))
8342 0 : set_immediate_dominator (CDI_DOMINATORS, rec, first_bb);
8343 0 : }
8344 :
8345 : /* This function creates recovery code for INSN. If MUTATE_P is nonzero,
8346 : INSN is a simple check, that should be converted to branchy one. */
8347 : static void
8348 0 : create_check_block_twin (rtx_insn *insn, bool mutate_p)
8349 : {
8350 0 : basic_block rec;
8351 0 : rtx_insn *label, *check, *twin;
8352 0 : rtx check_pat;
8353 0 : ds_t fs;
8354 0 : sd_iterator_def sd_it;
8355 0 : dep_t dep;
8356 0 : dep_def _new_dep, *new_dep = &_new_dep;
8357 0 : ds_t todo_spec;
8358 :
8359 0 : gcc_assert (ORIG_PAT (insn) != NULL_RTX);
8360 :
8361 0 : if (!mutate_p)
8362 0 : todo_spec = TODO_SPEC (insn);
8363 : else
8364 : {
8365 0 : gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn)
8366 : && (TODO_SPEC (insn) & SPECULATIVE) == 0);
8367 :
8368 0 : todo_spec = CHECK_SPEC (insn);
8369 : }
8370 :
8371 0 : todo_spec &= SPECULATIVE;
8372 :
8373 : /* Create recovery block. */
8374 0 : if (mutate_p || targetm.sched.needs_block_p (todo_spec))
8375 : {
8376 0 : rec = sched_create_recovery_block (NULL);
8377 0 : label = BB_HEAD (rec);
8378 : }
8379 : else
8380 : {
8381 0 : rec = EXIT_BLOCK_PTR_FOR_FN (cfun);
8382 0 : label = NULL;
8383 : }
8384 :
8385 : /* Emit CHECK. */
8386 0 : check_pat = targetm.sched.gen_spec_check (insn, label, todo_spec);
8387 :
8388 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8389 : {
8390 : /* To have mem_reg alive at the beginning of second_bb,
8391 : we emit check BEFORE insn, so insn after splitting
8392 : insn will be at the beginning of second_bb, which will
8393 : provide us with the correct life information. */
8394 0 : check = emit_jump_insn_before (check_pat, insn);
8395 0 : JUMP_LABEL (check) = label;
8396 0 : LABEL_NUSES (label)++;
8397 : }
8398 : else
8399 0 : check = emit_insn_before (check_pat, insn);
8400 :
8401 : /* Extend data structures. */
8402 0 : haifa_init_insn (check);
8403 :
8404 : /* CHECK is being added to current region. Extend ready list. */
8405 0 : gcc_assert (sched_ready_n_insns != -1);
8406 0 : sched_extend_ready_list (sched_ready_n_insns + 1);
8407 :
8408 0 : if (current_sched_info->add_remove_insn)
8409 0 : current_sched_info->add_remove_insn (insn, 0);
8410 :
8411 0 : RECOVERY_BLOCK (check) = rec;
8412 :
8413 0 : if (sched_verbose && spec_info->dump)
8414 0 : fprintf (spec_info->dump, ";;\t\tGenerated check insn : %s\n",
8415 0 : (*current_sched_info->print_insn) (check, 0));
8416 :
8417 0 : gcc_assert (ORIG_PAT (insn));
8418 :
8419 : /* Initialize TWIN (twin is a duplicate of original instruction
8420 : in the recovery block). */
8421 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8422 : {
8423 0 : sd_iterator_def sd_it;
8424 0 : dep_t dep;
8425 :
8426 0 : FOR_EACH_DEP (insn, SD_LIST_RES_BACK, sd_it, dep)
8427 0 : if ((DEP_STATUS (dep) & DEP_OUTPUT) != 0)
8428 : {
8429 0 : struct _dep _dep2, *dep2 = &_dep2;
8430 :
8431 0 : init_dep (dep2, DEP_PRO (dep), check, REG_DEP_TRUE);
8432 :
8433 0 : sd_add_dep (dep2, true);
8434 : }
8435 :
8436 0 : twin = emit_insn_after (ORIG_PAT (insn), BB_END (rec));
8437 0 : haifa_init_insn (twin);
8438 :
8439 0 : if (sched_verbose && spec_info->dump)
8440 : /* INSN_BB (insn) isn't determined for twin insns yet.
8441 : So we can't use current_sched_info->print_insn. */
8442 0 : fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n",
8443 0 : INSN_UID (twin), rec->index);
8444 : }
8445 : else
8446 : {
8447 0 : ORIG_PAT (check) = ORIG_PAT (insn);
8448 0 : HAS_INTERNAL_DEP (check) = 1;
8449 0 : twin = check;
8450 : /* ??? We probably should change all OUTPUT dependencies to
8451 : (TRUE | OUTPUT). */
8452 : }
8453 :
8454 : /* Copy all resolved back dependencies of INSN to TWIN. This will
8455 : provide correct value for INSN_TICK (TWIN). */
8456 0 : sd_copy_back_deps (twin, insn, true);
8457 :
8458 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8459 : /* In case of branchy check, fix CFG. */
8460 : {
8461 0 : basic_block first_bb, second_bb;
8462 0 : rtx_insn *jump;
8463 :
8464 0 : first_bb = BLOCK_FOR_INSN (check);
8465 0 : second_bb = sched_split_block (first_bb, check);
8466 :
8467 0 : sched_create_recovery_edges (first_bb, rec, second_bb);
8468 :
8469 0 : sched_init_only_bb (second_bb, first_bb);
8470 0 : sched_init_only_bb (rec, EXIT_BLOCK_PTR_FOR_FN (cfun));
8471 :
8472 0 : jump = BB_END (rec);
8473 0 : haifa_init_insn (jump);
8474 : }
8475 :
8476 : /* Move backward dependences from INSN to CHECK and
8477 : move forward dependences from INSN to TWIN. */
8478 :
8479 : /* First, create dependencies between INSN's producers and CHECK & TWIN. */
8480 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
8481 : {
8482 0 : rtx_insn *pro = DEP_PRO (dep);
8483 0 : ds_t ds;
8484 :
8485 : /* If BEGIN_DATA: [insn ~~TRUE~~> producer]:
8486 : check --TRUE--> producer ??? or ANTI ???
8487 : twin --TRUE--> producer
8488 : twin --ANTI--> check
8489 :
8490 : If BEGIN_CONTROL: [insn ~~ANTI~~> producer]:
8491 : check --ANTI--> producer
8492 : twin --ANTI--> producer
8493 : twin --ANTI--> check
8494 :
8495 : If BE_IN_SPEC: [insn ~~TRUE~~> producer]:
8496 : check ~~TRUE~~> producer
8497 : twin ~~TRUE~~> producer
8498 : twin --ANTI--> check */
8499 :
8500 0 : ds = DEP_STATUS (dep);
8501 :
8502 0 : if (ds & BEGIN_SPEC)
8503 : {
8504 0 : gcc_assert (!mutate_p);
8505 0 : ds &= ~BEGIN_SPEC;
8506 : }
8507 :
8508 0 : init_dep_1 (new_dep, pro, check, DEP_TYPE (dep), ds);
8509 0 : sd_add_dep (new_dep, false);
8510 :
8511 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8512 : {
8513 0 : DEP_CON (new_dep) = twin;
8514 0 : sd_add_dep (new_dep, false);
8515 : }
8516 : }
8517 :
8518 : /* Second, remove backward dependencies of INSN. */
8519 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
8520 0 : sd_iterator_cond (&sd_it, &dep);)
8521 : {
8522 0 : if ((DEP_STATUS (dep) & BEGIN_SPEC)
8523 0 : || mutate_p)
8524 : /* We can delete this dep because we overcome it with
8525 : BEGIN_SPECULATION. */
8526 0 : sd_delete_dep (sd_it);
8527 : else
8528 0 : sd_iterator_next (&sd_it);
8529 : }
8530 :
8531 : /* Future Speculations. Determine what BE_IN speculations will be like. */
8532 0 : fs = 0;
8533 :
8534 : /* Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only
8535 : here. */
8536 :
8537 0 : gcc_assert (!DONE_SPEC (insn));
8538 :
8539 0 : if (!mutate_p)
8540 : {
8541 0 : ds_t ts = TODO_SPEC (insn);
8542 :
8543 0 : DONE_SPEC (insn) = ts & BEGIN_SPEC;
8544 0 : CHECK_SPEC (check) = ts & BEGIN_SPEC;
8545 :
8546 : /* Luckiness of future speculations solely depends upon initial
8547 : BEGIN speculation. */
8548 0 : if (ts & BEGIN_DATA)
8549 0 : fs = set_dep_weak (fs, BE_IN_DATA, get_dep_weak (ts, BEGIN_DATA));
8550 0 : if (ts & BEGIN_CONTROL)
8551 0 : fs = set_dep_weak (fs, BE_IN_CONTROL,
8552 : get_dep_weak (ts, BEGIN_CONTROL));
8553 : }
8554 : else
8555 0 : CHECK_SPEC (check) = CHECK_SPEC (insn);
8556 :
8557 : /* Future speculations: call the helper. */
8558 0 : process_insn_forw_deps_be_in_spec (insn, twin, fs);
8559 :
8560 0 : if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
8561 : {
8562 : /* Which types of dependencies should we use here is,
8563 : generally, machine-dependent question... But, for now,
8564 : it is not. */
8565 :
8566 0 : if (!mutate_p)
8567 : {
8568 0 : init_dep (new_dep, insn, check, REG_DEP_TRUE);
8569 0 : sd_add_dep (new_dep, false);
8570 :
8571 0 : init_dep (new_dep, insn, twin, REG_DEP_OUTPUT);
8572 0 : sd_add_dep (new_dep, false);
8573 : }
8574 : else
8575 : {
8576 0 : if (spec_info->dump)
8577 0 : fprintf (spec_info->dump, ";;\t\tRemoved simple check : %s\n",
8578 0 : (*current_sched_info->print_insn) (insn, 0));
8579 :
8580 : /* Remove all dependencies of the INSN. */
8581 0 : {
8582 0 : sd_it = sd_iterator_start (insn, (SD_LIST_FORW
8583 : | SD_LIST_BACK
8584 : | SD_LIST_RES_BACK));
8585 0 : while (sd_iterator_cond (&sd_it, &dep))
8586 0 : sd_delete_dep (sd_it);
8587 : }
8588 :
8589 : /* If former check (INSN) already was moved to the ready (or queue)
8590 : list, add new check (CHECK) there too. */
8591 0 : if (QUEUE_INDEX (insn) != QUEUE_NOWHERE)
8592 0 : try_ready (check);
8593 :
8594 : /* Remove old check from instruction stream and free its
8595 : data. */
8596 0 : sched_remove_insn (insn);
8597 : }
8598 :
8599 0 : init_dep (new_dep, check, twin, REG_DEP_ANTI);
8600 0 : sd_add_dep (new_dep, false);
8601 : }
8602 : else
8603 : {
8604 0 : init_dep_1 (new_dep, insn, check, REG_DEP_TRUE, DEP_TRUE | DEP_OUTPUT);
8605 0 : sd_add_dep (new_dep, false);
8606 : }
8607 :
8608 0 : if (!mutate_p)
8609 : /* Fix priorities. If MUTATE_P is nonzero, this is not necessary,
8610 : because it'll be done later in add_to_speculative_block. */
8611 : {
8612 0 : auto_vec<rtx_insn *> priorities_roots;
8613 :
8614 0 : clear_priorities (twin, &priorities_roots);
8615 0 : calc_priorities (priorities_roots);
8616 0 : }
8617 0 : }
8618 :
8619 : /* Removes dependency between instructions in the recovery block REC
8620 : and usual region instructions. It keeps inner dependences so it
8621 : won't be necessary to recompute them. */
8622 : static void
8623 0 : fix_recovery_deps (basic_block rec)
8624 : {
8625 0 : rtx_insn *note, *insn, *jump;
8626 0 : auto_vec<rtx_insn *, 10> ready_list;
8627 0 : auto_bitmap in_ready;
8628 :
8629 : /* NOTE - a basic block note. */
8630 0 : note = NEXT_INSN (BB_HEAD (rec));
8631 0 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8632 0 : insn = BB_END (rec);
8633 0 : gcc_assert (JUMP_P (insn));
8634 0 : insn = PREV_INSN (insn);
8635 :
8636 0 : do
8637 : {
8638 0 : sd_iterator_def sd_it;
8639 0 : dep_t dep;
8640 :
8641 0 : for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
8642 0 : sd_iterator_cond (&sd_it, &dep);)
8643 : {
8644 0 : rtx_insn *consumer = DEP_CON (dep);
8645 :
8646 0 : if (BLOCK_FOR_INSN (consumer) != rec)
8647 : {
8648 0 : sd_delete_dep (sd_it);
8649 :
8650 0 : if (bitmap_set_bit (in_ready, INSN_LUID (consumer)))
8651 0 : ready_list.safe_push (consumer);
8652 : }
8653 : else
8654 : {
8655 0 : gcc_assert ((DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE);
8656 :
8657 0 : sd_iterator_next (&sd_it);
8658 : }
8659 : }
8660 :
8661 0 : insn = PREV_INSN (insn);
8662 : }
8663 0 : while (insn != note);
8664 :
8665 : /* Try to add instructions to the ready or queue list. */
8666 0 : unsigned int i;
8667 0 : rtx_insn *temp;
8668 0 : FOR_EACH_VEC_ELT_REVERSE (ready_list, i, temp)
8669 0 : try_ready (temp);
8670 :
8671 : /* Fixing jump's dependences. */
8672 0 : insn = BB_HEAD (rec);
8673 0 : jump = BB_END (rec);
8674 :
8675 0 : gcc_assert (LABEL_P (insn));
8676 0 : insn = NEXT_INSN (insn);
8677 :
8678 0 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn));
8679 0 : add_jump_dependencies (insn, jump);
8680 0 : }
8681 :
8682 : /* Change pattern of INSN to NEW_PAT. Invalidate cached haifa
8683 : instruction data. */
8684 : static bool
8685 0 : haifa_change_pattern (rtx_insn *insn, rtx new_pat)
8686 : {
8687 0 : int t;
8688 :
8689 0 : t = validate_change (insn, &PATTERN (insn), new_pat, 0);
8690 0 : if (!t)
8691 : return false;
8692 :
8693 0 : update_insn_after_change (insn);
8694 0 : return true;
8695 : }
8696 :
8697 : /* -1 - can't speculate,
8698 : 0 - for speculation with REQUEST mode it is OK to use
8699 : current instruction pattern,
8700 : 1 - need to change pattern for *NEW_PAT to be speculative. */
8701 : int
8702 0 : sched_speculate_insn (rtx_insn *insn, ds_t request, rtx *new_pat)
8703 : {
8704 0 : gcc_assert (current_sched_info->flags & DO_SPECULATION
8705 : && (request & SPECULATIVE)
8706 : && sched_insn_is_legitimate_for_speculation_p (insn, request));
8707 :
8708 0 : if ((request & spec_info->mask) != request)
8709 : return -1;
8710 :
8711 0 : if (request & BE_IN_SPEC
8712 0 : && !(request & BEGIN_SPEC))
8713 : return 0;
8714 :
8715 0 : return targetm.sched.speculate_insn (insn, request, new_pat);
8716 : }
8717 :
8718 : static int
8719 0 : haifa_speculate_insn (rtx_insn *insn, ds_t request, rtx *new_pat)
8720 : {
8721 0 : gcc_assert (sched_deps_info->generate_spec_deps
8722 : && !IS_SPECULATION_CHECK_P (insn));
8723 :
8724 0 : if (HAS_INTERNAL_DEP (insn)
8725 0 : || SCHED_GROUP_P (insn))
8726 : return -1;
8727 :
8728 0 : return sched_speculate_insn (insn, request, new_pat);
8729 : }
8730 :
8731 : /* Print some information about block BB, which starts with HEAD and
8732 : ends with TAIL, before scheduling it.
8733 : I is zero, if scheduler is about to start with the fresh ebb. */
8734 : static void
8735 192 : dump_new_block_header (int i, basic_block bb, rtx_insn *head, rtx_insn *tail)
8736 : {
8737 192 : if (!i)
8738 190 : fprintf (sched_dump,
8739 : ";; ======================================================\n");
8740 : else
8741 2 : fprintf (sched_dump,
8742 : ";; =====================ADVANCING TO=====================\n");
8743 192 : fprintf (sched_dump,
8744 : ";; -- basic block %d from %d to %d -- %s reload\n",
8745 192 : bb->index, INSN_UID (head), INSN_UID (tail),
8746 192 : (reload_completed ? "after" : "before"));
8747 192 : fprintf (sched_dump,
8748 : ";; ======================================================\n");
8749 192 : fprintf (sched_dump, "\n");
8750 192 : }
8751 :
8752 : /* Unlink basic block notes and labels and saves them, so they
8753 : can be easily restored. We unlink basic block notes in EBB to
8754 : provide back-compatibility with the previous code, as target backends
8755 : assume, that there'll be only instructions between
8756 : current_sched_info->{head and tail}. We restore these notes as soon
8757 : as we can.
8758 : FIRST (LAST) is the first (last) basic block in the ebb.
8759 : NB: In usual case (FIRST == LAST) nothing is really done. */
8760 : void
8761 10610721 : unlink_bb_notes (basic_block first, basic_block last)
8762 : {
8763 : /* We DON'T unlink basic block notes of the first block in the ebb. */
8764 10610721 : if (first == last)
8765 : return;
8766 :
8767 14 : bb_header = XNEWVEC (rtx_insn *, last_basic_block_for_fn (cfun));
8768 :
8769 : /* Make a sentinel. */
8770 14 : if (last->next_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
8771 14 : bb_header[last->next_bb->index] = 0;
8772 :
8773 14 : first = first->next_bb;
8774 82 : do
8775 : {
8776 48 : rtx_insn *prev, *label, *note, *next;
8777 :
8778 48 : label = BB_HEAD (last);
8779 48 : if (LABEL_P (label))
8780 0 : note = NEXT_INSN (label);
8781 : else
8782 : note = label;
8783 48 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8784 :
8785 48 : prev = PREV_INSN (label);
8786 48 : next = NEXT_INSN (note);
8787 48 : gcc_assert (prev && next);
8788 :
8789 48 : SET_NEXT_INSN (prev) = next;
8790 48 : SET_PREV_INSN (next) = prev;
8791 :
8792 48 : bb_header[last->index] = label;
8793 :
8794 48 : if (last == first)
8795 : break;
8796 :
8797 34 : last = last->prev_bb;
8798 34 : }
8799 : while (1);
8800 : }
8801 :
8802 : /* Restore basic block notes.
8803 : FIRST is the first basic block in the ebb. */
8804 : static void
8805 10610721 : restore_bb_notes (basic_block first)
8806 : {
8807 10610721 : if (!bb_header)
8808 : return;
8809 :
8810 : /* We DON'T unlink basic block notes of the first block in the ebb. */
8811 14 : first = first->next_bb;
8812 : /* Remember: FIRST is actually a second basic block in the ebb. */
8813 :
8814 14 : while (first != EXIT_BLOCK_PTR_FOR_FN (cfun)
8815 62 : && bb_header[first->index])
8816 : {
8817 48 : rtx_insn *prev, *label, *note, *next;
8818 :
8819 48 : label = bb_header[first->index];
8820 48 : prev = PREV_INSN (label);
8821 48 : next = NEXT_INSN (prev);
8822 :
8823 48 : if (LABEL_P (label))
8824 0 : note = NEXT_INSN (label);
8825 : else
8826 : note = label;
8827 48 : gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
8828 :
8829 48 : bb_header[first->index] = 0;
8830 :
8831 48 : SET_NEXT_INSN (prev) = label;
8832 48 : SET_NEXT_INSN (note) = next;
8833 48 : SET_PREV_INSN (next) = note;
8834 :
8835 48 : first = first->next_bb;
8836 : }
8837 :
8838 14 : free (bb_header);
8839 14 : bb_header = 0;
8840 : }
8841 :
8842 : /* Helper function.
8843 : Fix CFG after both in- and inter-block movement of
8844 : control_flow_insn_p JUMP. */
8845 : static void
8846 1 : fix_jump_move (rtx_insn *jump)
8847 : {
8848 1 : basic_block bb, jump_bb, jump_bb_next;
8849 :
8850 1 : bb = BLOCK_FOR_INSN (PREV_INSN (jump));
8851 1 : jump_bb = BLOCK_FOR_INSN (jump);
8852 1 : jump_bb_next = jump_bb->next_bb;
8853 :
8854 1 : gcc_assert (common_sched_info->sched_pass_id == SCHED_EBB_PASS
8855 : || IS_SPECULATION_BRANCHY_CHECK_P (jump));
8856 :
8857 1 : if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next)))
8858 : /* if jump_bb_next is not empty. */
8859 0 : BB_END (jump_bb) = BB_END (jump_bb_next);
8860 :
8861 1 : if (BB_END (bb) != PREV_INSN (jump))
8862 : /* Then there are instruction after jump that should be placed
8863 : to jump_bb_next. */
8864 : BB_END (jump_bb_next) = BB_END (bb);
8865 : else
8866 : /* Otherwise jump_bb_next is empty. */
8867 0 : BB_END (jump_bb_next) = NEXT_INSN (BB_HEAD (jump_bb_next));
8868 :
8869 : /* To make assertion in move_insn happy. */
8870 1 : BB_END (bb) = PREV_INSN (jump);
8871 :
8872 1 : update_bb_for_insn (jump_bb_next);
8873 1 : }
8874 :
8875 : /* Fix CFG after interblock movement of control_flow_insn_p JUMP. */
8876 : static void
8877 0 : move_block_after_check (rtx_insn *jump)
8878 : {
8879 0 : basic_block bb, jump_bb, jump_bb_next;
8880 0 : vec<edge, va_gc> *t;
8881 :
8882 0 : bb = BLOCK_FOR_INSN (PREV_INSN (jump));
8883 0 : jump_bb = BLOCK_FOR_INSN (jump);
8884 0 : jump_bb_next = jump_bb->next_bb;
8885 :
8886 0 : update_bb_for_insn (jump_bb);
8887 :
8888 0 : gcc_assert (IS_SPECULATION_CHECK_P (jump)
8889 : || IS_SPECULATION_CHECK_P (BB_END (jump_bb_next)));
8890 :
8891 0 : unlink_block (jump_bb_next);
8892 0 : link_block (jump_bb_next, bb);
8893 :
8894 0 : t = bb->succs;
8895 0 : bb->succs = 0;
8896 0 : move_succs (&(jump_bb->succs), bb);
8897 0 : move_succs (&(jump_bb_next->succs), jump_bb);
8898 0 : move_succs (&t, jump_bb_next);
8899 :
8900 0 : df_mark_solutions_dirty ();
8901 :
8902 0 : common_sched_info->fix_recovery_cfg
8903 0 : (bb->index, jump_bb->index, jump_bb_next->index);
8904 0 : }
8905 :
8906 : /* Helper function for move_block_after_check.
8907 : This functions attaches edge vector pointed to by SUCCSP to
8908 : block TO. */
8909 : static void
8910 0 : move_succs (vec<edge, va_gc> **succsp, basic_block to)
8911 : {
8912 0 : edge e;
8913 0 : edge_iterator ei;
8914 :
8915 0 : gcc_assert (to->succs == 0);
8916 :
8917 0 : to->succs = *succsp;
8918 :
8919 0 : FOR_EACH_EDGE (e, ei, to->succs)
8920 0 : e->src = to;
8921 :
8922 0 : *succsp = 0;
8923 0 : }
8924 :
8925 : /* Remove INSN from the instruction stream.
8926 : INSN should have any dependencies. */
8927 : static void
8928 0 : sched_remove_insn (rtx_insn *insn)
8929 : {
8930 0 : sd_finish_insn (insn);
8931 :
8932 0 : change_queue_index (insn, QUEUE_NOWHERE);
8933 0 : current_sched_info->add_remove_insn (insn, 1);
8934 0 : delete_insn (insn);
8935 0 : }
8936 :
8937 : /* Clear priorities of all instructions, that are forward dependent on INSN.
8938 : Store in vector pointed to by ROOTS_PTR insns on which priority () should
8939 : be invoked to initialize all cleared priorities. */
8940 : static void
8941 0 : clear_priorities (rtx_insn *insn, rtx_vec_t *roots_ptr)
8942 : {
8943 0 : sd_iterator_def sd_it;
8944 0 : dep_t dep;
8945 0 : bool insn_is_root_p = true;
8946 :
8947 0 : gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED);
8948 :
8949 0 : FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
8950 : {
8951 0 : rtx_insn *pro = DEP_PRO (dep);
8952 :
8953 0 : if (INSN_PRIORITY_STATUS (pro) >= 0
8954 0 : && QUEUE_INDEX (insn) != QUEUE_SCHEDULED)
8955 : {
8956 : /* If DEP doesn't contribute to priority then INSN itself should
8957 : be added to priority roots. */
8958 0 : if (contributes_to_priority_p (dep))
8959 0 : insn_is_root_p = false;
8960 :
8961 0 : INSN_PRIORITY_STATUS (pro) = -1;
8962 0 : clear_priorities (pro, roots_ptr);
8963 : }
8964 : }
8965 :
8966 0 : if (insn_is_root_p)
8967 0 : roots_ptr->safe_push (insn);
8968 0 : }
8969 :
8970 : /* Recompute priorities of instructions, whose priorities might have been
8971 : changed. ROOTS is a vector of instructions whose priority computation will
8972 : trigger initialization of all cleared priorities. */
8973 : static void
8974 0 : calc_priorities (const rtx_vec_t &roots)
8975 : {
8976 0 : int i;
8977 0 : rtx_insn *insn;
8978 :
8979 0 : FOR_EACH_VEC_ELT (roots, i, insn)
8980 0 : priority (insn);
8981 0 : }
8982 :
8983 :
8984 : /* Add dependences between JUMP and other instructions in the recovery
8985 : block. INSN is the first insn the recovery block. */
8986 : static void
8987 0 : add_jump_dependencies (rtx_insn *insn, rtx_insn *jump)
8988 : {
8989 0 : do
8990 : {
8991 0 : insn = NEXT_INSN (insn);
8992 0 : if (insn == jump)
8993 : break;
8994 :
8995 0 : if (dep_list_size (insn, SD_LIST_FORW) == 0)
8996 : {
8997 0 : dep_def _new_dep, *new_dep = &_new_dep;
8998 :
8999 0 : init_dep (new_dep, insn, jump, REG_DEP_ANTI);
9000 0 : sd_add_dep (new_dep, false);
9001 : }
9002 : }
9003 : while (1);
9004 :
9005 0 : gcc_assert (!sd_lists_empty_p (jump, SD_LIST_BACK));
9006 0 : }
9007 :
9008 : /* Extend data structures for logical insn UID. */
9009 : void
9010 991728 : sched_extend_luids (void)
9011 : {
9012 991728 : int new_luids_max_uid = get_max_uid () + 1;
9013 :
9014 991728 : sched_luids.safe_grow_cleared (new_luids_max_uid, true);
9015 991728 : }
9016 :
9017 : /* Initialize LUID for INSN. */
9018 : void
9019 140442797 : sched_init_insn_luid (rtx_insn *insn)
9020 : {
9021 140442797 : int i = INSN_P (insn) ? 1 : common_sched_info->luid_for_non_insn (insn);
9022 24316821 : int luid;
9023 :
9024 24316821 : if (i >= 0)
9025 : {
9026 140441304 : luid = sched_max_luid;
9027 140441304 : sched_max_luid += i;
9028 : }
9029 : else
9030 : luid = -1;
9031 :
9032 140442797 : SET_INSN_LUID (insn, luid);
9033 140442797 : }
9034 :
9035 : /* Initialize luids for BBS.
9036 : The hook common_sched_info->luid_for_non_insn () is used to determine
9037 : if notes, labels, etc. need luids. */
9038 : void
9039 989608 : sched_init_luids (const bb_vec_t &bbs)
9040 : {
9041 989608 : int i;
9042 989608 : basic_block bb;
9043 :
9044 989608 : sched_extend_luids ();
9045 12612551 : FOR_EACH_VEC_ELT (bbs, i, bb)
9046 : {
9047 10633335 : rtx_insn *insn;
9048 :
9049 151074833 : FOR_BB_INSNS (bb, insn)
9050 140441498 : sched_init_insn_luid (insn);
9051 : }
9052 989608 : }
9053 :
9054 : /* Free LUIDs. */
9055 : void
9056 989608 : sched_finish_luids (void)
9057 : {
9058 989608 : sched_luids.release ();
9059 989608 : sched_max_luid = 1;
9060 989608 : }
9061 :
9062 : /* Return logical uid of INSN. Helpful while debugging. */
9063 : int
9064 0 : insn_luid (rtx_insn *insn)
9065 : {
9066 0 : return INSN_LUID (insn);
9067 : }
9068 :
9069 : /* Extend per insn data in the target. */
9070 : void
9071 1982011 : sched_extend_target (void)
9072 : {
9073 1982011 : if (targetm.sched.h_i_d_extended)
9074 0 : targetm.sched.h_i_d_extended ();
9075 1982011 : }
9076 :
9077 : /* Extend global scheduler structures (those, that live across calls to
9078 : schedule_block) to include information about just emitted INSN. */
9079 : static void
9080 989608 : extend_h_i_d (void)
9081 : {
9082 989608 : int reserve = (get_max_uid () + 1 - h_i_d.length ());
9083 989608 : if (reserve > 0
9084 1978636 : && ! h_i_d.space (reserve))
9085 : {
9086 989028 : h_i_d.safe_grow_cleared (3U * get_max_uid () / 2, true);
9087 989028 : sched_extend_target ();
9088 : }
9089 989608 : }
9090 :
9091 : /* Initialize h_i_d entry of the INSN with default values.
9092 : Values, that are not explicitly initialized here, hold zero. */
9093 : static void
9094 140441498 : init_h_i_d (rtx_insn *insn)
9095 : {
9096 140441498 : if (INSN_LUID (insn) > 0)
9097 : {
9098 140440005 : INSN_COST (insn) = -1;
9099 140440005 : QUEUE_INDEX (insn) = QUEUE_NOWHERE;
9100 140440005 : INSN_TICK (insn) = INVALID_TICK;
9101 140440005 : INSN_EXACT_TICK (insn) = INVALID_TICK;
9102 140440005 : INTER_TICK (insn) = INVALID_TICK;
9103 140440005 : TODO_SPEC (insn) = HARD_DEP;
9104 140440005 : INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status
9105 140440005 : = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
9106 140440005 : INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status
9107 140440005 : = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
9108 : }
9109 140441498 : }
9110 :
9111 : /* Initialize haifa_insn_data for BBS. */
9112 : void
9113 989608 : haifa_init_h_i_d (const bb_vec_t &bbs)
9114 : {
9115 989608 : int i;
9116 989608 : basic_block bb;
9117 :
9118 989608 : extend_h_i_d ();
9119 12612551 : FOR_EACH_VEC_ELT (bbs, i, bb)
9120 : {
9121 10633335 : rtx_insn *insn;
9122 :
9123 151074833 : FOR_BB_INSNS (bb, insn)
9124 140441498 : init_h_i_d (insn);
9125 : }
9126 989608 : }
9127 :
9128 : /* Finalize haifa_insn_data. */
9129 : void
9130 989027 : haifa_finish_h_i_d (void)
9131 : {
9132 989027 : int i;
9133 989027 : haifa_insn_data_t data;
9134 989027 : reg_use_data *use, *next_use;
9135 989027 : reg_set_data *set, *next_set;
9136 :
9137 325461387 : FOR_EACH_VEC_ELT (h_i_d, i, data)
9138 : {
9139 324472360 : free (data->max_reg_pressure);
9140 324472360 : free (data->reg_pressure);
9141 324474849 : for (use = data->reg_use_list; use != NULL; use = next_use)
9142 : {
9143 2489 : next_use = use->next_insn_use;
9144 2489 : free (use);
9145 : }
9146 324474683 : for (set = data->reg_set_list; set != NULL; set = next_set)
9147 : {
9148 2323 : next_set = set->next_insn_set;
9149 2323 : free (set);
9150 : }
9151 :
9152 : }
9153 989027 : h_i_d.release ();
9154 989027 : }
9155 :
9156 : /* Init data for the new insn INSN. */
9157 : static void
9158 0 : haifa_init_insn (rtx_insn *insn)
9159 : {
9160 0 : gcc_assert (insn != NULL);
9161 :
9162 0 : sched_extend_luids ();
9163 0 : sched_init_insn_luid (insn);
9164 0 : sched_extend_target ();
9165 0 : sched_deps_init (false);
9166 0 : extend_h_i_d ();
9167 0 : init_h_i_d (insn);
9168 :
9169 0 : if (adding_bb_to_current_region_p)
9170 : {
9171 0 : sd_init_insn (insn);
9172 :
9173 : /* Extend dependency caches by one element. */
9174 0 : extend_dependency_caches (1, false);
9175 : }
9176 0 : if (sched_pressure != SCHED_PRESSURE_NONE)
9177 0 : init_insn_reg_pressure_info (insn);
9178 0 : }
9179 :
9180 : /* Init data for the new basic block BB which comes after AFTER. */
9181 : static void
9182 1 : haifa_init_only_bb (basic_block bb, basic_block after)
9183 : {
9184 1 : gcc_assert (bb != NULL);
9185 :
9186 1 : sched_init_bbs ();
9187 :
9188 1 : if (common_sched_info->add_block)
9189 : /* This changes only data structures of the front-end. */
9190 1 : common_sched_info->add_block (bb, after);
9191 1 : }
9192 :
9193 : /* A generic version of sched_split_block (). */
9194 : basic_block
9195 67 : sched_split_block_1 (basic_block first_bb, rtx after)
9196 : {
9197 67 : edge e;
9198 :
9199 67 : e = split_block (first_bb, after);
9200 67 : gcc_assert (e->src == first_bb);
9201 :
9202 : /* sched_split_block emits note if *check == BB_END. Probably it
9203 : is better to rip that note off. */
9204 :
9205 67 : return e->dest;
9206 : }
9207 :
9208 : /* A generic version of sched_create_empty_bb (). */
9209 : basic_block
9210 0 : sched_create_empty_bb_1 (basic_block after)
9211 : {
9212 0 : return create_empty_bb (after);
9213 : }
9214 :
9215 : /* Insert PAT as an INSN into the schedule and update the necessary data
9216 : structures to account for it. */
9217 : rtx_insn *
9218 0 : sched_emit_insn (rtx pat)
9219 : {
9220 0 : rtx_insn *insn = emit_insn_before (pat, first_nonscheduled_insn ());
9221 0 : haifa_init_insn (insn);
9222 :
9223 0 : if (current_sched_info->add_remove_insn)
9224 0 : current_sched_info->add_remove_insn (insn, 0);
9225 :
9226 0 : (*current_sched_info->begin_schedule_ready) (insn);
9227 0 : scheduled_insns.safe_push (insn);
9228 :
9229 0 : last_scheduled_insn = insn;
9230 0 : return insn;
9231 : }
9232 :
9233 : /* This function returns a candidate satisfying dispatch constraints from
9234 : the ready list. */
9235 :
9236 : static rtx_insn *
9237 16 : ready_remove_first_dispatch (struct ready_list *ready)
9238 : {
9239 16 : int i;
9240 16 : rtx_insn *insn = ready_element (ready, 0);
9241 :
9242 16 : if (ready->n_ready == 1
9243 0 : || !INSN_P (insn)
9244 0 : || INSN_CODE (insn) < 0
9245 0 : || !active_insn_p (insn)
9246 16 : || targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW))
9247 16 : return ready_remove_first (ready);
9248 :
9249 0 : for (i = 1; i < ready->n_ready; i++)
9250 : {
9251 0 : insn = ready_element (ready, i);
9252 :
9253 0 : if (!INSN_P (insn)
9254 0 : || INSN_CODE (insn) < 0
9255 0 : || !active_insn_p (insn))
9256 0 : continue;
9257 :
9258 0 : if (targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW))
9259 : {
9260 : /* Return ith element of ready. */
9261 0 : insn = ready_remove (ready, i);
9262 0 : return insn;
9263 : }
9264 : }
9265 :
9266 0 : if (targetm.sched.dispatch (NULL, DISPATCH_VIOLATION))
9267 0 : return ready_remove_first (ready);
9268 :
9269 0 : for (i = 1; i < ready->n_ready; i++)
9270 : {
9271 0 : insn = ready_element (ready, i);
9272 :
9273 0 : if (!INSN_P (insn)
9274 0 : || INSN_CODE (insn) < 0
9275 0 : || !active_insn_p (insn))
9276 0 : continue;
9277 :
9278 : /* Return i-th element of ready. */
9279 0 : if (targetm.sched.dispatch (insn, IS_CMP))
9280 0 : return ready_remove (ready, i);
9281 : }
9282 :
9283 0 : return ready_remove_first (ready);
9284 : }
9285 :
9286 : /* Get number of ready insn in the ready list. */
9287 :
9288 : int
9289 0 : number_in_ready (void)
9290 : {
9291 0 : return ready.n_ready;
9292 : }
9293 :
9294 : /* Get number of ready's in the ready list. */
9295 :
9296 : rtx_insn *
9297 279706540 : get_ready_element (int i)
9298 : {
9299 279706540 : return ready_element (&ready, i);
9300 : }
9301 :
9302 : #endif /* INSN_SCHEDULING */
|
