571 / 583 lines covered (97.9%)
1unit module BDD::Behave::Runner;23use BDD::Behave::Colors;4use BDD::Behave::Failure;5use BDD::Behave::Failures;6use BDD::Behave::Formatter;7use BDD::Behave::Formatter::Tree;8use BDD::Behave::SpecTree;910need BDD::Behave::Mock::Stub;11need BDD::Behave::LetRuntime;12need BDD::Behave::Benchmark;13need BDD::Behave::Benchmark::Baseline;14need BDD::Behave::Benchmark::Format;15need BDD::Behave::Configuration;16need BDD::Behave::Time;1718constant Suite = BDD::Behave::SpecTree::Suite;19constant ExampleGroup = BDD::Behave::SpecTree::ExampleGroup;20constant Example = BDD::Behave::SpecTree::Example;21constant LetRuntime = BDD::Behave::LetRuntime::LetRuntime;22constant BenchmarkResult = BDD::Behave::Benchmark::BenchmarkResult;23constant BaselineEntry = BDD::Behave::Benchmark::Baseline::BaselineEntry;24constant Configuration = BDD::Behave::Configuration::Configuration;2526our class AttemptResult {27 has Str $.outcome is rw;28 has %.failure-info is rw;29 has Int $.new-failures is rw = 0;30 has @.captured-matchers is rw;31 has Bool $.auto is rw = False;32 has Str $.description is rw;33 has Real $.duration is rw;34 has Bool $.continuation-called is rw = False;35}3637our class RetryRecord {38 has Str $.description is required;39 has Str $.location is required;40 has Int $.attempts is required;41 has Int $.max-attempts is required;42 has Str $.outcome is required;43}4445our class RunResult {46 has Int $.total is rw = 0;47 has Int $.passed is rw = 0;48 has Int $.failed is rw = 0;49 has Int $.pending is rw = 0;50 has Int $.skipped is rw = 0;51 has @.errors is rw;52 has @.retry-records is rw;5354 method add-pass {55 $!total++;56 $!passed++;57 }5859 method add-fail($error) {60 $!total++;61 $!failed++;62 @!errors.push($error);63 }6465 method add-pending {66 $!total++;67 $!pending++;68 }6970 method add-skipped {71 $!total++;72 $!skipped++;73 }7475 method add-retry(RetryRecord $record) {76 @!retry-records.push($record);77 }7879 method success {80 $!failed == 0;81 }82}8384our class Runner {85 has @!description-stack;86 has RunResult $.result .= new;87 has BDD::Behave::Formatter $.formatter;88 has @.include-tags;89 has @.exclude-tags;90 has @.example-patterns;91 has @.only-locations;92 has @.execution-order;93 has @.timed-examples;94 has $.aggregate-failures = False;95 has Bool $!focus-mode = False;96 has Bool $!top-level = False;97 has Str $.order = 'defined';98 has Int $.seed;99 has Bool $.show-seed = False;100 has Int $!rng-state;101 has Int $.fail-fast = 0;102 has Int $.retry = 0;103 has Bool $.aborted = False;104 has Real $.slow-threshold = 0;105 has Int $.profile-limit = 0;106 has Bool $.benchmark-mode = False;107 has Bool $.benchmark-quiet = False;108 has Int $.benchmark-iterations = 1;109 has IO::Path $.benchmark-baseline;110 has IO::Path $.benchmark-save;111 has Real $.benchmark-threshold = 0.10;112 has Str $.benchmark-format = 'text';113 has IO::Path $.benchmark-output;114 has @.benchmark-summaries;115 has @.benchmark-regressions;116 has Configuration $.config;117 has %!helper-cache;118 has @!effective-match-filters;119120 submethod TWEAK {121 $!formatter //= BDD::Behave::Formatter::Tree.new;122123 die "order must be 'random' or 'defined' (got: '$!order')"124 unless $!order eq 'random' | 'defined';125126 die "fail-fast must be 0 or a positive integer (got: $!fail-fast)"127 if $!fail-fast < 0;128129 die "retry must be 0 or a positive integer (got: $!retry)"130 if $!retry < 0;131132 die "slow-threshold must be 0 or positive (got: $!slow-threshold)"133 if $!slow-threshold < 0;134135 die "profile-limit must be 0 or positive (got: $!profile-limit)"136 if $!profile-limit < 0;137138 die "benchmark-iterations must be a positive integer (got: $!benchmark-iterations)"139 if $!benchmark-iterations < 1;140141 die "benchmark-threshold must be 0 or positive (got: $!benchmark-threshold)"142 if $!benchmark-threshold < 0;143144 die "benchmark-format must be 'text' or 'json' (got: '$!benchmark-format')"145 unless $!benchmark-format eq 'text' | 'json';146147 if $!order eq 'random' {148 $!seed //= (1 .. 2_147_483_646).pick;149 }150151 if $!seed.defined {152 $!rng-state = $!seed % 2_147_483_647;153 $!rng-state = 1 if $!rng-state == 0;154 }155 }156157 method should-abort(--> Bool) {158 $!fail-fast > 0 && $!result.failed >= $!fail-fast;159 }160161 method advance-rng(--> Int) {162 $!rng-state = ($!rng-state * 48271) % 2_147_483_647;163 $!rng-state;164 }165166 method effective-order($container --> Str) {167 if $container ~~ ExampleGroup {168 my $override = $container.effective-metadata-value('order');169 return $override if $override.defined;170 }171 $!order;172 }173174 method shuffled-children($container --> List) {175 my @items = $container.children.list;176 my $effective-order = self.effective-order($container);177 return @items.List if $effective-order eq 'defined' || @items.elems <= 1;178 my @result = @items;179 for (1 ..^ @result.elems).reverse -> $i {180 my $j = self.advance-rng % ($i + 1);181 @result[$i, $j] = @result[$j, $i];182 }183 @result.List;184 }185186 method run(Suite $suite) {187 $!top-level = (CALLERS::<$*BEHAVE-TOP-LEVEL-RUN> // False).so;188 my $*BEHAVE-TOP-LEVEL-RUN = False;189190 $!focus-mode = self.has-focus($suite);191 self.compute-match-filters($suite);192 self.run-config-hooks('before-all');193 LEAVE { self.run-config-hooks('after-all') }194 {195 my %helpers := self.helper-snapshot;196 my $*BEHAVE-HELPERS = %helpers;197 self.run-suite($suite);198 }199 self.execute-benchmark-mode($suite) if $!benchmark-mode;200 self.print-summary;201 $!result;202 }203204 method helper-snapshot(--> Hash) {205 return %() unless $!config.defined;206 my %h;207 for $!config.includes.list -> $entry {208 %!helper-cache{$entry.key} //= $entry.class.new;209 %h{$entry.key} = %!helper-cache{$entry.key};210 }211 %h;212 }213214 method run-config-hooks(Str $phase, $example = Nil) {215 return unless $!config.defined;216 for $!config.hooks-for($phase).list -> $hook {217 next if $example.defined && !$hook.matches-example($example);218 try {219 ($hook.block)();220 CATCH {221 default {222 warn "Config $phase hook failed: {.message}";223 }224 }225 }226 }227 }228229 method run-config-around-each(Example $example, &core) {230 return core() unless $!config.defined;231 my @arounds = $!config.hooks-for('around-each').list232 .grep({ .matches-example($example) });233 return core() unless @arounds.elems;234 my &chain = &core;235 for @arounds.reverse -> $hook {236 my &next = &chain;237 &chain = sub { ($hook.block)(&next) };238 }239 chain();240 }241242 method compute-match-filters(Suite $suite) {243 @!effective-match-filters = [];244 return unless $!config.defined;245 for $!config.match-filters.list -> $pair {246 my $key = $pair.key;247 my $expected = $pair.value;248 if self.suite-has-metadata-match($suite, $key, $expected) {249 @!effective-match-filters.push: $pair;250 }251 }252 }253254 method suite-has-metadata-match($node, Str $key, $expected --> Bool) {255 given $node {256 when Example {257 return self.example-metadata-matches($node, $key, $expected);258 }259 when ExampleGroup | Suite {260 for $node.children.list -> $child {261 return True if self.suite-has-metadata-match($child, $key, $expected);262 }263 }264 }265 False;266 }267268 method example-metadata-matches(Example $example, Str $key, $expected --> Bool) {269 my $actual = $example.effective-metadata-value($key);270 return False unless $actual.defined;271 if $expected ~~ Bool {272 return $expected ?? ?$actual !! !$actual;273 }274 $actual eq $expected;275 }276277 method execute-benchmark-mode(Suite $suite) {278 my @benchmarked = self.collect-benchmarked-examples($suite);279280 if $!benchmark-iterations > 1 {281 for @benchmarked -> $example {282 for 2 .. $!benchmark-iterations {283 self.silent-rerun($example);284 }285 }286 }287288 @!benchmark-summaries = self.aggregate-benchmark-summaries(@benchmarked);289290 if !$!benchmark-quiet && $!benchmark-baseline.defined {291 @!benchmark-regressions =292 self.compare-with-baseline(@!benchmark-summaries, $!benchmark-baseline);293 }294295 if !$!benchmark-quiet && $!benchmark-save.defined {296 self.save-benchmark-baseline(@!benchmark-summaries, $!benchmark-save);297 }298 }299300 method collect-benchmarked-examples($node --> List) {301 my @found;302 given $node {303 when Example {304 @found.push: $node if $node.benchmarks.elems;305 }306 default {307 for $node.children -> $child {308 @found.append: self.collect-benchmarked-examples($child);309 }310 }311 }312 @found.List;313 }314315 method silent-rerun(Example $example) {316 my $saved-result = $!result;317 my @saved-execution-order = @!execution-order;318 my @saved-timed-examples = @!timed-examples;319320 $!result = RunResult.new;321 @!execution-order = [];322 @!timed-examples = [];323324 my $sink = open '/dev/null', :w;325 {326 my $*OUT = $sink;327 try {328 self.handle-example($example);329 CATCH { default { } }330 }331 }332 $sink.close;333334 $!result = $saved-result;335 @!execution-order = @saved-execution-order;336 @!timed-examples = @saved-timed-examples;337 }338339 method aggregate-benchmark-summaries(@examples --> List) {340 my @summaries;341 for @examples -> $example {342 my $description = self.full-nested-description($example);343 my %by-key;344 for $example.benchmarks.list -> $bench {345 %by-key{$bench.key}.push: $bench;346 }347 for %by-key.pairs.sort(*.key) -> $pair {348 my $key = $pair.key;349 my @runs = $pair.value.list;350 my @all-timings = @runs.map(*.timings.flat).flat.list;351 my $iterations = [+] @runs.map(*.iterations);352 my $label = @runs[0].label;353 my $position = @runs[0].position;354 my %summary = self.compute-summary(@all-timings);355 @summaries.push: %(356 example => $example,357 description => $description,358 key => $key,359 label => $label,360 position => $position,361 runs => @runs.elems,362 iterations => $iterations,363 timings => @all-timings,364 |%summary,365 );366 }367 }368 @summaries.sort({ $^a<description> cmp $^b<description> || $^a<key> cmp $^b<key> }).List;369 }370371 method compute-summary(@timings --> Hash) {372 return %( min => Real, max => Real, mean => Real, median => Real, total => Real )373 unless @timings.elems;374 my @sorted = @timings.sort;375 my $total = [+] @timings;376 my $mean = $total / @timings.elems;377 my $n = @sorted.elems;378 my $median = $n %% 2379 ?? (@sorted[$n div 2 - 1] + @sorted[$n div 2]) / 2380 !! @sorted[$n div 2];381 %(382 min => @sorted[0],383 max => @sorted[*-1],384 mean => $mean,385 median => $median,386 total => $total,387 );388 }389390 method compare-with-baseline(@summaries, IO::Path $path --> List) {391 my @entries = BDD::Behave::Benchmark::Baseline::load($path);392 my %index = BDD::Behave::Benchmark::Baseline::index-by-key(@entries);393 my @regressions;394 for @summaries -> %s {395 my $entry = %index{%s<description>}{%s<key>};396 next unless $entry.defined;397 my $delta-pct = $entry.median == 0 ?? 0398 !! (%s<median> - $entry.median) / $entry.median;399 my %row = (|%s,400 baseline-median => $entry.median,401 baseline-mean => $entry.mean,402 delta-pct => $delta-pct,403 regression => $delta-pct > $!benchmark-threshold);404 @regressions.push: %row;405 }406 @regressions.List;407 }408409 method save-benchmark-baseline(@summaries, IO::Path $path --> Nil) {410 my BaselineEntry @entries = @summaries.map: -> %s {411 BaselineEntry.new(412 :description(%s<description>),413 :key(%s<key>),414 :iterations(%s<iterations>),415 :min(%s<min>.Real),416 :max(%s<max>.Real),417 :mean(%s<mean>.Real),418 :median(%s<median>.Real),419 :total(%s<total>.Real),420 );421 };422 BDD::Behave::Benchmark::Baseline::save($path, @entries);423 }424425 method has-focus($node --> Bool) {426 return True if $node.focused;427 if $node ~~ Suite | ExampleGroup {428 for $node.children -> $child {429 return True if self.has-focus($child);430 }431 }432 False;433 }434435 method run-suite(Suite $suite) {436 # A suite is a top-level container for a file437 # Walk all its children (groups and examples)438 for self.shuffled-children($suite) -> $child {439 last if self.should-abort;440 given $child {441 when ExampleGroup { self.run-group($child) if self.group-matches($child) }442 when Example { self.handle-example($child) if self.example-matches($child) }443 }444 }445 $!aborted = True if self.should-abort;446 }447448 method run-group(ExampleGroup $group) {449 $!formatter.group-start($group);450451 @!description-stack.push($group.description);452453 my Int $group-stub-snapshot = BDD::Behave::Mock::Stub::StubRegistry.active-count;454 LEAVE {455 BDD::Behave::Mock::Stub::StubRegistry.clear-since($group-stub-snapshot)456 if $group-stub-snapshot.defined;457 }458459 my $group-skipped = $group.effective-skipped;460461 if $group-skipped {462 self.run-group-body($group);463 } else {464 my @around-hooks = self.runnable-around-all-hooks($group);465466 my $continuation-called = False;467 my &core = sub {468 $continuation-called = True;469 self.run-hooks($group, 'before-all');470 self.run-group-body($group);471 self.run-hooks($group, 'after-all');472 };473474 if @around-hooks.elems {475 my &chain = &core;476 for @around-hooks.reverse -> $hook {477 my &next = &chain;478 &chain = sub { ($hook.callback)(&next) };479 }480481 try {482 chain();483 CATCH {484 default {485 warn "around-all hook failed in {$group.description}: {.message}";486 }487 }488 }489490 unless $continuation-called {491 self.mark-around-all-skipped($group);492 }493 } else {494 core();495 }496 }497498 @!description-stack.pop;499 $!formatter.group-end($group);500 }501502 method run-group-body(ExampleGroup $group) {503 for self.shuffled-children($group) -> $child {504 last if self.should-abort;505 given $child {506 when ExampleGroup { self.run-group($child) if self.group-matches($child) }507 when Example { self.handle-example($child) if self.example-matches($child) }508 }509 }510 }511512 method handle-example(Example $example) {513 if $example.effective-skipped {514 self.print-skipped($example);515 return;516 }517518 my $max-attempts = self.resolve-max-attempts($example);519 my Int $attempt-num = 0;520 my AttemptResult $result;521522 loop {523 $attempt-num++;524 my Int $failure-snapshot = Failures.list.elems;525 $result = self.attempt-example($example, :first-attempt($attempt-num == 1));526527 last if $result.outcome ne 'fail';528 last if $attempt-num >= $max-attempts;529530 Failures.list.splice($failure-snapshot, Failures.list.elems - $failure-snapshot);531 $!formatter.example-retry($example, :attempt($attempt-num), :max-attempts($max-attempts));532 }533534 self.commit-attempt-result($example, $result, :attempts($attempt-num), :$max-attempts);535 }536537 method resolve-max-attempts(Example $example --> Int) {538 my $meta-retry = $example.effective-metadata-value('retry');539 my $configured = $meta-retry.defined ?? max($meta-retry.Int, 0) !! $!retry;540 1 + $configured;541 }542543 method attempt-example(Example $example, Bool :$first-attempt = True --> AttemptResult) {544 my @lets = $example.get-metadata('lets', :default([])).flat.List;545 my $*LET-RUNTIME = LetRuntime.new(:definitions(@lets));546547 my Int $stub-snapshot = BDD::Behave::Mock::Stub::StubRegistry.active-count;548 LEAVE {549 if $stub-snapshot.defined {550 BDD::Behave::Mock::Stub::StubRegistry.clear-since($stub-snapshot);551 }552 }553554 my $result = AttemptResult.new;555 my @around-hooks = self.matching-around-each-hooks($example);556557 my &core = sub {558 $result.continuation-called = True;559 self.run-each-and-example-attempt($example, $result, :$first-attempt);560 };561562 my $reached-end = False;563 if !@around-hooks.elems {564 core();565 $reached-end = True;566 } else {567 my &chain = &core;568 for @around-hooks.reverse -> $hook {569 my &next = &chain;570 &chain = sub { ($hook.callback)(&next) };571 }572573 try {574 chain();575 $reached-end = True;576 CATCH {577 default {578 if $result.continuation-called {579 warn "around-each hook raised after example: {.message}";580 $reached-end = True;581 } else {582 $result.outcome = 'fail';583 $result.failure-info = %(584 description => self.full-description($example),585 file => $example.file,586 line => $example.line,587 exception => $_,588 );589 }590 }591 }592 }593 }594595 if !$result.continuation-called && $reached-end && !$result.outcome.defined {596 $result.outcome = 'around-skipped';597 }598599 $result;600 }601602 method run-each-and-example-attempt(Example $example, $result, Bool :$first-attempt) {603 self.run-config-hooks('before-each', $example);604 for self.ancestor-groups($example) -> $ancestor {605 self.run-each-hooks($ancestor, 'before-each', $example);606 }607608 self.run-config-around-each(609 $example,610 { self.run-example-attempt($example, $result, :$first-attempt) },611 );612613 for self.ancestor-groups($example).reverse -> $ancestor {614 self.run-each-hooks($ancestor, 'after-each', $example);615 }616 self.run-config-hooks('after-each', $example);617 }618619 method matching-around-each-hooks(Example $example --> List) {620 my @hooks;621 for self.ancestor-groups($example) -> $group {622 for $group.hooks('around-each') -> $hook {623 @hooks.push($hook) if $hook.matches($example);624 }625 }626 @hooks.List;627 }628629 method runnable-around-all-hooks(ExampleGroup $group --> List) {630 my @hooks;631 my @runnable;632 my $runnable-collected = False;633 for $group.hooks('around-all') -> $hook {634 if $hook.has-filter {635 unless $runnable-collected {636 @runnable = self.runnable-examples($group);637 $runnable-collected = True;638 }639 next unless @runnable.first({ $hook.matches($_) }).defined;640 }641 @hooks.push($hook);642 }643 @hooks.List;644 }645646 method print-around-skipped(Example $example) {647 $!formatter.example-around-skipped($example);648 $!result.add-skipped;649 }650651 method mark-around-all-skipped(ExampleGroup $group) {652 $!formatter.group-around-skipped($group);653 for self.runnable-examples($group) -> $example {654 next if $example.effective-skipped;655 $!result.add-skipped;656 }657 }658659 method print-skipped(Example $example) {660 $!formatter.example-skipped($example);661 $!result.add-skipped;662 }663664 method example-matches(Example $example --> Bool) {665 my @tags = $example.effective-tags;666667 if @!exclude-tags.elems668 && @tags.first({ $_ ∈ @!exclude-tags }).defined {669 return False;670 }671672 if $!focus-mode673 && !$example.effective-focused674 && !$example.effective-skipped {675 return False;676 }677678 if @!include-tags.elems679 && !@tags.first({ $_ ∈ @!include-tags }).defined {680 return False;681 }682683 return False unless self.description-matches($example);684 return False unless self.location-matches($example);685 return False unless self.config-metadata-matches($example);686 return False unless self.match-filter-matches($example);687688 True;689 }690691 method config-metadata-matches(Example $example --> Bool) {692 return True unless $!config.defined;693 for $!config.metadata-filters.kv -> $key, $expected {694 return False unless self.example-metadata-matches($example, $key, $expected);695 }696 for $!config.metadata-exclude-filters.kv -> $key, $expected {697 return False if self.example-metadata-matches($example, $key, $expected);698 }699 True;700 }701702 method match-filter-matches(Example $example --> Bool) {703 return True unless @!effective-match-filters.elems;704 for @!effective-match-filters.list -> $pair {705 return False706 unless self.example-metadata-matches($example, $pair.key, $pair.value);707 }708 True;709 }710711 method location-matches(Example $example --> Bool) {712 return True unless @!only-locations.elems;713 for @!only-locations -> $pattern {714 return True if self.location-matches-node($example, $pattern);715 for $example.ancestry -> $node {716 next unless $node ~~ ExampleGroup;717 return True if self.location-matches-node($node, $pattern);718 }719 }720 False;721 }722723 method location-matches-node($node, Str $pattern --> Bool) {724 return False unless $pattern.contains(':');725 my $idx = $pattern.rindex(':');726 my $pattern-path = $pattern.substr(0, $idx);727 my $pattern-line = $pattern.substr($idx + 1);728729 return False unless $node.line.Str eq $pattern-line;730 self!path-matches($node.file.Str, $pattern-path);731 }732733 method location-matches-pattern(Str $example-loc, Str $pattern --> Bool) {734 return False unless $pattern.contains(':');735 my $idx = $pattern.rindex(':');736 my $pattern-path = $pattern.substr(0, $idx);737 my $pattern-line = $pattern.substr($idx + 1);738739 my $ex-idx = $example-loc.rindex(':');740 return False unless $ex-idx.defined;741 my $example-path = $example-loc.substr(0, $ex-idx);742 my $example-line = $example-loc.substr($ex-idx + 1);743744 return False unless $example-line eq $pattern-line;745 self!path-matches($example-path, $pattern-path);746 }747748 method !path-matches(Str $node-path, Str $pattern-path --> Bool) {749 return True if $node-path eq $pattern-path;750 return True if $node-path.IO.absolute eq $pattern-path.IO.absolute;751 return True if $node-path.ends-with('/' ~ $pattern-path);752 return True if $node-path.IO.basename eq $pattern-path;753 False;754 }755756 method group-matches(ExampleGroup $group --> Bool) {757 return True unless @!include-tags.elems758 || @!exclude-tags.elems759 || @!example-patterns.elems760 || @!only-locations.elems761 || $!focus-mode;762763 for $group.children -> $child {764 given $child {765 when Example { return True if self.example-matches($child) }766 when ExampleGroup { return True if self.group-matches($child) }767 }768 }769 False;770 }771772 method description-matches(Example $example --> Bool) {773 return True unless @!example-patterns.elems;774 my $description = self.full-nested-description($example);775 for @!example-patterns -> $pattern {776 return True if self.match-pattern($description, $pattern);777 }778 False;779 }780781 method full-nested-description(Example $example --> Str) {782 my @parts = $example.ancestry.grep(ExampleGroup).map(*.description);783 @parts.push($example.description);784 @parts.join(' ');785 }786787 method match-pattern(Str $description, Str $pattern --> Bool) {788 if $pattern.chars > 2789 && $pattern.starts-with('/')790 && $pattern.ends-with('/') {791 my $body = $pattern.substr(1, $pattern.chars - 2);792 my $rx = / <{ $body }> /;793 return so $description.match($rx);794 }795 $description.contains($pattern);796 }797798 method resolve-auto-aggregation(Example $example --> List) {799 my $value = $example.effective-metadata-value('aggregate-failures');800 $value = $!aggregate-failures unless $value.defined;801802 return (False, Str).List unless $value.defined;803 return (False, Str).List if $value === False;804 if $value ~~ Str {805 return (False, Str).List unless $value.chars;806 return (True, $value).List;807 }808 (True, Str).List;809 }810811 method run-example-attempt(Example $example, $result, Bool :$first-attempt) {812 my Bool $auto = ?$example.get-metadata('auto-description', :default(False));813 $result.auto = $auto;814 $result.description = $example.description;815816 if $example.pending {817 $result.outcome = 'pending';818 return;819 }820821 @!execution-order.push("{$example.file}:{$example.line}") if $first-attempt;822823 my ($auto-agg-on, $auto-agg-label) = self.resolve-auto-aggregation($example);824 my $initial-failure-count = Failures.list.elems;825826 $!formatter.example-start($example, :$auto) if $first-attempt && !$auto;827828 my @captured-matchers;829 my $error;830 my $started = now;831 $example.started-at = $started;832833 my sub run-body() {834 my $*BEHAVE-AUTO-MATCHERS = $auto ?? @captured-matchers !! Array;835 my $*BEHAVE-AGGREGATION-LABEL = $auto-agg-on ?? ($auto-agg-label // Str) !! Str;836 my $*BEHAVE-AGGREGATING = ?$auto-agg-on;837 my $*BEHAVE-CURRENT-DESCRIPTION = self.full-description($example);838 try {839 $example.execute;840 CATCH {841 when X::BDD::Behave::ExpectationFailed {842 }843 default {844 if $auto-agg-on {845 my $msg = "exception in {self.full-description($example)}: " ~ .message;846 Failures.list.push(Failure.new(847 :file($example.file.Str),848 :line($example.line),849 :message($msg),850 :aggregation-label($auto-agg-label // Str),851 ));852 } else {853 $error = $_;854 }855 }856 }857 }858 }859860 my $freeze-meta = $example.effective-metadata-value('freeze-time');861862 if !$freeze-meta.defined || ($freeze-meta ~~ Bool && !$freeze-meta) {863 run-body();864 } elsif $freeze-meta === True {865 BDD::Behave::Time::freeze-time({ run-body() });866 } else {867 BDD::Behave::Time::freeze-time($freeze-meta, { run-body() });868 }869870 my $finished = now;871 $example.finished-at = $finished;872 $example.duration = ($finished - $started).Real;873 $result.duration = $example.duration;874 $result.captured-matchers = @captured-matchers if $auto;875876 if $auto {877 my $derived = self.derive-auto-description(@captured-matchers);878 $result.description = $derived if $derived.defined;879 }880881 if $error.defined {882 $result.outcome = 'fail';883 $result.failure-info = %(884 description => self.full-description($example),885 file => $example.file,886 line => $example.line,887 example-line => $example.line,888 exception => $error,889 message => $error.message,890 );891892 # Only the top-level Runner (bin/behave sets $*BEHAVE-TOP-LEVEL-RUN893 # before entering the Runner) records exception failures in the global894 # Failures.list so the end-of-run summary can render them. Nested runs —895 # synthetic suites built inside spec bodies — would otherwise pollute896 # the outer runner's failure delta accounting.897 if $!top-level {898 Failures.list.push(Failure.new(899 :file($example.file.Str),900 :line($example.line),901 :message("exception in {self.full-description($example)}: " ~ $error.message),902 :from-runner-exception,903 ));904 }905906 return;907 }908909 $result.new-failures = Failures.list.elems - $initial-failure-count;910911 if $result.new-failures > 0 {912 $result.outcome = 'fail';913914 my @new = Failures.list[$initial-failure-count .. Failures.list.elems - 1];915 my @expectation-failures = @new.grep(!*.from-runner-exception);916 my @runner-exceptions = @new.grep(*.from-runner-exception);917 my $primary-failure = Failures.list[$initial-failure-count];918919 $result.failure-info = %(920 description => self.full-description($example),921 file => $example.file,922 line => ($primary-failure.defined ?? $primary-failure.line !! $example.line),923 example-line => $example.line,924 expectations => @expectation-failures.map(-> $fl {925 my %rec = (926 file => $fl.file,927 line => $fl.line,928 negated => $fl.negated,929 given => ($fl.given.defined ?? $fl.given.gist !! Str),930 expected => ($fl.expected.defined ?? $fl.expected.gist !! Str),931 );932 my $message = $fl.message // compose-failure-message(%rec<given>, %rec<expected>, $fl.negated);933 %rec<message> = $message if $message.chars;934 %rec<aggregation_label> = $fl.aggregation-label if $fl.aggregation-label.defined;935 %rec;936 }).List,937 );938939 if @runner-exceptions.elems {940 $result.failure-info<message> = @runner-exceptions[0].message;941 }942 } else {943 $result.outcome = 'pass';944 }945 }946947 method commit-attempt-result(948 Example $example,949 AttemptResult $result,950 Int :$attempts = 1,951 Int :$max-attempts = 1,952 ) {953 given $result.outcome {954 when 'pending' {955 $!formatter.example-pending($example);956 $!result.add-pending;957 }958 when 'around-skipped' {959 self.print-around-skipped($example);960 }961 when 'pass' {962 @!timed-examples.push: %(963 example => $example,964 description => self.full-description($example),965 duration => $example.duration,966 );967 if $result.auto {968 $!formatter.example-auto-description(969 $example, :description($result.description),970 );971 }972 $!result.add-pass;973 $!formatter.example-pass($example);974 self.record-retry($example, $result, :$attempts, :$max-attempts) if $attempts > 1;975 self.maybe-print-slow($example);976 }977 when 'fail' {978 @!timed-examples.push: %(979 example => $example,980 description => self.full-description($example),981 duration => $example.duration,982 );983 if $result.auto {984 $!formatter.example-auto-description(985 $example, :description($result.description),986 );987 }988 $!result.add-fail($result.failure-info);989 $!formatter.example-fail($example, :failure-info($result.failure-info));990 self.record-retry($example, $result, :$attempts, :$max-attempts) if $attempts > 1;991 self.maybe-print-slow($example);992 }993 }994 }995996 method record-retry(997 Example $example, AttemptResult $result,998 Int :$attempts!, Int :$max-attempts!,999 ) {1000 my $record = RetryRecord.new(1001 :description(self.full-description($example)),1002 :location("{$example.file}:{$example.line}"),1003 :$attempts,1004 :$max-attempts,1005 :outcome($result.outcome),1006 );1007 $!result.add-retry($record);1008 }10091010 method maybe-print-slow(Example $example) {1011 return unless $!slow-threshold > 0;1012 return unless $example.duration.defined;1013 return unless $example.duration >= $!slow-threshold;1014 $!formatter.example-slow($example, :threshold($!slow-threshold));1015 }10161017 method derive-auto-description(@captured) {1018 return Nil unless @captured.elems;1019 my %first = @captured[0];1020 my $matcher = %first<matcher>;1021 my $negated = ?%first<negated>;1022 my $desc = $matcher.description;1023 return Nil unless $desc.defined && $desc.chars;1024 my $verb = $negated ?? 'should not' !! 'should';1025 "{$verb} {$desc}";1026 }10271028 method ancestor-groups($node) {1029 $node.ancestry.grep(ExampleGroup).List;1030 }10311032 method run-hooks(ExampleGroup $group, Str $phase) {1033 my @hooks = $group.hooks($phase);1034 return unless @hooks.elems;1035 my @runnable;1036 my $runnable-collected = False;1037 for @hooks -> $hook {1038 if $hook.has-filter {1039 unless $runnable-collected {1040 @runnable = self.runnable-examples($group);1041 $runnable-collected = True;1042 }1043 next unless @runnable.first({ $hook.matches($_) }).defined;1044 }1045 try {1046 ($hook.callback)();1047 CATCH {1048 default {1049 warn "Hook $phase failed in {$group.description}: {$_.message}";1050 }1051 }1052 }1053 }1054 }10551056 method run-each-hooks(ExampleGroup $group, Str $phase, Example $example) {1057 for $group.hooks($phase) -> $hook {1058 next unless $hook.matches($example);1059 try {1060 ($hook.callback)();1061 CATCH {1062 default {1063 warn "Hook $phase failed in {$group.description}: {$_.message}";1064 }1065 }1066 }1067 }1068 }10691070 method runnable-examples(ExampleGroup $group --> List) {1071 my @examples;1072 for $group.children -> $child {1073 given $child {1074 when Example {1075 @examples.push($child)1076 if self.example-matches($child) && !$child.effective-skipped;1077 }1078 when ExampleGroup {1079 @examples.append: self.runnable-examples($child);1080 }1081 }1082 }1083 @examples.List;1084 }10851086 method full-description(Example $example) {1087 my @parts = @!description-stack.clone;1088 @parts.push($example.description);1089 @parts.join(' ');1090 }10911092 method print-summary {1093 $!formatter.run-summary(1094 $!result,1095 :$!aborted, :$!fail-fast, :$!order, :$!seed, :$!show-seed,1096 );10971098 $!formatter.retry-summary($!result.retry-records)1099 if $!result.retry-records.elems;11001101 $!formatter.profile-summary(@!timed-examples, :limit($!profile-limit))1102 if $!profile-limit > 0;1103 $!formatter.benchmark-summary-section(1104 @!benchmark-summaries, @!benchmark-regressions,1105 :threshold($!benchmark-threshold),1106 :format($!benchmark-format),1107 :output($!benchmark-output),1108 :runner(self),1109 ) if $!benchmark-mode && !$!benchmark-quiet;1110 }11111112 method print-benchmark-summary(1113 @summaries = @!benchmark-summaries,1114 @regressions = @!benchmark-regressions,1115 Real :$threshold = $!benchmark-threshold,1116 Str :$format = $!benchmark-format,1117 IO::Path :$output = $!benchmark-output,1118 ) {1119 $!formatter.benchmark-summary-section(1120 @summaries, @regressions,1121 :$threshold, :$format, :$output, :runner(self),1122 );1123 }11241125 method render-benchmark-output(1126 @summaries, @regressions,1127 Real :$threshold = $!benchmark-threshold,1128 Str :$format = $!benchmark-format,1129 --> Str1130 ) {1131 given $format {1132 when 'json' {1133 BDD::Behave::Benchmark::Format::to-json-document(1134 @summaries, @regressions, $threshold,1135 );1136 }1137 default {1138 self.render-benchmark-text(@summaries, @regressions, :$threshold);1139 }1140 }1141 }11421143 method render-benchmark-text(1144 @summaries, @regressions,1145 Real :$threshold = $!benchmark-threshold,1146 --> Str1147 ) {1148 my @blocks;1149 @blocks.push: self.render-bench-summary-table(@summaries);1150 if @regressions.elems {1151 @blocks.push: '';1152 @blocks.push: self.render-bench-comparison-table(@regressions, :$threshold);1153 }1154 @blocks.join("\n");1155 }11561157 method render-bench-summary-table(@summaries --> Str) {1158 my $count = @summaries.elems;1159 my $heading = "Benchmarks ($count measurement" ~ ($count == 1 ?? '' !! 's') ~ '):';11601161 my @headers = <DESCRIPTION KEY ITER MIN(s) MAX(s) MEAN(s) MEDIAN(s)>;1162 my @aligns = <left left right right right right right>;1163 my @rows;1164 for @summaries -> %s {1165 @rows.push: [1166 %s<description>,1167 %s<key>,1168 %s<iterations>.Str,1169 sprintf('%.6f', %s<min>),1170 sprintf('%.6f', %s<max>),1171 sprintf('%.6f', %s<mean>),1172 sprintf('%.6f', %s<median>),1173 ];1174 }1175 my @widths = BDD::Behave::Benchmark::Format::column-widths(@headers, @rows);1176 $heading ~ "\n" ~1177 BDD::Behave::Benchmark::Format::render-table(@headers, @rows, @widths, @aligns);1178 }11791180 method render-bench-comparison-table(@regressions, Real :$threshold = $!benchmark-threshold --> Str) {1181 my @hits = @regressions.grep(*<regression>);1182 my $threshold-pct = sprintf '%.1f%%', $threshold * 100;1183 my $heading = @hits.elems1184 ?? red("Benchmark regressions ({@hits.elems}, threshold $threshold-pct):")1185 !! "Benchmark comparison (no regressions; threshold $threshold-pct):";11861187 my @headers = <DESCRIPTION KEY BASELINE CURRENT DELTA>;1188 my @aligns = <left left right right left>;1189 my @rows;1190 for @regressions -> %r {1191 @rows.push: [1192 %r<description>,1193 %r<key>,1194 sprintf('%.6fs', %r<baseline-median>),1195 sprintf('%.6fs', %r<median>),1196 self.render-delta-cell(%r),1197 ];1198 }1199 my @widths = BDD::Behave::Benchmark::Format::column-widths(@headers, @rows);1200 $heading ~ "\n" ~1201 BDD::Behave::Benchmark::Format::render-table(@headers, @rows, @widths, @aligns);1202 }12031204 method render-delta-cell(%r --> Str) {1205 my $delta-pct = %r<delta-pct>;1206 my $sign = $delta-pct >= 0 ?? '+' !! '';1207 my $body = sprintf '%s%.1f%%', $sign, $delta-pct * 100;1208 my ($arrow, $colored);1209 if %r<regression> {1210 $arrow = '↑';1211 $colored = red("$arrow $body REGRESSION");1212 } elsif $delta-pct < -$!benchmark-threshold {1213 $arrow = '↓';1214 $colored = green("$arrow $body");1215 } elsif $delta-pct > 0 {1216 $arrow = '↑';1217 $colored = "$arrow $body";1218 } elsif $delta-pct < 0 {1219 $arrow = '↓';1220 $colored = "$arrow $body";1221 } else {1222 $arrow = '→';1223 $colored = "$arrow $body";1224 }1225 $colored;1226 }12271228 method print-profile(Int $limit = $!profile-limit, @records = @!timed-examples) {1229 $!formatter.profile-summary(@records, :$limit);1230 }1231}