From eebfba035b96346e49def5d82fdb2c6743419749 Mon Sep 17 00:00:00 2001 From: Jacek Gebal Date: Wed, 23 Oct 2019 01:44:31 +0100 Subject: [PATCH 1/6] Added support for nested contexts. TODO - review & update documentation & examples. --- source/core/ut_suite_builder.pkb | 61 ++++--- test/ut3_tester/core/test_suite_builder.pkb | 176 +++++++++++++++----- test/ut3_tester/core/test_suite_builder.pks | 3 + test/ut3_user/api/test_ut_run.pkb | 2 +- 4 files changed, 171 insertions(+), 71 deletions(-) diff --git a/source/core/ut_suite_builder.pkb b/source/core/ut_suite_builder.pkb index 101ab2179..fc3e72e60 100644 --- a/source/core/ut_suite_builder.pkb +++ b/source/core/ut_suite_builder.pkb @@ -625,7 +625,7 @@ create or replace package body ut_suite_builder is a_suite.path := lower(coalesce(a_suite.path, a_suite.object_name)); end; - procedure add_suite_tests( + procedure add_tests_to_items( a_suite in out nocopy ut_suite, a_annotations t_annotations_info, a_suite_items in out nocopy ut_suite_items @@ -738,7 +738,8 @@ create or replace package body ut_suite_builder is procedure get_suite_contexts_items( a_suite in out nocopy ut_suite, a_annotations in out nocopy t_annotations_info, - a_suite_items out nocopy ut_suite_items + a_suite_items out nocopy ut_suite_items, + a_parent_context_pos in integer := 0 ) is l_context_pos t_annotation_position; l_end_context_pos t_annotation_position; @@ -755,35 +756,43 @@ create or replace package body ut_suite_builder is return; end if; - l_context_pos := a_annotations.by_name( gc_context).first; + l_context_pos := a_annotations.by_name( gc_context).next(a_parent_context_pos); while l_context_pos is not null loop l_end_context_pos := get_endcontext_position(l_context_pos, a_annotations.by_name ); + + l_context_name := coalesce( a_annotations.by_line( l_context_pos ).text, gc_context||'_'||l_context_no ); + l_context := ut_suite_context(a_suite.object_owner, a_suite.object_name, l_context_name, l_context_pos ); + l_context.path := a_suite.path||'.'||l_context_name; + l_context.description := a_annotations.by_line( l_context_pos ).text; + l_context.parse_time := a_annotations.parse_time; + + --if nested context found + if a_annotations.by_name(gc_context).next(l_context_pos) < l_end_context_pos or l_end_context_pos is null then + get_suite_contexts_items( l_context, a_annotations, l_context_items, l_context_pos ); + l_end_context_pos := get_endcontext_position(l_context_pos, a_annotations.by_name ); + else + l_context_items := ut_suite_items(); + end if; - exit when l_end_context_pos is null; + if l_end_context_pos is null then + a_suite.put_warning( + 'Missing "--%endcontext" annotation for a "--%context" annotation. The end of package specification is effective end of context.'|| get_object_reference( a_suite, null, l_context_pos ) + ); + l_end_context_pos := a_annotations.by_line.last; + end if; - l_context_items := ut_suite_items(); --create a sub-set of annotations to process as sub-suite (context) - l_ctx_annotations := get_annotations_in_context( a_annotations, l_context_pos, l_end_context_pos); + l_ctx_annotations := get_annotations_in_context( a_annotations, l_context_pos, l_end_context_pos); - l_context_name := coalesce( - l_ctx_annotations.by_line( l_context_pos ).text - , gc_context||'_'||l_context_no - ); if l_context_names.exists(l_context_name) then add_annotation_ignored_warning( a_suite, 'context', 'Context name must be unique in a suite. Context and all of it''s content ignored.', l_context_pos ); else l_context_names(l_context_name) := true; - l_context := ut_suite_context(a_suite.object_owner, a_suite.object_name, l_context_name, l_context_pos ); - - l_context.path := a_suite.path||'.'||l_context_name; - l_context.description := l_ctx_annotations.by_line( l_context_pos ).text; - l_context.parse_time := a_annotations.parse_time; - warning_on_duplicate_annot( l_context, l_ctx_annotations.by_name, gc_context ); - add_suite_tests( l_context, l_ctx_annotations, l_context_items ); + add_tests_to_items( l_context, l_ctx_annotations, l_context_items ); add_items_to_list(a_suite_items, l_context_items); a_suite_items.extend; a_suite_items(a_suite_items.last) := l_context; @@ -798,27 +807,17 @@ create or replace package body ut_suite_builder is end loop; end; - procedure warning_on_incomplete_context( + procedure warning_on_floating_endcontext( a_suite in out nocopy ut_suite, a_package_ann_index tt_annotations_by_name ) is l_annotation_pos t_annotation_position; begin - if a_package_ann_index.exists(gc_context) then - l_annotation_pos := a_package_ann_index(gc_context).first; - while l_annotation_pos is not null loop - add_annotation_ignored_warning( - a_suite, gc_context, 'Invalid annotation %%%. Cannot find following "--%endcontext".', - l_annotation_pos - ); - l_annotation_pos := a_package_ann_index(gc_context).next(l_annotation_pos); - end loop; - end if; if a_package_ann_index.exists(gc_endcontext) then l_annotation_pos := a_package_ann_index(gc_endcontext).first; while l_annotation_pos is not null loop add_annotation_ignored_warning( - a_suite, gc_endcontext, 'Invalid annotation %%%. Cannot find preceding "--%context".', + a_suite, gc_endcontext, 'Extra %%% annotation found. Cannot find corresponding "--%context".', l_annotation_pos ); l_annotation_pos := a_package_ann_index(gc_endcontext).next(l_annotation_pos); @@ -891,10 +890,10 @@ create or replace package body ut_suite_builder is build_suitepath( l_suite, l_annotations ); get_suite_contexts_items( l_suite, l_annotations, a_suite_items ); --create suite tests and add - add_suite_tests( l_suite, l_annotations, a_suite_items ); + add_tests_to_items( l_suite, l_annotations, a_suite_items ); --by this time all contexts were consumed and l_annotations should not have any context/endcontext annotation in it. - warning_on_incomplete_context( l_suite, l_annotations.by_name ); + warning_on_floating_endcontext( l_suite, l_annotations.by_name ); a_suite_items.extend; a_suite_items( a_suite_items.last) := l_suite; diff --git a/test/ut3_tester/core/test_suite_builder.pkb b/test/ut3_tester/core/test_suite_builder.pkb index b49ecb2da..514becc1c 100644 --- a/test/ut3_tester/core/test_suite_builder.pkb +++ b/test/ut3_tester/core/test_suite_builder.pkb @@ -674,6 +674,99 @@ create or replace package body test_suite_builder is ); end; + procedure nested_contexts is + l_actual clob; + l_annotations ut3.ut_annotations; + begin + --Arrange + l_annotations := ut3.ut_annotations( + ut3.ut_annotation( 1, 'suite','Cool', null), + ut3.ut_annotation( 2, 'beforeall',null, 'suite_level_beforeall'), + ut3.ut_annotation( 3, 'test','In suite', 'suite_level_test'), + ut3.ut_annotation( 4, 'context','a_context', null), + ut3.ut_annotation( 5, 'displayname','A context', null), + ut3.ut_annotation( 6, 'beforeall',null, 'context_setup'), + ut3.ut_annotation( 7, 'test', 'First test in context', 'first_test_in_a_context'), + ut3.ut_annotation( 8, 'context','a_nested_context', null), + ut3.ut_annotation( 9, 'displayname','A nested context', null), + ut3.ut_annotation(10, 'beforeall',null, 'nested_context_setup'), + ut3.ut_annotation(11, 'test', 'Test in nested context', 'test_in_nested_context'), + ut3.ut_annotation(12, 'endcontext',null, null), + ut3.ut_annotation(13, 'context','nested_context_2', null), + ut3.ut_annotation(14, 'test', 'Test in nested context', 'test_in_nested_context_2'), + ut3.ut_annotation(15, 'context','a_nested_context_3', null), + ut3.ut_annotation(16, 'test', 'Test in nested context', 'test_in_nested_context_3'), + ut3.ut_annotation(17, 'endcontext',null, null), + ut3.ut_annotation(18, 'endcontext',null, null), + ut3.ut_annotation(19, 'test', 'Second test in context', 'second_test_in_a_context'), + ut3.ut_annotation(20, 'endcontext',null, null) + ); + --Act + l_actual := invoke_builder_for_annotations(l_annotations, 'SOME_PACKAGE'); + --Assert + ut.expect(l_actual).to_be_like( + ''|| + '' || + '%' || + '%' || + '' || + '%a_contextA contextsome_package.a_context' || + '%' || + '' || + '%nested_context_2nested_context_2some_package.a_context.nested_context_2' || + '%' || + '' || + '%a_nested_context_3a_nested_context_3some_package.a_context.nested_context_2.a_nested_context_3' || + '%' || + '' || + '%test_in_nested_context_3Test in nested contextsome_package.a_context.nested_context_2.a_nested_context_3.test_in_nested_context_3' || + '%' || + '' || + '' || + '%' || + '' || + '%test_in_nested_context_2Test in nested contextsome_package.a_context.nested_context_2.test_in_nested_context_2' || + '%' || + '' || + '' || + '%' || + '' || + '%a_nested_contextA nested contextsome_package.a_context.a_nested_context' || + '%' || + '' || + '%test_in_nested_contextTest in nested contextsome_package.a_context.a_nested_context.test_in_nested_context' || + '%' || + '' || + '' || + '%some_packagenested_context_setup' || + '%' || + '%' || + '' || + '%first_test_in_a_contextFirst test in contextsome_package.a_context.first_test_in_a_context' || + '%' || + '' || + '%second_test_in_a_contextSecond test in contextsome_package.a_context.second_test_in_a_context' || + '%' || + '' || + '' || + '%some_packagecontext_setup' || + '%' || + '' || + '' || + '' || + '%suite_level_testIn suitesome_package.suite_level_test' || + '%' || + '' || + '' || + '%some_packagesuite_level_beforeall' || + '%' || + '' || + ''|| + '' + ); + end; + + procedure before_after_in_context is l_actual clob; l_annotations ut3.ut_annotations; @@ -783,7 +876,7 @@ create or replace package body test_suite_builder is ut3.ut_annotation(1, 'suite','Cool', null), ut3.ut_annotation(2, 'beforeall',null, 'suite_level_beforeall'), ut3.ut_annotation(3, 'test','In suite', 'suite_level_test'), - ut3.ut_annotation(4, 'context','A context', null), + ut3.ut_annotation(4, 'context','a_context', null), ut3.ut_annotation(5, 'beforeall',null, 'context_setup'), ut3.ut_annotation(7, 'test', 'In context', 'test_in_a_context') ); @@ -791,25 +884,30 @@ create or replace package body test_suite_builder is l_actual := invoke_builder_for_annotations(l_annotations, 'SOME_PACKAGE'); --Assert ut.expect(l_actual).to_be_like( - '%Invalid annotation "--\%context". Cannot find following "--\%endcontext". Annotation ignored.%at package "UT3_TESTER.SOME_PACKAGE", line 4%' + '%Missing "--\%endcontext" annotation for a "--\%context" annotation. The end of context considered be end of package.%at package "UT3_TESTER.SOME_PACKAGE", line 4%' ,'\' ); ut.expect(l_actual).to_be_like( ''|| '' || - '%' || - '' || - '%suite_level_testIn suitesome_package.suite_level_test' || - '%' || - '' || - '%test_in_a_contextIn contextsome_package.test_in_a_context' || - '%' || - '' || - '' || - '%some_packagesuite_level_beforeall' || - '%some_packagecontext_setup' || - '%' || - '' || + '%' || + '%a_contexta_contextsome_package.a_context' || + '%' || + '' || + '%test_in_a_contextIn contextsome_package.a_context.test_in_a_context' || + '%' || + '' || + '' || + '%some_packagecontext_setup' || + '%' || + '%' || + '%suite_level_testIn suitesome_package.suite_level_test' || + '%' || + '' || + '' || + '%some_packagesuite_level_beforeall' || + '%' || + '' || ''|| '' ); @@ -835,7 +933,7 @@ create or replace package body test_suite_builder is l_actual := invoke_builder_for_annotations(l_annotations, 'SOME_PACKAGE'); --Assert ut.expect(l_actual).to_be_like( - '%Invalid annotation "--\%endcontext". Cannot find preceding "--\%context". Annotation ignored.%at package "UT3_TESTER.SOME_PACKAGE", line 9%' + '%Extra "--\%endcontext" annotation found. Cannot find corresponding "--\%context". Annotation ignored.%at package "UT3_TESTER.SOME_PACKAGE", line 9%' ,'\' ); ut.expect(l_actual).to_be_like( @@ -850,7 +948,7 @@ create or replace package body test_suite_builder is '%' || '' || '' || - '%some_packagecontext_setup' || + '%some_packagecontext_setup' || '%' || '' || '' || @@ -859,7 +957,7 @@ create or replace package body test_suite_builder is '%' || '' || '' || - '%some_packagesuite_level_beforeall' || + '%some_packagesuite_level_beforeall' || '%' || '' || ''|| @@ -1152,7 +1250,7 @@ create or replace package body test_suite_builder is '%testtag%'|| '%%' ); - + end; procedure suite_tag_annotation is @@ -1173,9 +1271,9 @@ create or replace package body test_suite_builder is '%suitetag%'|| '%%' ); - + end; - + procedure test_tags_annotation is l_actual clob; l_annotations ut3.ut_annotations; @@ -1195,7 +1293,7 @@ create or replace package body test_suite_builder is '%testtagtesttag2testtag3%'|| '%%' ); - + end; procedure suite_tags_annotation is @@ -1216,7 +1314,7 @@ create or replace package body test_suite_builder is '%suitetagsuitetag1suitetag2%'|| '%%' ); - + end; procedure test_2line_tags_annotation is @@ -1239,7 +1337,7 @@ create or replace package body test_suite_builder is '%testtagtesttag2%'|| '%%' ); - + end; procedure suite_2line_tags_annotation is @@ -1261,7 +1359,7 @@ create or replace package body test_suite_builder is '%suitetagsuitetag1%'|| '%%' ); - + end; procedure test_empty_tag is @@ -1280,9 +1378,9 @@ create or replace package body test_suite_builder is '%%"--%tags" annotation requires a tag value populated. Annotation ignored.%%'|| '%%' ); - + end; - + procedure suite_empty_tag is l_actual clob; l_annotations ut3.ut_annotations; @@ -1299,7 +1397,7 @@ create or replace package body test_suite_builder is '%"--%tags" annotation requires a tag value populated. Annotation ignored.%%'|| '%%' ); - + end; procedure test_duplicate_tag is @@ -1322,9 +1420,9 @@ create or replace package body test_suite_builder is '%testtagtesttag1testtag2%'|| '%%' ); - + end; - + procedure suite_duplicate_tag is l_actual clob; l_annotations ut3.ut_annotations; @@ -1344,7 +1442,7 @@ create or replace package body test_suite_builder is '%suitetagsuitetag1suitetag2%'|| '%%' ); - + end; procedure test_empty_tag_between is @@ -1366,9 +1464,9 @@ create or replace package body test_suite_builder is '%testtagtesttag1%'|| '%%' ); - + end; - + procedure suite_empty_tag_between is l_actual clob; l_annotations ut3.ut_annotations; @@ -1387,8 +1485,8 @@ create or replace package body test_suite_builder is '%suitetagsuitetag1%'|| '%%' ); - - end; + + end; procedure test_special_char_tag is l_actual clob; @@ -1409,9 +1507,9 @@ create or replace package body test_suite_builder is '%#?$%^&*!|\/@][%'|| '%%' ); - + end; - + procedure suite_special_char_tag is l_actual clob; l_annotations ut3.ut_annotations; @@ -1430,8 +1528,8 @@ create or replace package body test_suite_builder is '%#?$%^&*!|\/@][%'|| '%%' ); - - end; + + end; end test_suite_builder; / diff --git a/test/ut3_tester/core/test_suite_builder.pks b/test/ut3_tester/core/test_suite_builder.pks index 721db153f..688cc342c 100644 --- a/test/ut3_tester/core/test_suite_builder.pks +++ b/test/ut3_tester/core/test_suite_builder.pks @@ -116,6 +116,9 @@ create or replace package test_suite_builder is --%test(Creates nested suite for content between context/endcontext annotations) procedure suite_from_context; + --%test(Creates nested contexts inside a context) + procedure nested_contexts; + --%test(Associates before/after all/each to tests in context only) procedure before_after_in_context; diff --git a/test/ut3_user/api/test_ut_run.pkb b/test/ut3_user/api/test_ut_run.pkb index 329b71357..9160814dd 100644 --- a/test/ut3_user/api/test_ut_run.pkb +++ b/test/ut3_user/api/test_ut_run.pkb @@ -688,7 +688,7 @@ Failures:% select * bulk collect into l_results from table(ut3.ut.run('bad_annotations')); l_actual := ut3_tester_helper.main_helper.table_to_clob(l_results); - ut.expect(l_actual).to_be_like('%Invalid annotation "--%context". Cannot find following "--%endcontext". Annotation ignored.% + ut.expect(l_actual).to_be_like('%Missing "--%endcontext" annotation for a "--%context" annotation. The end of context considered be end of package.% %1 tests, 0 failed, 0 errored, 0 disabled, 1 warning(s)%'); end; From c5f81eb1b25729166f13182cdd508c4860cd6fcb Mon Sep 17 00:00:00 2001 From: Jacek Gebal Date: Wed, 23 Oct 2019 02:23:12 +0100 Subject: [PATCH 2/6] Fixing failing tests --- source/core/ut_suite_builder.pkb | 2 +- test/ut3_tester/core/test_suite_builder.pkb | 2 +- test/ut3_tester/core/test_suite_builder.pks | 2 +- test/ut3_user/api/test_ut_run.pkb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/core/ut_suite_builder.pkb b/source/core/ut_suite_builder.pkb index fc3e72e60..ebdd4a57c 100644 --- a/source/core/ut_suite_builder.pkb +++ b/source/core/ut_suite_builder.pkb @@ -777,7 +777,7 @@ create or replace package body ut_suite_builder is if l_end_context_pos is null then a_suite.put_warning( - 'Missing "--%endcontext" annotation for a "--%context" annotation. The end of package specification is effective end of context.'|| get_object_reference( a_suite, null, l_context_pos ) + 'Missing "--%endcontext" annotation for a "--%context" annotation. The end of package is considered end of context.'|| get_object_reference( a_suite, null, l_context_pos ) ); l_end_context_pos := a_annotations.by_line.last; end if; diff --git a/test/ut3_tester/core/test_suite_builder.pkb b/test/ut3_tester/core/test_suite_builder.pkb index 514becc1c..68d3f47f8 100644 --- a/test/ut3_tester/core/test_suite_builder.pkb +++ b/test/ut3_tester/core/test_suite_builder.pkb @@ -884,7 +884,7 @@ create or replace package body test_suite_builder is l_actual := invoke_builder_for_annotations(l_annotations, 'SOME_PACKAGE'); --Assert ut.expect(l_actual).to_be_like( - '%Missing "--\%endcontext" annotation for a "--\%context" annotation. The end of context considered be end of package.%at package "UT3_TESTER.SOME_PACKAGE", line 4%' + '%Missing "--\%endcontext" annotation for a "--\%context" annotation. The end of package is considered end of context.%at package "UT3_TESTER.SOME_PACKAGE", line 4%' ,'\' ); ut.expect(l_actual).to_be_like( diff --git a/test/ut3_tester/core/test_suite_builder.pks b/test/ut3_tester/core/test_suite_builder.pks index 688cc342c..c4e945598 100644 --- a/test/ut3_tester/core/test_suite_builder.pks +++ b/test/ut3_tester/core/test_suite_builder.pks @@ -125,7 +125,7 @@ create or replace package test_suite_builder is --%test(Propagates beforeeach/aftereach to context) procedure before_after_out_of_context; - --%test(Does not create context and gives warning when endcontext is missing) + --%test(Gives warning when endcontext is missing) procedure context_without_endcontext; --%test(Gives warning if --%endcontext is missing a preceding --%context) diff --git a/test/ut3_user/api/test_ut_run.pkb b/test/ut3_user/api/test_ut_run.pkb index 9160814dd..6febdd0c1 100644 --- a/test/ut3_user/api/test_ut_run.pkb +++ b/test/ut3_user/api/test_ut_run.pkb @@ -688,7 +688,7 @@ Failures:% select * bulk collect into l_results from table(ut3.ut.run('bad_annotations')); l_actual := ut3_tester_helper.main_helper.table_to_clob(l_results); - ut.expect(l_actual).to_be_like('%Missing "--%endcontext" annotation for a "--%context" annotation. The end of context considered be end of package.% + ut.expect(l_actual).to_be_like('%Missing "--%endcontext" annotation for a "--%context" annotation. The end of package is considered end of context.% %1 tests, 0 failed, 0 errored, 0 disabled, 1 warning(s)%'); end; From e1dfd05429ec432f5f3eb9c2338fb88c969e43b1 Mon Sep 17 00:00:00 2001 From: Jacek Gebal Date: Thu, 24 Oct 2019 00:02:26 +0100 Subject: [PATCH 3/6] Updated documentation. Added validation of context name. --- docs/userguide/annotations.md | 118 +++++++++++++++++--- source/core/ut_suite_builder.pkb | 10 ++ test/ut3_tester/core/test_suite_builder.pkb | 54 +++++++-- test/ut3_tester/core/test_suite_builder.pks | 3 + 4 files changed, 162 insertions(+), 23 deletions(-) diff --git a/docs/userguide/annotations.md b/docs/userguide/annotations.md index c7dea5865..5f51c4229 100644 --- a/docs/userguide/annotations.md +++ b/docs/userguide/annotations.md @@ -997,9 +997,9 @@ In most of the cases, the code to be tested is consisting of PLSQL packages cont When creating test suites, it's quite common to maintain `one to one` relationship between test suite packages and tested code. When it comes to test procedures themselves, it is best practice to have one test procedure for one tested behavior of the code that is tested. -The relationship between test procedure and tested procedure/function will be therefore `many to one` in most of the cases. +The relationship between test procedure and tested code will be therefore `many to one` or `many to many` in most of the cases. -With this comes a challenge. How to group tests, related to one tested procedure, so that it is obvious that they relate to the same code. +With this comes a challenge. How to group tests, related to one tested behavior, so that it is obvious that they relate to the same thing. This is where utPLSQL contexts come handy. @@ -1008,18 +1008,22 @@ Contexts allow for creating sub-suites within a suite package and they allow for In essence, context behaves like a suite within a suite. Context have following characteristics: -- start with the `--%context` annotation and ends with `--%endcontext` -- can have a name provided as parameter for example `--%context(remove_rooms_by_name)` -- when no name is provided for context, the context is names `context_N` where `N` is the number of the context in suite -- can have their own `--%beforeall`, `--%beforeeach`, `--%afterall` and `--%aftereach` procedures -- `--%beforeall`, `--%beforeeach`, `--%afterall` and `--%aftereach` procedures defined at suite level, propagate to context -- test suite package can have multiple contexts in it -- contexts cannot be nested - +- context starts with the `--%context` annotation and ends with `--%endcontext` +- can have a name provided as parameter for example `--%context(remove_rooms_by_name)`. This is different than with `suite` and `test` annotations, where name is taken from test `package/procedure` +- when no name is provided for context, the context is named `context_N` where `N` is the number of the context in suite or parent context +- context name must be unique within it's parent (suite or parent context) +- if context name is not unique within it's parent, context and it's entire content is excluded from execution +- context name should not contain spaces or special characters +- context name cannot contain a `.` (hard stop) character +- contexts can be nested, so a context can be nested within another context +- suite/context can have multiple nested sibling contexts in it +- contexts can have their own `--%beforeall`, `--%beforeeach`, `--%afterall` and `--%aftereach` procedures +- `--%beforeall`, `--%beforeeach`, `--%afterall` and `--%aftereach` procedures defined at ancestor level, propagate to context +- if `--%endcontext` is missing for a context, the context spans to the end of package specification The below example illustrates usage of `--%context` for separating tests for individual procedures of package. -Tested tables and code +Sample tables and code ```sql create table rooms ( room_key number primary key, @@ -1078,8 +1082,8 @@ end; Below test suite defines: - `--%beforeall` outside of context, that will be executed before all tests -- `--%context(remove_rooms_by_name)` to group tests for `remove_rooms_by_name` procedure -- `--%context(add_rooms_content)` to group tests for `add_rooms_content` procedure +- `--%context(remove_rooms_by_name)` to group tests related to `remove_rooms_by_name` functionality +- `--%context(add_rooms_content)` to group tests related to `add_rooms_content` functionality ```sql create or replace package test_rooms_management is @@ -1103,7 +1107,6 @@ create or replace package test_rooms_management is --%endcontext - --%context(add_rooms_content) --%displayname(Add content to a room) @@ -1221,6 +1224,93 @@ Finished in .035261 seconds 5 tests, 0 failed, 0 errored, 0 disabled, 0 warning(s) ``` +Example of nested contexts test suite specification. +*Source - [slide 145](https://www.slideshare.net/Kevlin/structure-and-interpretation-of-test-cases/145?src=clipshare) of Structure and Interpretation of Test Cases by Kevlin Henney* + +```sql +create or replace package queue_spec as + --%suite(Queue specification) + + --%context(a_new_queue) + --%displayname(A new queue) + + --%test(Is empty) + procedure is_empty; + --%test(Preserves positive bounding capacity) + procedure positive_bounding_capacity; + --%test(Cannot be created with non positive bounding capacity) + procedure non_positive_bounding_cap; + --%endcontext + --%context(an_empty_queue) + --%displayname(An empty queue) + + --%test(Dequeues an empty value) + procedure deq_empty_value; + --%test(Remains empty when null enqueued) + procedure empty_with_null_enq; + --%test(Becomes non empty when non null value enqueued) + procedure non_empty_after_enq; + --%endcontext + --%context(a_non_empty_queue) + --%displayname(A non empty queue) + + --%context(that_is_not_full) + --%displayname(that is not full) + + --%test(Becomes longer when non null value enqueued) + procedure grow_on_enq_non_null; + --%test(Becomes full when enqueued up to capacity) + procedure full_on_enq_to_cap; + --%endcontext + --%context(that_is_full) + --%displayname(That is full) + + --%test(Ignores further enqueued values) + procedure full_ignore_enq; + --%test(Becomes non full when dequeued) + procedure non_full_on_deq; + --%endcontext + + --%test(Dequeues values in order enqueued) + procedure dequeue_ordered; + --%test(Remains unchanged when null enqueued) + procedure no_change_on_null_enq; + --%endcontext +end; +``` + + +When such specification gets executed `ut.run('queue_spec'')` (without body created) you will see the nesting of tests within contexts. +``` +Queue specification + An empty queue + Dequeues an empty value [.014 sec] (FAILED - 1) + Remains empty when null enqueued [.004 sec] (FAILED - 2) + Becomes non empty when non null value enqueued [.005 sec] (FAILED - 3) + A non empty queue + that is not full + Becomes longer when non null value enqueued [.005 sec] (FAILED - 4) + Becomes full when enqueued up to capacity [.005 sec] (FAILED - 5) + That is full + Ignores further enqueued values [.004 sec] (FAILED - 6) + Becomes non full when dequeued [.005 sec] (FAILED - 7) + Dequeues values in order enqueued [.006 sec] (FAILED - 8) + Remains unchanged when null enqueued [.004 sec] (FAILED - 9) + A new queue + Is empty [.007 sec] (FAILED - 10) + Preserves positive bounding capacity [.006 sec] (FAILED - 11) + Cannot be created with non positive bounding capacity [.005 sec] (FAILED - 12) +Failures: + 1) deq_empty_value + ORA-04067: not executed, package body "UT3.QUEUE_SPEC" does not exist + ORA-06508: PL/SQL: could not find program unit being called: "UT3.QUEUE_SPEC" + ORA-06512: at line 6 +... +Finished in .088573 seconds +12 tests, 0 failed, 12 errored, 0 disabled, 0 warning(s) +``` + +Suite nesting allows for organizing tests into human-readable specification of behavior. ### Tags diff --git a/source/core/ut_suite_builder.pkb b/source/core/ut_suite_builder.pkb index 9207f782f..4afbce48c 100644 --- a/source/core/ut_suite_builder.pkb +++ b/source/core/ut_suite_builder.pkb @@ -774,6 +774,16 @@ create or replace package body ut_suite_builder is l_end_context_pos := get_endcontext_position(l_context_pos, a_annotations.by_name ); l_context_name := coalesce( a_annotations.by_line( l_context_pos ).text, gc_context||'_'||l_context_no ); + if regexp_like( l_context_name, '\.' ) or l_context_name is null then + if regexp_like( l_context_name, '\.' ) then + a_suite.put_warning( + 'Invalid value "'||l_context_name||'" for context name. The name cannot contain "." (hard stop) character.' || + ' Context name ignored and fallback to auto-name "'||gc_context||'_'||l_context_no||'" ' || + get_object_reference( a_suite, null, l_context_pos ) + ); + end if; + l_context_name := gc_context||'_'||l_context_no; + end if; l_context := ut_suite_context(a_suite.object_owner, a_suite.object_name, l_context_name, l_context_pos ); l_context.path := a_suite.path||'.'||l_context_name; l_context.description := a_annotations.by_line( l_context_pos ).text; diff --git a/test/ut3_tester/core/test_suite_builder.pkb b/test/ut3_tester/core/test_suite_builder.pkb index d4879cb83..82d7ec840 100644 --- a/test/ut3_tester/core/test_suite_builder.pkb +++ b/test/ut3_tester/core/test_suite_builder.pkb @@ -965,7 +965,6 @@ create or replace package body test_suite_builder is ); end; - --%test(Gives warning when two contexts have the same name) procedure duplicate_context_name is l_actual clob; l_annotations ut3.ut_annotations; @@ -1022,6 +1021,43 @@ create or replace package body test_suite_builder is ); end; + procedure hard_stop_in_ctx_name is + l_actual clob; + l_annotations ut3.ut_annotations; + l_bad_name varchar2(100); + begin + --Arrange + l_bad_name := 'Context with invalid name. Should fail'; + l_annotations := ut3.ut_annotations( + ut3.ut_annotation(1, 'suite','Cool', null), + ut3.ut_annotation(4, 'context','Context with invalid name. Should fail', null), + ut3.ut_annotation(7, 'test', 'In context', 'test_in_a_context'), + ut3.ut_annotation(13, 'endcontext',null, null) + ); + --Act + l_actual := invoke_builder_for_annotations(l_annotations, 'SOME_PACKAGE'); + --Assert + ut.expect(l_actual).to_be_like( + '%Invalid value "'||l_bad_name||'" for context name. The name cannot contain "." (hard stop) character. Context name ignored and fallback to auto-name "context_1"%' + ); + ut.expect(l_actual).to_be_like( + ''|| + '' || + '%' || + '' || + '%context_1Context with invalid name. Should failsome_package.context_1' || + '%' || + '' || + '%test_in_a_contextIn contextsome_package.context_1.test_in_a_context' || + '%' || + '%' || + '%' || + '%' || + ''|| + '' + ); + end; + procedure throws_value_empty is l_actual clob; l_annotations ut3.ut_annotations; @@ -1551,13 +1587,13 @@ create or replace package body test_suite_builder is ); ut.expect(l_actual).to_be_like( '%Invalid value "bad tag" for "--%tags" annotation.'|| - ' See documentation for details on valid tag values. Annotation value ignored. -at package "UT3_TESTER.SOME_PACKAGE", line 3%' + ' See documentation for details on valid tag values. Annotation value ignored.' || + '%at package "UT3_TESTER.SOME_PACKAGE", line 3%' ); ut.expect(l_actual).to_be_like( '%Invalid value "bad tag" for "--%tags" annotation.'|| - ' See documentation for details on valid tag values. Annotation value ignored. -at package "UT3_TESTER.SOME_PACKAGE.TEST_PROCEDURE", line 9%' + ' See documentation for details on valid tag values. Annotation value ignored.' || + '%at package "UT3_TESTER.SOME_PACKAGE.TEST_PROCEDURE", line 9%' ); end; @@ -1581,13 +1617,13 @@ at package "UT3_TESTER.SOME_PACKAGE.TEST_PROCEDURE", line 9 ); ut.expect(l_actual).to_be_like( '%Invalid value "-invalid_tag" for "--%tags" annotation.'|| - ' See documentation for details on valid tag values. Annotation value ignored. -at package "UT3_TESTER.SOME_PACKAGE", line 3%' + ' See documentation for details on valid tag values. Annotation value ignored.' || + '%at package "UT3_TESTER.SOME_PACKAGE", line 3%' ); ut.expect(l_actual).to_be_like( '%Invalid value "-invalid_tag" for "--%tags" annotation.'|| - ' See documentation for details on valid tag values. Annotation value ignored. -at package "UT3_TESTER.SOME_PACKAGE.TEST_PROCEDURE", line 9%' + ' See documentation for details on valid tag values. Annotation value ignored.' || + '%at package "UT3_TESTER.SOME_PACKAGE.TEST_PROCEDURE", line 9%' ); end; diff --git a/test/ut3_tester/core/test_suite_builder.pks b/test/ut3_tester/core/test_suite_builder.pks index 3fd08607f..c9683cd9e 100644 --- a/test/ut3_tester/core/test_suite_builder.pks +++ b/test/ut3_tester/core/test_suite_builder.pks @@ -134,6 +134,9 @@ create or replace package test_suite_builder is --%test(Gives warning when two contexts have the same name and ignores duplicated context) procedure duplicate_context_name; + --%test(Fallback to default naming and gives warning when context name contains "." character) + procedure hard_stop_in_ctx_name; + --%endcontext --%context(throws) From 0070cf8ed0bf6ed33cf395f19640d7d4823fcaf2 Mon Sep 17 00:00:00 2001 From: Jacek Gebal Date: Thu, 24 Oct 2019 00:04:58 +0100 Subject: [PATCH 4/6] changed hard stop to "full stop/period" --- docs/userguide/annotations.md | 2 +- source/core/ut_suite_builder.pkb | 2 +- test/ut3_tester/core/test_suite_builder.pkb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/userguide/annotations.md b/docs/userguide/annotations.md index 5f51c4229..51d94215d 100644 --- a/docs/userguide/annotations.md +++ b/docs/userguide/annotations.md @@ -1014,7 +1014,7 @@ Context have following characteristics: - context name must be unique within it's parent (suite or parent context) - if context name is not unique within it's parent, context and it's entire content is excluded from execution - context name should not contain spaces or special characters -- context name cannot contain a `.` (hard stop) character +- context name cannot contain a `.` (full stop/period) character - contexts can be nested, so a context can be nested within another context - suite/context can have multiple nested sibling contexts in it - contexts can have their own `--%beforeall`, `--%beforeeach`, `--%afterall` and `--%aftereach` procedures diff --git a/source/core/ut_suite_builder.pkb b/source/core/ut_suite_builder.pkb index 4afbce48c..197bec926 100644 --- a/source/core/ut_suite_builder.pkb +++ b/source/core/ut_suite_builder.pkb @@ -777,7 +777,7 @@ create or replace package body ut_suite_builder is if regexp_like( l_context_name, '\.' ) or l_context_name is null then if regexp_like( l_context_name, '\.' ) then a_suite.put_warning( - 'Invalid value "'||l_context_name||'" for context name. The name cannot contain "." (hard stop) character.' || + 'Invalid value "'||l_context_name||'" for context name. The name cannot contain "." (full stop/period) character.' || ' Context name ignored and fallback to auto-name "'||gc_context||'_'||l_context_no||'" ' || get_object_reference( a_suite, null, l_context_pos ) ); diff --git a/test/ut3_tester/core/test_suite_builder.pkb b/test/ut3_tester/core/test_suite_builder.pkb index 82d7ec840..97de59c11 100644 --- a/test/ut3_tester/core/test_suite_builder.pkb +++ b/test/ut3_tester/core/test_suite_builder.pkb @@ -1038,7 +1038,7 @@ create or replace package body test_suite_builder is l_actual := invoke_builder_for_annotations(l_annotations, 'SOME_PACKAGE'); --Assert ut.expect(l_actual).to_be_like( - '%Invalid value "'||l_bad_name||'" for context name. The name cannot contain "." (hard stop) character. Context name ignored and fallback to auto-name "context_1"%' + '%Invalid value "'||l_bad_name||'" for context name. The name cannot contain "." (full stop/period) character. Context name ignored and fallback to auto-name "context_1"%' ); ut.expect(l_actual).to_be_like( ''|| From 8f0b267086cc84feef0156a97d9f07626b73b1d1 Mon Sep 17 00:00:00 2001 From: Jacek Gebal Date: Thu, 24 Oct 2019 00:11:51 +0100 Subject: [PATCH 5/6] Cleanup --- source/core/ut_suite_builder.pkb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/core/ut_suite_builder.pkb b/source/core/ut_suite_builder.pkb index 197bec926..4ab68afa6 100644 --- a/source/core/ut_suite_builder.pkb +++ b/source/core/ut_suite_builder.pkb @@ -829,7 +829,7 @@ create or replace package body ut_suite_builder is end loop; end; - procedure warning_on_floating_endcontext( + procedure warning_on_extra_endcontext( a_suite in out nocopy ut_suite, a_package_ann_index tt_annotations_by_name ) is @@ -915,7 +915,7 @@ create or replace package body ut_suite_builder is add_tests_to_items( l_suite, l_annotations, a_suite_items ); --by this time all contexts were consumed and l_annotations should not have any context/endcontext annotation in it. - warning_on_floating_endcontext( l_suite, l_annotations.by_name ); + warning_on_extra_endcontext( l_suite, l_annotations.by_name ); a_suite_items.extend; a_suite_items( a_suite_items.last) := l_suite; From be39586c8ee666100d267cf8e389f3f0cdb875b7 Mon Sep 17 00:00:00 2001 From: Jacek Gebal Date: Sun, 27 Oct 2019 12:28:57 +0000 Subject: [PATCH 6/6] Added support for `--%name` annotation, to name the contexts. Changed behavior of ``--%context` annotation. The value f annotation now only indicates the context description. Resolves #1016 --- docs/userguide/annotations.md | 161 ++++++++-- source/core/ut_suite_builder.pkb | 120 +++++--- test/ut3_tester/core/test_suite_builder.pkb | 291 +++++++++++++++--- test/ut3_tester/core/test_suite_builder.pks | 57 ++-- test/ut3_tester/core/test_suite_manager.pkb | 4 +- test/ut3_user/api/test_ut_run.pkb | 5 +- test/ut3_user/reporters.pkb | 4 +- .../reporters/test_realtime_reporter.pkb | 17 +- 8 files changed, 506 insertions(+), 153 deletions(-) diff --git a/docs/userguide/annotations.md b/docs/userguide/annotations.md index ddfec86f1..f8fbaa25d 100644 --- a/docs/userguide/annotations.md +++ b/docs/userguide/annotations.md @@ -21,9 +21,9 @@ We strongly recommend putting package level annotations at the very top of packa | --- | --- | --- | | `--%suite()` | Package | Mandatory. Marks package as a test suite. Optional suite description can be provided (see `displayname`). | | `--%suitepath()` | Package | Similar to java package. The annotation allows logical grouping of suites into hierarchies. | -| `--%displayname()` | Package/procedure | Human-readable and meaningful description of a context/suite/test. Provides description to a `context` when used within `context`. When used with `test` or `suite` annotation, overrides the `` provided with `suite`/`test`. | +| `--%displayname()` | Package/procedure | Human-readable and meaningful description of a context/suite/test. Overrides the `` provided with `suite`/`test`/`context` annotation. This annotation is redundant and might be removed in future releases. | | `--%test()` | Procedure | Denotes that the annotated procedure is a unit test procedure. Optional test description can by provided (see `displayname`). | -| `--%throws([,...])`| Procedure | Denotes that the annotated test procedure must throw one of the exceptions provided. Supported forms of exceptions are: numeric literals, numeric contant names, exception constant names, predefined Oracle exception names. | +| `--%throws([,...])`| Procedure | Denotes that the annotated test procedure must throw one of the exceptions provided. Supported forms of exceptions are: numeric literals, numeric constant names, exception constant names, predefined Oracle exception names. | | `--%beforeall` | Procedure | Denotes that the annotated procedure should be executed once before all elements of the suite. | | `--%beforeall([[.].][,...])` | Package | Denotes that the mentioned procedure(s) should be executed once before all elements of the suite. | | `--%afterall` | Procedure | Denotes that the annotated procedure should be executed once after all elements of the suite. | @@ -36,7 +36,8 @@ We strongly recommend putting package level annotations at the very top of packa | `--%aftertest([[.].][,...])` | Procedure | Denotes that mentioned procedure(s) should be executed after the annotated `%test` procedure. | | `--%rollback()` | Package/procedure | Defines transaction control. Supported values: `auto`(default) - a savepoint is created before invocation of each "before block" is and a rollback to specific savepoint is issued after each "after" block; `manual` - rollback is never issued automatically. Property can be overridden for child element (test in suite) | | `--%disabled` | Package/procedure | Used to disable a suite or a test. Disabled suites/tests do not get executed, they are however marked and reported as disabled in a test run. | -| `--%context()` | Package | Denotes start of a named context (sub-suite) in a suite package | +| `--%context()` | Package | Denotes start of a named context (sub-suite) in a suite package an optional description for context can be provided. | +| `--%name()` | Package | Denotes name for a context. Must be placed after the context annotation and before start of nested context. | | `--%endcontext` | Package | Denotes end of a nested context (sub-suite) in a suite package | | `--%tags` | Package/procedure | Used to label a test or a suite for purpose of identification | @@ -1008,14 +1009,15 @@ Contexts allow for creating sub-suites within a suite package and they allow for In essence, context behaves like a suite within a suite. Context have following characteristics: -- context starts with the `--%context` annotation and ends with `--%endcontext` -- can have a name provided as parameter for example `--%context(remove_rooms_by_name)`. This is different than with `suite` and `test` annotations, where name is taken from test `package/procedure` -- when no name is provided for context, the context is named `context_N` where `N` is the number of the context in suite or parent context -- context name must be unique within it's parent (suite or parent context) +- context starts with the `--%context` annotation and ends with `--%endcontext`. Everything placed between those two annotations belongs to that context +- can have a description provided as parameter for example `--%context(Some interesting stuff)`. +- can have a name provided with `--%name` annotation. This is different than with `suite` and `test` annotations, where name is taken from `package/procedure` name. +- contexts can be nested, you can place a context inside another context +- when no name is provided for context, the context is named `context_N` where `N` is the number of the context in suite or parent context. +- context name must be unique within it's parent (suite / parent context) - if context name is not unique within it's parent, context and it's entire content is excluded from execution - context name should not contain spaces or special characters - context name cannot contain a `.` (full stop/period) character -- contexts can be nested, so a context can be nested within another context - suite/context can have multiple nested sibling contexts in it - contexts can have their own `--%beforeall`, `--%beforeeach`, `--%afterall` and `--%aftereach` procedures - `--%beforeall`, `--%beforeeach`, `--%afterall` and `--%aftereach` procedures defined at ancestor level, propagate to context @@ -1231,8 +1233,7 @@ Example of nested contexts test suite specification. create or replace package queue_spec as --%suite(Queue specification) - --%context(a_new_queue) - --%displayname(A new queue) + --%context(A new queue) --%test(Is empty) procedure is_empty; @@ -1241,8 +1242,7 @@ create or replace package queue_spec as --%test(Cannot be created with non positive bounding capacity) procedure non_positive_bounding_cap; --%endcontext - --%context(an_empty_queue) - --%displayname(An empty queue) + --%context(An empty queue) --%test(Dequeues an empty value) procedure deq_empty_value; @@ -1251,19 +1251,16 @@ create or replace package queue_spec as --%test(Becomes non empty when non null value enqueued) procedure non_empty_after_enq; --%endcontext - --%context(a_non_empty_queue) - --%displayname(A non empty queue) + --%context(A non empty queue) - --%context(that_is_not_full) - --%displayname(that is not full) + --%context(that is not full) --%test(Becomes longer when non null value enqueued) procedure grow_on_enq_non_null; --%test(Becomes full when enqueued up to capacity) procedure full_on_enq_to_cap; --%endcontext - --%context(that_is_full) - --%displayname(That is full) + --%context(that is full) --%test(Ignores further enqueued values) procedure full_ignore_enq; @@ -1312,6 +1309,122 @@ Finished in .088573 seconds Suite nesting allows for organizing tests into human-readable specification of behavior. +### Name +The `--%name` annotation is currently only used only for naming a context. +If a context doesn't have explicit name specified, then the name is given automatically by framework. + +The automatic name will be `context_#n` where `n` is a context number within a suite/parent context. + +The `--%name` can be useful when you would like to run only a specific context or its items by `suitepath`. + +Consider the below example. + +```sql +create or replace package queue_spec as + --%suite(Queue specification) + + --%context(A new queue) + + --%test(Cannot be created with non positive bounding capacity) + procedure non_positive_bounding_cap; + --%endcontext + --%context(An empty queue) + + --%test(Becomes non empty when non null value enqueued) + procedure non_empty_after_enq; + --%endcontext + --%context(A non empty queue) + + --%context(that is not full) + + --%test(Becomes full when enqueued up to capacity) + procedure full_on_enq_to_cap; + --%endcontext + --%context(that is full) + + --%test(Becomes non full when dequeued) + procedure non_full_on_deq; + --%endcontext + + --%endcontext +end; +``` + +In the above code, suitepaths, context names and context descriptions will be as follows. + +| suitepath | description | name | +|-----------|------------|------| +| queue_spec | Queue specification | queue_spec | +| queue_spec.context_#1 | A new queue | context_#1 | +| queue_spec.context_#2 | An empty queue | context_#2 | +| queue_spec.context_#3 | A non empty queue | context_#3 | +| queue_spec.context_#3.context_#1 | that is not full | context_#1 | +| queue_spec.context_#3.context_#2 | that is full | context_#2 | + +In order to run only the tests for the context `A non empty queue that is not full` you will need to call utPLSQL as below: +```sql + exec ut.run(':queue_spec.context_#3.context_#1'); +``` + +You can use `--%name` annotation to explicitly name contexts on suitepath. +```sql +create or replace package queue_spec as + --%suite(Queue specification) + + --%context(A new queue) + --%name(a_new_queue) + + --%test(Cannot be created with non positive bounding capacity) + procedure non_positive_bounding_cap; + --%endcontext + --%context(An empty queue) + --%name(an_empty_queue) + + --%test(Becomes non empty when non null value enqueued) + procedure non_empty_after_enq; + --%endcontext + --%context(A non empty queue) + --%name(a_non_empty_queue) + + --%context(that is not full) + --%name(that_is_not_full) + + --%test(Becomes full when enqueued up to capacity) + procedure full_on_enq_to_cap; + --%endcontext + --%context(that is full) + --%name(that_is_full) + + --%test(Becomes non full when dequeued) + procedure non_full_on_deq; + --%endcontext + + --%endcontext +end; +``` + +In the above code, suitepaths, context names and context descriptions will be as follows. + +| suitepath | description | name | +|-----------|------------|------| +| queue_spec | Queue specification | queue_spec | +| queue_spec.a_new_queue | A new queue | a_new_queue | +| queue_spec.an_empty_queue | An empty queue | an_empty_queue | +| queue_spec.a_non_empty_queue | A non empty queue | a_non_empty_queue | +| queue_spec.a_non_empty_queue.that_is_not_full | that is not full | that_is_not_full | +| queue_spec.a_non_empty_queue.that_is_full | that is full | that_is_full | + + +The `--%name` annotation is only relevant for: +- running subsets of tests by given context suitepath +- some of test reports, like `ut_junit_reporter` that use suitepath or test-suite element names (not descriptions) for reporting + +#### Name naming convention + +The value of `--%name` annotation must follow the following naming rules: +- cannot contain spaces +- cannot contain a `.` (full stop/dot) +- is case-insensitive ### Tags @@ -1440,8 +1553,9 @@ If you want to create tests for your application it is recommended to structure * Payments recognition * Payments set off -The `%suitepath` annotation is used for such grouping. Even though test packages are defined in a flat structure the `%suitepath` is used by the framework to form them into a hierarchical structure. Your payments recognition test package might look like: +The `--%suitepath` annotation is used for such grouping. Even though test packages are defined in a flat structure the `--%suitepath` is used by the framework to form them into a hierarchical structure. +Your payments recognition test package might look like: ```sql create or replace package test_payment_recognition as @@ -1476,8 +1590,8 @@ create or replace package test_payment_set_off as end test_payment_set_off; ``` -When you execute tests for your application, the framework constructs a test suite for each test package. Then it combines suites into grouping suites by the `%suitepath` annotation value so that the fully qualified path to the `recognize_by_num` procedure is `USER:payments.test_payment_recognition.test_recognize_by_num`. If any of its expectations fails then the test is marked as failed, also the `test_payment_recognition` suite, the parent suite `payments` and the whole run is marked as failed. -The test report indicates which expectation has failed on the payments module. The payments recognition submodule is causing the failure as `recognize_by_num` has not met the expectations of the test. Grouping tests into modules and submodules using the `%suitepath` annotation allows you to logically organize your project's flat structure of packages into functional groups. +When you execute tests for your application, the framework constructs a test suite for each test package. Then it combines suites into grouping suites by the `--%suitepath` annotation value so that the fully qualified path to the `recognize_by_num` procedure is `USER:payments.test_payment_recognition.test_recognize_by_num`. If any of its expectations fails then the test is marked as failed, also the `test_payment_recognition` suite, the parent suite `payments` and the whole run is marked as failed. +The test report indicates which expectation has failed on the payments module. The payments recognition submodule is causing the failure as `recognize_by_num` has not met the expectations of the test. Grouping tests into modules and submodules using the `--%suitepath` annotation allows you to logically organize your project's flat structure of packages into functional groups. An additional advantage of such grouping is the fact that every element level of the grouping can be an actual unit test package containing a common module level setup for all of the submodules. So in addition to the packages mentioned above you could have the following package. ```sql @@ -1493,9 +1607,10 @@ create or replace package payments as end payments; ``` -A `%suitepath` can be provided in three ways: + +When executing tests, `path` for executing tests can be provided in three ways: * schema - execute all tests in the schema -* [schema]:suite1[.suite2][.suite3]...[.procedure] - execute all tests in all suites from suite1[.suite2][.suite3]...[.procedure] path. If schema is not provided, then the current schema is used. Example: `:all.rooms_tests` +* [schema]:suite1[.suite2][.suite3]...[.procedure] - execute all tests by `suitepath` in all suites on path suite1[.suite2][.suite3]...[.procedure]. If schema is not provided, then the current schema is used. Example: `:all.rooms_tests` * [schema.]package[.procedure] - execute all tests in the specified test package. The whole hierarchy of suites in the schema is built before all before/after hooks or part suites for the provided suite package are executed as well. Example: `tests.test_contact.test_last_name_validator` or simply `test_contact.test_last_name_validator` if `tests` is the current schema. diff --git a/source/core/ut_suite_builder.pkb b/source/core/ut_suite_builder.pkb index 4ab68afa6..b3c6f81dd 100644 --- a/source/core/ut_suite_builder.pkb +++ b/source/core/ut_suite_builder.pkb @@ -36,6 +36,7 @@ create or replace package body ut_suite_builder is gc_throws constant t_annotation_name := 'throws'; gc_rollback constant t_annotation_name := 'rollback'; gc_context constant t_annotation_name := 'context'; + gc_name constant t_annotation_name := 'name'; gc_endcontext constant t_annotation_name := 'endcontext'; type tt_annotations is table of t_annotation_name; @@ -57,6 +58,7 @@ create or replace package body ut_suite_builder is gc_throws, gc_rollback, gc_context, + gc_name, gc_endcontext ); @@ -747,21 +749,47 @@ create or replace package body ut_suite_builder is return l_result; end; - procedure get_suite_contexts_items( - a_suite in out nocopy ut_suite, + procedure get_context_items( + a_parent in out nocopy ut_suite, a_annotations in out nocopy t_annotations_info, a_suite_items out nocopy ut_suite_items, a_parent_context_pos in integer := 0 ) is - l_context_pos t_annotation_position; - l_end_context_pos t_annotation_position; - l_context_name t_object_name; - l_ctx_annotations t_annotations_info; - l_context ut_suite_context; - l_context_no binary_integer := 1; - l_context_items ut_suite_items; + l_context_pos t_annotation_position; + l_next_context_pos t_annotation_position; + l_end_context_pos t_annotation_position; + l_ctx_annotations t_annotations_info; + l_context ut_suite_context; + l_context_no binary_integer := 1; + l_context_items ut_suite_items; type tt_context_names is table of boolean index by t_object_name; - l_context_names tt_context_names; + l_used_context_names tt_context_names; + l_context_name t_object_name; + l_default_context_name t_object_name; + function get_context_name( + a_parent in out nocopy ut_suite, + a_context_names in tt_annotation_texts, + a_start_position binary_integer, + a_end_position binary_integer + ) return varchar2 is + l_result t_annotation_name; + l_found boolean; + l_annotation_pos binary_integer; + begin + l_annotation_pos := a_context_names.first; + while l_annotation_pos is not null loop + if l_annotation_pos > a_start_position and l_annotation_pos < a_end_position then + if l_found then + add_annotation_ignored_warning(a_parent, gc_name,'Duplicate annotation %%%.', l_annotation_pos); + else + l_result := a_context_names(l_annotation_pos); + end if; + l_found := true; + end if; + l_annotation_pos := a_context_names.next(l_annotation_pos); + end loop; + return l_result; + end; begin a_suite_items := ut_suite_items(); if not a_annotations.by_name.exists(gc_context) then @@ -771,35 +799,57 @@ create or replace package body ut_suite_builder is l_context_pos := a_annotations.by_name( gc_context).next(a_parent_context_pos); while l_context_pos is not null loop + l_default_context_name := 'nested_context_#'||l_context_no; l_end_context_pos := get_endcontext_position(l_context_pos, a_annotations.by_name ); - - l_context_name := coalesce( a_annotations.by_line( l_context_pos ).text, gc_context||'_'||l_context_no ); - if regexp_like( l_context_name, '\.' ) or l_context_name is null then - if regexp_like( l_context_name, '\.' ) then - a_suite.put_warning( - 'Invalid value "'||l_context_name||'" for context name. The name cannot contain "." (full stop/period) character.' || - ' Context name ignored and fallback to auto-name "'||gc_context||'_'||l_context_no||'" ' || - get_object_reference( a_suite, null, l_context_pos ) + + l_next_context_pos := a_annotations.by_name(gc_context).next(l_context_pos); + if a_annotations.by_name.exists(gc_name) then + l_context_name := + get_context_name( + a_parent, + a_annotations.by_name( gc_name ), + l_context_pos, + least( + coalesce( l_end_context_pos, a_annotations.by_line.last ), + coalesce( l_next_context_pos, a_annotations.by_line.last ) + ) + ); + end if; + if not regexp_like( l_context_name, '^(\w|[$#])+$' ) or l_context_name is null then + if not regexp_like( l_context_name, '^(\w|[$#])+$' ) then + a_parent.put_warning( + 'Invalid value "'||l_context_name||'" for context name.' || + ' Context name ignored and fallback to auto-name "'||l_default_context_name||'" ' || + get_object_reference( a_parent, null, l_context_pos ) ); end if; - l_context_name := gc_context||'_'||l_context_no; + l_context_name := l_default_context_name; + end if; + if l_used_context_names.exists(l_context_name) then + add_annotation_ignored_warning( + a_parent, gc_name, + 'Context name "'||l_context_name||'" already used in this scope. Name must be unique.' || + ' Using fallback name '||l_default_context_name||'.', l_context_pos ); + l_context_name := l_default_context_name; end if; - l_context := ut_suite_context(a_suite.object_owner, a_suite.object_name, l_context_name, l_context_pos ); - l_context.path := a_suite.path||'.'||l_context_name; - l_context.description := a_annotations.by_line( l_context_pos ).text; + l_used_context_names(l_context_name) := true; + + l_context := ut_suite_context(a_parent.object_owner, a_parent.object_name, l_context_name, l_context_pos ); + l_context.path := a_parent.path||'.'||l_context_name; + l_context.description := coalesce( a_annotations.by_line( l_context_pos ).text, l_context_name ); l_context.parse_time := a_annotations.parse_time; --if nested context found - if a_annotations.by_name(gc_context).next(l_context_pos) < l_end_context_pos or l_end_context_pos is null then - get_suite_contexts_items( l_context, a_annotations, l_context_items, l_context_pos ); + if l_next_context_pos < l_end_context_pos or l_end_context_pos is null then + get_context_items( l_context, a_annotations, l_context_items, l_context_pos ); l_end_context_pos := get_endcontext_position(l_context_pos, a_annotations.by_name ); else l_context_items := ut_suite_items(); end if; if l_end_context_pos is null then - a_suite.put_warning( - 'Missing "--%endcontext" annotation for a "--%context" annotation. The end of package is considered end of context.'|| get_object_reference( a_suite, null, l_context_pos ) + a_parent.put_warning( + 'Missing "--%endcontext" annotation for a "--%context" annotation. The end of package is considered end of context.'|| get_object_reference( a_parent, null, l_context_pos ) ); l_end_context_pos := a_annotations.by_line.last; end if; @@ -807,18 +857,12 @@ create or replace package body ut_suite_builder is --create a sub-set of annotations to process as sub-suite (context) l_ctx_annotations := get_annotations_in_context( a_annotations, l_context_pos, l_end_context_pos); - if l_context_names.exists(l_context_name) then - add_annotation_ignored_warning( a_suite, 'context', 'Context name must be unique in a suite. Context and all of it''s content ignored.', l_context_pos ); - else - l_context_names(l_context_name) := true; - - warning_on_duplicate_annot( l_context, l_ctx_annotations.by_name, gc_context ); + warning_on_duplicate_annot( l_context, l_ctx_annotations.by_name, gc_context ); - add_tests_to_items( l_context, l_ctx_annotations, l_context_items ); - add_items_to_list(a_suite_items, l_context_items); - a_suite_items.extend; - a_suite_items(a_suite_items.last) := l_context; - end if; + add_tests_to_items( l_context, l_ctx_annotations, l_context_items ); + add_items_to_list(a_suite_items, l_context_items); + a_suite_items.extend; + a_suite_items(a_suite_items.last) := l_context; -- remove annotations within context after processing them delete_annotations_range(a_annotations, l_context_pos, l_end_context_pos); @@ -910,7 +954,7 @@ create or replace package body ut_suite_builder is warning_on_duplicate_annot( l_suite, l_annotations.by_name, gc_suite ); build_suitepath( l_suite, l_annotations ); - get_suite_contexts_items( l_suite, l_annotations, a_suite_items ); + get_context_items( l_suite, l_annotations, a_suite_items ); --create suite tests and add add_tests_to_items( l_suite, l_annotations, a_suite_items ); diff --git a/test/ut3_tester/core/test_suite_builder.pkb b/test/ut3_tester/core/test_suite_builder.pkb index 97de59c11..9b2c26567 100644 --- a/test/ut3_tester/core/test_suite_builder.pkb +++ b/test/ut3_tester/core/test_suite_builder.pkb @@ -635,8 +635,8 @@ create or replace package body test_suite_builder is ut3.ut_annotation(1, 'suite','Cool', null), ut3.ut_annotation(2, 'beforeall',null, 'suite_level_beforeall'), ut3.ut_annotation(3, 'test','In suite', 'suite_level_test'), - ut3.ut_annotation(4, 'context','a_context', null), - ut3.ut_annotation(5, 'displayname','A context', null), + ut3.ut_annotation(4, 'context','A context', null), + ut3.ut_annotation(5, 'name','a_context', null), ut3.ut_annotation(6, 'beforeall',null, 'context_setup'), ut3.ut_annotation(7, 'test', 'In context', 'test_in_a_context'), ut3.ut_annotation(8, 'endcontext',null, null) @@ -683,23 +683,24 @@ create or replace package body test_suite_builder is ut3.ut_annotation( 1, 'suite','Cool', null), ut3.ut_annotation( 2, 'beforeall',null, 'suite_level_beforeall'), ut3.ut_annotation( 3, 'test','In suite', 'suite_level_test'), - ut3.ut_annotation( 4, 'context','a_context', null), - ut3.ut_annotation( 5, 'displayname','A context', null), + ut3.ut_annotation( 4, 'context','A context', null), + ut3.ut_annotation( 5, 'name','a_context', null), ut3.ut_annotation( 6, 'beforeall',null, 'context_setup'), ut3.ut_annotation( 7, 'test', 'First test in context', 'first_test_in_a_context'), - ut3.ut_annotation( 8, 'context','a_nested_context', null), - ut3.ut_annotation( 9, 'displayname','A nested context', null), + ut3.ut_annotation( 8, 'context','A nested context', null), + ut3.ut_annotation( 9, 'name','a_nested_context', null), ut3.ut_annotation(10, 'beforeall',null, 'nested_context_setup'), ut3.ut_annotation(11, 'test', 'Test in nested context', 'test_in_nested_context'), ut3.ut_annotation(12, 'endcontext',null, null), - ut3.ut_annotation(13, 'context','nested_context_2', null), - ut3.ut_annotation(14, 'test', 'Test in nested context', 'test_in_nested_context_2'), - ut3.ut_annotation(15, 'context','a_nested_context_3', null), - ut3.ut_annotation(16, 'test', 'Test in nested context', 'test_in_nested_context_3'), - ut3.ut_annotation(17, 'endcontext',null, null), - ut3.ut_annotation(18, 'endcontext',null, null), - ut3.ut_annotation(19, 'test', 'Second test in context', 'second_test_in_a_context'), - ut3.ut_annotation(20, 'endcontext',null, null) + ut3.ut_annotation(13, 'context',null, null), + ut3.ut_annotation(14, 'name','nested_context_2', null), + ut3.ut_annotation(15, 'test', 'Test in nested context', 'test_in_nested_context_2'), + ut3.ut_annotation(16, 'context','a_nested_context_3', null), + ut3.ut_annotation(17, 'test', 'Test in nested context', 'test_in_nested_context_3'), + ut3.ut_annotation(18, 'endcontext',null, null), + ut3.ut_annotation(19, 'endcontext',null, null), + ut3.ut_annotation(20, 'test', 'Second test in context', 'second_test_in_a_context'), + ut3.ut_annotation(21, 'endcontext',null, null) ); --Act l_actual := invoke_builder_for_annotations(l_annotations, 'SOME_PACKAGE'); @@ -716,10 +717,10 @@ create or replace package body test_suite_builder is '%nested_context_2nested_context_2some_package.a_context.nested_context_2' || '%' || '' || - '%a_nested_context_3a_nested_context_3some_package.a_context.nested_context_2.a_nested_context_3' || + '%nested_context_#1a_nested_context_3some_package.a_context.nested_context_2.nested_context_#1' || '%' || '' || - '%test_in_nested_context_3Test in nested contextsome_package.a_context.nested_context_2.a_nested_context_3.test_in_nested_context_3' || + '%test_in_nested_context_3Test in nested contextsome_package.a_context.nested_context_2.nested_context_#1.test_in_nested_context_3' || '%' || '' || '' || @@ -775,7 +776,7 @@ create or replace package body test_suite_builder is l_annotations := ut3.ut_annotations( ut3.ut_annotation(1, 'suite', 'Cool', null), ut3.ut_annotation(2, 'test', 'In suite', 'suite_level_test'), - ut3.ut_annotation(3, 'context', 'a_context', null), + ut3.ut_annotation(3, 'context', 'A context', null), ut3.ut_annotation(4, 'beforeall', 'context_beforeall', null), ut3.ut_annotation(5, 'beforeeach', null, 'context_beforeeach'), ut3.ut_annotation(6, 'test', 'In context', 'test_in_a_context'), @@ -791,7 +792,7 @@ create or replace package body test_suite_builder is '' || '%' || '%' || - '%a_context' || + '%nested_context_#1A contextsome_package.nested_context_#1' || '%' || '%' || '%test_in_a_context' || @@ -827,7 +828,7 @@ create or replace package body test_suite_builder is ut3.ut_annotation(2, 'beforeall',null, 'suite_level_beforeall'), ut3.ut_annotation(3, 'beforeeach',null, 'suite_level_beforeeach'), ut3.ut_annotation(4, 'test','In suite', 'suite_level_test'), - ut3.ut_annotation(5, 'context','a_context', null), + ut3.ut_annotation(5, 'context',null, null), ut3.ut_annotation(6, 'test', 'In context', 'test_in_a_context'), ut3.ut_annotation(7, 'endcontext',null, null), ut3.ut_annotation(8, 'aftereach',null, 'suite_level_aftereach'), @@ -841,7 +842,7 @@ create or replace package body test_suite_builder is '' || '%' || '%' || - '%a_context' || + '%nested_context_#1nested_context_#1some_package.nested_context_#1' || '%' || '%' || '%test_in_a_context' || @@ -876,8 +877,9 @@ create or replace package body test_suite_builder is ut3.ut_annotation(1, 'suite','Cool', null), ut3.ut_annotation(2, 'beforeall',null, 'suite_level_beforeall'), ut3.ut_annotation(3, 'test','In suite', 'suite_level_test'), - ut3.ut_annotation(4, 'context','a_context', null), - ut3.ut_annotation(5, 'beforeall',null, 'context_setup'), + ut3.ut_annotation(4, 'context','Some context', null), + ut3.ut_annotation(5, 'name','a_context', null), + ut3.ut_annotation(6, 'beforeall',null, 'context_setup'), ut3.ut_annotation(7, 'test', 'In context', 'test_in_a_context') ); --Act @@ -891,7 +893,7 @@ create or replace package body test_suite_builder is ''|| '' || '%' || - '%a_contexta_contextsome_package.a_context' || + '%a_contextSome contextsome_package.a_context' || '%' || '' || '%test_in_a_contextIn contextsome_package.a_context.test_in_a_context' || @@ -922,8 +924,8 @@ create or replace package body test_suite_builder is ut3.ut_annotation(1, 'suite','Cool', null), ut3.ut_annotation(2, 'beforeall',null, 'suite_level_beforeall'), ut3.ut_annotation(3, 'test','In suite', 'suite_level_test'), - ut3.ut_annotation(4, 'context','a_context', null), - ut3.ut_annotation(5, 'displayname','A context', null), + ut3.ut_annotation(4, 'context','A context', null), + ut3.ut_annotation(5, 'name','a_context', null), ut3.ut_annotation(6, 'beforeall',null, 'context_setup'), ut3.ut_annotation(7, 'test', 'In context', 'test_in_a_context'), ut3.ut_annotation(8, 'endcontext',null, null), @@ -974,13 +976,13 @@ create or replace package body test_suite_builder is ut3.ut_annotation(1, 'suite','Cool', null), ut3.ut_annotation(2, 'beforeall',null, 'suite_level_beforeall'), ut3.ut_annotation(3, 'test','In suite', 'suite_level_test'), - ut3.ut_annotation(4, 'context','a_context', null), - ut3.ut_annotation(5, 'displayname','A context', null), + ut3.ut_annotation(4, 'context','A context', null), + ut3.ut_annotation(5, 'name','a_context', null), ut3.ut_annotation(6, 'beforeall',null, 'context_setup'), ut3.ut_annotation(7, 'test', 'In context', 'test_in_a_context'), ut3.ut_annotation(8, 'endcontext',null, null), - ut3.ut_annotation(9, 'context','a_context', null), - ut3.ut_annotation(10, 'displayname','A context', null), + ut3.ut_annotation(9, 'context','A context', null), + ut3.ut_annotation(10, 'name','a_context', null), ut3.ut_annotation(11, 'beforeall',null, 'setup_in_duplicated_context'), ut3.ut_annotation(12, 'test', 'In duplicated context', 'test_in_duplicated_context'), ut3.ut_annotation(13, 'endcontext',null, null) @@ -989,31 +991,43 @@ create or replace package body test_suite_builder is l_actual := invoke_builder_for_annotations(l_annotations, 'SOME_PACKAGE'); --Assert ut.expect(l_actual).to_be_like( - '%Context name must be unique in a suite. Context and all of it's content ignored.%at package "UT3_TESTER.SOME_PACKAGE", line 9%' + '%Context name "a_context" already used in this scope. Name must be unique. Using fallback name nested_context_#2.%%' ,'\' ); ut.expect(l_actual).to_be_like( ''|| '' || '%' || - '' || - '%a_contextA contextsome_package.a_context' || - '%' || - '' || - '%test_in_a_contextIn contextsome_package.a_context.test_in_a_context' || - '%' || - '' || - '' || - '%some_packagecontext_setup' || - '%' || - '' || - '' || - '' || - '%suite_level_testIn suitesome_package.suite_level_test' || - '%' || + '' || + '%nested_context_#2A contextsome_package.nested_context_#2' || + '%' || + '' || + '%test_in_duplicated_contextIn duplicated contextsome_package.nested_context_#2.test_in_duplicated_context' || + '%' || + '' || + '' || + '%some_packagesetup_in_duplicated_context' || + '%' || + '' || + '' || + '' || + '%a_contextA contextsome_package.a_context' || + '%' || + '' || + '%test_in_a_contextIn contextsome_package.a_context.test_in_a_context' || + '%' || + '' || + '' || + '%some_packagecontext_setup' || + '%' || + '' || + '' || + '' || + '%suite_level_testIn suitesome_package.suite_level_test' || + '%' || '' || '' || - '%some_packagesuite_level_beforeall' || + '%some_packagesuite_level_beforeall' || '%' || '' || ''|| @@ -1027,10 +1041,11 @@ create or replace package body test_suite_builder is l_bad_name varchar2(100); begin --Arrange - l_bad_name := 'Context with invalid name. Should fail'; + l_bad_name := 'ctx_with_dot.in_it'; l_annotations := ut3.ut_annotations( ut3.ut_annotation(1, 'suite','Cool', null), - ut3.ut_annotation(4, 'context','Context with invalid name. Should fail', null), + ut3.ut_annotation(4, 'context',null, null), + ut3.ut_annotation(5, 'name',l_bad_name, null), ut3.ut_annotation(7, 'test', 'In context', 'test_in_a_context'), ut3.ut_annotation(13, 'endcontext',null, null) ); @@ -1038,17 +1053,17 @@ create or replace package body test_suite_builder is l_actual := invoke_builder_for_annotations(l_annotations, 'SOME_PACKAGE'); --Assert ut.expect(l_actual).to_be_like( - '%Invalid value "'||l_bad_name||'" for context name. The name cannot contain "." (full stop/period) character. Context name ignored and fallback to auto-name "context_1"%' + '%Invalid value "'||l_bad_name||'" for context name. Context name ignored and fallback to auto-name "nested_context_#1"%' ); ut.expect(l_actual).to_be_like( ''|| '' || '%' || '' || - '%context_1Context with invalid name. Should failsome_package.context_1' || + '%nested_context_#1nested_context_#1some_package.nested_context_#1' || '%' || '' || - '%test_in_a_contextIn contextsome_package.context_1.test_in_a_context' || + '%test_in_a_contextIn contextsome_package.nested_context_#1.test_in_a_context' || '%' || '%' || '%' || @@ -1058,6 +1073,180 @@ create or replace package body test_suite_builder is ); end; + procedure name_with_spaces_invalid is + l_actual clob; + l_annotations ut3.ut_annotations; + l_bad_name varchar2(100); + begin + --Arrange + l_bad_name := 'context name with spaces'; + l_annotations := ut3.ut_annotations( + ut3.ut_annotation(1, 'suite','Cool', null), + ut3.ut_annotation(4, 'context',null, null), + ut3.ut_annotation(5, 'name',l_bad_name, null), + ut3.ut_annotation(7, 'test', 'In context', 'test_in_a_context'), + ut3.ut_annotation(13, 'endcontext',null, null) + ); + --Act + l_actual := invoke_builder_for_annotations(l_annotations, 'SOME_PACKAGE'); + --Assert + ut.expect(l_actual).to_be_like( + '%Invalid value "'||l_bad_name||'" for context name. Context name ignored and fallback to auto-name "nested_context_#1"%' + ); + ut.expect(l_actual).to_be_like( + ''|| + '' || + '%' || + '' || + '%nested_context_#1nested_context_#1some_package.nested_context_#1' || + '%' || + '' || + '%test_in_a_contextIn contextsome_package.nested_context_#1.test_in_a_context' || + '%' || + '%' || + '%' || + '%' || + ''|| + '' + ); + end; + + procedure duplicate_name_annotation is + l_actual clob; + l_annotations ut3.ut_annotations; + begin + --Arrange + l_annotations := ut3.ut_annotations( + ut3.ut_annotation(1, 'suite','Cool', null), + ut3.ut_annotation(4, 'context','A context', null), + ut3.ut_annotation(5, 'name','a_context_name', null), + ut3.ut_annotation(6, 'name','a_newer_context_name', null), + ut3.ut_annotation(7, 'test', 'In context', 'test_in_a_context'), + ut3.ut_annotation(8, 'endcontext',null, null), + ut3.ut_annotation(12, 'test', 'In suite', 'suite_level_test') + ); + --Act + l_actual := invoke_builder_for_annotations(l_annotations, 'SOME_PACKAGE'); + --Assert + ut.expect(l_actual).to_be_like( + '%Duplicate annotation "--%name". Annotation ignored.%%' + ,'\' + ); + ut.expect(l_actual).to_be_like( + ''|| + '' || + '%' || + '' || + '%a_context_nameA contextsome_package.a_context_name' || + '%' || + '' || + '%test_in_a_contextIn contextsome_package.a_context_name.test_in_a_context' || + '%' || + '' || + '' || + '' || + '' || + '' || + '%suite_level_testIn suitesome_package.suite_level_test' || + '%' || + '' || + '' || + '' || + ''|| + '' + ); + end; + + procedure name_outside_of_context is + l_actual clob; + l_annotations ut3.ut_annotations; + begin + --Arrange + l_annotations := ut3.ut_annotations( + ut3.ut_annotation(1, 'suite','Cool', null), + ut3.ut_annotation(3, 'name','a_context_name', null), + ut3.ut_annotation(4, 'context','A context', null), + ut3.ut_annotation(7, 'test', 'In context', 'test_in_a_context'), + ut3.ut_annotation(8, 'endcontext',null, null), + ut3.ut_annotation(12, 'test', 'In suite', 'suite_level_test') + ); + --Act + l_actual := invoke_builder_for_annotations(l_annotations, 'SOME_PACKAGE'); + --Assert + ut.expect(l_actual).to_be_like( + '%%' + ,'\' + ); + ut.expect(l_actual).to_be_like( + ''|| + '' || + '%' || + '' || + '%nested_context_#1A contextsome_package.nested_context_#1' || + '%' || + '' || + '%test_in_a_contextIn contextsome_package.nested_context_#1.test_in_a_context' || + '%' || + '' || + '' || + '' || + '' || + '' || + '%suite_level_testIn suitesome_package.suite_level_test' || + '%' || + '' || + '' || + '' || + ''|| + '' + ); + end; + + procedure name_empty_value is + l_actual clob; + l_annotations ut3.ut_annotations; + begin + --Arrange + l_annotations := ut3.ut_annotations( + ut3.ut_annotation(1, 'suite','Cool', null), + ut3.ut_annotation(4, 'context','A context', null), + ut3.ut_annotation(5, 'name',null, null), + ut3.ut_annotation(7, 'test', 'In context', 'test_in_a_context'), + ut3.ut_annotation(8, 'endcontext',null, null), + ut3.ut_annotation(12, 'test', 'In suite', 'suite_level_test') + ); + --Act + l_actual := invoke_builder_for_annotations(l_annotations, 'SOME_PACKAGE'); + --Assert + ut.expect(l_actual).to_be_like( + '%%' + ,'\' + ); + ut.expect(l_actual).to_be_like( + ''|| + '' || + '%' || + '' || + '%nested_context_#1A contextsome_package.nested_context_#1' || + '%' || + '' || + '%test_in_a_contextIn contextsome_package.nested_context_#1.test_in_a_context' || + '%' || + '' || + '' || + '' || + '' || + '' || + '%suite_level_testIn suitesome_package.suite_level_test' || + '%' || + '' || + '' || + '' || + ''|| + '' + ); + end; + procedure throws_value_empty is l_actual clob; l_annotations ut3.ut_annotations; diff --git a/test/ut3_tester/core/test_suite_builder.pks b/test/ut3_tester/core/test_suite_builder.pks index c9683cd9e..f1fa7f157 100644 --- a/test/ut3_tester/core/test_suite_builder.pks +++ b/test/ut3_tester/core/test_suite_builder.pks @@ -2,8 +2,7 @@ create or replace package test_suite_builder is --%suite(suite_builder) --%suitepath(utplsql.ut3_tester.core) - --%context(suite) - --%displayname(--%suite annotation) + --%context(--%suite annotation) --%test(Sets suite name from package name and leaves description empty) procedure no_suite_description; @@ -16,8 +15,7 @@ create or replace package test_suite_builder is --%endcontext - --%context(displayname) - --%displayname(--%displayname annotation) + --%context(--%displayname annotation) --%test(Overrides suite description using first --%displayname annotation) procedure suite_descr_from_displayname; @@ -30,8 +28,7 @@ create or replace package test_suite_builder is --%endcontext - --%context(test) - --%displayname(--%test annotation) + --%context(--%test annotation) --%test(Creates a test item for procedure annotated with --%test annotation) procedure test_annotation; @@ -44,8 +41,7 @@ create or replace package test_suite_builder is --%endcontext - --%context(suitepath) - --%displayname(--%suitepath annotation) + --%context(--%suitepath annotation) --%test(Sets suite path using first --%suitepath annotation) procedure suitepath_from_non_empty_path; @@ -61,8 +57,7 @@ create or replace package test_suite_builder is --%endcontext - --%context(rollback) - --%displayname(--%rollback annotation) + --%context--%rollback annotation) --%test(Sets rollback type using first --%rollback annotation) procedure rollback_type_valid; @@ -78,8 +73,7 @@ create or replace package test_suite_builder is --%endcontext - --%context(before_after_all_each) - --%displayname(--%before/after all/each annotations) + --%context(--%before/after all/each annotations) --%test(Supports multiple before/after all/each procedure level definitions) procedure multiple_before_after; @@ -110,11 +104,10 @@ create or replace package test_suite_builder is --%endcontext - --%context(context) - --%displayname(--%context annotation) + --%context(--%context annotation) - --%test(Creates nested suite for content between context/endcontext annotations) - procedure suite_from_context; + --%test(Creates nested suite for content between context/endcontext annotations) + procedure suite_from_context; --%test(Creates nested contexts inside a context) procedure nested_contexts; @@ -131,16 +124,31 @@ create or replace package test_suite_builder is --%test(Gives warning if --%endcontext is missing a preceding --%context) procedure endcontext_without_context; - --%test(Gives warning when two contexts have the same name and ignores duplicated context) + --%test(Gives warning when two contexts have the same name and falls back to default context name) procedure duplicate_context_name; - --%test(Fallback to default naming and gives warning when context name contains "." character) + --%endcontext + + --%context(--%name annotation) + + --%test(Falls back to default context name and gives warning when context name contains "." character) procedure hard_stop_in_ctx_name; + --%test(Falls back to default context name and gives warning when name contains spaces) + procedure name_with_spaces_invalid; + + --%test(Raises warning when more than one name annotation used ) + procedure duplicate_name_annotation; + + --%test(Is ignored when used outside of context - no warning given) + procedure name_outside_of_context; + + --%test(Is ignored when name value is empty) + procedure name_empty_value; + --%endcontext - --%context(throws) - --%displayname(--%throws annotation) + --%context(--%throws annotation) --%test(Gives warning if --%throws annotation has no value) procedure throws_value_empty; @@ -150,8 +158,7 @@ create or replace package test_suite_builder is --%endcontext - --%context(beforetest_aftertest) - --%displayname(--%beforetest/aftertest annotation) + --%context(--%beforetest/aftertest annotation) --%test(Supports multiple occurrences of beforetest/aftertest for a test) procedure before_aftertest_multi; @@ -167,8 +174,7 @@ create or replace package test_suite_builder is --%endcontext - --%context(unknown_annotation) - --%displayname(--%bad_annotation) + --%context(--%bad_annotation) --%test(Gives warning when unknown procedure level annotation passed) procedure test_bad_procedure_annotation; @@ -178,8 +184,7 @@ create or replace package test_suite_builder is --%endcontext - --%context(tags_annotation) - --%displayname(--%tag_annotation) + --%context(--%tag_annotation) --%test(Build suite test with tag) procedure test_tag_annotation; diff --git a/test/ut3_tester/core/test_suite_manager.pkb b/test/ut3_tester/core/test_suite_manager.pkb index 6f4237f9c..e693771b4 100644 --- a/test/ut3_tester/core/test_suite_manager.pkb +++ b/test/ut3_tester/core/test_suite_manager.pkb @@ -259,8 +259,8 @@ end test_package_3;]'; gv_glob_val number; - --%context(some_context) - --%displayname(Some context description) + --%context(Some context description) + --%name(some_context) --%test --%displayname(Test1 from test package 1) diff --git a/test/ut3_user/api/test_ut_run.pkb b/test/ut3_user/api/test_ut_run.pkb index 46d7d6f80..feac3fe6e 100644 --- a/test/ut3_user/api/test_ut_run.pkb +++ b/test/ut3_user/api/test_ut_run.pkb @@ -1080,9 +1080,8 @@ Failures:% --%beforeall procedure before_suite; - --%context(some_context) - - --%displayname(context description) + --%context(context description) + --%name(some_context) --%beforeall procedure before_context; diff --git a/test/ut3_user/reporters.pkb b/test/ut3_user/reporters.pkb index 6d6d573aa..9f571a589 100644 --- a/test/ut3_user/reporters.pkb +++ b/test/ut3_user/reporters.pkb @@ -14,8 +14,8 @@ as --%beforeeach procedure beforeeach; - --%context(some_context) - --%displayname(A description of some context) + --%context(A description of some context) + --%name(some_context) --%test --%beforetest(beforetest) diff --git a/test/ut3_user/reporters/test_realtime_reporter.pkb b/test/ut3_user/reporters/test_realtime_reporter.pkb index 8eb490a11..155925f0f 100644 --- a/test/ut3_user/reporters/test_realtime_reporter.pkb +++ b/test/ut3_user/reporters/test_realtime_reporter.pkb @@ -9,7 +9,8 @@ create or replace package body test_realtime_reporter as --%suite(suite ) --%suitepath(realtime_reporting) - --%context(test context) + --%context + --%name(test_context) --%test(test 1 - OK) procedure test_1_ok; @@ -164,12 +165,12 @@ create or replace package body test_realtime_reporter as select 'post-test' as event_type, 'realtime_reporting.check_realtime_reporting2.test_5' as item_id from dual union all select 'post-suite' as event_type, 'realtime_reporting.check_realtime_reporting2' as item_id from dual union all select 'pre-suite' as event_type, 'realtime_reporting.check_realtime_reporting1' as item_id from dual union all - select 'pre-suite' as event_type, 'realtime_reporting.check_realtime_reporting1.test context' as item_id from dual union all - select 'pre-test' as event_type, 'realtime_reporting.check_realtime_reporting1.test context.test_1_ok' as item_id from dual union all - select 'post-test' as event_type, 'realtime_reporting.check_realtime_reporting1.test context.test_1_ok' as item_id from dual union all - select 'pre-test' as event_type, 'realtime_reporting.check_realtime_reporting1.test context.test_2_nok' as item_id from dual union all - select 'post-test' as event_type, 'realtime_reporting.check_realtime_reporting1.test context.test_2_nok' as item_id from dual union all - select 'post-suite' as event_type, 'realtime_reporting.check_realtime_reporting1.test context' as item_id from dual union all + select 'pre-suite' as event_type, 'realtime_reporting.check_realtime_reporting1.test_context' as item_id from dual union all + select 'pre-test' as event_type, 'realtime_reporting.check_realtime_reporting1.test_context.test_1_ok' as item_id from dual union all + select 'post-test' as event_type, 'realtime_reporting.check_realtime_reporting1.test_context.test_1_ok' as item_id from dual union all + select 'pre-test' as event_type, 'realtime_reporting.check_realtime_reporting1.test_context.test_2_nok' as item_id from dual union all + select 'post-test' as event_type, 'realtime_reporting.check_realtime_reporting1.test_context.test_2_nok' as item_id from dual union all + select 'post-suite' as event_type, 'realtime_reporting.check_realtime_reporting1.test_context' as item_id from dual union all select 'post-suite' as event_type, 'realtime_reporting.check_realtime_reporting1' as item_id from dual union all select 'post-suite' as event_type, 'realtime_reporting' as item_id from dual union all select 'post-run' as event_type, null as item_id from dual; @@ -310,7 +311,7 @@ create or replace package body test_realtime_reporter as into l_actual from table(g_events) t where t.event_doc.extract('/event[@type="post-test"]/test/@id').getstringval() - = 'realtime_reporting.check_realtime_reporting1.test context.test_2_nok'; + = 'realtime_reporting.check_realtime_reporting1.test_context.test_2_nok'; ut.expect(l_actual).to_equal(l_expected); end single_failed_message;