`in` operator seems not to be supported
Alexander-Serov opened this issue · comments
Consider a simple function:
def detector(s: str):
if 'py' in s:
return True
else:
return False
Pynguin test generation fails on it with an unclear AttributeError message:
pynguin \
--output-path ./pynguin --project-path . \
--module-name pynguin-example \
-v
[11:59:10] INFO Start Pynguin Test Generation… generator.py:110
INFO Collecting static constants from module under test generator.py:209
INFO No constants found generator.py:212
INFO Setting up runtime collection of constants generator.py:221
INFO Stop Pynguin Test Generation… generator.py:113
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /Users/user/miniforge3/envs/pynguin/bin/pynguin:8 in <module> │
│ │
│ 5 from pynguin.cli import main │
│ 6 if __name__ == '__main__': │
│ 7 │ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) │
│ ❱ 8 │ sys.exit(main()) │
│ 9 │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/cli.py:190 in main │
│ │
│ 187 │ set_configuration(parsed.config) │
│ 188 │ if console is not None: │
│ 189 │ │ with console.status("Running Pynguin..."): │
│ ❱ 190 │ │ │ return run_pynguin().value │
│ 191 │ else: │
│ 192 │ │ return run_pynguin().value │
│ 193 │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/generator.py:111 in │
│ run_pynguin │
│ │
│ 108 │ """ │
│ 109 │ try: │
│ 110 │ │ _LOGGER.info("Start Pynguin Test Generation…") │
│ ❱ 111 │ │ return _run() │
│ 112 │ finally: │
│ 113 │ │ _LOGGER.info("Stop Pynguin Test Generation…") │
│ 114 │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/generator.py:496 in │
│ _run │
│ │
│ 493 │
│ 494 │
│ 495 def _run() -> ReturnCode: │
│ ❱ 496 │ if (setup_result := _setup_and_check()) is None: │
│ 497 │ │ return ReturnCode.SETUP_FAILED │
│ 498 │ executor, test_cluster, constant_provider = setup_result │
│ 499 │ # traces slices for test cases after execution │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/generator.py:253 in │
│ _setup_and_check │
│ │
│ 250 │ │ set(config.configuration.statistics_output.coverage_metrics), │
│ 251 │ │ dynamic_constant_provider, │
│ 252 │ ) │
│ ❱ 253 │ if not _load_sut(tracer): │
│ 254 │ │ return None │
│ 255 │ if not _setup_report_dir(): │
│ 256 │ │ return None │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/generator.py:164 in │
│ _load_sut │
│ │
│ 161 │ try: │
│ 162 │ │ # We need to set the current thread ident so the import trace is recorded. │
│ 163 │ │ tracer.current_thread_identifier = threading.current_thread().ident │
│ ❱ 164 │ │ importlib.import_module(config.configuration.module_name) │
│ 165 │ except ImportError as ex: │
│ 166 │ │ # A module could not be imported because some dependencies │
│ 167 │ │ # are missing or it is malformed │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/importlib/__init__.py:126 in import_module │
│ │
│ 123 │ │ │ if character != '.': │
│ 124 │ │ │ │ break │
│ 125 │ │ │ level += 1 │
│ ❱ 126 │ return _bootstrap._gcd_import(name[level:], package, level) │
│ 127 │
│ 128 │
│ 129 _RELOADING = {} │
│ <frozen importlib._bootstrap>:1050 in _gcd_import │
│ <frozen importlib._bootstrap>:1027 in _find_and_load │
│ <frozen importlib._bootstrap>:1006 in _find_and_load_unlocked │
│ <frozen importlib._bootstrap>:688 in _load_unlocked │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/instrumentation/machi │
│ nery.py:56 in exec_module │
│ │
│ 53 │ │
│ 54 │ def exec_module(self, module): │
│ 55 │ │ self._tracer.reset() │
│ ❱ 56 │ │ super().exec_module(module) │
│ 57 │ │ self._tracer.store_import_trace() │
│ 58 │ │
│ 59 │ def get_code(self, fullname) -> CodeType: │
│ <frozen importlib._bootstrap_external>:879 in exec_module │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/instrumentation/machi │
│ nery.py:71 in get_code │
│ │
│ 68 │ │ """ │
│ 69 │ │ to_instrument = cast(CodeType, super().get_code(fullname)) │
│ 70 │ │ assert to_instrument, "Failed to get code object of module." │
│ ❱ 71 │ │ return self._transformer.instrument_module(to_instrument) │
│ 72 │
│ 73 │
│ 74 def build_transformer( │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/instrumentation/instr │
│ umentation.py:205 in instrument_module │
│ │
│ 202 │ │ │ │ # Abort instrumentation, since we have already │
│ 203 │ │ │ │ # instrumented this code object. │
│ 204 │ │ │ │ assert False, "Tried to instrument already instrumented module." │
│ ❱ 205 │ │ return self._instrument_code_recursive(module_code) │
│ 206 │ │
│ 207 │ def _instrument_code_recursive( │
│ 208 │ │ self, │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/instrumentation/instr │
│ umentation.py:244 in _instrument_code_recursive │
│ │
│ 241 │ │ for adapter in self._instrumentation_adapters: │
│ 242 │ │ │ adapter.visit_entry_node(real_entry_node.basic_block, code_object_id) │
│ 243 │ │ self._instrument_cfg(cfg, code_object_id) │
│ ❱ 244 │ │ return self._instrument_inner_code_objects( │
│ 245 │ │ │ cfg.bytecode_cfg().to_code(), code_object_id │
│ 246 │ │ ) │
│ 247 │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/instrumentation/instr │
│ umentation.py:265 in _instrument_inner_code_objects │
│ │
│ 262 │ │ │ if isinstance(const, CodeType): │
│ 263 │ │ │ │ # The const is an inner code object │
│ 264 │ │ │ │ new_consts.append( │
│ ❱ 265 │ │ │ │ │ self._instrument_code_recursive( │
│ 266 │ │ │ │ │ │ const, parent_code_object_id=parent_code_object_id │
│ 267 │ │ │ │ │ ) │
│ 268 │ │ │ │ ) │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/instrumentation/instr │
│ umentation.py:243 in _instrument_code_recursive │
│ │
│ 240 │ │ assert real_entry_node.basic_block is not None, "Basic block cannot be None." │
│ 241 │ │ for adapter in self._instrumentation_adapters: │
│ 242 │ │ │ adapter.visit_entry_node(real_entry_node.basic_block, code_object_id) │
│ ❱ 243 │ │ self._instrument_cfg(cfg, code_object_id) │
│ 244 │ │ return self._instrument_inner_code_objects( │
│ 245 │ │ │ cfg.bytecode_cfg().to_code(), code_object_id │
│ 246 │ │ ) │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/instrumentation/instr │
│ umentation.py:290 in _instrument_cfg │
│ │
│ 287 │ │ │ │ node.basic_block is not None │
│ 288 │ │ │ ), "Non artificial node does not have a basic block." │
│ 289 │ │ │ for adapter in self._instrumentation_adapters: │
│ ❱ 290 │ │ │ │ adapter.visit_node(cfg, code_object_id, node, node.basic_block) │
│ 291 │
│ 292 │
│ 293 class BranchCoverageInstrumentation(InstrumentationAdapter): │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/instrumentation/instr │
│ umentation.py:345 in visit_node │
│ │
│ 342 │ │ │ │ │ code_object_id=code_object_id, │
│ 343 │ │ │ │ ) │
│ 344 │ │ │ elif maybe_jump.is_cond_jump(): │
│ ❱ 345 │ │ │ │ predicate_id = self._instrument_cond_jump( │
│ 346 │ │ │ │ │ code_object_id=code_object_id, │
│ 347 │ │ │ │ │ maybe_compare_idx=maybe_compare_idx, │
│ 348 │ │ │ │ │ jump=maybe_jump, │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/instrumentation/instr │
│ umentation.py:387 in _instrument_cond_jump │
│ │
│ 384 │ │ │ and maybe_compare.opcode in op.OP_COMPARE │
│ 385 │ │ ): │
│ 386 │ │ │ assert maybe_compare_idx is not None │
│ ❱ 387 │ │ │ return self._instrument_compare_based_conditional_jump( │
│ 388 │ │ │ │ block=block, │
│ 389 │ │ │ │ code_object_id=code_object_id, │
│ 390 │ │ │ │ compare_idx=maybe_compare_idx, │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/instrumentation/instr │
│ umentation.py:481 in _instrument_compare_based_conditional_jump │
│ │
│ 478 │ │ │ │ # bytecode library. │
│ 479 │ │ │ │ compare = Compare.IS_NOT if operation.arg else Compare.IS │
│ 480 │ │ │ case "CONTAINS_OP": │
│ ❱ 481 │ │ │ │ compare = Compare.NOT_IN if operation.arg else Compare.IN │
│ 482 │ │ │ case _: │
│ 483 │ │ │ │ raise RuntimeError(f"Unknown comparison OP {operation}") │
│ 484 │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/enum.py:437 in __getattr__ │
│ │
│ 434 │ │ try: │
│ 435 │ │ │ return cls._member_map_[name] │
│ 436 │ │ except KeyError: │
│ ❱ 437 │ │ │ raise AttributeError(name) from None │
│ 438 │ │
│ 439 │ def __getitem__(cls, name): │
│ 440 │ │ return cls._member_map_[name] │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
AttributeError: IN
From what I understand, this functionality is still in development, which is fine since it's a free project. :) But perhaps, at least give the user a clearer message? I was kind of confused by the message AttributeError: IN.
And it also seems to me that in is a very popular operator, so please consider it a feature request. :)
Other than that, thanks and keep up the good job!
Thanks for your feedback. This problem is related to the library that we use for bytecode instrumentation, as Pynguin only works with a certain version of that library. Can you try to manually install bytecode = 0.13 in your virtual env? That should fix this problem.
Thanks for the fast answer @Wooza!
This did allow me to advance further indeed. However, this simple example still fails (supposedly, after test generation is finished and while trying to interpret variable s in the definition?):
> pynguin \
--output-path ./pynguin --project-path . \
--module-name pynguin-example \
-v
[12:39:20] INFO Start Pynguin Test Generation… generator.py:110
INFO Collecting static constants from module under test generator.py:209
INFO No constants found generator.py:212
INFO Setting up runtime collection of constants generator.py:221
INFO Analyzed project to create test cluster module.py:1186
INFO Modules: 1 module.py:1187
INFO Functions: 1 module.py:1188
INFO Classes: 11 module.py:1189
INFO Using seed 1672745959264509000 generator.py:195
INFO Using strategy: Algorithm.DYNAMOSA generationalgorithmfactory.py:272
INFO Instantiated 3 fitness functions generationalgorithmfactory.py:363
INFO Using CoverageArchive generationalgorithmfactory.py:316
INFO Using selection function: Selection.TOURNAMENT_SELECTION generationalgorithmfactory.py:291
INFO No stopping condition configured! generationalgorithmfactory.py:95
INFO Using fallback timeout of 600 seconds generationalgorithmfactory.py:96
INFO Using crossover function: SinglePointRelativeCrossOver generationalgorithmfactory.py:304
INFO Using ranking function: RankBasedPreferenceSorting generationalgorithmfactory.py:324
INFO Start generating test cases generator.py:507
INFO Initial Population, Coverage: 0.666667 searchobserver.py:66
INFO Iteration: 1, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 2, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 3, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 4, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 5, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 6, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 7, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 8, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 9, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 10, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 11, Coverage: 0.666667 searchobserver.py:70
[12:39:21] INFO Iteration: 12, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 13, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 14, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 15, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 16, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 17, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 18, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 19, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 20, Coverage: 0.666667 searchobserver.py:70
INFO Iteration: 21, Coverage: 1.000000 searchobserver.py:70
INFO Algorithm stopped before using all resources. generator.py:510
INFO Stop generating test cases generator.py:515
INFO Start generating assertions generator.py:588
INFO Setup mutation controller mutationadapter.py:68
INFO Build AST for pynguin-example mutationadapter.py:54
INFO Mutate module pynguin-example mutationadapter.py:56
INFO Generated 4 mutants mutationadapter.py:64
INFO Running tests on mutant 1/4 assertiongenerator.py:290
INFO Running tests on mutant 2/4 assertiongenerator.py:290
INFO Running tests on mutant 3/4 assertiongenerator.py:290
INFO Running tests on mutant 4/4 assertiongenerator.py:290
INFO Mutant 0 killed by Test(s): 0, 1 assertiongenerator.py:369
INFO Mutant 1 killed by Test(s): 0, 1 assertiongenerator.py:369
INFO Mutant 2 killed by Test(s): 1 assertiongenerator.py:369
INFO Mutant 3 killed by Test(s): 0, 1 assertiongenerator.py:369
INFO Number of Surviving Mutant(s): 0 (Mutants: ) assertiongenerator.py:381
INFO Calculating resulting FinalBranchCoverage generator.py:428
INFO Stop Pynguin Test Generation… generator.py:113
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /Users/user/miniforge3/envs/pynguin/bin/pynguin:8 in <module> │
│ │
│ 5 from pynguin.cli import main │
│ 6 if __name__ == '__main__': │
│ 7 │ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) │
│ ❱ 8 │ sys.exit(main()) │
│ 9 │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/cli.py:190 in main │
│ │
│ 187 │ set_configuration(parsed.config) │
│ 188 │ if console is not None: │
│ 189 │ │ with console.status("Running Pynguin..."): │
│ ❱ 190 │ │ │ return run_pynguin().value │
│ 191 │ else: │
│ 192 │ │ return run_pynguin().value │
│ 193 │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/generator.py:111 in │
│ run_pynguin │
│ │
│ 108 │ """ │
│ 109 │ try: │
│ 110 │ │ _LOGGER.info("Start Pynguin Test Generation…") │
│ ❱ 111 │ │ return _run() │
│ 112 │ finally: │
│ 113 │ │ _LOGGER.info("Stop Pynguin Test Generation…") │
│ 114 │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/generator.py:533 in │
│ _run │
│ │
│ 530 │ │ config.configuration.test_case_output.export_strategy │
│ 531 │ │ == config.ExportStrategy.PY_TEST │
│ 532 │ ): │
│ ❱ 533 │ │ _export_chromosome(generation_result) │
│ 534 │ │
│ 535 │ if config.configuration.statistics_output.create_coverage_report: │
│ 536 │ │ coverage_report = get_coverage_report( │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/generator.py:690 in │
│ _export_chromosome │
│ │
│ 687 │ ) │
│ 688 │ export_visitor = export.PyTestChromosomeToAstVisitor() │
│ 689 │ chromosome.accept(export_visitor) │
│ ❱ 690 │ export.save_module_to_file( │
│ 691 │ │ export_visitor.to_module(), │
│ 692 │ │ target_file, │
│ 693 │ │ config.configuration.test_case_output.format_with_black, │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/pynguin/generation/export.py: │
│ 194 in save_module_to_file │
│ │
│ 191 │ │ │ # so we only import it if we need it. │
│ 192 │ │ │ import black # pylint:disable=import-outside-toplevel │
│ 193 │ │ │ │
│ ❱ 194 │ │ │ output = black.format_str(output, mode=black.FileMode()) │
│ 195 │ │ file.write(output) │
│ 196 │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/black/__init__.py:1073 in │
│ format_str │
│ │
│ 1070 │ │ hey │
│ 1071 │ │
│ 1072 │ """ │
│ ❱ 1073 │ dst_contents = _format_str_once(src_contents, mode=mode) │
│ 1074 │ # Forced second pass to work around optional trailing commas (becoming │
│ 1075 │ # forced trailing commas on pass 2) interacting differently with optional │
│ 1076 │ # parentheses. Admittedly ugly. │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/black/__init__.py:1083 in │
│ _format_str_once │
│ │
│ 1080 │
│ 1081 │
│ 1082 def _format_str_once(src_contents: str, *, mode: Mode) -> str: │
│ ❱ 1083 │ src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions) │
│ 1084 │ dst_blocks: List[LinesBlock] = [] │
│ 1085 │ if mode.target_versions: │
│ 1086 │ │ versions = mode.target_versions │
│ │
│ /Users/user/miniforge3/envs/pynguin/lib/python3.10/site-packages/black/parsing.py:127 in │
│ lib2to3_parse │
│ │
│ 124 │ │ │ msg = f"{original_msg}\n{PY2_HINT}" │
│ 125 │ │ │ raise InvalidInput(msg) from None │
│ 126 │ │ │
│ ❱ 127 │ │ raise exc from None │
│ 128 │ │
│ 129 │ if isinstance(result, Leaf): │
│ 130 │ │ result = Node(syms.file_input, [result]) │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
InvalidInput: Cannot parse: 2:14: import pynguin-example as module_0
(Location 2:14 is the s symbol in the def)
Pynguin tries to format the generated tests using black. I think location 2:14 refers to the - in the string import pynguin-example as module_0. It seems like you tried to apply Pynguin on a module called on pynguin-example. AFAIK, module names (i.e. things that can be imported in Python) are not allowed to contain a - in their name, which is why black fails to parse the generated tests. Renaming your file accordingly should fix that problem as well.
Oh, nice catch, thank you! Maybe add a clearer message for when the formatting fails, but it solves my issue, thanks!
And one more question, not related @Wooza,
Would you know what the message AttributeError: 'NoneType' object has no attribute 'NDArray' could mean? I am trying to create tests for code including pd.DataFrames. I am guessing this also means that testing code including pandas DataFrames is not currently supported?