bug in 0.85.0 unable to complete AWS scans
shaunwebb-db opened this issue · comments
Title: bug in 0.85.0 unable to complete AWS scans
Description:
0.85.0 bug during AWS scans, unable to complete scan due to the exception being thrown. This consistently fails.
Reverting to 0.84.0 resolves the issues.
Versions used
python - 3.11.3
cartography - 0.85.0
Logs:
ERROR:cartography.sync:Unhandled exception during sync stage 'aws'
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/cartography/sync.py", line 109, in run
stage_func(neo4j_session, config)
File "/usr/local/lib/python3.11/site-packages/cartography/util.py", line 178, in timed
return method(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/cartography/intel/aws/__init__.py", line 298, in start_aws_ingestion
sync_successful = _sync_multiple_accounts(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/cartography/intel/aws/__init__.py", line 196, in _sync_multiple_accounts
raise Exception('\n'.join(exception_tracebacks))
Exception: 2023-09-25 17:45:40.060351 - Exception for account ID: REDACTED
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 337, in fix_parameters
dehydrated, = dehydrator.dehydrate([parameters])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 406, in dehydrate
return tuple(map(dehydrate_, values))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 402, in dehydrate_
return {key: dehydrate_(value) for key, value in obj.items()}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 402, in <dictcomp>
return {key: dehydrate_(value) for key, value in obj.items()}
^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 397, in dehydrate_
return list(map(dehydrate_, obj))
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 402, in dehydrate_
return {key: dehydrate_(value) for key, value in obj.items()}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 402, in <dictcomp>
return {key: dehydrate_(value) for key, value in obj.items()}
^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 404, in dehydrate_
raise TypeError(obj)
TypeError: ('redacted....amazonaws.com',)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/cartography/intel/aws/__init__.py", line 169, in _sync_multiple_accounts
_sync_one_account(
File "/usr/local/lib/python3.11/site-packages/cartography/intel/aws/__init__.py", line 64, in _sync_one_account
RESOURCE_FUNCTIONS[func_name](**sync_args)
File "/usr/local/lib/python3.11/site-packages/cartography/util.py", line 178, in timed
return method(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/cartography/intel/aws/ec2/network_interfaces.py", line 245, in sync_network_interfaces
load_network_data(
File "/usr/local/lib/python3.11/site-packages/cartography/intel/aws/ec2/network_interfaces.py", line 220, in load_network_data
load_network_interfaces(neo4j_session, network_interface_list, region, current_aws_account_id, update_tag)
File "/usr/local/lib/python3.11/site-packages/cartography/util.py", line 178, in timed
return method(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/cartography/intel/aws/ec2/network_interfaces.py", line 134, in load_network_interfaces
load(
File "/usr/local/lib/python3.11/site-packages/cartography/client/core/tx.py", line 254, in load
load_graph_data(neo4j_session, ingestion_query, dict_list, **kwargs)
File "/usr/local/lib/python3.11/site-packages/cartography/client/core/tx.py", line 210, in load_graph_data
neo4j_session.write_transaction(
File "/usr/local/lib/python3.11/site-packages/neo4j/work/simple.py", line 434, in write_transaction
return self._run_transaction(WRITE_ACCESS, transaction_function, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/work/simple.py", line 327, in _run_transaction
result = transaction_function(tx, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/cartography/client/core/tx.py", line 191, in write_list_of_dicts_tx
tx.run(query, kwargs)
File "/usr/local/lib/python3.11/site-packages/neo4j/work/transaction.py", line 143, in run
result._tx_ready_run(query, parameters)
File "/usr/local/lib/python3.11/site-packages/neo4j/work/result.py", line 70, in _tx_ready_run
self._run(query, parameters, None, None, None, None)
File "/usr/local/lib/python3.11/site-packages/neo4j/work/result.py", line 77, in _run
parameters = DataDehydrator.fix_parameters(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 340, in fix_parameters
raise TypeError("Parameters of type {} are not supported".format(type(value).__name__))
TypeError: Parameters of type tuple are not supported
Traceback (most recent call last):
File "/usr/local/bin/cartography", line 8, in <module>
sys.exit(main())
^^^^^^
File "/usr/local/lib/python3.11/site-packages/cartography/cli.py", line 709, in main
sys.exit(CLI(prog='cartography').main(argv))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/cartography/cli.py", line 690, in main
return cartography.sync.run_with_config(self.sync, config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/cartography/sync.py", line 187, in run_with_config
return sync.run(neo4j_driver, config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/cartography/sync.py", line 109, in run
stage_func(neo4j_session, config)
File "/usr/local/lib/python3.11/site-packages/cartography/util.py", line 178, in timed
return method(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/cartography/intel/aws/__init__.py", line 298, in start_aws_ingestion
sync_successful = _sync_multiple_accounts(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/cartography/intel/aws/__init__.py", line 196, in _sync_multiple_accounts
raise Exception('\n'.join(exception_tracebacks))
Exception: 2023-09-25 17:45:40.060351 - Exception for account ID: REDACTED
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 337, in fix_parameters
dehydrated, = dehydrator.dehydrate([parameters])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 406, in dehydrate
return tuple(map(dehydrate_, values))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 402, in dehydrate_
return {key: dehydrate_(value) for key, value in obj.items()}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 402, in <dictcomp>
return {key: dehydrate_(value) for key, value in obj.items()}
^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 397, in dehydrate_
return list(map(dehydrate_, obj))
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 402, in dehydrate_
return {key: dehydrate_(value) for key, value in obj.items()}
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 402, in <dictcomp>
return {key: dehydrate_(value) for key, value in obj.items()}
^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 404, in dehydrate_
raise TypeError(obj)
TypeError: ('redacted..amazonaws.com',)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/cartography/intel/aws/__init__.py", line 169, in _sync_multiple_accounts
_sync_one_account(
File "/usr/local/lib/python3.11/site-packages/cartography/intel/aws/__init__.py", line 64, in _sync_one_account
RESOURCE_FUNCTIONS[func_name](**sync_args)
File "/usr/local/lib/python3.11/site-packages/cartography/util.py", line 178, in timed
return method(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/cartography/intel/aws/ec2/network_interfaces.py", line 245, in sync_network_interfaces
load_network_data(
File "/usr/local/lib/python3.11/site-packages/cartography/intel/aws/ec2/network_interfaces.py", line 220, in load_network_data
load_network_interfaces(neo4j_session, network_interface_list, region, current_aws_account_id, update_tag)
File "/usr/local/lib/python3.11/site-packages/cartography/util.py", line 178, in timed
return method(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/cartography/intel/aws/ec2/network_interfaces.py", line 134, in load_network_interfaces
load(
File "/usr/local/lib/python3.11/site-packages/cartography/client/core/tx.py", line 254, in load
load_graph_data(neo4j_session, ingestion_query, dict_list, **kwargs)
File "/usr/local/lib/python3.11/site-packages/cartography/client/core/tx.py", line 210, in load_graph_data
neo4j_session.write_transaction(
File "/usr/local/lib/python3.11/site-packages/neo4j/work/simple.py", line 434, in write_transaction
return self._run_transaction(WRITE_ACCESS, transaction_function, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/work/simple.py", line 327, in _run_transaction
result = transaction_function(tx, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/cartography/client/core/tx.py", line 191, in write_list_of_dicts_tx
tx.run(query, kwargs)
File "/usr/local/lib/python3.11/site-packages/neo4j/work/transaction.py", line 143, in run
result._tx_ready_run(query, parameters)
File "/usr/local/lib/python3.11/site-packages/neo4j/work/result.py", line 70, in _tx_ready_run
self._run(query, parameters, None, None, None, None)
File "/usr/local/lib/python3.11/site-packages/neo4j/work/result.py", line 77, in _run
parameters = DataDehydrator.fix_parameters(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/neo4j/data.py", line 340, in fix_parameters
raise TypeError("Parameters of type {} are not supported".format(type(value).__name__))
TypeError: Parameters of type tuple are not supported
Thanks for the bug report! Fixed in #1251 -- I'll cut a patch release right now.