Encoding error with python
jdjenontin opened this issue · comments
The python example does not work with verses with special characters (Non ascii)
Add encoding to correct
with open(data, 'r', encoding='utf8') as kjv_bible:
kjv = yaml.safe_load(kjv_bible)
print(kjv['books']['John'][3][16])
Hi @jdjenontin
Thank you for testing and reporting.
Can you share more details on:
- version of Python
- the error you had
Python 311
Code Sample
def load_bible():
with open('data\lsg.yml', 'r') as file:
bible_data = yaml.safe_load(file)
return bible_data
load_bible()Tracelog
Traceback (most recent call last):
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\reader.py", line 89, in peek
return self.buffer[self.pointer+index]
~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
IndexError: string index out of range
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "c:\Users\udjen\Workspace\bible\main.py", line 12, in <module>
load_bible()
File "c:\Users\udjen\Workspace\bible\main.py", line 8, in load_bible
bible_data = yaml.safe_load(file)
^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\__init__.py", line 125, in safe_load
return load(stream, SafeLoader)
^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\__init__.py", line 81, in load
return loader.get_single_data()
^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\constructor.py", line 49, in get_single_data
node = self.get_single_node()
^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\composer.py", line 36, in get_single_node
document = self.compose_document()
^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\composer.py", line 55, in compose_document
node = self.compose_node(None, None)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\composer.py", line 84, in compose_node
node = self.compose_mapping_node(anchor)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\composer.py", line 133, in compose_mapping_node
item_value = self.compose_node(node, item_key)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\composer.py", line 84, in compose_node
node = self.compose_mapping_node(anchor)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\composer.py", line 133, in compose_mapping_node
item_value = self.compose_node(node, item_key)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\composer.py", line 84, in compose_node
node = self.compose_mapping_node(anchor)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\composer.py", line 133, in compose_mapping_node
item_value = self.compose_node(node, item_key)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\composer.py", line 84, in compose_node
node = self.compose_mapping_node(anchor)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\composer.py", line 133, in compose_mapping_node
item_value = self.compose_node(node, item_key)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\composer.py", line 64, in compose_node
if self.check_event(AliasEvent):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\parser.py", line 98, in check_event
self.current_event = self.state()
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\scanner.py", line 666, in fetch_flow_scalar
self.tokens.append(self.scan_flow_scalar(style))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\scanner.py", line 1152, in scan_flow_scalar
chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\scanner.py", line 1190, in scan_flow_scalar_non_spaces
while self.peek(length) not in '\'\"\\\0 \t\r\n\x85\u2028\u2029':
^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\reader.py", line 91, in peek
self.update(index+1)
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\reader.py", line 153, in update
self.update_raw()
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\site-packages\yaml\reader.py", line 178, in update_raw data = self.stream.read(size)
^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\udjen\AppData\Local\Programs\Python\Python311\Lib\encodings\cp1252.py", line 23, in decode
return codecs.charmap_decode(input,self.errors,decoding_table)[0]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
UnicodeDecodeError: 'charmap' codec can't decode byte 0x9d in position 7417: character maps to <undefined>
Thanks again @jdjenontin
Closing this as solved by #2