Error decoding the grib file when reading/streaming the data from S3 bucket

[Kashyap Maheshwari]

I am generating the data in grib format using the client and then saving the generated file in the AWS S3 bucket. Now, when I am trying to read the file from the S3 bucket, I am getting an error.

Client code

c = cdsapi.Client(url=URL, key=KEY)
c.retrieve(
    'reanalysis-era5-land',
    {
        'variable': 'soil_temperature_level_4',
        'year': year,
        'month': [
            '01', '02', '03',
            '04', '05', '06',
            '07', '08', '09',
            '10', '11', '12',
        ],
        'day': [
            '01', '02', '03',
            '04', '05', '06',
            '07', '08', '09',
            '10', '11', '12',
            '13', '14', '15',
            '16', '17', '18',
            '19', '20', '21',
            '22', '23', '24',
            '25', '26', '27',
            '28', '29', '30',
            '31',
        ],
        'time': [
            '00:00', '01:00', '02:00',
            '03:00', '04:00', '05:00',
            '06:00', '07:00', '08:00',
            '09:00', '10:00', '11:00',
            '12:00', '13:00', '14:00',
            '15:00', '16:00', '17:00',
            '18:00', '19:00', '20:00',
            '21:00', '22:00', '23:00',
        ],
        'area': [
            np.floor(location[1]*100)/100,
            np.floor(location[0]*100)/100,
            np.ceil(location[1]*100)/100,
            np.ceil(location[0]*100)/100,
        ],
        'format': FORMAT,
    })

Code for reading the file from S3

def get_grib_from_s3(grib_file):
        s3_client = boto3.client('s3')
        bucket = <my_bucket_name>
        try:
            resp = s3_client.get_object(
                Bucket=bucket,
                Key=grib_file
            )
            return resp
        except s3_client.exceptions.NoSuchKey as e:
            raise HTTPException(
                status_code=404,
                detail=e
            )

GRIB_FILE_NAME = <filename>.grib
a = get_grib_from_s3(GRIB_FILE_NAME)

and then I try to decode the streaming value as follows:

a['Body'].read().decode('utf-8')

When I do this, I get the following error:

Traceback (most recent call last):
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/uvicorn/protocols/http/h11_impl.py", line 396, in run_asgi
    result = await app(self.scope, self.receive, self.send)
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/uvicorn/middleware/proxy_headers.py", line 45, in __call__
    return await self.app(scope, receive, send)
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/fastapi/applications.py", line 199, in __call__
    await super().__call__(scope, receive, send)
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/starlette/applications.py", line 112, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/starlette/middleware/errors.py", line 181, in __call__
    raise exc from None
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/starlette/middleware/errors.py", line 159, in __call__
    await self.app(scope, receive, _send)
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/starlette/middleware/cors.py", line 78, in __call__
    await self.app(scope, receive, send)
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/starlette/exceptions.py", line 82, in __call__
    raise exc from None
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/starlette/exceptions.py", line 71, in __call__
    await self.app(scope, receive, sender)
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/starlette/routing.py", line 580, in __call__
    await route.handle(scope, receive, send)
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/starlette/routing.py", line 241, in handle
    await self.app(scope, receive, send)
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/starlette/routing.py", line 52, in app
    response = await func(request)
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/fastapi/routing.py", line 202, in app
    dependant=dependant, values=values, is_coroutine=is_coroutine
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/fastapi/routing.py", line 150, in run_endpoint_function
    return await run_in_threadpool(dependant.call, **values)
  File "/home/energycortex/.local/share/virtualenvs/wärmenachfragetool-i62m6NeI/lib/python3.7/site-packages/starlette/concurrency.py", line 40, in run_in_threadpool
    return await loop.run_in_executor(None, func, *args)
  File "/usr/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "./app/api/v1/slp.py", line 37, in get_warm_slp
    temps = HeatSLPFunctions.load_temperature_data(db, year, location)
  File "./app/services/heat_demand_funcs.py", line 185, in load_temperature_data
    print(a['Body'].read().decode('utf-8'))
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 11: invalid start byte