Update bolides package to read new detection pipeline data strcutures
jcsmithhere opened this issue · comments
Jeff Smith commented
Update bolides to import detection pipeline runs using new validator data structures.
Example of error:
KeyError Traceback (most recent call last)
Cell In[12], line 1
----> 1 bdf = BolideDataFrame(source='glm-pipeline', files=['./bolide_database.fs'])
File ~/dev_atap/bolides/bolides/bdf.py:140, in BolideDataFrame.__init__(self, *args, **kwargs)
138 else:
139 min_confidence = 0
--> 140 init_gdf = pipeline(files=files, min_confidence=min_confidence)
142 init_gdf['source'] = source
144 # rearrange columns, respecting original order if csv or pickle
File ~/dev_atap/bolides/bolides/sources.py:106, in pipeline(files, min_confidence)
103 def pipeline(files, min_confidence=0):
105 from .pipeline_utils import dict_from_zodb
--> 106 dict_of_lists = dict_from_zodb(files=files, min_confidence=min_confidence)
108 df = pd.DataFrame(dict_of_lists)
110 column_translation = {'avgLon': 'longitude', 'avgLat': 'latitude', 'bolideTime': 'datetime',
111 'timeDuration': 'duration', 'goesSatellite': 'detectedBy'}
File ~/dev_atap/bolides/bolides/pipeline_utils.py:34, in dict_from_zodb(files, min_confidence)
31 d = dict_from_obj(d, broken)
33 # only add to list of dicts if the confidence is above the threshold
---> 34 if d['confidence'] > min_confidence:
35 # create dict containing basic data
36 data0 = {'_id': ID, 'confidence': d['confidence'],
37 'method': d['confidenceSource'], 'comments': d['howFound'],
38 'yaw_flip_flag': d['yaw_flip_flag']}
39 # get dict from the attributes and values of the features object
KeyError: 'confidence'