Compatibility with ES 7

[Alberto Baselga]

Hi,
Currently gcsbeat is incompatible with ES7, raising a ' mapper_parsing_exception' every time it tries to upload content:

2019-12-03T00:07:46.182+0100 ERROR pipeline/output.go:74 Failed to connect: Connection marked as failed because the onConnect callback failed: Error loading Elasticsearch template: could not load template. Elasticsearh returned: couldn't load template: couldn't load json. Error: 400 Bad Request: {"error":{"root_cause":[{"type":"mapper_parsing_exception","reason":"Root mapping definition has unsupported parameters: [doc : {_meta={version=7.0.0-alpha1}, dynamic_templates=[{fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {strings_as_keyword={mapping={ignore_above=1024, type=keyword}, match_mapping_type=string}}], properties={kubernetes={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}}}, node={properties={name={ignore_above=1024, type=keyword}}}, pod={properties={name={ignore_above=1024, type=keyword}}}, namespace={ignore_above=1024, type=keyword}, annotations={type=object}, labels={type=object}}}, @timestamp={type=date}, file={norms=false, type=text}, meta={properties={cloud={properties={machine_type={ignore_above=1024, type=keyword}, availability_zone={ignore_above=1024, type=keyword}, instance_id={ignore_above=1024, type=keyword}, instance_name={ignore_above=1024, type=keyword}, project_id={ignore_above=1024, type=keyword}, provider={ignore_above=1024, type=keyword}, region={ignore_above=1024, type=keyword}}}}}, line={type=long}, beat={properties={hostname={ignore_above=1024, type=keyword}, timezone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}}}, json={type=object}, error={properties={code={type=long}, type={ignore_above=1024, type=keyword}, message={norms=false, type=text}}}, event={norms=false, type=text}, fields={type=object}, docker={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, id={ignore_above=1024, type=keyword}, labels={type=object}}}}}, tags={ignore_above=1024, type=keyword}}, date_detection=false}]"}],"type":"mapper_parsing_exception","reason":"Failed to parse mapping [_doc]: Root mapping definition has unsupported parameters: [doc : {_meta={version=7.0.0-alpha1}, dynamic_templates=[{fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {strings_as_keyword={mapping={ignore_above=1024, type=keyword}, match_mapping_type=string}}], properties={kubernetes={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}}}, node={properties={name={ignore_above=1024, type=keyword}}}, pod={properties={name={ignore_above=1024, type=keyword}}}, namespace={ignore_above=1024, type=keyword}, annotations={type=object}, labels={type=object}}}, @timestamp={type=date}, file={norms=false, type=text}, meta={properties={cloud={properties={machine_type={ignore_above=1024, type=keyword}, availability_zone={ignore_above=1024, type=keyword}, instance_id={ignore_above=1024, type=keyword}, instance_name={ignore_above=1024, type=keyword}, project_id={ignore_above=1024, type=keyword}, provider={ignore_above=1024, type=keyword}, region={ignore_above=1024, type=keyword}}}}}, line={type=long}, beat={properties={hostname={ignore_above=1024, type=keyword}, timezone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}}}, json={type=object}, error={properties={code={type=long}, type={ignore_above=1024, type=keyword}, message={norms=false, type=text}}}, event={norms=false, type=text}, fields={type=object}, docker={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, id={ignore_above=1024, type=keyword}, labels={type=object}}}}}, tags={ignore_above=1024, type=keyword}}, date_detection=false}]","caused_by":{"type":"mapper_parsing_exception","reason":"Root mapping definition has unsupported parameters: [doc : {_meta={version=7.0.0-alpha1}, dynamic_templates=[{fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {strings_as_keyword={mapping={ignore_above=1024, type=keyword}, match_mapping_type=string}}], properties={kubernetes={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}}}, node={properties={name={ignore_above=1024, type=keyword}}}, pod={properties={name={ignore_above=1024, type=keyword}}}, namespace={ignore_above=1024, type=keyword}, annotations={type=object}, labels={type=object}}}, @timestamp={type=date}, file={norms=false, type=text}, meta={properties={cloud={properties={machine_type={ignore_above=1024, type=keyword}, availability_zone={ignore_above=1024, type=keyword}, instance_id={ignore_above=1024, type=keyword}, instance_name={ignore_above=1024, type=keyword}, project_id={ignore_above=1024, type=keyword}, provider={ignore_above=1024, type=keyword}, region={ignore_above=1024, type=keyword}}}}}, line={type=long}, beat={properties={hostname={ignore_above=1024, type=keyword}, timezone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}}}, json={type=object}, error={properties={code={type=long}, type={ignore_above=1024, type=keyword}, message={norms=false, type=text}}}, event={norms=false, type=text}, fields={type=object}, docker={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, id={ignore_above=1024, type=keyword}, labels={type=object}}}}}, tags={ignore_above=1024, type=keyword}}, date_detection=false}]"}},"status":400}. Response body: {"error":{"root_cause":[{"type":"mapper_parsing_exception","reason":"Root mapping definition has unsupported parameters: [doc : {_meta={version=7.0.0-alpha1}, dynamic_templates=[{fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {strings_as_keyword={mapping={ignore_above=1024, type=keyword}, match_mapping_type=string}}], properties={kubernetes={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}}}, node={properties={name={ignore_above=1024, type=keyword}}}, pod={properties={name={ignore_above=1024, type=keyword}}}, namespace={ignore_above=1024, type=keyword}, annotations={type=object}, labels={type=object}}}, @timestamp={type=date}, file={norms=false, type=text}, meta={properties={cloud={properties={machine_type={ignore_above=1024, type=keyword}, availability_zone={ignore_above=1024, type=keyword}, instance_id={ignore_above=1024, type=keyword}, instance_name={ignore_above=1024, type=keyword}, project_id={ignore_above=1024, type=keyword}, provider={ignore_above=1024, type=keyword}, region={ignore_above=1024, type=keyword}}}}}, line={type=long}, beat={properties={hostname={ignore_above=1024, type=keyword}, timezone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}}}, json={type=object}, error={properties={code={type=long}, type={ignore_above=1024, type=keyword}, message={norms=false, type=text}}}, event={norms=false, type=text}, fields={type=object}, docker={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, id={ignore_above=1024, type=keyword}, labels={type=object}}}}}, tags={ignore_above=1024, type=keyword}}, date_detection=false}]"}],"type":"mapper_parsing_exception","reason":"Failed to parse mapping [_doc]: Root mapping definition has unsupported parameters: [doc : {_meta={version=7.0.0-alpha1}, dynamic_templates=[{fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {strings_as_keyword={mapping={ignore_above=1024, type=keyword}, match_mapping_type=string}}], properties={kubernetes={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}}}, node={properties={name={ignore_above=1024, type=keyword}}}, pod={properties={name={ignore_above=1024, type=keyword}}}, namespace={ignore_above=1024, type=keyword}, annotations={type=object}, labels={type=object}}}, @timestamp={type=date}, file={norms=false, type=text}, meta={properties={cloud={properties={machine_type={ignore_above=1024, type=keyword}, availability_zone={ignore_above=1024, type=keyword}, instance_id={ignore_above=1024, type=keyword}, instance_name={ignore_above=1024, type=keyword}, project_id={ignore_above=1024, type=keyword}, provider={ignore_above=1024, type=keyword}, region={ignore_above=1024, type=keyword}}}}}, line={type=long}, beat={properties={hostname={ignore_above=1024, type=keyword}, timezone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}}}, json={type=object}, error={properties={code={type=long}, type={ignore_above=1024, type=keyword}, message={norms=false, type=text}}}, event={norms=false, type=text}, fields={type=object}, docker={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, id={ignore_above=1024, type=keyword}, labels={type=object}}}}}, tags={ignore_above=1024, type=keyword}}, date_detection=false}]","caused_by":{"type":"mapper_parsing_exception","reason":"Root mapping definition has unsupported parameters: [doc : {_meta={version=7.0.0-alpha1}, dynamic_templates=[{fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {fields={path_match=fields.*, mapping={type=keyword}, match_mapping_type=string}}, {docker.container.labels={path_match=docker.container.labels.*, mapping={type=keyword}, match_mapping_type=string}}, {strings_as_keyword={mapping={ignore_above=1024, type=keyword}, match_mapping_type=string}}], properties={kubernetes={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}}}, node={properties={name={ignore_above=1024, type=keyword}}}, pod={properties={name={ignore_above=1024, type=keyword}}}, namespace={ignore_above=1024, type=keyword}, annotations={type=object}, labels={type=object}}}, @timestamp={type=date}, file={norms=false, type=text}, meta={properties={cloud={properties={machine_type={ignore_above=1024, type=keyword}, availability_zone={ignore_above=1024, type=keyword}, instance_id={ignore_above=1024, type=keyword}, instance_name={ignore_above=1024, type=keyword}, project_id={ignore_above=1024, type=keyword}, provider={ignore_above=1024, type=keyword}, region={ignore_above=1024, type=keyword}}}}}, line={type=long}, beat={properties={hostname={ignore_above=1024, type=keyword}, timezone={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, version={ignore_above=1024, type=keyword}}}, json={type=object}, error={properties={code={type=long}, type={ignore_above=1024, type=keyword}, message={norms=false, type=text}}}, event={norms=false, type=text}, fields={type=object}, docker={properties={container={properties={image={ignore_above=1024, type=keyword}, name={ignore_above=1024, type=keyword}, id={ignore_above=1024, type=keyword}, labels={type=object}}}}}, tags={ignore_above=1024, type=keyword}}, date_detection=false}]"}},"status":400}. Template is: {"index_patterns":["gcsbeat-7.0.0-alpha1-*"],"mappings":{"doc":{"_meta":{"version":"7.0.0-alpha1"},"date_detection":false,"dynamic_templates":[{"fields":{"mapping":{"type":"keyword"},"match_mapping_type":"string","path_match":"fields.*"}},{"docker.container.labels":{"mapping":{"type":"keyword"},"match_mapping_type":"string","path_match":"docker.container.labels.*"}},{"fields":{"mapping":{"type":"keyword"},"match_mapping_type":"string","path_match":"fields.*"}},{"docker.container.labels":{"mapping":{"type":"keyword"},"match_mapping_type":"string","path_match":"docker.container.labels.*"}},{"fields":{"mapping":{"type":"keyword"},"match_mapping_type":"string","path_match":"fields.*"}},{"docker.container.labels":{"mapping":{"type":"keyword"},"match_mapping_type":"string","path_match":"docker.container.labels.*"}},{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"properties":{"@timestamp":{"type":"date"},"beat":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword"},"timezone":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"}}},"docker":{"properties":{"container":{"properties":{"id":{"ignore_above":1024,"type":"keyword"},"image":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"},"name":{"ignore_above":1024,"type":"keyword"}}}}},"error":{"properties":{"code":{"type":"long"},"message":{"norms":false,"type":"text"},"type":{"ignore_above":1024,"type":"keyword"}}},"event":{"norms":false,"type":"text"},"fields":{"type":"object"},"file":{"norms":false,"type":"text"},"json":{"type":"object"},"kubernetes":{"properties":{"annotations":{"type":"object"},"container":{"properties":{"image":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword"}}},"labels":{"type":"object"},"namespace":{"ignore_above":1024,"type":"keyword"},"node":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"pod":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}}}},"line":{"type":"long"},"meta":{"properties":{"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"instance_id":{"ignore_above":1024,"type":"keyword"},"instance_name":{"ignore_above":1024,"type":"keyword"},"machine_type":{"ignore_above":1024,"type":"keyword"},"project_id":{"ignore_above":1024,"type":"keyword"},"provider":{"ignore_above":1024,"type":"keyword"},"region":{"ignore_above":1024,"type":"keyword"}}}}},"tags":{"ignore_above":1024,"type":"keyword"}}}},"order":1,"settings":{"index":{"mapping":{"total_fields":{"limit":10000}},"number_of_routing_shards":30,"refresh_interval":"5s"}}}