[BUG] Custom ISM policy isnt injected

[Ľuboš Hozzán]

Describe the bug
On fresh start (DataPrepper against fresh empty initialized OpenSearch DB) we expecting, that DataPrepper inject our custom ISM policy + custom index template.

From logs from DataPrepper I can see, that the index template is managed, but ISM policy not.

Expected behavior
If custom ISM policy is used, must be showed in ISM management GUI and ISM must managing the OTel indices.

Environment (please complete the following information):

• DataPrepper Docker image v 2.6.2
• OpenSearch Docker image v2.11.1

Additional context

/usr/share/data-prepper/config/data-prepper-config.yaml

ssl: false

/usr/share/data-prepper/otel-span-index-template.json

{
  "index_patterns": ["otel-v1-apm-span-*"],
  "version": 1,
  "template": {
    "settings": {
      "plugins.index_state_management.rollover_alias": "otel-v1-apm-span"
    }
  }
}

/usr/share/data-prepper/otel-span-ism-policy.json

{
  "policy": {
    "policy_id": "otel-span",
    "description": "Managing raw spans for trace analytics",
    "default_state": "current_write_index",
    "states": [
      {
        "name": "current_write_index",
        "actions": [
            {
                "rollover": {
                    "min_size": "10gb",
                    "min_index_age": "24h"
                }
            }
        ],
        "transitions": [
            {
                "state_name": "delete",
                "conditions": {
                    "min_index_age": "3d"
                }
            }
        ]
      },
      {
        "name": "delete",
        "actions": [
          {
            "delete": {}
          }
        ]
      }
    ],
    "ism_template": [
      {
        "index_patterns": ["otel-v1-apm-span-*"]
      }
    ]
  }
}

/usr/share/data-prepper/pipelines/pipelines.yaml

entry-pipeline:
  workers: 4
  delay: "100"
  source:
    otel_trace_source:
      ssl: true
      sslKeyCertChainFile: "/usr/share/data-prepper/server.crt"
      sslKeyFile: "/usr/share/data-prepper/server.key"
      port: 21890
      authentication:
        http_basic:
          username: REDACTED
          password: REDACTED
  buffer:
    bounded_blocking:
      buffer_size: 1024
      batch_size: 256
  sink:
    - pipeline:
        name: "raw-pipeline"
    - pipeline:
        name: "service-map-pipeline"

raw-pipeline:
  source:
    pipeline:
      name: "entry-pipeline"
  processor:
    - otel_traces:
  sink:
    - opensearch:
        #index_type: trace-analytics-raw
        index_type: custom
        index: otel-v1-apm-span
        hosts: [ https://os-endpoint.REDACTED:9200 ]
        cert: "/usr/share/data-prepper/root-ca.pem"
        template_file: "/usr/share/data-prepper/otel-span-index-template.json"
        ism_policy_file: "/usr/share/data-prepper/otel-span-ism-policy.json"
        username: REDACTED
        password: REDACTED

service-map-pipeline:
  delay: "100"
  source:
    pipeline:
      name: "entry-pipeline"
  processor:
    - service_map:
  sink:
    - opensearch:
        index_type: trace-analytics-service-map
        hosts: [ https://os-endpoint.REDACTED:9200 ]
        cert: "/usr/share/data-prepper/root-ca.pem"
        username: REDACTED
        password: REDACTED

[Ľuboš Hozzán]

Working fine in DataPrepper v 2.7.0, but I see different settings in template, when I get it from OpenSearch:

GET _template/otel-v1-apm-span-index-template

{
  "otel-v1-apm-span-index-template": {
    "order": 0,
    "version": 1,
    "index_patterns": [
      "otel-v1-apm-span-*"
    ],
    "settings": {
      "index": {
        "opendistro": {
          "index_state_management": {
            "rollover_alias": "otel-v1-apm-span"
          }
        }
      }
    },
    "mappings": {
      "_source": {
        "enabled": true
      },
      "dynamic_templates": [
        {
          "resource_attributes_map": {
            "path_match": "resource.attributes.*",
            "mapping": {
              "type": "keyword"
            }
          }
        },
        {
          "span_attributes_map": {
            "path_match": "span.attributes.*",
            "mapping": {
              "type": "keyword"
            }
          }
        }
      ],
      "date_detection": false,
      "properties": {
        "traceId": {
          "ignore_above": 256,
          "type": "keyword"
        },
        "kind": {
          "ignore_above": 128,
          "type": "keyword"
        },
        "traceGroupFields": {
          "type": "object",
          "properties": {
            "endTime": {
              "type": "date_nanos"
            },
            "durationInNanos": {
              "type": "long"
            },
            "statusCode": {
              "type": "integer"
            }
          }
        },
        "traceGroup": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "serviceName": {
          "type": "keyword"
        },
        "parentSpanId": {
          "ignore_above": 256,
          "type": "keyword"
        },
        "spanId": {
          "ignore_above": 256,
          "type": "keyword"
        },
        "name": {
          "ignore_above": 1024,
          "type": "keyword"
        },
        "startTime": {
          "type": "date_nanos"
        },
        "links": {
          "type": "nested"
        },
        "endTime": {
          "type": "date_nanos"
        },
        "durationInNanos": {
          "type": "long"
        },
        "events": {
          "type": "nested",
          "properties": {
            "time": {
              "type": "date_nanos"
            }
          }
        },
        "status": {
          "type": "object",
          "properties": {
            "code": {
              "type": "integer"
            },
            "message": {
              "type": "keyword"
            }
          }
        }
      }
    },
    "aliases": {}
  }
}

So, can somebody explain for me, what is correct?

Variant A (our settings):

...
      "template": {
        "settings": {
          "plugins.index_state_management.rollover_alias": "otel-v1-apm-span"
        }
      }
...

OR

Variant B (settings from OpenSearch after DataPrepper injection):

...
    "settings": {
      "index": {
        "opendistro": {
          "index_state_management": {
            "rollover_alias": "otel-v1-apm-span"
          }
        }
      }
    }
...

?

Thank you!