Pass-through object field type

Pass-through objects extend the functionality of objects by allowing to access their subfields without including the name of the pass-through object as prefix. For instance:

resp = client.indices.create(
    index="my-index-000001",
    mappings={
        "properties": {
            "attributes": {
                "type": "passthrough",
                "priority": 10,
                "properties": {
                    "id": {
                        "type": "keyword"
                    }
                }
            }
        }
    },
)
print(resp)

resp1 = client.index(
    index="my-index-000001",
    id="1",
    document={
        "attributes": {
            "id": "foo",
            "zone": 10
        }
    },
)
print(resp1)

resp2 = client.search(
    index="my-index-000001",
    query={
        "bool": {
            "must": [
                {
                    "match": {
                        "id": "foo"
                    }
                },
                {
                    "match": {
                        "zone": 10
                    }
                }
            ]
        }
    },
)
print(resp2)

resp3 = client.search(
    index="my-index-000001",
    query={
        "bool": {
            "must": [
                {
                    "match": {
                        "attributes.id": "foo"
                    }
                },
                {
                    "match": {
                        "attributes.zone": 10
                    }
                }
            ]
        }
    },
)
print(resp3)

const response = await client.indices.create({
  index: "my-index-000001",
  mappings: {
    properties: {
      attributes: {
        type: "passthrough",
        priority: 10,
        properties: {
          id: {
            type: "keyword",
          },
        },
      },
    },
  },
});
console.log(response);

const response1 = await client.index({
  index: "my-index-000001",
  id: 1,
  document: {
    attributes: {
      id: "foo",
      zone: 10,
    },
  },
});
console.log(response1);

const response2 = await client.search({
  index: "my-index-000001",
  query: {
    bool: {
      must: [
        {
          match: {
            id: "foo",
          },
        },
        {
          match: {
            zone: 10,
          },
        },
      ],
    },
  },
});
console.log(response2);

const response3 = await client.search({
  index: "my-index-000001",
  query: {
    bool: {
      must: [
        {
          match: {
            "attributes.id": "foo",
          },
        },
        {
          match: {
            "attributes.zone": 10,
          },
        },
      ],
    },
  },
});
console.log(response3);

PUT my-index-000001
{
  "mappings": {
    "properties": {
      "attributes": {
        "type": "passthrough", 
        "priority": 10,
        "properties": {
          "id": {
            "type": "keyword"
          }
        }
      }
    }
  }
}

PUT my-index-000001/_doc/1
{
  "attributes" : {  
    "id": "foo",
    "zone": 10
  }
}

GET my-index-000001/_search
{
  "query": {
    "bool": {
      "must": [
        { "match": { "id": "foo" }},  
        { "match": { "zone": 10 }}
      ]
    }
  }
}

GET my-index-000001/_search
{
  "query": {
    "bool": {
      "must": [
        { "match": { "attributes.id": "foo" }}, 
        { "match": { "attributes.zone": 10 }}
      ]
    }
  }
}

	An object is defined as pass-through. Its priority (required) is used for conflict resolution.
	Object contents get indexed as usual, including dynamic mappings.
	Sub-fields can be referenced in queries as if they’re defined at the root level.
	Sub-fields can also be referenced including the object name as prefix.

Conflict resolution

edit

It’s possible for conflicting names to arise, for fields that are defined within different scopes:

A pass-through object is defined next to a field that has the same name as one of the pass-through object sub-fields, e.g.

resp = client.index(
    index="my-index-000001",
    id="1",
    document={
        "attributes": {
            "id": "foo"
        },
        "id": "bar"
    },
)
print(resp)

const response = await client.index({
  index: "my-index-000001",
  id: 1,
  document: {
    attributes: {
      id: "foo",
    },
    id: "bar",
  },
});
console.log(response);

PUT my-index-000001/_doc/1
{
  "attributes" : {
    "id": "foo"
  },
  "id": "bar"
}

In this case, references to id point to the field at the root level, while field attributes.id can only be accessed using the full path.

Two (or more) pass-through objects are defined within the same object and contain fields with the same name, e.g.

resp = client.indices.create(
    index="my-index-000002",
    mappings={
        "properties": {
            "attributes": {
                "type": "passthrough",
                "priority": 10,
                "properties": {
                    "id": {
                        "type": "keyword"
                    }
                }
            },
            "resource.attributes": {
                "type": "passthrough",
                "priority": 20,
                "properties": {
                    "id": {
                        "type": "keyword"
                    }
                }
            }
        }
    },
)
print(resp)

const response = await client.indices.create({
  index: "my-index-000002",
  mappings: {
    properties: {
      attributes: {
        type: "passthrough",
        priority: 10,
        properties: {
          id: {
            type: "keyword",
          },
        },
      },
      "resource.attributes": {
        type: "passthrough",
        priority: 20,
        properties: {
          id: {
            type: "keyword",
          },
        },
      },
    },
  },
});
console.log(response);

PUT my-index-000002
{
  "mappings": {
    "properties": {
      "attributes": {
        "type": "passthrough",
        "priority": 10,
        "properties": {
          "id": {
            "type": "keyword"
          }
        }
      },
      "resource.attributes": {
        "type": "passthrough",
        "priority": 20,
        "properties": {
          "id": {
            "type": "keyword"
          }
        }
      }
    }
  }
}

In this case, param priority is used for conflict resolution, with the higher values taking precedence. In the example above, resource.attributes has higher priority than attributes, so references to id point to the field within resource.attributes. attributes.id can still be accessed using its full path.

Defining sub-fields as time-series dimensions

edit

It is possible to configure a pass-through field as a container for time-series dimensions. In this case, all sub-fields get annotated with the same parameter under the covers, and they’re also included in routing path and tsid calculations, thus simplifying the TSDS setup:

resp = client.indices.put_index_template(
    name="my-metrics",
    index_patterns=[
        "metrics-mymetrics-*"
    ],
    priority=200,
    data_stream={},
    template={
        "settings": {
            "index.mode": "time_series"
        },
        "mappings": {
            "properties": {
                "attributes": {
                    "type": "passthrough",
                    "priority": 10,
                    "time_series_dimension": True,
                    "properties": {
                        "host.name": {
                            "type": "keyword"
                        }
                    }
                },
                "cpu": {
                    "type": "integer",
                    "time_series_metric": "counter"
                }
            }
        }
    },
)
print(resp)

resp1 = client.index(
    index="metrics-mymetrics-test",
    document={
        "@timestamp": "2020-01-01T00:00:00.000Z",
        "attributes": {
            "host.name": "foo",
            "zone": "bar"
        },
        "cpu": 10
    },
)
print(resp1)

const response = await client.indices.putIndexTemplate({
  name: "my-metrics",
  index_patterns: ["metrics-mymetrics-*"],
  priority: 200,
  data_stream: {},
  template: {
    settings: {
      "index.mode": "time_series",
    },
    mappings: {
      properties: {
        attributes: {
          type: "passthrough",
          priority: 10,
          time_series_dimension: true,
          properties: {
            "host.name": {
              type: "keyword",
            },
          },
        },
        cpu: {
          type: "integer",
          time_series_metric: "counter",
        },
      },
    },
  },
});
console.log(response);

const response1 = await client.index({
  index: "metrics-mymetrics-test",
  document: {
    "@timestamp": "2020-01-01T00:00:00.000Z",
    attributes: {
      "host.name": "foo",
      zone: "bar",
    },
    cpu: 10,
  },
});
console.log(response1);

PUT _index_template/my-metrics
{
  "index_patterns": ["metrics-mymetrics-*"],
  "priority": 200,
  "data_stream": { },
  "template": {
    "settings": {
      "index.mode": "time_series"
    },
    "mappings": {
      "properties": {
        "attributes": {
          "type": "passthrough",
          "priority": 10,
          "time_series_dimension": true,
          "properties": {
            "host.name": {
              "type": "keyword"
            }
          }
        },
        "cpu": {
          "type": "integer",
          "time_series_metric": "counter"
        }
      }
    }
  }
}

POST metrics-mymetrics-test/_doc
{
  "@timestamp": "2020-01-01T00:00:00.000Z",
  "attributes" : {
    "host.name": "foo",
    "zone": "bar"
  },
  "cpu": 10
}

In the example above, attributes is defined as a dimension container. Its sub-fields host.name (static) and zone (dynamic) get included in the routing path and tsid, and can be referenced in queries without the attributes. prefix.

Sub-field auto-flattening

edit

Pass-through fields apply auto-flattening to sub-fields by default, to reduce dynamic mapping conflicts. As a consequence, no sub-object definitions are allowed within pass-through fields.

Parameters for `passthrough` fields

edit

The following parameters are accepted by passthrough fields:

`priority`	(Required) used for naming conflict resolution between pass-through fields. The field with the highest value wins. Accepts non-negative integer values.
`time_series_dimension`	Whether or not to treat sub-fields as time-series dimensions. Accepts `false` (default) or `true`.
`dynamic`	Whether or not new `properties` should be added dynamically to an existing object. Accepts `true` (default), `runtime`, `false` and `strict`.
`enabled`	Whether the JSON value given for the object field should be parsed and indexed (`true`, default) or completely ignored (`false`).
`properties`	The fields within the object, which can be of any data type, including `object`. New properties may be added to an existing object.

If you need to index arrays of objects instead of single objects, read Nested first.

« Object field type Percolator field type »

Pass-through object field type