Transforms APIs

Introduced 1.0

Creates a transform job.

Path parameters

Request body fields

You can specify the following options in the HTTP request body:

Sample Request

The following request creates a transform job with the id sample:

  1. PUT _plugins/_transform/sample
  2. {
  3.   "transform": {
  4.     "enabled": true,
  5.     "continuous": true,
  6.     "schedule": {
  7.       "interval": {
  8.         "period": 1,
  9.         "unit": "Minutes",
  10.         "start_time": 1602100553
  11.       }
  12.     },
  13.     "description": "Sample transform job",
  14.     "source_index": "sample_index",
  15.     "target_index": "sample_target",
  16.     "data_selection_query": {
  17.       "match_all": {}
  18.     },
  19.     "page_size": 1,
  20.     "groups": [
  21.       {
  22.         "terms": {
  23.           "source_field": "customer_gender",
  24.           "target_field": "gender"
  25.         }
  26.       },
  27.       {
  28.         "terms": {
  29.           "source_field": "day_of_week",
  30.           "target_field": "day"
  31.         }
  32.       }
  33.     ],
  34.     "aggregations": {
  35.       "quantity": {
  36.         "sum": {
  37.           "field": "total_quantity"
  38.         }
  39.       }
  40.     }
  41.   }
  42. }

Sample Response

  1. {
  2.   "_id": "sample",
  3.   "_version": 7,
  4.   "_seq_no": 13,
  5.   "_primary_term": 1,
  6.   "transform": {
  7.     "transform_id": "sample",
  8.     "schema_version": 7,
  9.     "continuous": true,
  10.     "schedule": {
  11.       "interval": {
  12.         "start_time": 1621467964243,
  13.         "period": 1,
  14.         "unit": "Minutes"
  15.       }
  16.     },
  17.     "metadata_id": null,
  18.     "updated_at": 1621467964243,
  19.     "enabled": true,
  20.     "enabled_at": 1621467964243,
  21.     "description": "Sample transform job",
  22.     "source_index": "sample_index",
  23.     "data_selection_query": {
  24.       "match_all": {
  25.         "boost": 1.0
  26.       }
  27.     },
  28.     "target_index": "sample_target",
  29.     "roles": [],
  30.     "page_size": 1,
  31.     "groups": [
  32.       {
  33.         "terms": {
  34.           "source_field": "customer_gender",
  35.           "target_field": "gender"
  36.         }
  37.       },
  38.       {
  39.         "terms": {
  40.           "source_field": "day_of_week",
  41.           "target_field": "day"
  42.         }
  43.       }
  44.     ],
  45.     "aggregations": {
  46.       "quantity": {
  47.         "sum": {
  48.           "field": "total_quantity"
  49.         }
  50.       }
  51.     }
  52.   }
  53. }

Update a transform job

Introduced 1.0

Updates the transform job if transform_id already exists. For this request you must specify the sequence number and primary term of the transform to be updated. To get these, use the API call.

Request format

  1. PUT _plugins/_transform/<transform_id>?if_seq_no=<seq_no>&if_primary_term=<primary_term>

The update operation supports the following query parameters:

Request body fields

You can update the following fields.

Sample Request

The following request updates a transform job with the id sample, sequence number 13, and primary term 1:

  1. PUT _plugins/_transform/sample?if_seq_no=13&if_primary_term=1
  2. {
  3.   "transform": {
  4.   "enabled": true,
  5.   "schedule": {
  6.     "interval": {
  7.     "period": 1,
  8.     "unit": "Minutes",
  9.     "start_time": 1602100553
  10.     }
  11.   },
  12.   "description": "Sample transform job",
  13.   "source_index": "sample_index",
  14.   "target_index": "sample_target",
  15.   "data_selection_query": {
  16.     "match_all": {}
  17.   },
  18.   "page_size": 1,
  19.   "groups": [
  20.     {
  21.     "terms": {
  22.       "source_field": "customer_gender",
  23.       "target_field": "gender"
  24.     }
  25.     },
  26.     {
  27.     "terms": {
  28.       "source_field": "day_of_week",
  29.       "target_field": "day"
  30.     }
  31.     }
  32.   ],
  33.   "aggregations": {
  34.     "quantity": {
  35.     "sum": {
  36.       "field": "total_quantity"
  37.     }
  38.     }
  39.   }
  40.   }
  41. }

Sample Response

  1. PUT _plugins/_transform/sample?if_seq_no=13&if_primary_term=1
  2. {
  3.   "transform": {
  4.     "enabled": true,
  5.     "schedule": {
  6.       "interval": {
  7.         "period": 1,
  8.         "unit": "Minutes",
  9.         "start_time": 1602100553
  10.       }
  11.     },
  12.     "source_index": "sample_index",
  13.     "target_index": "sample_target",
  14.     "data_selection_query": {
  15.       "match_all": {}
  16.     },
  17.     "page_size": 1,
  18.     "groups": [
  19.       {
  20.         "terms": {
  21.           "source_field": "customer_gender",
  22.           "target_field": "gender"
  23.         }
  24.       },
  26.         "terms": {
  27.           "source_field": "day_of_week",
  28.           "target_field": "day"
  29.         }
  30.       }
  31.     ],
  32.     "aggregations": {
  33.       "quantity": {
  34.         "sum": {
  35.           "field": "total_quantity"
  36.         }
  37.       }
  38.     }
  39.   }
  40. }

Returns a transform job’s details.

Request format

  1. GET _plugins/_transform/<transform_id>

Sample Request

The following request returns the details of the transform job with the id sample:

  1. GET _plugins/_transform/sample

Sample Response

  1. {
  2.   "_id": "sample",
  3.   "_version": 7,
  4.   "_seq_no": 13,
  5.   "_primary_term": 1,
  6.   "transform": {
  7.     "transform_id": "sample",
  8.     "schema_version": 7,
  9.     "continuous": true,
  10.     "schedule": {
  11.       "interval": {
  12.         "start_time": 1621467964243,
  13.         "period": 1,
  14.         "unit": "Minutes"
  15.       }
  16.     },
  17.     "metadata_id": null,
  18.     "updated_at": 1621467964243,
  19.     "enabled": true,
  20.     "enabled_at": 1621467964243,
  21.     "description": "Sample transform job",
  22.     "source_index": "sample_index",
  23.     "data_selection_query": {
  24.       "match_all": {
  25.         "boost": 1.0
  26.       }
  27.     },
  28.     "target_index": "sample_target",
  29.     "roles": [],
  30.     "page_size": 1,
  31.     "groups": [
  32.       {
  33.         "terms": {
  34.           "source_field": "customer_gender",
  35.           "target_field": "gender"
  36.         }
  37.       },
  38.       {
  39.         "terms": {
  40.           "source_field": "day_of_week",
  41.           "target_field": "day"
  42.         }
  43.       }
  44.     ],
  45.     "aggregations": {
  46.       "quantity": {
  47.         "sum": {
  48.           "field": "total_quantity"
  49.         }
  50.       }
  51.     }
  52.   }
  53. }

You can also get details of all transform jobs by omitting transform_id.

Sample Request

The following request returns the details of all transform jobs:

Sample Response

  1. {
  2.   "total_transforms": 1,
  3.   "transforms": [
  4.     {
  5.       "_id": "sample",
  6.       "_seq_no": 13,
  7.       "_primary_term": 1,
  8.       "transform": {
  9.         "transform_id": "sample",
  10.         "schema_version": 7,
  11.         "continuous": true,
  12.         "schedule": {
  13.           "interval": {
  14.             "start_time": 1621467964243,
  15.             "period": 1,
  16.             "unit": "Minutes"
  17.           }
  18.         },
  19.         "metadata_id": null,
  20.         "updated_at": 1621467964243,
  21.         "enabled": true,
  22.         "enabled_at": 1621467964243,
  23.         "description": "Sample transform job",
  24.         "source_index": "sample_index",
  25.         "data_selection_query": {
  26.           "match_all": {
  27.             "boost": 1.0
  28.           }
  29.         },
  30.         "target_index": "sample_target",
  31.         "roles": [],
  32.         "page_size": 1,
  33.         "groups": [
  34.           {
  35.             "terms": {
  36.               "source_field": "customer_gender",
  37.               "target_field": "gender"
  38.             }
  39.           },
  40.           {
  41.             "terms": {
  42.               "source_field": "day_of_week",
  43.               "target_field": "day"
  44.             }
  45.           }
  46.         ],
  47.         "aggregations": {
  48.           "quantity": {
  49.             "sum": {
  50.               "field": "total_quantity"
  51.             }
  52.           }
  53.         }
  54.       }
  55.     }
  56.   ]
  57. }

Query parameters

You can specify the following GET API operation’s query parameters to filter the results.

Sample Request

The following request returns two results starting from transform 8:

  1. GET _plugins/_transform?size=2&from=8

Sample Response

  1. {
  2.   "total_transforms": 18,
  3.   "transforms": [
  4.     {
  5.       "_id": "sample8",
  6.       "_seq_no": 93,
  7.       "_primary_term": 1,
  8.       "transform": {
  9.         "transform_id": "sample8",
  10.         "schema_version": 7,
  11.         "schedule": {
  12.           "interval": {
  13.             "start_time": 1622063596812,
  14.             "period": 1,
  15.             "unit": "Minutes"
  16.           }
  17.         },
  18.         "metadata_id": "y4hFAB2ZURQ2dzY7BAMxWA",
  19.         "updated_at": 1622063657233,
  20.         "enabled": false,
  21.         "enabled_at": null,
  22.         "description": "Sample transform job",
  23.         "source_index": "sample_index3",
  24.         "data_selection_query": {
  25.           "match_all": {
  26.             "boost": 1.0
  27.           }
  28.         },
  29.         "target_index": "sample_target3",
  30.         "roles": [],
  31.         "page_size": 1,
  32.         "groups": [
  33.           {
  34.             "terms": {
  35.               "source_field": "customer_gender",
  36.               "target_field": "gender"
  37.             }
  38.           },
  39.           {
  40.             "terms": {
  41.               "source_field": "day_of_week",
  42.               "target_field": "day"
  43.             }
  44.           }
  45.         ],
  46.         "aggregations": {
  47.             "sum": {
  48.               "field": "total_quantity"
  49.             }
  50.           }
  51.         }
  53.     },
  54.     {
  55.       "_id": "sample9",
  56.       "_seq_no": 98,
  57.       "_primary_term": 1,
  58.       "transform": {
  59.         "transform_id": "sample9",
  60.         "schema_version": 7,
  61.         "schedule": {
  62.           "interval": {
  63.             "start_time": 1622063598065,
  64.             "period": 1,
  65.             "unit": "Minutes"
  66.           }
  67.         },
  68.         "metadata_id": "x8tCIiYMTE3veSbIJkit5A",
  69.         "updated_at": 1622063658388,
  70.         "enabled": false,
  71.         "enabled_at": null,
  72.         "description": "Sample transform job",
  73.         "source_index": "sample_index4",
  74.         "data_selection_query": {
  75.           "match_all": {
  76.             "boost": 1.0
  77.           }
  78.         },
  79.         "target_index": "sample_target4",
  80.         "roles": [],
  81.         "page_size": 1,
  82.         "groups": [
  83.           {
  84.             "terms": {
  85.               "source_field": "customer_gender",
  86.               "target_field": "gender"
  87.             }
  88.           },
  89.           {
  90.             "terms": {
  91.               "source_field": "day_of_week",
  92.               "target_field": "day"
  93.             }
  94.           }
  95.         ],
  96.         "aggregations": {
  97.           "quantity": {
  98.             "sum": {
  99.               "field": "total_quantity"
  100.             }
  101.           }
  102.         }
  103.       }
  104.     }
  105.   ]
  106. }

Start a transform job

Introduced 1.0

Transform jobs created using the API are automatically enabled, but if you ever need to enable a job, you can use the start API operation.

  1. POST _plugins/_transform/<transform_id>/_start

Sample Request

The following request starts the transform job with the ID sample:

  1. POST _plugins/_transform/sample/_start

Sample Response

  1. {
  2.   "acknowledged": true
  3. }

Stops a transform job.

Request format

  1. POST _plugins/_transform/<transform_id>/_stop

Sample Request

The following request stops the transform job with the ID sample:

  1. POST _plugins/_transform/sample/_stop

Sample Response

Get the status of a transform job

Introduced 1.0

Returns the status and metadata of a transform job.

Request format

  1. GET _plugins/_transform/<transform_id>/_explain

Sample Request

The following request returns the details of the transform job with the ID sample:

  1. GET _plugins/_transform/sample/_explain

Sample Response

  1. {
  2.   "sample": {
  3.     "metadata_id": "PzmjweME5xbgkenl9UpsYw",
  4.     "transform_metadata": {
  5.       "continuous_stats": {
  6.         "last_timestamp": 1621883525672,
  7.         "documents_behind": {
  8.           "sample_index": 72
  9.           }
  10.       },
  11.       "transform_id": "sample",
  12.       "last_updated_at": 1621883525873,
  13.       "status": "finished",
  14.       "failure_reason": "null",
  15.       "stats": {
  16.         "pages_processed": 0,
  17.         "documents_processed": 0,
  18.         "documents_indexed": 0,
  19.         "index_time_in_millis": 0,
  20.         "search_time_in_millis": 0
  21.       }
  22.     }
  23.   }
  24. }

Introduced 1.0

Returns a preview of what a transformed index would look like.

Sample Request

  1. POST _plugins/_transform/_preview
  3. {
  4.   "transform": {
  5.   "enabled": false,
  6.   "schedule": {
  7.     "interval": {
  8.     "period": 1,
  9.     "unit": "Minutes",
  10.     "start_time": 1602100553
  11.     }
  12.   },
  13.   "description": "test transform",
  14.   "source_index": "sample_index",
  15.   "target_index": "sample_target",
  16.   "data_selection_query": {
  17.     "match_all": {}
  18.   },
  19.   "page_size": 10,
  20.   "groups": [
  21.     {
  22.     "terms": {
  23.       "source_field": "customer_gender",
  24.       "target_field": "gender"
  25.     }
  26.     },
  27.     {
  28.     "terms": {
  29.       "source_field": "day_of_week",
  30.       "target_field": "day"
  31.     }
  32.     }
  33.   ],
  34.   "aggregations": {
  35.     "quantity": {
  36.     "sum": {
  37.       "field": "total_quantity"
  38.     }
  39.     }
  40.   }
  41.   }
  42. }

Sample Response

  1. {
  2.   "documents" : [
  3.   {
  4.     "quantity" : 862.0,
  5.     "gender" : "FEMALE",
  6.     "day" : "Friday"
  7.   },
  8.   {
  9.     "quantity" : 682.0,
  10.     "gender" : "FEMALE",
  11.     "day" : "Monday"
  12.   },
  13.   {
  14.     "quantity" : 772.0,
  15.     "gender" : "FEMALE",
  16.     "day" : "Saturday"
  17.   },
  18.   {
  19.     "quantity" : 669.0,
  20.     "gender" : "FEMALE",
  21.     "day" : "Sunday"
  22.   },
  23.   {
  24.     "quantity" : 887.0,
  25.     "gender" : "FEMALE",
  26.     "day" : "Thursday"
  27.   }
  28.   ]
  29. }

Delete a transform job

Introduced 1.0

Deletes a transform job. This operation does not delete the source or target indexes.

Request format

  1. DELETE _plugins/_transform/<transform_id>

Sample Request

  1. DELETE _plugins/_transform/sample

Sample Response

  1. {
  2.   "took": 205,
  3.   "errors": false,
  4.   "items": [
  5.     {
  6.       "delete": {
  7.         "_index": ".opensearch-ism-config",
  8.         "_id": "sample",
  9.         "_version": 4,
  10.         "result": "deleted",
  11.         "forced_refresh": true,
  12.         "_shards": {
  13.           "total": 2,
  14.           "successful": 1,
  15.           "failed": 0
  16.         },
  17.         "_seq_no": 6,
  18.         "_primary_term": 1,
  19.         "status": 200
  20.       }
  21.     }