Chunk text using the specified chunker

import requests

url = "https://{APPLICATION_ID}.applications.lucidworks.com/ai/async-chunking/{CHUNKER}/{MODEL_ID}"

payload = {
    "batch": [{ "text": "The itsy bitsy spider climbed up the waterspout.
Down came the rain.
And washed the spider out.
Out came the sun.
And dried up all the rain.
And the itsy bitsy spider climbed up the spout again." }],
    "useCaseConfig": { "dataType": "passage" },
    "modelConfig": { "vectorQuantizationMethod": "max-scale" }
}
headers = {
    "Authorization": "<authorization>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

HttpResponse<String> response = Unirest.post("https://{APPLICATION_ID}.applications.lucidworks.com/ai/async-chunking/{CHUNKER}/{MODEL_ID}")
  .header("Authorization", "<authorization>")
  .header("Content-Type", "application/json")
  .body("{\n  \"batch\": [\n    {\n      \"text\": \"The itsy bitsy spider climbed up the waterspout.\\nDown came the rain.\\nAnd washed the spider out.\\nOut came the sun.\\nAnd dried up all the rain.\\nAnd the itsy bitsy spider climbed up the spout again.\"\n    }\n  ],\n  \"useCaseConfig\": {\n    \"dataType\": \"passage\"\n  },\n  \"modelConfig\": {\n    \"vectorQuantizationMethod\": \"max-scale\"\n  }\n}")
  .asString();

const options = {
  method: 'POST',
  headers: {Authorization: '<authorization>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    batch: [
      {
        text: 'The itsy bitsy spider climbed up the waterspout.\nDown came the rain.\nAnd washed the spider out.\nOut came the sun.\nAnd dried up all the rain.\nAnd the itsy bitsy spider climbed up the spout again.'
      }
    ],
    useCaseConfig: {dataType: 'passage'},
    modelConfig: {vectorQuantizationMethod: 'max-scale'}
  })
};

fetch('https://{APPLICATION_ID}.applications.lucidworks.com/ai/async-chunking/{CHUNKER}/{MODEL_ID}', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

require 'uri'
require 'net/http'

url = URI("https://{APPLICATION_ID}.applications.lucidworks.com/ai/async-chunking/{CHUNKER}/{MODEL_ID}")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = '<authorization>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"batch\": [\n    {\n      \"text\": \"The itsy bitsy spider climbed up the waterspout.\\nDown came the rain.\\nAnd washed the spider out.\\nOut came the sun.\\nAnd dried up all the rain.\\nAnd the itsy bitsy spider climbed up the spout again.\"\n    }\n  ],\n  \"useCaseConfig\": {\n    \"dataType\": \"passage\"\n  },\n  \"modelConfig\": {\n    \"vectorQuantizationMethod\": \"max-scale\"\n  }\n}"

response = http.request(request)
puts response.read_body

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://{APPLICATION_ID}.applications.lucidworks.com/ai/async-chunking/{CHUNKER}/{MODEL_ID}"

	payload := strings.NewReader("{\n  \"batch\": [\n    {\n      \"text\": \"The itsy bitsy spider climbed up the waterspout.\\nDown came the rain.\\nAnd washed the spider out.\\nOut came the sun.\\nAnd dried up all the rain.\\nAnd the itsy bitsy spider climbed up the spout again.\"\n    }\n  ],\n  \"useCaseConfig\": {\n    \"dataType\": \"passage\"\n  },\n  \"modelConfig\": {\n    \"vectorQuantizationMethod\": \"max-scale\"\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "<authorization>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://{APPLICATION_ID}.applications.lucidworks.com/ai/async-chunking/{CHUNKER}/{MODEL_ID}",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'batch' => [
        [
                'text' => 'The itsy bitsy spider climbed up the waterspout.
Down came the rain.
And washed the spider out.
Out came the sun.
And dried up all the rain.
And the itsy bitsy spider climbed up the spout again.'
        ]
    ],
    'useCaseConfig' => [
        'dataType' => 'passage'
    ],
    'modelConfig' => [
        'vectorQuantizationMethod' => 'max-scale'
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: <authorization>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

curl --request POST \
  --url https://{APPLICATION_ID}.applications.lucidworks.com/ai/async-chunking/{CHUNKER}/{MODEL_ID} \
  --header 'Authorization: <authorization>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "batch": [
    {
      "text": "The itsy bitsy spider climbed up the waterspout.\nDown came the rain.\nAnd washed the spider out.\nOut came the sun.\nAnd dried up all the rain.\nAnd the itsy bitsy spider climbed up the spout again."
    }
  ],
  "useCaseConfig": {
    "dataType": "passage"
  },
  "modelConfig": {
    "vectorQuantizationMethod": "max-scale"
  }
}
'

{
  "chunkingId": "09c5f8f1-e790-4a06-9e4e-60f577792043",
  "status": "SUBMITTED"
}

{
  "chunkingId": "441eb3be-7de6-470a-8141-e416a15c7db1",
  "status": "READY",
  "message": "<string>"
}

Chunk text using the specified chunker

This is an all-purpose chunking endpoint you can use by specifying the chunker and modelId to use.

The POST request submits text to be split into chunks based on the CHUNKER (chunking strategy) and MODEL_ID sent in the request.

The API responds with the following information:

chunkingId that is a unique UUID for the submitted chunking task, and can be used later to retrieve the results.
status that indicates the current state of the chunking task.

IMPORTANT: The available use cases by chunker are detailed in their own section in this specification.

POST

async-chunking

{CHUNKER}

{MODEL_ID}

Chunk text using the specified chunker

import requests

url = "https://{APPLICATION_ID}.applications.lucidworks.com/ai/async-chunking/{CHUNKER}/{MODEL_ID}"

payload = {
    "batch": [{ "text": "The itsy bitsy spider climbed up the waterspout.
Down came the rain.
And washed the spider out.
Out came the sun.
And dried up all the rain.
And the itsy bitsy spider climbed up the spout again." }],
    "useCaseConfig": { "dataType": "passage" },
    "modelConfig": { "vectorQuantizationMethod": "max-scale" }
}
headers = {
    "Authorization": "<authorization>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

HttpResponse<String> response = Unirest.post("https://{APPLICATION_ID}.applications.lucidworks.com/ai/async-chunking/{CHUNKER}/{MODEL_ID}")
  .header("Authorization", "<authorization>")
  .header("Content-Type", "application/json")
  .body("{\n  \"batch\": [\n    {\n      \"text\": \"The itsy bitsy spider climbed up the waterspout.\\nDown came the rain.\\nAnd washed the spider out.\\nOut came the sun.\\nAnd dried up all the rain.\\nAnd the itsy bitsy spider climbed up the spout again.\"\n    }\n  ],\n  \"useCaseConfig\": {\n    \"dataType\": \"passage\"\n  },\n  \"modelConfig\": {\n    \"vectorQuantizationMethod\": \"max-scale\"\n  }\n}")
  .asString();

const options = {
  method: 'POST',
  headers: {Authorization: '<authorization>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    batch: [
      {
        text: 'The itsy bitsy spider climbed up the waterspout.\nDown came the rain.\nAnd washed the spider out.\nOut came the sun.\nAnd dried up all the rain.\nAnd the itsy bitsy spider climbed up the spout again.'
      }
    ],
    useCaseConfig: {dataType: 'passage'},
    modelConfig: {vectorQuantizationMethod: 'max-scale'}
  })
};

fetch('https://{APPLICATION_ID}.applications.lucidworks.com/ai/async-chunking/{CHUNKER}/{MODEL_ID}', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

require 'uri'
require 'net/http'

url = URI("https://{APPLICATION_ID}.applications.lucidworks.com/ai/async-chunking/{CHUNKER}/{MODEL_ID}")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = '<authorization>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"batch\": [\n    {\n      \"text\": \"The itsy bitsy spider climbed up the waterspout.\\nDown came the rain.\\nAnd washed the spider out.\\nOut came the sun.\\nAnd dried up all the rain.\\nAnd the itsy bitsy spider climbed up the spout again.\"\n    }\n  ],\n  \"useCaseConfig\": {\n    \"dataType\": \"passage\"\n  },\n  \"modelConfig\": {\n    \"vectorQuantizationMethod\": \"max-scale\"\n  }\n}"

response = http.request(request)
puts response.read_body

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://{APPLICATION_ID}.applications.lucidworks.com/ai/async-chunking/{CHUNKER}/{MODEL_ID}"

	payload := strings.NewReader("{\n  \"batch\": [\n    {\n      \"text\": \"The itsy bitsy spider climbed up the waterspout.\\nDown came the rain.\\nAnd washed the spider out.\\nOut came the sun.\\nAnd dried up all the rain.\\nAnd the itsy bitsy spider climbed up the spout again.\"\n    }\n  ],\n  \"useCaseConfig\": {\n    \"dataType\": \"passage\"\n  },\n  \"modelConfig\": {\n    \"vectorQuantizationMethod\": \"max-scale\"\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "<authorization>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://{APPLICATION_ID}.applications.lucidworks.com/ai/async-chunking/{CHUNKER}/{MODEL_ID}",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'batch' => [
        [
                'text' => 'The itsy bitsy spider climbed up the waterspout.
Down came the rain.
And washed the spider out.
Out came the sun.
And dried up all the rain.
And the itsy bitsy spider climbed up the spout again.'
        ]
    ],
    'useCaseConfig' => [
        'dataType' => 'passage'
    ],
    'modelConfig' => [
        'vectorQuantizationMethod' => 'max-scale'
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: <authorization>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

curl --request POST \
  --url https://{APPLICATION_ID}.applications.lucidworks.com/ai/async-chunking/{CHUNKER}/{MODEL_ID} \
  --header 'Authorization: <authorization>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "batch": [
    {
      "text": "The itsy bitsy spider climbed up the waterspout.\nDown came the rain.\nAnd washed the spider out.\nOut came the sun.\nAnd dried up all the rain.\nAnd the itsy bitsy spider climbed up the spout again."
    }
  ],
  "useCaseConfig": {
    "dataType": "passage"
  },
  "modelConfig": {
    "vectorQuantizationMethod": "max-scale"
  }
}
'

{
  "chunkingId": "09c5f8f1-e790-4a06-9e4e-60f577792043",
  "status": "SUBMITTED"
}

{
  "chunkingId": "441eb3be-7de6-470a-8141-e416a15c7db1",
  "status": "READY",
  "message": "<string>"
}

Headers

Authorization

string

required

Bearer token used for authentication. Format: Authorization: Bearer ACCESS_TOKEN.

Content-Type

string

The content type of the request body.

Example:

"application/json"

Path Parameters

MODEL_ID

string

required

Unique identifier for the model.

Example:

"gte-small"

CHUNKER

enum<string>

required

Chunking strategy for the way text is split.

Available options:

dynamic-newline,

dynamic-sentence,

regex-splitter,

semantic,

sentence

Body

application/json

batch

object[]

The batch of key:value pairs used in the chunking request.

Show child attributes

modelConfig

ModelConfig · object

Provides fields and values that specify ranges for tokens.

Show child attributes

useCaseConfig

UseCaseConfigChunking · object

Show child attributes

chunkerConfig

object

This parameter contains fields that determine how the text is split and the size of text chunks created.

IMPORTANT: Specific information is detailed in each use case.

Response

This is the response to the POST chunking request submitted for a specific chunker and modelId.

chunkingId

string<uuid>

The universal unique identifier (UUID) returned in the POST request. This UUID is required in the GET request to retrieve results.

Example:

"441eb3be-7de6-470a-8141-e416a15c7db1"

status

string

The current status of the request. Allowed values are:

SUBMITTED - The POST request was successful and the response has returned the chunkingId and status that is used by the GET request.
ERROR - An error was generated when the GET request was sent.
READY - The results associated with the chunkingId are available and ready to be retrieved.
RETRIEVED - The results associated with the chunkingId are returned successfully when the GET request was sent.

Example:

"SUBMITTED"

Authentication

Async Chunking

Async Prediction

Commerce Studio Rules & Rewrites

Platform Signals

Prediction

Prompt Preview

Tokenization

Use Case

Typeahead

Chunk text using the specified chunker

Headers

Path Parameters

Body

Response