> ## Documentation Index > Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt > Use this file to discover all available pages before exploring further. # LWAI Chunker Stage > Index pipeline stage configuration specifications export const schema = { "type": "object", "title": "LWAI Chunker Stage", "description": "Splits large text into smaller semantic chunks and generates vector embeddings for each chunk using a Lucidworks AI model. Reads the input text from a pipeline context variable and writes the resulting chunks and vectors to configurable destination fields. Use this stage to prepare long-form content for vector search.", "required": ["accountName", "chunkingStrategy", "modelName", "inputContextVariable", "outputContextVariable"], "properties": { "skip": { "type": "boolean", "title": "Skip This Stage", "description": "Controls whether this stage executes during pipeline processing at runtime. When set to `true`, the stage is completely bypassed and documents pass through unchanged to the next stage. Useful for A/B testing, gradual rollouts, or temporarily disabling a stage without removing it from the pipeline.", "default": false, "hints": ["advanced"] }, "label": { "type": "string", "title": "Label", "description": "Human-readable identifier displayed in the Fusion Admin UI, monitoring dashboards, and log messages. Use descriptive labels like `Parse Product PDFs` to aid debugging and team collaboration. Labels appear in performance metrics and error reports, making it easier to identify which stage failed.", "hints": ["advanced"], "maxLength": 255 }, "condition": { "type": "string", "title": "Condition", "description": "JavaScript expression evaluated at runtime on each document to conditionally execute this stage. The expression must return `true` to execute or `false` to skip. Access document fields using `doc.getFieldValue('fieldName')` and request parameters via `request.getFirstParam('paramName')`. For example, `doc.getFieldValue('type') === 'premium'` executes this stage only for premium content.", "hints": ["code", "code/javascript", "advanced"] }, "accountName": { "type": "string", "title": "Account Name", "description": "Specifies the Lucidworks AI account name registered in the Lucidworks AI Gateway Service. This value must match an existing account. An invalid name causes all chunking calls to fail. Use the account selector to choose from configured accounts.", "hints": ["enumUrl:/api/query-stages/lwai-accounts"] }, "chunkingStrategy": { "type": "string", "title": "Chunking Strategy", "description": "Specifies the text chunking algorithm used to split the input into segments, such as sentence-based or token-count-based strategies. The available strategies depend on the configured Lucidworks AI account. Choose a strategy appropriate for the document type and downstream retrieval model.", "hints": ["enumUrl:/api/query-stages/lwai-chunking-strategies?account=${accountName}"] }, "modelName": { "type": "string", "title": "Model for Vectorization", "description": "Specifies the Lucidworks AI embedding model used to vectorize each text chunk after splitting. The model must be registered under the selected account with an `embedding` use case. An incompatible model type causes vectorization errors.", "hints": ["enumUrl:/api/query-stages/lwai-model?account=${accountName}&useCase=embedding"] }, "inputContextVariable": { "type": "string", "title": "Input context variable", "description": "Specifies the name of the pipeline context variable whose value contains the large text to be chunked. Supports template expressions such as `` for dynamic resolution. The variable must be populated by an upstream stage before this stage executes." }, "outputContextVariable": { "type": "string", "title": "Destination Field Name & Context Output", "description": "Specifies the destination field name and context variable for the chunk vector output. Must contain `*_chunk_vector_*` in the name and target a dense vector field type in the Solr schema. The value populates both the document field and the pipeline context under this key." }, "outputTextSpans": { "type": "string", "title": "Destination Field Name for Text Spans", "description": "Specifies the document field where the character span positions `[start, stop]` for each text chunk are stored, for example `body_spans_ss`. Use this field to map chunks back to their source positions in the original document. Leave empty to skip span output." }, "outputTextChunks": { "type": "string", "title": "Destination Field Name for Text Chunks (not the vectors)", "description": "Specifies the document field where the raw text of each chunk is stored, for example `body_chunks_ss`. This field contains the human-readable text segments before vectorization. Leave empty to skip plain-text chunk output." }, "chunkerConfig": { "type": "array", "title": "Chunker Configuration", "description": "Sends additional key-value parameters to the Lucidworks AI chunker alongside the standard configuration. Use this to pass chunker-specific options such as maximum chunk size or overlap. Each entry requires a `key` and an optional `value`.", "minItems": 0, "items": { "type": "object", "required": ["key"], "properties": { "key": { "type": "string", "title": "Parameter Name" }, "value": { "type": "string", "title": "Parameter Value" } } } }, "modelConfig": { "type": "array", "title": "Model Configuration", "description": "Sends additional key-value configuration parameters to the Lucidworks AI embedding model alongside the vectorization request. Use this to supply model-specific options not covered by the standard configuration. Each entry requires a `key` and an optional `value`.", "minItems": 0, "items": { "type": "object", "required": ["key"], "properties": { "key": { "type": "string", "title": "Parameter Name" }, "value": { "type": "string", "title": "Parameter Value" } } } }, "maxTries": { "type": "integer", "title": "Maximum Asynchronous Call Tries", "description": "Sets the maximum number of attempts for each asynchronous Lucidworks AI API call. Increase this value to add retry resilience for transient network or service errors. Each retry is attempted immediately after a failure with no delay.", "default": 1, "minimum": 1, "exclusiveMinimum": false }, "failOnError": { "type": "boolean", "title": "Fail on Error", "description": "Controls whether a chunking or vectorization error causes the entire pipeline to fail. When `true`, any error throws an exception and stops the document. When `false`, errors are logged and the document continues. Set to `true` in production to surface integration issues immediately.", "default": false } }, "category": "AI", "categoryPriority": 10, "unsafe": false }; export const SchemaParamFields = ({schema}) => { const sanitize = str => { if (typeof str !== "string") return str; return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'"); }; const renderMd = str => { const s = sanitize(str); const text = (/[.!?]\)*$/).test(s) ? s : `${s}.`; return text.split(/(\*\*[^*]+\*\*|_[^_]+_|`[^`]+`)/g).map((part, i) => { if (part.startsWith("**")) return {part.slice(2, -2)}; if (part.startsWith("_")) return {part.slice(1, -1)}; if (part.startsWith("`")) return {part.slice(1, -1)}; return part; }); }; const {description, properties = {}, required: requiredProps = []} = schema; const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]); const renderProp = ([name, prop]) => { const isRequired = requiredProps.includes(name); const hasDefault = prop.default !== undefined; const rawDefault = prop.default; const hints = prop.hints || []; const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"'))); const postBadges = []; if (prop.title) { postBadges.push(<>API property: {name}); } const constraints = []; if (prop.minimum !== undefined && prop.maximum !== undefined) { constraints.push(`Range: ${prop.minimum} – ${prop.maximum}`); } else if (prop.minimum !== undefined) { constraints.push(`Min: ${prop.minimum}`); } else if (prop.maximum !== undefined) { constraints.push(`Max: ${prop.maximum}`); } if (prop.minLength !== undefined && prop.maxLength !== undefined) { constraints.push(`Length: ${prop.minLength} – ${prop.maxLength}`); } else if (prop.minLength !== undefined) { constraints.push(`Min length: ${prop.minLength}`); } else if (prop.maxLength !== undefined) { constraints.push(`Max length: ${prop.maxLength}`); } const fieldProps = { key: name, body: prop.title || name, type: prop.type, ...postBadges.length > 0 && ({ post: postBadges }), ...isRequired && ({ required: true }), ...!isComplexDefault && hasDefault ? { default: sanitize(String(rawDefault)) } : {} }; const isObject = prop.type === "object" && prop.properties; const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties; return {prop.description &&

{renderMd(prop.description)}

} {prop.enum &&

Allowed values: {prop.enum.map((v, i) => <>{i > 0 && ", "}{String(v)})}

} {constraints.length > 0 &&

{constraints.join(" · ")}

} {isComplexDefault &&

Default: 

              
                {JSON.stringify(rawDefault, null, 2)}
} {isArrayOfObjects && } {isObject && } ; }; return
{description &&

{renderMd(description)}

} {visibleProps.map(renderProp)}
; }; export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => { const [isLoaded, setIsLoaded] = useState(false); useEffect(() => { const timer = setTimeout(() => { setIsLoaded(true); }, 500); return () => clearTimeout(timer); }, []); return
{isLoaded && ` }} />} Powered by Lucidworks Agent Studio
; }; [localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/index-pipeline-stages/lwai-chunker-stage [mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/index-pipeline-stages/lwai-chunker-stage [old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/uiycyd Lucidworks Search 5.9.12 and later integrates with Lucidworks AI to perform [chunking](/docs/lucidworks-search/11-vector-search/chunking). When you include chunking in your index pipeline, Lucidworks AI automatically splits large documents into smaller, more focused segments. This approach is especially powerful when paired with Neural Hybrid Search to surface the most relevant chunks instead of entire documents. Chunking also improves the accuracy of AI assistants by delivering semantically rich training data in precise, context-aware pieces. This stage performs chunking [asynchronously](/docs/lucidworks-search/09-developer-documentation/config-specs/index-pipeline-stages/overview#run-stages-asynchronously) and stores those vectors in Solr. Click your use case below to see examples of how chunking can enhance the search experience: * Break product descriptions into focused chunks so customers can find relevant details faster. * Reduce support tickets by training AI assistants on semantically-segmented help articles for more accurate answers. * Split multimedia transcripts (like product videos or webinars) into meaningful chunks so customers can find answers in content they wouldn't normally read. * Break down long technical specs so buyers can validate requirements without reading full documents. * Improve Request For Quote (RFQ) matching by extracting key details so sales teams can respond faster and more accurately. * Surface relevant regulatory or compliance info within dense documents for more efficient legal reviews. * Chunk large case studies, policies, or contracts into semantically useful segments so employees get faster answers. * Boost AI assistant quality by providing smaller, context-rich units of information so responses are more precise. * Improve onboarding by connecting new hires with task-relevant excerpts from training content, runbooks, and policy documents. This feature is available starting in Lucidworks Search 5.9.12 and in all subsequent Lucidworks Search 5.9 releases. ## Prerequisites To use this stage, non-admin Fusion users must be granted the `PUT,POST,GET:/LWAI-ACCOUNT-NAME/**` permission in Fusion, which is the Lucidworks AI API Account Name defined in [Lucidworks AI Gateway](/docs/lw-platform/lw-ai/lw-ai-gateway) when this stage is configured. Click **Get Started** below to see how to enable chunking in Fusion: