> ## Documentation Index
> Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt
> Use this file to discover all available pages before exploring further.

# XML Transformation

> Index pipeline stage configuration specifications

export const schema = {
  "type": "object",
  "title": "XML Transformation",
  "description": "Transforms XML content stored in a document field into one or more pipeline documents by applying XPath expressions to extract field values. Configure a root XPath to define the document boundaries and define mapping rules to extract values from each matching node into named fields. When `splitOnRoot` is enabled and the root XPath matches multiple nodes, each match becomes a separate output document.",
  "required": ["rootXPath"],
  "properties": {
    "skip": {
      "type": "boolean",
      "title": "Skip This Stage",
      "description": "Controls whether this stage executes during pipeline processing at runtime. When set to `true`, the stage is completely bypassed and documents pass through unchanged to the next stage. Useful for A/B testing, gradual rollouts, or temporarily disabling a stage without removing it from the pipeline.",
      "default": false,
      "hints": ["advanced"]
    },
    "label": {
      "type": "string",
      "title": "Label",
      "description": "Human-readable identifier displayed in the Fusion Admin UI, monitoring dashboards, and log messages. Use descriptive labels like `Parse Product PDFs` to aid debugging and team collaboration. Labels appear in performance metrics and error reports, making it easier to identify which stage failed.",
      "hints": ["advanced"],
      "maxLength": 255
    },
    "condition": {
      "type": "string",
      "title": "Condition",
      "description": "JavaScript expression evaluated at runtime on each document to conditionally execute this stage. The expression must return `true` to execute or `false` to skip. Access document fields using `doc.getFieldValue('fieldName')` and request parameters via `request.getFirstParam('paramName')`. For example, `doc.getFieldValue('type') === 'premium'` executes this stage only for premium content.",
      "hints": ["code", "code/javascript", "advanced"]
    },
    "rootXPath": {
      "type": "string",
      "title": "Root XPath",
      "description": "Specifies the XPath expression that defines the root element or elements within the XML body used to scope document extraction. Each node matching this expression is treated as the root of one extracted document. When multiple nodes match and `splitOnRoot` is enabled, each match produces a separate pipeline document."
    },
    "splitOnRoot": {
      "type": "boolean",
      "title": "New Document per Root XPath Match?",
      "description": "Controls whether each match of the root XPath expression produces a separate pipeline document. When `true`, a root XPath matching ten elements generates ten documents. When `false`, all matches are merged into a single document. Defaults to `true` for backward compatibility.",
      "default": true
    },
    "parentIdField": {
      "type": "string",
      "title": "Parent ID Field Name",
      "description": "Specifies the field name used to store the parent document's ID on each child document created by the XML split. Set this field when downstream stages or the index schema need to link child documents back to their parent. Leave empty when parent-child relationships do not need to be tracked."
    },
    "bodyField": {
      "type": "string",
      "title": "Body Field Name",
      "description": "Specifies the name of the document field that contains the XML content to be transformed by this stage. The field value must be a valid XML string. Malformed XML causes the stage to fail. Change this only when the XML content is stored in a field other than `body`.",
      "default": "body"
    },
    "outputXMLFragments": {
      "type": "boolean",
      "title": "Output XML Fragments as Strings",
      "description": "Controls whether XPath matches that select XML nodes are output as serialized XML strings or as plain text content. When `true`, the entire matched node including its tags is written as a string. When `false`, only the text content of the node is extracted. Use `true` when downstream stages need to parse or display the XML structure.",
      "default": false
    },
    "mappings": {
      "type": "array",
      "title": "XPath Mappings",
      "description": "The XPath rules to apply to extract content from the designated Body field.  Extractions are added on to the document.",
      "items": {
        "type": "object",
        "required": ["xpath", "field"],
        "properties": {
          "xpath": {
            "type": "string",
            "title": "Value Expression",
            "description": "Specifies the XPath expression or literal string used to extract the value written to the target field. A literal string is used as-is. An XPath expression can be relative to the root element or absolute from the document root. When the expression matches multiple nodes and `multivalue` is `false`, only the first match is used."
          },
          "field": {
            "type": "string",
            "title": "Field Expression",
            "description": "Specifies the XPath expression or literal string used to determine the target field name for the extracted value. A literal string names the field directly. An XPath expression dynamically resolves the field name from the XML content. When both `field` and `xpath` contain multiple XPath matches, they are paired in order."
          },
          "multivalue": {
            "type": "boolean",
            "title": "Multi Value",
            "description": "Controls whether all matches of the value expression are written to the target field or only the first match. When `true`, all matching values are added to the field as a multi-valued list. When `false` and multiple field and value expressions match, the stage pairs them as field-value tuples.",
            "default": false
          }
        }
      }
    },
    "metadata": {
      "type": "array",
      "title": "Additional Metadata",
      "description": "Defines additional static key-value pairs added as fields to every document produced by the XML transformation. Use this to attach metadata such as source system identifiers or processing timestamps that are not present in the XML content. Each entry requires a field name and a value, which may be a literal string or an XPath expression.",
      "items": {
        "type": "object",
        "required": ["field", "value"],
        "properties": {
          "field": {
            "type": "string",
            "title": "Field",
            "description": "Specifies the document field name where the metadata value is written. This is a static field name added to every document produced by the transformation. Use descriptive names that do not conflict with XPath-mapped fields."
          },
          "value": {
            "type": "string",
            "title": "Value",
            "description": "Specifies the value written to the metadata field. May be a literal string or an XPath expression. Literal strings take precedence over XPath expressions when both could apply. Use an XPath expression to extract a value from the XML content that applies to all produced documents."
          }
        }
      }
    },
    "keepParent": {
      "type": "boolean",
      "title": "Keep Parent Document",
      "description": "Controls whether the original parent document is retained in the pipeline output alongside any child documents created by the XML split. When `true`, the parent and all child documents are emitted. When `false`, only the child documents extracted from the body field are output. Disable to avoid indexing the raw parent document when it has been fully decomposed into children.",
      "default": false
    }
  },
  "category": "Document Transformation",
  "categoryPriority": 9,
  "unsafe": false
};

export const SchemaParamFields = ({schema}) => {
  const sanitize = str => {
    if (typeof str !== "string") return str;
    return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'");
  };
  const renderMd = str => {
    const s = sanitize(str);
    const text = (/[.!?]\)*$/).test(s) ? s : `${s}.`;
    return text.split(/(\*\*[^*]+\*\*|_[^_]+_|`[^`]+`)/g).map((part, i) => {
      if (part.startsWith("**")) return <strong key={i}>{part.slice(2, -2)}</strong>;
      if (part.startsWith("_")) return <em key={i}>{part.slice(1, -1)}</em>;
      if (part.startsWith("`")) return <code key={i}>{part.slice(1, -1)}</code>;
      return part;
    });
  };
  const {description, properties = {}, required: requiredProps = []} = schema;
  const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]);
  const renderProp = ([name, prop]) => {
    const isRequired = requiredProps.includes(name);
    const hasDefault = prop.default !== undefined;
    const rawDefault = prop.default;
    const hints = prop.hints || [];
    const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"')));
    const postBadges = [];
    if (prop.title) {
      postBadges.push(<><span className="text-stone-400 dark:text-stone-500">API property: </span>{name}</>);
    }
    const constraints = [];
    if (prop.minimum !== undefined && prop.maximum !== undefined) {
      constraints.push(`Range: ${prop.minimum} – ${prop.maximum}`);
    } else if (prop.minimum !== undefined) {
      constraints.push(`Min: ${prop.minimum}`);
    } else if (prop.maximum !== undefined) {
      constraints.push(`Max: ${prop.maximum}`);
    }
    if (prop.minLength !== undefined && prop.maxLength !== undefined) {
      constraints.push(`Length: ${prop.minLength} – ${prop.maxLength}`);
    } else if (prop.minLength !== undefined) {
      constraints.push(`Min length: ${prop.minLength}`);
    } else if (prop.maxLength !== undefined) {
      constraints.push(`Max length: ${prop.maxLength}`);
    }
    const fieldProps = {
      key: name,
      body: prop.title || name,
      type: prop.type,
      ...postBadges.length > 0 && ({
        post: postBadges
      }),
      ...isRequired && ({
        required: true
      }),
      ...!isComplexDefault && hasDefault ? {
        default: sanitize(String(rawDefault))
      } : {}
    };
    const isObject = prop.type === "object" && prop.properties;
    const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties;
    return <ParamField {...fieldProps}>
        {prop.description && <p>{renderMd(prop.description)}</p>}

        {prop.enum && <p>
            Allowed values: 
            {prop.enum.map((v, i) => <>{i > 0 && ", "}<code key={i}>{String(v)}</code></>)}
          </p>}

        {constraints.length > 0 && <p className="text-stone-500 dark:text-stone-400 text-sm">
            {constraints.join(" · ")}
          </p>}

        {isComplexDefault && <div className="flex">
            <p>
              <strong>Default:</strong>
            </p>
            <pre className="!my-0">
              <code>
                {JSON.stringify(rawDefault, null, 2)}
              </code>
            </pre>
          </div>}

        {isArrayOfObjects && <Expandable title="item properties">
            <SchemaParamFields schema={{
      properties: prop.items.properties,
      required: prop.items.required
    }} />
          </Expandable>}

        {isObject && <Expandable title="properties">
            <SchemaParamFields schema={{
      properties: prop.properties,
      required: prop.required
    }} />
          </Expandable>}
      </ParamField>;
  };
  return <div>
      {description && <p>{renderMd(description)}</p>}

      {visibleProps.map(renderProp)}
    </div>;
};

export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => {
  const [isLoaded, setIsLoaded] = useState(false);
  useEffect(() => {
    const timer = setTimeout(() => {
      setIsLoaded(true);
    }, 500);
    return () => clearTimeout(timer);
  }, []);
  return <div className="lw-template-container">
      <Card title={title} icon={icon}>
        {isLoaded && <span dangerouslySetInnerHTML={{
    __html: `<lw-template id="a029c1a9-28be-427e-b0e1-5d918920246a"></lw-template
            >`
  }} />}
        <Link href={linkHref} className="agent-studio-link text-left text-gray-600 gap-2 dark:text-gray-400 text-sm font-medium flex flex-row items-center hover:text-primary dark:hover:text-primary-light group-hover:text-primary group-hover:dark:text-primary-light">Powered by Lucidworks Agent Studio</Link>
      </Card>
    </div>;
};

[localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/index-pipeline-stages/xml-transformation

[mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/index-pipeline-stages/xml-transformation

[old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/kixq4z

The XML Transformation stage (previously called the XML Transform Stage) allows you to process an XML document into one or more Solr documents and to specify mappings between elements and document fields.
A common use case for an XML Transformation stage in a pipeline is when the XML document is a container-like document which contains a set of inner elements, each of which should be treated as a separate document.
A parent ID field can be used to relate these multiple documents back to the containing document.

<LwTemplate />

## Pipeline Configuration

The default XML processing provided by the [Apache Tika parser](/docs/lucidworks-search/09-developer-documentation/config-specs/parsers/apache-tika-parser) extracts all text from an XML into a single document field called `content`.
This not only flattens the document contents, it loses all information about the containing elements in the document.
To process XML documents using an XML Transformation stage, the index pipeline must have as its initial processing stage an Apache Tika parser which is configured to pass the document through to the XML Transformation stage *as raw XML*, via the following configuration:

* UI checkbox "Add original document content" **unchecked** / REST API property "addOriginalContent" set to **false**
* UI checkbox "Return parsed content as XML or HTML" **checked** / REST API property "keepOriginalStructure" set to **true**
* UI checkbox "Return original XML and HTML instead of Tika XML output" **checked** / REST API property "returnXml" set to **true**

With this configuration, the Apache Tika parser decodes the raw input stream of bytes into a string containing the entire XML document which is returned in the PipelineDocument field `body`.

The pipeline must have a Field Mapping stage after the XML Transformation stage, before the Solr Indexer stage. The Field Mapping stage is used to remove the following fields from the document:

* *raw-content*
* Content-Type
* Content-Length
* parsing
* parsing\_time

## XML Transforms

The XML Transformation stage uses a Solr
[XPathRecordReader](http://lucene.apache.org/solr/6_1_0/solr-dataimporthandler/org/apache/solr/handler/dataimport/XPathRecordReader.html)
which is a streaming XML parser that supports *only a limited subset of XPath selectors*.
It provides exact matching on element attributes and it can only extract the element text, not attribute values.

Examples of allowed XPath specifications where "a", "b", "c" are any element tags, likewise "attrName" is any attribute name:

```java theme={"dark"}
/a/b/c
/a/b/c[@attrName='someValue']
/a/b/c[@attrName=]/d
/a/b/c/@attrName
//b//...
```

<Note>
  When specifying the list of `mappings`, for each mapping, the specification for the `xpath` attribute must include the full path, i.e., the `xpath` attribute will include the `rootXPath`. See the example configuration below.
</Note>

## Example Stage Specification

*Definition of an XML-Transformation stage that extracts elements from a MEDLINE/Pubmed article abstract:*

```json wrap  expandable  theme={"dark"}
{ "type" : "xml-transform",
  "id" : "n0j2a9k9",
  "rootXPath" : "/MedlineCitationSet/MedlineCitation",
  "bodyField" : "body",
  "mappings" : [ {
      "xpath" : "/MedlineCitationSet/MedlineCitation/Article/ArticleTitle",
      "field" : "article-title_txt",
      "multivalue" : false
  }, {
      "xpath" : "/MedlineCitationSet/MedlineCitation/Article/Abstract/AbstractText",
      "field" : "article-abstract_txt",
      "multivalue" : true
  }, {
      "xpath" : "/MedlineCitationSet/MedlineCitation/MeshHeadingList/MeshHeading/DescriptorName",
      "field" : "mesh-heading_txt",
      "multivalue" : true
  }, {
      "xpath" : "/MedlineCitationSet/MedlineCitation/PMID",
      "field" : "pmid_txt",
      "multivalue" : false
  } ],
  "keepParent" : false,
  "skip" : false,
  "label" : "medline_xml_transform",
}
```

*Template for a minimal index pipeline that includes an XML-Transformation stage. Replace the XPath and field names in the XML-Transformation stage according to your data.*

```json wrap  theme={"dark"}
{
    "id" : "xml-pipeline-default",
    "stages" : [ {
    "type" : "tika-parser",
    "includeImages" : false,
    "flattenCompound" : false,
    "addFailedDocs" : false,
    "addOriginalContent" : false,
    "contentField" : "_raw_content_",
    "returnXml" : true,
    "keepOriginalStructure" : true,
    "extractHtmlLinks" : false,
    "extractOtherLinks" : false,
    "csvParsing" : false,
    "skip" : false,
    "label" : "tika",
    "sourceField" : "_raw_content_"
    }, {
    "type" : "xml-transformation",
    "rootXPath" : "/ROOTS/ROOT",
    "bodyField" : "body",
    "mappings" : [ {
        "xpath" : "/ROOTS/ROOT/element",
        "field" : "element-field_t",
        "multivalue" : false
    } ],
    "keepParent" : false,
    "skip" : false,
    "label" : "xml"
    }, {
    "type" : "field-mapping",
    "mappings" : [ {
        "source" : "parsing",
        "operation" : "delete"
    }, {
        "source" : "parsing_time",
        "operation" : "delete"
    }, {
        "source" : "Content-Type",
        "operation" : "delete"
    }, {
        "source" : "Content-Length",
        "operation" : "delete"
    } ],
    "skip" : false,
    "label" : "field mapping"
    }, {
    "type" : "solr-index",
    "enforceSchema" : true,
    "bufferDocsForSolr" : false,
    "skip" : false,
    "label" : "solr-index"
    } ]
}
```

## Configuration

<Tip>
  When entering configuration values in the UI, use *unescaped* characters, such as `\t` for the tab character. When entering configuration values in the API, use *escaped* characters, such as `\\t` for the tab character.
</Tip>

<SchemaParamFields schema={schema} />