> ## Documentation Index
> Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Text

> Parser stage configuration specifications

export const schema = {
  "type": "object",
  "title": "Text",
  "description": "Parses plain text content with configurable character encoding detection, optional line splitting, whitespace trimming, and comment processing. Suitable for log files, configuration files, and any text-based content. Processes the file as a single document or splits it into per-line documents depending on configuration.",
  "required": ["charset", "ignoreBOM", "type"],
  "properties": {
    "id": {
      "type": "string",
      "title": "Parser ID",
      "default": "d0522cb8-92d3-47cb-97c3-b30e51376273"
    },
    "label": {
      "type": "string",
      "title": "Label",
      "description": "Human-readable identifier displayed in the Fusion Admin UI, monitoring dashboards, and log messages. Use descriptive labels like `Parse Product PDFs` to aid debugging and team collaboration. Labels appear in performance metrics and error reports, making it easier to identify which stage failed.",
      "maxLength": 255
    },
    "enabled": {
      "type": "boolean",
      "title": "Enable this Parser Stage",
      "default": true,
      "description": "Controls whether this parser stage is active and available for use. When `false`, the stage is completely inactive regardless of other settings. When `true`, the stage runs according to its other configuration options."
    },
    "mediaTypes": {
      "type": "array",
      "title": "Media Types to match",
      "description": "Specifies the media types this parser stage handles. Documents with a matching media type are routed to this stage for parsing. See `inheritMediaTypes` to combine this list with the stage's built-in defaults.",
      "items": {
        "type": "string",
        "pattern": "^[^\\/]+\\/[^\\/]+$",
        "format": "rfc2646"
      }
    },
    "inheritMediaTypes": {
      "type": "boolean",
      "title": "Match default media types in this Parser Stage",
      "description": "Controls whether this stage combines its built-in default media types with those in `mediaTypes`. When `true`, both lists are merged. When `false`, only the `mediaTypes` list is used and must contain at least one entry. Set to `false` to override the default media types entirely.",
      "default": true
    },
    "ignoredMediaTypes": {
      "type": "array",
      "title": "Media Types to ignore",
      "description": "Specifies media types this parser stage excludes from processing. Documents matching an ignored media type are skipped even if they match `mediaTypes`. Use this to carve out exceptions from a broadly matched media type set.",
      "items": {
        "type": "string",
        "pattern": "^[^\\/]+\\/[^\\/]+$",
        "format": "rfc2646"
      }
    },
    "pathPatterns": {
      "type": "array",
      "title": "File names to parse",
      "description": "Restricts this parser stage to files whose names match the specified pattern. Use forward slashes (`/`) to join archive names with entry names when matching files inside archives. If no pattern is specified, the stage applies to all matching media types.",
      "items": {
        "type": "object",
        "properties": {
          "syntax": {
            "type": "string",
            "title": "Pattern type",
            "description": "glob uses bash shell-style wildcards and regex uses Java (PCRE-style) regex.",
            "enum": ["glob", "regex"],
            "default": "glob"
          },
          "pattern": {
            "type": "string",
            "title": "File name or pattern",
            "description": "glob examples are \"z.txt\" or \"*.md\" or \"/a/*/b/f.txt\". regex examples are \"z.txt$\" or \".*\\.txt$\" or \"^/a/[^\\/]*/b/f.txt$\"."
          }
        }
      }
    },
    "errorHandling": {
      "type": "string",
      "title": "Error Handling",
      "enum": ["ignore", "log", "fail", "mark"],
      "default": "mark"
    },
    "outputFieldPrefix": {
      "type": "string",
      "title": "Prefix parsed fields with",
      "description": "Sets a string prefix applied to all fields extracted by this parser, useful for namespacing or avoiding field name collisions. For example, `tika_` produces fields like `tika_title` and `tika_author`. Leave empty to apply no prefix.",
      "maxLength": 20,
      "pattern": "^$|^[A-Za-z_][A-Za-z0-9_\\-\\.]+$"
    },
    "charset": {
      "type": "string",
      "title": "Character Set",
      "description": "Specifies the character encoding used to read file content. Common values include `UTF-8`, `ISO-8859-1`, and `Windows-1252`. If incorrect, text may appear garbled or cause parsing errors.",
      "default": "detect"
    },
    "ignoreBOM": {
      "type": "boolean",
      "title": "Ignore BOM",
      "description": "Controls whether the parser ignores the Byte-Order Mark (BOM) at the start of the file. When `true`, BOM is always ignored and the configured charset is used. When `false`, a valid BOM overrides the configured charset.",
      "default": false
    },
    "splitLines": {
      "type": "boolean",
      "title": "Split lines",
      "description": "Controls whether each line in the text file is created as a separate indexed document. When `true`, each line becomes an individual document. When `false`, the entire file content is stored in a single document's output field.",
      "default": false
    },
    "skipHeaderLines": {
      "type": "integer",
      "title": "Skip header lines",
      "description": "Sets the number of lines to skip from the start of the file before processing content. Use this to ignore file headers, metadata, or boilerplate that should not be indexed. Only applies when `splitLines` is enabled.",
      "default": 0
    },
    "trimWhitespace": {
      "type": "boolean",
      "title": "Trim whitespace",
      "description": "Controls whether leading and trailing whitespace is removed from each line before indexing. When `true`, lines like `  text  ` become `text`. When `false`, whitespace is preserved as-is in the indexed content.",
      "default": false
    },
    "skipEmptyLines": {
      "type": "boolean",
      "title": "Skip empty lines",
      "description": "Controls whether blank lines are skipped rather than processed into documents. When `true`, lines containing only whitespace are ignored. When `false`, empty lines are processed and may generate documents with empty content.",
      "default": false
    },
    "outputField": {
      "type": "string",
      "title": "Output field",
      "description": "Sets the name of the field where extracted text content is stored in the indexed document. For example, the default `body` field stores the full text from the parsed file. Change this to direct content into a different field name.",
      "default": "body",
      "minLength": 1
    },
    "maxLength": {
      "type": "integer",
      "title": "Maximum length",
      "description": "Sets the maximum number of characters read from the text file into the output field. Content beyond this limit is truncated. Set to `-1` to read the entire file regardless of size.",
      "default": 1048576,
      "maximum": 2147483647,
      "exclusiveMaximum": false,
      "minimum": 0,
      "exclusiveMinimum": false
    },
    "maxLineLength": {
      "type": "integer",
      "title": "Maximum line length",
      "description": "Sets the maximum number of characters allowed in any individual line when `splitLines` is enabled. Lines exceeding this limit trigger an error or truncation based on error handling settings. Protects against malformed files with extremely long lines.",
      "default": 1048576,
      "maximum": 2147483647,
      "exclusiveMaximum": false,
      "minimum": 0,
      "exclusiveMinimum": false
    },
    "commentField": {
      "type": "string",
      "title": "Comment field",
      "description": "Sets the name of the field where comment lines are stored when `commentHandling` is set to `as_field`. For example, the default `comment` field stores extracted comment text separately from the main content. Used when preserving inline metadata or annotations from comment lines.",
      "default": "comment",
      "minLength": 1
    },
    "comment": {
      "type": "string",
      "title": "Comment character",
      "description": "Character sequence that marks the beginning of a comment line in the text file. Lines starting with these characters are treated according to the commentHandling setting. Common alternatives include \"//\", \";\", or \"--\".",
      "default": "#",
      "minLength": 1
    },
    "commentHandling": {
      "type": "string",
      "title": "Comment Handling",
      "description": "Determines how lines identified by the comment character are processed. Use `include` to preserve comments in the text content, `ignore` to skip comment lines entirely, or `as_field` to extract comments to `commentField` and remove them from main content. Choose `as_field` to preserve annotations separately or `ignore` to clean the content.",
      "enum": ["ignore", "include", "as_field"],
      "default": "include"
    },
    "type": {
      "type": "string",
      "enum": ["text"],
      "default": "text"
    }
  },
  "additionalProperties": false,
  "category": "Other",
  "categoryPriority": 1,
  "unsafe": false
};

export const SchemaParamFields = ({schema}) => {
  const sanitize = str => {
    if (typeof str !== "string") return str;
    return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'");
  };
  const renderMd = str => {
    const s = sanitize(str);
    const text = (/[.!?]\)*$/).test(s) ? s : `${s}.`;
    return text.split(/(\*\*[^*]+\*\*|_[^_]+_|`[^`]+`)/g).map((part, i) => {
      if (part.startsWith("**")) return <strong key={i}>{part.slice(2, -2)}</strong>;
      if (part.startsWith("_")) return <em key={i}>{part.slice(1, -1)}</em>;
      if (part.startsWith("`")) return <code key={i}>{part.slice(1, -1)}</code>;
      return part;
    });
  };
  const {description, properties = {}, required: requiredProps = []} = schema;
  const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]);
  const renderProp = ([name, prop]) => {
    const isRequired = requiredProps.includes(name);
    const hasDefault = prop.default !== undefined;
    const rawDefault = prop.default;
    const hints = prop.hints || [];
    const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"')));
    const postBadges = [];
    if (prop.title) {
      postBadges.push(<><span className="text-stone-400 dark:text-stone-500">API property: </span>{name}</>);
    }
    const constraints = [];
    if (prop.minimum !== undefined && prop.maximum !== undefined) {
      constraints.push(`Range: ${prop.minimum} – ${prop.maximum}`);
    } else if (prop.minimum !== undefined) {
      constraints.push(`Min: ${prop.minimum}`);
    } else if (prop.maximum !== undefined) {
      constraints.push(`Max: ${prop.maximum}`);
    }
    if (prop.minLength !== undefined && prop.maxLength !== undefined) {
      constraints.push(`Length: ${prop.minLength} – ${prop.maxLength}`);
    } else if (prop.minLength !== undefined) {
      constraints.push(`Min length: ${prop.minLength}`);
    } else if (prop.maxLength !== undefined) {
      constraints.push(`Max length: ${prop.maxLength}`);
    }
    const fieldProps = {
      key: name,
      body: prop.title || name,
      type: prop.type,
      ...postBadges.length > 0 && ({
        post: postBadges
      }),
      ...isRequired && ({
        required: true
      }),
      ...!isComplexDefault && hasDefault ? {
        default: sanitize(String(rawDefault))
      } : {}
    };
    const isObject = prop.type === "object" && prop.properties;
    const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties;
    return <ParamField {...fieldProps}>
        {prop.description && <p>{renderMd(prop.description)}</p>}

        {prop.enum && <p>
            Allowed values: 
            {prop.enum.map((v, i) => <>{i > 0 && ", "}<code key={i}>{String(v)}</code></>)}
          </p>}

        {constraints.length > 0 && <p className="text-stone-500 dark:text-stone-400 text-sm">
            {constraints.join(" · ")}
          </p>}

        {isComplexDefault && <div className="flex">
            <p>
              <strong>Default:</strong>
            </p>
            <pre className="!my-0">
              <code>
                {JSON.stringify(rawDefault, null, 2)}
              </code>
            </pre>
          </div>}

        {isArrayOfObjects && <Expandable title="item properties">
            <SchemaParamFields schema={{
      properties: prop.items.properties,
      required: prop.items.required
    }} />
          </Expandable>}

        {isObject && <Expandable title="properties">
            <SchemaParamFields schema={{
      properties: prop.properties,
      required: prop.required
    }} />
          </Expandable>}
      </ParamField>;
  };
  return <div>
      {description && <p>{renderMd(description)}</p>}

      {visibleProps.map(renderProp)}
    </div>;
};

export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => {
  const [isLoaded, setIsLoaded] = useState(false);
  useEffect(() => {
    const timer = setTimeout(() => {
      setIsLoaded(true);
    }, 500);
    return () => clearTimeout(timer);
  }, []);
  return <div className="lw-template-container">
      <Card title={title} icon={icon}>
        {isLoaded && <span dangerouslySetInnerHTML={{
    __html: `<lw-template id="a029c1a9-28be-427e-b0e1-5d918920246a"></lw-template
            >`
  }} />}
        <Link href={linkHref} className="agent-studio-link text-left text-gray-600 gap-2 dark:text-gray-400 text-sm font-medium flex flex-row items-center hover:text-primary dark:hover:text-primary-light group-hover:text-primary group-hover:dark:text-primary-light">Powered by Lucidworks Agent Studio</Link>
      </Card>
    </div>;
};

[localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/parsers/text-parser

[mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/parsers/text-parser

[old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/70dpnu

The Plain Text parser can split a text file by lines or consume it into a single document.

<LwTemplate />

Options for treatment of this filetype include:

* Plain Text Parser Fields
* Number of header rows to skip
* Split on line end or not
* Comment character
* Skip empty lines
* Charset

<Tip>
  When entering configuration values in the UI, use *unescaped* characters, such as `\t` for the tab character. When entering configuration values in the API, use *escaped* characters, such as `\\t` for the tab character.
</Tip>

<SchemaParamFields schema={schema} />