> ## Documentation Index > Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt > Use this file to discover all available pages before exploring further. # CSV > Parser stage configuration specifications export const schema = { "type": "object", "title": "CSV", "description": "Parses CSV (Comma-Separated Values) content, converting tabular data into indexed documents where each row typically becomes a separate document. Supports custom delimiters, quote characters, header detection, and automatic format detection. Column headers map to field names in the indexed documents.", "required": ["charset", "ignoreBOM", "type"], "properties": { "id": { "type": "string", "title": "Parser ID", "default": "2150fab3-495b-4582-b8ad-5985cca3aea5" }, "label": { "type": "string", "title": "Label", "description": "Human-readable identifier displayed in the Fusion Admin UI, monitoring dashboards, and log messages. Use descriptive labels like `Parse Product PDFs` to aid debugging and team collaboration. Labels appear in performance metrics and error reports, making it easier to identify which stage failed.", "maxLength": 255 }, "enabled": { "type": "boolean", "title": "Enable this Parser Stage", "default": true, "description": "Controls whether this parser stage is active and available for use. When `false`, the stage is completely inactive regardless of other settings. When `true`, the stage runs according to its other configuration options." }, "mediaTypes": { "type": "array", "title": "Media Types to match", "description": "Specifies the media types this parser stage handles. Documents with a matching media type are routed to this stage for parsing. See `inheritMediaTypes` to combine this list with the stage's built-in defaults.", "items": { "type": "string", "pattern": "^[^\\/]+\\/[^\\/]+$", "format": "rfc2646" } }, "inheritMediaTypes": { "type": "boolean", "title": "Match default media types in this Parser Stage", "description": "Controls whether this stage combines its built-in default media types with those in `mediaTypes`. When `true`, both lists are merged. When `false`, only the `mediaTypes` list is used and must contain at least one entry. Set to `false` to override the default media types entirely.", "default": true }, "ignoredMediaTypes": { "type": "array", "title": "Media Types to ignore", "description": "Specifies media types this parser stage excludes from processing. Documents matching an ignored media type are skipped even if they match `mediaTypes`. Use this to carve out exceptions from a broadly matched media type set.", "items": { "type": "string", "pattern": "^[^\\/]+\\/[^\\/]+$", "format": "rfc2646" } }, "pathPatterns": { "type": "array", "title": "File names to parse", "description": "Restricts this parser stage to files whose names match the specified pattern. Use forward slashes (`/`) to join archive names with entry names when matching files inside archives. If no pattern is specified, the stage applies to all matching media types.", "items": { "type": "object", "properties": { "syntax": { "type": "string", "title": "Pattern type", "description": "glob uses bash shell-style wildcards and regex uses Java (PCRE-style) regex.", "enum": ["glob", "regex"], "default": "glob" }, "pattern": { "type": "string", "title": "File name or pattern", "description": "glob examples are \"z.txt\" or \"*.md\" or \"/a/*/b/f.txt\". regex examples are \"z.txt$\" or \".*\\.txt$\" or \"^/a/[^\\/]*/b/f.txt$\"." } } } }, "errorHandling": { "type": "string", "title": "Error Handling", "enum": ["ignore", "log", "fail", "mark"], "default": "mark" }, "outputFieldPrefix": { "type": "string", "title": "Prefix parsed fields with", "description": "Sets a string prefix applied to all fields extracted by this parser, useful for namespacing or avoiding field name collisions. For example, `tika_` produces fields like `tika_title` and `tika_author`. Leave empty to apply no prefix.", "maxLength": 20, "pattern": "^$|^[A-Za-z_][A-Za-z0-9_\\-\\.]+$" }, "charset": { "type": "string", "title": "Character Set", "description": "Specifies the character encoding used to read file content. Common values include `UTF-8`, `ISO-8859-1`, and `Windows-1252`. If incorrect, text may appear garbled or cause parsing errors.", "default": "detect" }, "ignoreBOM": { "type": "boolean", "title": "Ignore BOM", "description": "Controls whether the parser ignores the Byte-Order Mark (BOM) at the start of the file. When `true`, BOM is always ignored and the configured charset is used. When `false`, a valid BOM overrides the configured charset.", "default": false }, "delimiter": { "type": "string", "title": "Delimiter", "description": "Character or string that separates field values in each row. Commonly used values: comma `,` (default for CSV), pipe `|`, semicolon `;`, tab `\\t`, or any custom string. If not specified, auto-detection will attempt to identify the delimiter from the file content.", "minLength": 1 }, "quote": { "type": "string", "title": "Quote", "description": "Character used to enclose field values that contain delimiters, line breaks, or other special characters. When auto-detection is disabled, defaults to double quote (\"). Common alternatives include single quote (').", "maxLength": 1 }, "quoteEscape": { "type": "string", "title": "Quote escape", "description": "Character used to escape quote characters within quoted fields. For example, to include a quote in a quoted field, it's typically doubled: \"He said \"\"Hello\"\"\" becomes: He said \"Hello\". When auto-detection is disabled, defaults to double quote (\").", "maxLength": 1 }, "autoDetect": { "type": "boolean", "title": "Auto-detect CSV Format", "description": "Automatically detect CSV format settings by analyzing file content. When enabled (default), the parser attempts to identify the delimiter, quote character, quote escape character, and comment character from the data. Disabling requires manual configuration of these settings.", "default": true }, "trimWhitespace": { "type": "boolean", "title": "Trim whitespace", "description": "Remove leading and trailing whitespace (spaces, tabs) from field values. When enabled (default), values like \" text \" become \"text\". Useful for cleaning data but may alter intentional spacing.", "default": true }, "hasHeaders": { "type": "boolean", "title": "Headers in file", "description": "Interpret the first row of the CSV file as column headers, which become field names in indexed documents. When enabled (default), the first row is not indexed as data. When disabled, generic column names are generated (column_0, column_1, etc.) or the headers list must be provided.", "default": true }, "headers": { "type": "array", "title": "Header list", "description": "Explicitly specify column header names as an array of strings. When provided, these names override any headers in the file itself (if `hasHeaders` is `true`) or provide names when `hasHeaders` is `false`. Each string becomes a field name for the corresponding column position.", "items": { "type": "string" } }, "skipEmptyLines": { "type": "boolean", "title": "Skip empty lines", "description": "Ignore blank lines in the CSV file rather than creating documents for them. When enabled (default), lines containing only whitespace or no content are skipped. When disabled, empty lines are processed and may generate documents with all fields empty or null.", "default": true }, "lineSeparator": { "type": "string", "title": "Line Separator", "description": "Character or string that separates rows in the CSV file. Common values: newline `\\n` (Unix/Linux/Mac), carriage return + newline `\\r\\n` (Windows), or custom separators. If not specified, standard line breaks are detected automatically.", "minLength": 1 }, "nullValue": { "type": "string", "title": "Null value", "description": "Replacement value for null or missing field values in the CSV data. When specified, any null fields are replaced with this string before indexing. If not specified, null values remain null in the indexed documents." }, "emptyValue": { "type": "string", "title": "Empty string replacement", "description": "Replacement value for empty string field values (fields with no content) in the CSV data. When specified, empty strings are replaced with this value before indexing. If not specified, empty strings remain as empty strings." }, "includeRowNumber": { "type": "boolean", "title": "Include row number", "description": "Adds a field containing the source row number from the original CSV file to each indexed document. Facilitates debugging, error tracking, and correlating indexed documents with source data. Useful for identifying which CSV line produced a given document when troubleshooting parsing errors.", "default": true }, "comment": { "type": "string", "title": "Comment character", "description": "Character that marks the beginning of a comment line in the CSV file. Lines starting with this character are treated according to the commentHandling setting. When auto-detection is disabled, defaults to hash (#).", "maxLength": 1 }, "commentHandling": { "type": "string", "title": "Comment Handling", "description": "Determines how the parser processes comment lines identified by the comment character. Use `ignore` to skip comment lines entirely, `as_field` to add comment text as a field in the next data row's document, or `as_document` to create separate documents for comment lines. Choose `as_field` or `as_document` to preserve metadata or annotations embedded in CSV files.", "enum": ["ignore", "as_field", "as_document"], "default": "ignore" }, "maxRowLength": { "type": "integer", "title": "Maximum line length", "description": "Maximum number of characters allowed for a single row in the CSV file. Rows exceeding this limit will trigger an error or be truncated based on error handling settings. Prevents memory issues from malformed files with extremely long lines.", "default": 10485760, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": 0, "exclusiveMinimum": false }, "maxNumColumns": { "type": "integer", "title": "Maximum number of columns", "description": "Maximum number of columns (fields) allowed in any row of the CSV file. Rows with more columns than this limit will trigger an error based on error handling settings. Protects against malformed data and memory exhaustion.", "default": 1000, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": 0, "exclusiveMinimum": false }, "maxColumnChars": { "type": "integer", "title": "Maximum number or characters per column", "description": "Maximum number of characters allowed in any individual column value (field). Values exceeding this limit will trigger an error or be truncated based on error handling settings. Prevents memory issues from unexpectedly large field values.", "default": 10485760, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": 0, "exclusiveMinimum": false }, "columnHandling": { "type": "string", "title": "Column mismatch handling", "description": "Determines how the parser handles rows with column count mismatches relative to the header row. Use `error` for strict validation that fails mismatched rows, `align` to pad missing columns with `fillValue` or truncate extra columns, or `default` to process rows as-is. Choose `align` for automatic correction of minor formatting inconsistencies.", "enum": ["error", "align", "default"], "default": "default" }, "fillValue": { "type": "string", "title": "Column fill value", "description": "Value inserted into missing columns when columnHandling is set to 'align'. When a row has fewer columns than expected, this value fills the gap. Use empty string for silent padding, or a distinct value like \"MISSING\" for tracking data quality.", "default": "" }, "type": { "type": "string", "enum": ["csv"], "default": "csv" } }, "additionalProperties": false, "category": "Other", "categoryPriority": 1, "unsafe": false }; export const SchemaParamFields = ({schema}) => { const sanitize = str => { if (typeof str !== "string") return str; return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'"); }; const renderMd = str => { const s = sanitize(str); const text = (/[.!?]\)*$/).test(s) ? s : `${s}.`; return text.split(/(\*\*[^*]+\*\*|_[^_]+_|`[^`]+`)/g).map((part, i) => { if (part.startsWith("**")) return {part.slice(2, -2)}; if (part.startsWith("_")) return {part.slice(1, -1)}; if (part.startsWith("`")) return {part.slice(1, -1)}; return part; }); }; const {description, properties = {}, required: requiredProps = []} = schema; const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]); const renderProp = ([name, prop]) => { const isRequired = requiredProps.includes(name); const hasDefault = prop.default !== undefined; const rawDefault = prop.default; const hints = prop.hints || []; const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"'))); const postBadges = []; if (prop.title) { postBadges.push(<>API property: {name}); } const constraints = []; if (prop.minimum !== undefined && prop.maximum !== undefined) { constraints.push(`Range: ${prop.minimum} – ${prop.maximum}`); } else if (prop.minimum !== undefined) { constraints.push(`Min: ${prop.minimum}`); } else if (prop.maximum !== undefined) { constraints.push(`Max: ${prop.maximum}`); } if (prop.minLength !== undefined && prop.maxLength !== undefined) { constraints.push(`Length: ${prop.minLength} – ${prop.maxLength}`); } else if (prop.minLength !== undefined) { constraints.push(`Min length: ${prop.minLength}`); } else if (prop.maxLength !== undefined) { constraints.push(`Max length: ${prop.maxLength}`); } const fieldProps = { key: name, body: prop.title || name, type: prop.type, ...postBadges.length > 0 && ({ post: postBadges }), ...isRequired && ({ required: true }), ...!isComplexDefault && hasDefault ? { default: sanitize(String(rawDefault)) } : {} }; const isObject = prop.type === "object" && prop.properties; const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties; return {prop.description &&

{renderMd(prop.description)}

} {prop.enum &&

Allowed values: {prop.enum.map((v, i) => <>{i > 0 && ", "}{String(v)})}

} {constraints.length > 0 &&

{constraints.join(" · ")}

} {isComplexDefault &&

Default: 

              
                {JSON.stringify(rawDefault, null, 2)}
} {isArrayOfObjects && } {isObject && } ; }; return
{description &&

{renderMd(description)}

} {visibleProps.map(renderProp)}
; }; export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => { const [isLoaded, setIsLoaded] = useState(false); useEffect(() => { const timer = setTimeout(() => { setIsLoaded(true); }, 500); return () => clearTimeout(timer); }, []); return
{isLoaded && ` }} />} Powered by Lucidworks Agent Studio
; }; [localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/parsers/csv-parser [mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/parsers/csv-parser [old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/sdlzci This parser breaks down incoming CSV files into the most efficient components for Lucidworks Search to index. It produces one new document per row from the CSV input, excluding comment rows and header rows. If your CSV file contains a column named `id`, this column is consumed to populate the document's unique identifier (Solr's `uniqueKey`) and is not available as a stored field. This behavior occurs because the default value of the parser's **Document ID Source Field** parameter is also `id`. When a CSV column matches this parameter: * The column's value is used to generate the document ID. * The column does not appear in the indexed document as a field. If you need to preserve your `id` column data as a regular field, use one of these options: * Change the column header from `id` to another name such as `record_id` or `item_id`. This is the simplest solution. * In the CSV parser stage configuration, set the **Prefix parsed fields with** parameter to a value such as `csv_`. This makes the `id` column appear as `csv_id` in your indexed documents. * In the Index Workbench's parser configuration, set the **Document ID Source Field** to a different column name. This allows `id` to be treated as a normal field, but you must specify a different column to use as the document identifier. See [Parsers Overview](/docs/lucidworks-search/04-move-data-in/parsers/overview) for information about configuring the **Document ID Source Field** parameter. When entering configuration values in the UI, use *unescaped* characters, such as `\t` for the tab character. When entering configuration values in the API, use *escaped* characters, such as `\\t` for the tab character.