> ## Documentation Index > Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt > Use this file to discover all available pages before exploring further. # Grok > Parser stage configuration specifications export const schema = { "type": "object", "title": "Grok", "description": "Parses semi-structured text content using Grok patterns, which combine regular expressions with predefined patterns for common data formats. Ideal for parsing log files (Apache, nginx, syslog), extracting structured fields from unstructured text, and handling formats with consistent patterns but varying values. Supports custom Grok definitions and pattern composition.", "required": ["charset", "ignoreBOM", "type"], "properties": { "id": { "type": "string", "title": "Parser ID", "default": "a0defe60-37f0-4091-8b22-90bf802fefc0" }, "label": { "type": "string", "title": "Label", "description": "Human-readable identifier displayed in the Fusion Admin UI, monitoring dashboards, and log messages. Use descriptive labels like `Parse Product PDFs` to aid debugging and team collaboration. Labels appear in performance metrics and error reports, making it easier to identify which stage failed.", "maxLength": 255 }, "enabled": { "type": "boolean", "title": "Enable this Parser Stage", "default": true, "description": "Controls whether this parser stage is active and available for use. When `false`, the stage is completely inactive regardless of other settings. When `true`, the stage runs according to its other configuration options." }, "mediaTypes": { "type": "array", "title": "Media Types to match", "description": "Specifies the media types this parser stage handles. Documents with a matching media type are routed to this stage for parsing. See `inheritMediaTypes` to combine this list with the stage's built-in defaults.", "items": { "type": "string", "pattern": "^[^\\/]+\\/[^\\/]+$", "format": "rfc2646" } }, "inheritMediaTypes": { "type": "boolean", "title": "Match default media types in this Parser Stage", "description": "Controls whether this stage combines its built-in default media types with those in `mediaTypes`. When `true`, both lists are merged. When `false`, only the `mediaTypes` list is used and must contain at least one entry. Set to `false` to override the default media types entirely.", "default": true }, "ignoredMediaTypes": { "type": "array", "title": "Media Types to ignore", "description": "Specifies media types this parser stage excludes from processing. Documents matching an ignored media type are skipped even if they match `mediaTypes`. Use this to carve out exceptions from a broadly matched media type set.", "items": { "type": "string", "pattern": "^[^\\/]+\\/[^\\/]+$", "format": "rfc2646" } }, "pathPatterns": { "type": "array", "title": "File names to parse", "description": "Restricts this parser stage to files whose names match the specified pattern. Use forward slashes (`/`) to join archive names with entry names when matching files inside archives. If no pattern is specified, the stage applies to all matching media types.", "items": { "type": "object", "properties": { "syntax": { "type": "string", "title": "Pattern type", "description": "glob uses bash shell-style wildcards and regex uses Java (PCRE-style) regex.", "enum": ["glob", "regex"], "default": "glob" }, "pattern": { "type": "string", "title": "File name or pattern", "description": "glob examples are \"z.txt\" or \"*.md\" or \"/a/*/b/f.txt\". regex examples are \"z.txt$\" or \".*\\.txt$\" or \"^/a/[^\\/]*/b/f.txt$\"." } } } }, "errorHandling": { "type": "string", "title": "Error Handling", "enum": ["ignore", "log", "fail", "mark"], "default": "mark" }, "outputFieldPrefix": { "type": "string", "title": "Prefix parsed fields with", "description": "Sets a string prefix applied to all fields extracted by this parser, useful for namespacing or avoiding field name collisions. For example, `tika_` produces fields like `tika_title` and `tika_author`. Leave empty to apply no prefix.", "maxLength": 20, "pattern": "^$|^[A-Za-z_][A-Za-z0-9_\\-\\.]+$" }, "charset": { "type": "string", "title": "Character Set", "description": "Specifies the character encoding used to read file content. Common values include `UTF-8`, `ISO-8859-1`, and `Windows-1252`. If incorrect, text may appear garbled or cause parsing errors.", "default": "detect" }, "ignoreBOM": { "type": "boolean", "title": "Ignore BOM", "description": "Controls whether the parser ignores the Byte-Order Mark (BOM) at the start of the file. When `true`, BOM is always ignored and the configured charset is used. When `false`, a valid BOM overrides the configured charset.", "default": false }, "grokDefinition": { "type": "string", "title": "Grok Definition", "description": "Defines custom Grok patterns that extend or override the built-in pattern library. Specify reusable named patterns in the format `PATTERN_NAME regex_expression`, one per line. Reference them in `grokPattern` using `%{PATTERN_NAME}`. Use this to define domain-specific patterns not covered by the built-in library.", "hints": ["code/javascript"] }, "grokPattern": { "type": "string", "title": "Grok Pattern", "description": "Specifies the Grok pattern expression used to parse each line of text and extract named fields. Combine literal text with pattern placeholders in the format `%{PATTERN:field_name}`. Matched groups become document fields. For example, `%{IP:client_ip} %{WORD:method}` extracts an IP address and HTTP method from each log line.", "hints": ["code/javascript"] }, "type": { "type": "string", "enum": ["grok"], "default": "grok" } }, "additionalProperties": false, "category": "Other", "categoryPriority": 1, "unsafe": false }; export const SchemaParamFields = ({schema}) => { const sanitize = str => { if (typeof str !== "string") return str; return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'"); }; const renderMd = str => { const s = sanitize(str); const text = (/[.!?]\)*$/).test(s) ? s : `${s}.`; return text.split(/(\*\*[^*]+\*\*|_[^_]+_|`[^`]+`)/g).map((part, i) => { if (part.startsWith("**")) return {part.slice(2, -2)}; if (part.startsWith("_")) return {part.slice(1, -1)}; if (part.startsWith("`")) return {part.slice(1, -1)}; return part; }); }; const {description, properties = {}, required: requiredProps = []} = schema; const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]); const renderProp = ([name, prop]) => { const isRequired = requiredProps.includes(name); const hasDefault = prop.default !== undefined; const rawDefault = prop.default; const hints = prop.hints || []; const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"'))); const postBadges = []; if (prop.title) { postBadges.push(<>API property: {name}); } const constraints = []; if (prop.minimum !== undefined && prop.maximum !== undefined) { constraints.push(`Range: ${prop.minimum} – ${prop.maximum}`); } else if (prop.minimum !== undefined) { constraints.push(`Min: ${prop.minimum}`); } else if (prop.maximum !== undefined) { constraints.push(`Max: ${prop.maximum}`); } if (prop.minLength !== undefined && prop.maxLength !== undefined) { constraints.push(`Length: ${prop.minLength} – ${prop.maxLength}`); } else if (prop.minLength !== undefined) { constraints.push(`Min length: ${prop.minLength}`); } else if (prop.maxLength !== undefined) { constraints.push(`Max length: ${prop.maxLength}`); } const fieldProps = { key: name, body: prop.title || name, type: prop.type, ...postBadges.length > 0 && ({ post: postBadges }), ...isRequired && ({ required: true }), ...!isComplexDefault && hasDefault ? { default: sanitize(String(rawDefault)) } : {} }; const isObject = prop.type === "object" && prop.properties; const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties; return {prop.description &&

{renderMd(prop.description)}

} {prop.enum &&

Allowed values: {prop.enum.map((v, i) => <>{i > 0 && ", "}{String(v)})}

} {constraints.length > 0 &&

{constraints.join(" · ")}

} {isComplexDefault &&

Default: 

              
                {JSON.stringify(rawDefault, null, 2)}
} {isArrayOfObjects && } {isObject && } ; }; return
{description &&

{renderMd(description)}

} {visibleProps.map(renderProp)}
; }; export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => { const [isLoaded, setIsLoaded] = useState(false); useEffect(() => { const timer = setTimeout(() => { setIsLoaded(true); }, 500); return () => clearTimeout(timer); }, []); return
{isLoaded && ` }} />} Powered by Lucidworks Agent Studio
; }; [localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/parsers/grok-parser/overview [mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/parsers/grok-parser/overview [old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/7z9z1h The Grok parser stage uses [Java Grok](https://github.com/thekrakken/java-grok) and Grok patterns (a specific kind of regex matching) to parse log files and similar text files that have line-oriented, semi-structured data. Parsing a text file with the Grok parser lets you give more structure to semi-structured data and extract more information. ## Whether the Grok stage parses a file Before a Grok parser stage parses a file, the file must meet criteria regarding the media type and file name. ### Media type The Grok parser stage parses files that have media types that match either the default media types *or* media types that you specify. Select or unselect **Use default media types for this parser stage**: * **Selected.** The Grok parser stage parses files that have one of the default media types (`text/plain` or `text/x-log`), as well as files that have media types that you specify under **Media Types for this Parser Stage**. * **Unselected.** The Grok parser stage *only* parses files that have one of the media types that you specify under **Media Types for this Parser Stage**. ### File name Optionally, you can specify a file name or file name pattern that a file must match for the Grok parser stage to parse the file. | Field | Description | | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Pattern Type | `glob`. Use `bash` shell wildcards. Examples include `z.txt`, `*.md`, and `/a/*/b/f.txt`. `regex`. Use Java regular expressions (PCRE; Perl-compatible regular expressions). Examples include `z.txt$`, `.*\.txt$`, and `^/a/[^\/]*/b/f.txt$`. | | File Name or Pattern | Name of the file or a pattern for the file name. The parser parses matching files. | ## Grok patterns Grok patterns are regular expressions written in the language of the [Oniguruma regular expression library](https://github.com/kkos/oniguruma), which has [this syntax](https://github.com/kkos/oniguruma/blob/master/doc/RE). You configure a Grok parsing stage to use predefined [Grok patterns](/docs/lucidworks-search/09-developer-documentation/config-specs/parsers/grok-parser/grok-patterns) (about 300 patterns are available) and/or Grok pattern definitions that you write yourself. * **Use predefined patterns.** Under the **Grok Pattern** part of the Grok parser stage configuration, specify a single top-level Grok pattern by name, for example, `REDISLOG`. * **Write your own Grok pattern definition(s).** (*optional*) Write one or more Grok pattern definitions, and then enter them in the **Grok Definition** part of the Grok parser stage configuration. ## Parsing rules These are rules that affect the results of parsing: * **Precedence in the event of identical names.** If the name of a custom Grok pattern definition that you provide is identical to the name of a predefined pattern definition, then your definition is used. * **Invalid patterns.** If a pattern is not syntactically valid, then the full text of the row being parsed is treated as a single field. * **Pattern does not match any data.** If a pattern does not match any data, then the full text of the row being parsed is treated as a single field. * **Line by line.** Parsing is line by line. If data has a multiline structure, the parser does not capture the relationship between lines. When entering configuration values in the UI, use *unescaped* characters, such as `\t` for the tab character. When entering configuration values in the API, use *escaped* characters, such as `\\t` for the tab character.