> ## Documentation Index > Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt > Use this file to discover all available pages before exploring further. # Cluster Labeling Jobs export const schema = { "type": "object", "title": "Cluster Labeling", "description": "Attaches representative keyword labels to existing document clusters or categories. Use this job when you already have cluster assignments and want to surface the most distinctive terms per cluster. To create new clusters, use the Document Clustering job instead.", "required": ["id", "trainingCollection", "fieldToVectorize", "dataFormat", "clusterIdField", "outputCollection", "type"], "properties": { "id": { "type": "string", "title": "Spark Job ID", "description": "Sets the unique identifier for this Spark job, used to reference it in the API. Allowed characters: a-z, A-Z, dash (`-`), and underscore (`_`). Must start with a letter.", "maxLength": 63, "pattern": "[a-zA-Z][_\\-a-zA-Z0-9]*[a-zA-Z0-9]?" }, "sparkConfig": { "type": "array", "title": "Spark Settings", "description": "Sets Spark configuration key-value pairs applied to this job's execution context. Use entries like `spark.executor.memory=4g` to tune resource allocation beyond cluster defaults.", "hints": ["advanced"], "items": { "type": "object", "required": ["key"], "properties": { "key": { "type": "string", "title": "Parameter Name" }, "value": { "type": "string", "title": "Parameter Value" } } } }, "trainingCollection": { "type": "string", "title": "Training Collection", "description": "Specifies the Solr collection containing documents with predefined cluster or category assignments. The cluster ID field must be populated in this collection before running the job.", "minLength": 1 }, "fieldToVectorize": { "type": "string", "title": "Field to detect keywords from", "description": "Specifies the field from which keywords are extracted for each cluster. Text in this field is tokenized and analyzed to discover the most representative terms per cluster.", "minLength": 1 }, "dataFormat": { "type": "string", "title": "Data format", "description": "Specifies the Spark-compatible format of the training data, such as `solr`, `parquet`, or `orc`. This controls which Spark connector loads the training DataFrame.", "default": "solr", "minLength": 1 }, "trainingDataFrameConfigOptions": { "type": "object", "title": "Dataframe Config Options", "description": "Sets additional Spark DataFrame loading options as key-value pairs for the training data source. Use connector-specific options such as `zkHost` when reading from non-default Solr configurations.", "properties": {}, "additionalProperties": { "type": "string" }, "hints": ["advanced"] }, "trainingDataFilterQuery": { "type": "string", "title": "Training data filter query", "description": "Filters training data using a Solr query before extracting cluster keywords. Use `*:*` to include all documents or a more specific query to restrict the input to a relevant subset.", "default": "*:*", "hints": ["advanced"] }, "sparkSQL": { "type": "string", "title": "Spark SQL filter query", "description": "Filters input training data using a Spark SQL expression before processing. The input data is registered as `spark_input`, allowing standard SQL `WHERE` clauses and joins.", "default": "SELECT * from spark_input", "hints": ["code/sql", "advanced"] }, "trainingDataSamplingFraction": { "type": "number", "title": "Training data sampling fraction", "description": "Sets the fraction of training data sampled before keyword extraction. Values less than `1.0` speed up processing on very large collections at the cost of keyword coverage.", "default": 1, "hints": ["advanced"], "maximum": 1, "exclusiveMaximum": false }, "randomSeed": { "type": "integer", "title": "Random seed", "description": "Sets the random seed used during data sampling to produce deterministic results. Fix this value to reproduce identical keyword outputs across multiple runs.", "default": 1234, "hints": ["advanced"] }, "outputCollection": { "type": "string", "title": "Output Collection", "description": "Specifies the Solr collection where the cluster label output documents are written. Each output document includes the cluster ID and associated keyword fields.", "minLength": 1 }, "overwriteOutput": { "type": "boolean", "title": "Overwrite Output", "description": "Controls whether the output collection is overwritten when the job runs. This hidden advanced field defaults to `true`. Set to `false` if you need to preserve existing output documents.", "default": true, "hints": ["hidden", "advanced"] }, "dataOutputFormat": { "type": "string", "title": "Data output format", "description": "Specifies the Spark-compatible format for writing the cluster label output, such as `solr` or `parquet`. Must match the target storage system used for the output collection.", "default": "solr", "hints": ["advanced"], "minLength": 1 }, "sourceFields": { "type": "string", "title": "Fields to Load", "description": "Specifies the field names to load from the input source, limiting which columns are read into the DataFrame. Leave unset to load all fields. Restrict to improve read performance on wide schemas.", "hints": ["advanced"] }, "partitionCols": { "type": "string", "title": "Partition fields", "description": "Specifies a comma-delimited list of column names used to partition the output DataFrame before writing to non-Solr storage. Partition columns improve downstream query performance by enabling file pruning.", "hints": ["advanced"] }, "writeOptions": { "type": "array", "title": "Write Options", "description": "Sets additional key-value options passed to the Spark writer when writing output to Solr or other sinks. Use entries like `commitWithin=10000` to control Solr commit behavior.", "hints": ["advanced"], "items": { "type": "object", "required": ["key"], "properties": { "key": { "type": "string", "title": "Parameter Name" }, "value": { "type": "string", "title": "Parameter Value" } } } }, "readOptions": { "type": "array", "title": "Read Options", "description": "Sets additional key-value options passed to the Spark reader when loading input from Solr or other sources. Use these to configure connector-specific behavior such as `rows` or `zkHost`.", "hints": ["advanced"], "items": { "type": "object", "required": ["key"], "properties": { "key": { "type": "string", "title": "Parameter Name" }, "value": { "type": "string", "title": "Parameter Value" } } } }, "modelId": { "type": "string", "title": "Model ID", "description": "Sets the unique identifier for the cluster labeling model. Defaults to the Spark job ID if not provided. Set explicitly to reuse or reference the model across multiple runs.", "hints": ["advanced"], "minLength": 1 }, "clusterIdField": { "type": "string", "title": "Existing Document Category Field", "description": "Specifies the field in the training collection that contains the existing cluster or category assignment for each document. This field must be populated before running the job.", "minLength": 1 }, "analyzerConfig": { "type": "string", "title": "Lucene Analyzer Schema", "description": "Sets the Lucene text analyzer configuration applied to the keyword extraction field. The default uses standard tokenization with lowercasing, KStem stemming, and English stop word filtering.", "default": "{ \"analyzers\": [{ \"name\": \"StdTokLowerStop\",\"charFilters\": [ { \"type\": \"htmlstrip\" } ],\"tokenizer\": { \"type\": \"standard\" },\"filters\": [{ \"type\": \"lowercase\" },{ \"type\": \"KStem\" },{ \"type\": \"length\", \"min\": \"2\", \"max\": \"32767\" },{ \"type\": \"fusionstop\", \"ignoreCase\": \"true\", \"format\": \"snowball\", \"words\": \"org/apache/lucene/analysis/snowball/english_stop.txt\" }] }],\"fields\": [{ \"regex\": \".+\", \"analyzer\": \"StdTokLowerStop\" } ]}", "hints": ["lengthy", "code/json"], "minLength": 1 }, "clusterLabelField": { "type": "string", "title": "Top Unique Terms Field Name", "description": "Specifies the output field name where the top unique keywords for each cluster are stored. Maps the `cluster_label` field by default. Change this to avoid conflicts with existing schema fields.", "default": "cluster_label" }, "freqTermField": { "type": "string", "title": "Top Frequent Terms Field Name", "description": "Specifies the output field name where the most frequent terms for each cluster are stored. Maps the `freq_terms` field by default. Change this to avoid conflicts with existing schema fields.", "default": "freq_terms" }, "minDF": { "type": "number", "title": "Min Doc Support", "description": "Sets the minimum document support threshold for a term to be considered as a cluster keyword. Values less than `1.0` indicate a percentage of documents. Values greater than `1.0` indicate an exact count.", "default": 5 }, "maxDF": { "type": "number", "title": "Max Doc Support", "description": "Sets the maximum document support threshold for a term to be considered as a cluster keyword. Terms appearing in too many documents are excluded as common words that don't distinguish clusters.", "default": 0.75 }, "norm": { "type": "integer", "title": "Vector normalization", "description": "Selects the p-norm used to normalize document feature vectors before keyword extraction. Set to `-1` to disable normalization. Typical values are `1` (Manhattan) or `2` (Euclidean).", "enum": [-1, 0, 1, 2], "default": 2, "hints": ["advanced"] }, "numKeywordsPerLabel": { "type": "integer", "title": "Number of Keywords for Each Cluster", "description": "Sets the number of distinctive keywords extracted to label each cluster. Increase this to provide richer cluster descriptions. The keywords are stored in `clusterLabelField`.", "default": 5 }, "stopwordsList": { "type": "array", "title": "List of stopwords", "description": "Lists stopword blob resources defined in the Lucene analyzer configuration. This field is read-only and hidden. Stopwords are controlled via the `analyzerConfig` field.", "hints": ["readonly", "hidden"], "items": { "type": "string", "minLength": 1, "reference": "blob", "blobType": "file:spark" } }, "type": { "type": "string", "title": "Spark Job Type", "enum": ["cluster_labeling"], "default": "cluster_labeling", "hints": ["readonly"] } }, "additionalProperties": true, "category": "Other", "categoryPriority": 1, "propertyGroups": [{ "label": "Input/Output Parameters", "properties": ["trainingCollection", "outputCollection", "dataFormat", "trainingDataFilterQuery", "readOptions", "writeOptions", "trainingDataFrameConfigOptions", "trainingDataSamplingFraction", "randomSeed"] }, { "label": "Field Parameters", "properties": ["fieldToVectorize", "sourceFields", "clusterIdField", "freqTermField", "clusterLabelField"] }, { "label": "Model Tuning Parameters", "properties": ["maxDF", "minDF", "norm", "numKeywordsPerLabel"] }, { "label": "Featurization Parameters", "properties": ["analyzerConfig"] }, { "label": "Misc. Parameters", "properties": ["modelId"] }] }; export const SchemaParamFields = ({schema}) => { const sanitize = str => { if (typeof str !== "string") return str; return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'"); }; const renderMd = str => { const s = sanitize(str); const text = (/[.!?]\)*$/).test(s) ? s : `${s}.`; return text.split(/(\*\*[^*]+\*\*|_[^_]+_|`[^`]+`)/g).map((part, i) => { if (part.startsWith("**")) return {part.slice(2, -2)}; if (part.startsWith("_")) return {part.slice(1, -1)}; if (part.startsWith("`")) return {part.slice(1, -1)}; return part; }); }; const {description, properties = {}, required: requiredProps = []} = schema; const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]); const renderProp = ([name, prop]) => { const isRequired = requiredProps.includes(name); const hasDefault = prop.default !== undefined; const rawDefault = prop.default; const hints = prop.hints || []; const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"'))); const postBadges = []; if (prop.title) { postBadges.push(<>API property: {name}); } const constraints = []; if (prop.minimum !== undefined && prop.maximum !== undefined) { constraints.push(`Range: ${prop.minimum} – ${prop.maximum}`); } else if (prop.minimum !== undefined) { constraints.push(`Min: ${prop.minimum}`); } else if (prop.maximum !== undefined) { constraints.push(`Max: ${prop.maximum}`); } if (prop.minLength !== undefined && prop.maxLength !== undefined) { constraints.push(`Length: ${prop.minLength} – ${prop.maxLength}`); } else if (prop.minLength !== undefined) { constraints.push(`Min length: ${prop.minLength}`); } else if (prop.maxLength !== undefined) { constraints.push(`Max length: ${prop.maxLength}`); } const fieldProps = { key: name, body: prop.title || name, type: prop.type, ...postBadges.length > 0 && ({ post: postBadges }), ...isRequired && ({ required: true }), ...!isComplexDefault && hasDefault ? { default: sanitize(String(rawDefault)) } : {} }; const isObject = prop.type === "object" && prop.properties; const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties; return {prop.description &&

{renderMd(prop.description)}

} {prop.enum &&

Allowed values: {prop.enum.map((v, i) => <>{i > 0 && ", "}{String(v)})}

} {constraints.length > 0 &&

{constraints.join(" · ")}

} {isComplexDefault &&

Default: 

              
                {JSON.stringify(rawDefault, null, 2)}
} {isArrayOfObjects && } {isObject && } ; }; return
{description &&

{renderMd(description)}

} {visibleProps.map(renderProp)}
; }; export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => { const [isLoaded, setIsLoaded] = useState(false); useEffect(() => { const timer = setTimeout(() => { setIsLoaded(true); }, 500); return () => clearTimeout(timer); }, []); return
{isLoaded && ` }} />} Powered by Lucidworks Agent Studio
; }; [localhost link]: http://localhost:3000/docs/5/fusion/reference/config-ref/jobs/cluster-labeling [mintlify link]: https://doc.lucidworks.com/docs/5/fusion/reference/config-ref/jobs/cluster-labeling [old doc.lw link]: https://doc.lucidworks.com/fusion/5.9/577 Cluster labeling jobs are run against your data collections, and are used: * When clusters or well-defined document categories already exist * When you want to discover and attach keywords to see representative words within existing clusters To create a cluster labeling job, sign in to Fusion and click **Collections > Jobs**. Then click **Add+** and in the Clustering and Outlier Analysis Jobs section, select **Cluster Labeling**. You can enter basic and advanced parameters to configure the job. If the field has a default value, it is populated when you click to add the job. ## Basic parameters To enter advanced parameters in the UI, click **Advanced**. Those parameters are described in [the advanced parameters section](#advanced-parameters). * **Spark job ID.** The unique ID for the Spark job that references this job in the API. This is the `id` field in the configuration file. Required field. * **Input/Output Parameters.** This section includes these parameters: * **Training collection.** The Solr collection that contains documents associated with defined categories or clusters. The job will be run against this information. This is the `trainingCollection` field in the configuration file. Required field. * **Output collection.** The Solr collection where the job output is stored. The job will write the output to this collection. This is the `outputCollection` field in the configuration file. Required field. * **Data format.** The format that contains training data. The format must be compatible with Spark and options include `solr`, `parquet`, and `orc`. Required field. * **Field Parameters.** This section includes these parameters: * **Field to detect keywords from.** The field that contains the data that the job will use to discover keywords for the cluster. This is the `fieldToVectorize` field in the configuration file. Required field. * **Existing document category field.** The field that contains existing cluster IDs or document categories. This is the `clusterIdField` field in the configuration file. Required field. * **Top frequent terms field name.** The field where the job output stores top frequent terms in each cluster. Terms may overlap with other clusters. This is the `freqTermField` field in the configuration file. Optional field. * **Top unique terms field name.** The field where the job output stores the top frequent terms that, for the most part, are unique in each cluster. This is the `clusterLabelField` field in the configuration file. Optional field. * **Model Tuning Parameters.** This section includes these parameters: * **Max doc support.** The maximum number of documents that can contain the term. Values that are `<1.0` indicate a percentage, `1.0` is 100 percent, and `>1.0` indicates the exact number. This is the `maxDF` field in the configuration file. Optional field. * **Min doc support.** The minimum of documents that must contain the term. Values that are `<1.0` indicate a percentage, `1.0` is 100 percent, and `>1.0` indicates the exact number. This is the `minDF` field in the configuration file. Optional field. * **Number of keywords for each cluster.** The number of keywords required to label each cluster. This is the `numKeywordsPerLabel` field in the configuration file. Optional field. * **Featurization Parameters.** This section includes this parameter: * **Lucene analyzer schema.** This is the JSON-encoded Lucene text analyzer schema used for tokenization. This is the `analyzerConfig` field in the configuration file. Optional field. ## Advanced parameters If you click the **Advanced** toggle, the following optional fields are displayed in the UI. * **Spark Settings.** The Spark configuration settings include: * **Spark SQL filter query.** This field contains the Spark SQL query that filters your input data. For example, `SELECT * from spark_input` registers the input data as `spark_input`. This is the `sparkSQL` field in the configuration file. * **Data output format.** The format for the job output. The format must be compatible with Spark and options include `solr` and `parquet`. This is the `dataOutputFormat` field in the configuration file. * **Partition fields.** If the job output is written to non-Solr sources, this field contains a comma-delimited list of column names that partition the dataframe before the external output is written. This is the `partitionCols` field in the configuration file. * **Read Options.** This section lets you enter `parameter name:parameter value` options to use when reading input from Solr or other sources. This is the `readOptions` field in the configuration file. * **Write Options.** This section lets you enter `parameter name:parameter value` options to use when writing output to Solr or other sources. This is the `writeOptions` field in the configuration file. * **Dataframe config options.** This section includes these parameters: * **Property name:property value.** Each entry defines an additional Spark dataframe loading configuration option. This is the `trainingDataFrameConfigOptions` field in the configuration file. * **Training data sampling fraction.** This is the fractional amount of the training data the job will use. This is the `trainingDataSamplingFraction` field in the configuration file. * **Random seed.** This value is used in any deterministic pseudorandom number generation to group documents into clusters based on similarities in their content. This is the `randomSeed` field in the configuration file. * **Field Parameters.** The advanced option adds this parameter: * **Fields to load.** This field contains a comma-delimited list of Solr fields to load. If blank, the job selects the required fields to load at runtime. This is the `sourceFields` field in the configuration file. * **Miscellaneous Parameters.** This section includes this parameter: * **Model ID.** The unique identifier for the model to be trained. If no value is entered, the `Spark Job ID` is used. This is the `modelId` field in the configuration file. To create new clusters, use the [Document Clustering job](/docs/5/fusion/reference/config-ref/jobs/document-clustering). ## Configuration properties