> ## Documentation Index > Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt > Use this file to discover all available pages before exploring further. # Document Clustering > Job configuration specifications export const schema = { "type": "object", "title": "Document Clustering", "description": "Groups documents into clusters based on text similarity and attaches keyword labels to each cluster. Configure the training collection, vectorization field, and cluster count settings to run the clustering job. Use the Cluster Labeling job instead when you already have predefined clusters and only need to label them.", "required": ["id", "trainingCollection", "fieldToVectorize", "dataFormat", "outputCollection", "uidField", "type"], "properties": { "id": { "type": "string", "title": "Spark Job ID", "description": "Sets the unique identifier for this Spark job, used to reference it in the API. Allowed characters: a-z, A-Z, dash (`-`), and underscore (`_`). Must start with a letter.", "maxLength": 63, "pattern": "[a-zA-Z][_\\-a-zA-Z0-9]*[a-zA-Z0-9]?" }, "sparkConfig": { "type": "array", "title": "Spark Settings", "description": "Sets Spark configuration key-value pairs applied to this job's execution context. Use entries like `spark.executor.memory=4g` to tune resource allocation beyond cluster defaults.", "hints": ["advanced"], "items": { "type": "object", "required": ["key"], "properties": { "key": { "type": "string", "title": "Parameter Name" }, "value": { "type": "string", "title": "Parameter Value" } } } }, "trainingCollection": { "type": "string", "title": "Training Collection", "description": "Specifies the Solr collection containing the documents to cluster. All documents in this collection are vectorized using the `fieldToVectorize` field and grouped into clusters.", "minLength": 1 }, "fieldToVectorize": { "type": "string", "title": "Field to Vectorize", "description": "Specifies the Solr field containing the text content used to compute document similarities for clustering. Combine multiple fields with weights using the syntax `field1:weight1,field2:weight2`.", "minLength": 1 }, "dataFormat": { "type": "string", "title": "Data format", "description": "Specifies the Spark-compatible format of the training data, such as `solr`, `parquet`, or `orc`. This controls which Spark connector loads the training DataFrame.", "default": "solr", "minLength": 1 }, "trainingDataFrameConfigOptions": { "type": "object", "title": "Dataframe Config Options", "description": "Sets additional Spark DataFrame loading options as key-value pairs for the training data source. Use connector-specific options such as `zkHost` when reading from non-default Solr configurations.", "properties": {}, "additionalProperties": { "type": "string" }, "hints": ["advanced"] }, "trainingDataFilterQuery": { "type": "string", "title": "Training data filter query", "description": "Filters training data using a Solr query before clustering. Use `*:*` to cluster all documents or a more specific query to restrict clustering to a relevant subset.", "default": "*:*", "hints": ["advanced"] }, "sparkSQL": { "type": "string", "title": "Spark SQL filter query", "description": "Filters input training data using a Spark SQL expression before processing. The input data is registered as `spark_input`, allowing standard SQL `WHERE` clauses and joins.", "default": "SELECT * from spark_input", "hints": ["code/sql", "advanced"] }, "trainingDataSamplingFraction": { "type": "number", "title": "Training data sampling fraction", "description": "Sets the fraction of training data sampled before clustering. Values less than `1.0` speed up clustering on very large collections at the cost of cluster representativeness.", "default": 1, "hints": ["advanced"], "maximum": 1, "exclusiveMaximum": false }, "randomSeed": { "type": "integer", "title": "Random seed", "description": "Sets the random seed used during clustering and sampling to produce deterministic results. Fix this value to reproduce identical cluster assignments across multiple runs.", "default": 1234, "hints": ["advanced"] }, "outputCollection": { "type": "string", "title": "Output Collection", "description": "Specifies the Solr collection where cluster-labeled output documents are written. Each output document receives a cluster ID and associated keyword label fields.", "minLength": 1 }, "overwriteOutput": { "type": "boolean", "title": "Overwrite Output", "description": "Controls whether the output collection is overwritten when the job runs. This hidden advanced field defaults to `true`. Set to `false` to preserve existing output documents.", "default": true, "hints": ["hidden", "advanced"] }, "dataOutputFormat": { "type": "string", "title": "Data output format", "description": "Specifies the Spark-compatible format for writing cluster-labeled output, such as `solr` or `parquet`. Must match the target storage system used for the output collection.", "default": "solr", "hints": ["advanced"], "minLength": 1 }, "sourceFields": { "type": "string", "title": "Fields to Load", "description": "Specifies the field names to load from the input source, limiting which columns are read into the DataFrame. Leave unset to load all fields. Restrict to improve read performance on wide schemas.", "hints": ["advanced"] }, "partitionCols": { "type": "string", "title": "Partition fields", "description": "Specifies a comma-delimited list of column names used to partition the output DataFrame before writing to non-Solr storage. Partition columns improve downstream query performance by enabling file pruning.", "hints": ["advanced"] }, "writeOptions": { "type": "array", "title": "Write Options", "description": "Sets additional key-value options passed to the Spark writer when writing output to Solr or other sinks. Use entries like `commitWithin=10000` to control Solr commit behavior.", "hints": ["advanced"], "items": { "type": "object", "required": ["key"], "properties": { "key": { "type": "string", "title": "Parameter Name" }, "value": { "type": "string", "title": "Parameter Value" } } } }, "readOptions": { "type": "array", "title": "Read Options", "description": "Sets additional key-value options passed to the Spark reader when loading input from Solr or other sources. Use these to configure connector-specific behavior such as `rows` or `zkHost`.", "hints": ["advanced"], "items": { "type": "object", "required": ["key"], "properties": { "key": { "type": "string", "title": "Parameter Name" }, "value": { "type": "string", "title": "Parameter Value" } } } }, "uidField": { "type": "string", "title": "ID Field Name", "description": "Specifies the field in the training collection that uniquely identifies each document. Maps the `id` field by default. This field is preserved in the output to link cluster assignments back to source documents.", "default": "id", "minLength": 1 }, "clusterIdField": { "type": "string", "title": "Output Field Name for Cluster Id", "description": "Specifies the output field name where the cluster assignment ID is written for each document. Maps the `cluster_id` field by default. Change to avoid conflicts with existing schema fields.", "default": "cluster_id" }, "clusterLabelField": { "type": "string", "title": "Detected Cluster Keywords Field Name", "description": "Specifies the output field name where the distinctive keywords for each document's cluster are written. Maps the `cluster_label` field by default. Change to avoid conflicts with existing schema fields.", "default": "cluster_label" }, "freqTermField": { "type": "string", "title": "Top Frequent Terms Field Name", "description": "Specifies the output field name where the most frequent terms for each cluster are written. Maps the `freq_terms` field by default. Change to avoid conflicts with existing schema fields.", "default": "freq_terms" }, "distToCenterField": { "type": "string", "title": "Output Field Name for doc distance to its cluster center", "description": "Specifies the output field name where each document's distance to its cluster center is written. Maps the `dist_to_center` field by default. Use this field to identify outlier documents within a cluster.", "default": "dist_to_center" }, "minDF": { "type": "number", "title": "Min Doc Support", "description": "Sets the minimum document support for a term to be considered in cluster keyword extraction. Values less than `1.0` indicate a percentage. Values greater than `1.0` indicate an exact document count.", "default": 5 }, "maxDF": { "type": "number", "title": "Max Doc Support", "description": "Sets the maximum document support for a term to be included in cluster keyword extraction. Terms appearing too frequently across all clusters are excluded as they don't distinguish individual clusters.", "default": 0.5 }, "kExact": { "type": "integer", "title": "Number of Clusters", "description": "Sets the exact number of clusters to create. When set to `0`, the job automatically selects the optimal number of clusters within the range defined by `kMin` and `kMax`.", "default": 0 }, "kMax": { "type": "integer", "title": "Max Possible Number of Clusters", "description": "Sets the maximum number of clusters the job will create when automatic cluster selection is enabled. The optimal number of clusters is chosen from the range `[kMin, kMax]`.", "default": 20 }, "kMin": { "type": "integer", "title": "Min Possible Number of Clusters", "description": "Sets the minimum number of clusters the job will create when automatic cluster selection is enabled. The optimal number of clusters is chosen from the range `[kMin, kMax]`.", "default": 2 }, "docLenTrim": { "type": "boolean", "title": "Find Extreme Length Doc Flag", "description": "Controls whether documents with extreme lengths (very short or very long) are separated into their own groups before clustering. Enabling this prevents outlier documents from distorting cluster boundaries.", "default": true }, "outlierTrim": { "type": "boolean", "title": "Find Outliers Flag", "description": "Controls whether outlier detection is applied before clustering. When `true`, documents that are far from any cluster centroid are identified and separated to improve the quality of the main clusters.", "default": true }, "shortLen": { "type": "number", "title": "Length Threshold for Short Doc", "description": "Sets the length threshold used to classify documents as very short. Values less than `1.0` indicate a percentage. Values greater than `1.0` indicate an exact character count.", "default": 5 }, "longLen": { "type": "number", "title": "Length Threshold for Long Doc", "description": "Sets the length threshold used to classify documents as very long. Values less than `1.0` indicate a percentage of maximum document length. Values greater than `1.0` indicate an exact character count.", "default": 0.99 }, "numKeywordsPerLabel": { "type": "integer", "title": "Number of Keywords for Each Cluster", "description": "Sets the number of distinctive keywords extracted to label each cluster. Increase this to provide richer cluster descriptions stored in the `clusterLabelField`.", "default": 5 }, "modelId": { "type": "string", "title": "Model ID", "description": "Sets the unique identifier for the document clustering model. Defaults to the Spark job ID if not provided. Set explicitly to reuse or reference the model across multiple runs.", "hints": ["advanced"], "minLength": 1 }, "w2vDimension": { "type": "integer", "title": "Word2Vec Dimension", "description": "Sets the number of Word2Vec embedding dimensions used to represent document text. Set to `0` to disable Word2Vec vectorization. Suggested values are in the range 100–150 for meaningful embeddings.", "default": 0, "hints": ["advanced"], "minimum": 0, "exclusiveMinimum": false }, "w2vWindowSize": { "type": "integer", "title": "Word2Vec Window Size", "description": "Sets the context window size used during Word2Vec training, controlling how many neighboring words are considered. Larger windows capture broader context. The minimum value is `3`.", "default": 8, "hints": ["advanced"], "minimum": 3, "exclusiveMinimum": false }, "norm": { "type": "integer", "title": "Vector normalization", "description": "Selects the p-norm used to normalize document feature vectors before clustering. Set to `-1` to disable normalization. Typical values are `1` (Manhattan) or `2` (Euclidean).", "enum": [-1, 0, 1, 2], "default": 2, "hints": ["advanced"] }, "analyzerConfig": { "type": "string", "title": "Lucene Analyzer Schema", "description": "Sets the Lucene text analyzer configuration applied to the vectorization field during featurization. The default uses standard tokenization with lowercasing, KStem stemming, digit removal, and English stop word filtering.", "default": "{ \"analyzers\": [{ \"name\": \"StdTokLowerStop\",\"charFilters\": [ { \"type\": \"htmlstrip\" } ],\"tokenizer\": { \"type\": \"standard\" },\"filters\": [{ \"type\": \"lowercase\" },{ \"type\": \"KStem\" },{ \"type\": \"patternreplace\", \"pattern\": \"^[\\\\d.]+$\", \"replacement\": \" \", \"replace\": \"all\" },{ \"type\": \"length\", \"min\": \"2\", \"max\": \"32767\" },{ \"type\": \"fusionstop\", \"ignoreCase\": \"true\", \"format\": \"snowball\", \"words\": \"org/apache/lucene/analysis/snowball/english_stop.txt\" }] }],\"fields\": [{ \"regex\": \".+\", \"analyzer\": \"StdTokLowerStop\" } ]}", "hints": ["code/json", "advanced", "lengthy"], "minLength": 1 }, "clusteringMethod": { "type": "string", "title": "Clustering Method (hierarchical or kmeans)", "description": "Selects the clustering algorithm used to group documents. Choose `hierarchical` for tree-based clustering that does not require a predefined cluster count. Choose `kmeans` for centroid-based clustering.", "default": "hierarchical", "hints": ["advanced"] }, "outlierK": { "type": "integer", "title": "Number of outlier groups", "description": "Sets the number of initial clusters used internally to identify outlier documents before the main clustering step. This value is used only when `outlierTrim` is enabled.", "default": 10, "hints": ["advanced"] }, "outlierThreshold": { "type": "number", "title": "Outlier cutoff", "description": "Sets the distance threshold below which a document is classified as an outlier relative to its nearest cluster centroid. Documents closer than this value to a centroid are considered inliers.", "default": 0.01, "hints": ["advanced"] }, "minDivisibleSize": { "type": "number", "title": "Minimum divisible cluster size", "description": "Sets the minimum number of documents a cluster must contain before it can be split further during hierarchical clustering. Values less than `1.0` indicate a fraction of total documents. Values greater than `1.0` indicate an exact count.", "default": 0, "hints": ["advanced"] }, "kDiscount": { "type": "number", "title": "Discount for K when choosing number of clusters", "description": "Applies a discount factor to favor larger or smaller values of K when selecting the optimal cluster count. Smaller values push the algorithm toward higher K within the `[kMin, kMax]` range.", "default": 0.7, "hints": ["advanced"] }, "stopwordsList": { "type": "array", "title": "List of stopwords", "description": "Lists stopword blob resources defined in the Lucene analyzer configuration. This field is read-only and hidden. Stopwords are controlled via the `analyzerConfig` field.", "hints": ["readonly", "hidden"], "items": { "type": "string", "minLength": 1, "reference": "blob", "blobType": "file:spark" } }, "type": { "type": "string", "title": "Spark Job Type", "enum": ["doc_clustering"], "default": "doc_clustering", "hints": ["readonly"] } }, "additionalProperties": true, "category": "Other", "categoryPriority": 1, "propertyGroups": [{ "label": "Input/Output Parameters", "properties": ["trainingCollection", "outputCollection", "dataFormat", "trainingDataFilterQuery", "readOptions", "writeOptions", "trainingDataFrameConfigOptions", "trainingDataSamplingFraction", "randomSeed"] }, { "label": "Field Parameters", "properties": ["fieldToVectorize", "sourceFields", "uidField", "clusterIdField", "freqTermField", "clusterLabelField", "distToCenterField"] }, { "label": "Model Tuning Parameters", "properties": ["clusteringMethod", "outlierTrim", "outlierK", "outlierThreshold", "kExact", "kMax", "kMin", "w2vDimension", "w2vWindowSize", "maxDF", "minDF", "norm", "numKeywordsPerLabel", "minDivisibleSize", "kDiscount"] }, { "label": "Featurization Parameters", "properties": ["analyzerConfig", "docLenTrim", "longLen", "shortLen"] }, { "label": "Misc. Parameters", "properties": ["modelId"] }] }; export const SchemaParamFields = ({schema}) => { const sanitize = str => { if (typeof str !== "string") return str; return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'"); }; const renderMd = str => { const s = sanitize(str); const text = (/[.!?]\)*$/).test(s) ? s : `${s}.`; return text.split(/(\*\*[^*]+\*\*|_[^_]+_|`[^`]+`)/g).map((part, i) => { if (part.startsWith("**")) return {part.slice(2, -2)}; if (part.startsWith("_")) return {part.slice(1, -1)}; if (part.startsWith("`")) return {part.slice(1, -1)}; return part; }); }; const {description, properties = {}, required: requiredProps = []} = schema; const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]); const renderProp = ([name, prop]) => { const isRequired = requiredProps.includes(name); const hasDefault = prop.default !== undefined; const rawDefault = prop.default; const hints = prop.hints || []; const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"'))); const postBadges = []; if (prop.title) { postBadges.push(<>API property: {name}); } const constraints = []; if (prop.minimum !== undefined && prop.maximum !== undefined) { constraints.push(`Range: ${prop.minimum} – ${prop.maximum}`); } else if (prop.minimum !== undefined) { constraints.push(`Min: ${prop.minimum}`); } else if (prop.maximum !== undefined) { constraints.push(`Max: ${prop.maximum}`); } if (prop.minLength !== undefined && prop.maxLength !== undefined) { constraints.push(`Length: ${prop.minLength} – ${prop.maxLength}`); } else if (prop.minLength !== undefined) { constraints.push(`Min length: ${prop.minLength}`); } else if (prop.maxLength !== undefined) { constraints.push(`Max length: ${prop.maxLength}`); } const fieldProps = { key: name, body: prop.title || name, type: prop.type, ...postBadges.length > 0 && ({ post: postBadges }), ...isRequired && ({ required: true }), ...!isComplexDefault && hasDefault ? { default: sanitize(String(rawDefault)) } : {} }; const isObject = prop.type === "object" && prop.properties; const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties; return {prop.description &&

{renderMd(prop.description)}

} {prop.enum &&

Allowed values: {prop.enum.map((v, i) => <>{i > 0 && ", "}{String(v)})}

} {constraints.length > 0 &&

{constraints.join(" · ")}

} {isComplexDefault &&

Default: 

              
                {JSON.stringify(rawDefault, null, 2)}
} {isArrayOfObjects && } {isObject && } ; }; return
{description &&

{renderMd(description)}

} {visibleProps.map(renderProp)}
; }; export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => { const [isLoaded, setIsLoaded] = useState(false); useEffect(() => { const timer = setTimeout(() => { setIsLoaded(true); }, 500); return () => clearTimeout(timer); }, []); return
{isLoaded && ` }} />} Powered by Lucidworks Agent Studio
; }; [localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/document-clustering [mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/document-clustering [old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/x7s5ub The Document Clustering job uses an unsupervised machine learning algorithm to group documents into clusters based on similarities in their content. You can enable more efficient document exploration by using these clusters as facets, high-level summaries or themes, or to recommend other documents from the same cluster. The job can automatically group similar documents in all kinds of content, such as clinical trials, legal documents, book reviews, blogs, scientific papers, and products. Searchable content (your primary collection) The job adds the following fields to the content documents: | | | | ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `cluster_id` | The IDs associated with each cluster so that we can easily identify the clusters by number. A negative number means the document is an outlier or extremely long. | | `cluster_label`, `cluster_label_txt` | Unique keywords assigned to each cluster so that there are no overlapping keywords between the clusters.
● The cluster label `long_doc` is applied to very lengthy documents.
● The cluster label `short_doc` is applied to very short documents.
● Outliers are grouped with cluster labels like `outlier_group0`, `outlier_group1`, and so on. | | `dist_to_center` | The document’s distance from its corresponding cluster center. The shorter the distance, the closer the document is to the center of the cluster. | | `freq_terms`, `freq_terms_txt` | The most frequent words in the cluster. | | `clustering_model_id` | The ID of the Document Clustering job that attached these fields. | The Document Clustering job is an end-to-end job that includes the following: * Document preprocessing * Separating out extremely lengthy documents and outliers (de-noise) * Automatic selection of the number of clusters * Extracting cluster keyword labels You can choose between multiple clustering and featurization methods to find the best combination of methods. The quick learning for **Configuring Fusion’s Document Clustering Job** focuses on how to configure parameters for the Document Clustering Job in the Fusion UI. The course for **Document Clustering** focuses on document clustering methods and job configuration. ## Cluster labels and frequent terms Cluster labels are the terms that best represent the documents in a given cluster. They are typically words that can be found in the documents closest to the cluster centroid, so the `dist_to_center` field can be used to sort documents by similarity to the `cluster_label` field. Frequent terms are the terms that appear most frequently in documents in a given cluster. Different clusters may have overlapping frequent terms. Some of the frequent terms may also appear in the cluster label. ## Configuration tips The minimum required fields to configure are straightforward: * **Training Collection** * **Output Collection** * **Field to Vectorize** When you first create a new Document Clustering job, set the **Training Collection** to point to a special collection that contains a sample set of documents. Set the **Output Collection** to a new, empty collection. That way, you can test the job quickly over a smaller input collection and you can clear the output collection after each test. When you create your sample collection and your test output collection, uncheck **Enable Signals** to prevent secondary collections from being created. When you are satisfied with the results of the job, set *both* the **Training Collection** and the **Output Collection** to your primary collection. It may take some time for the job to run over your entire primary collection. At the end, the new clustering fields are added to your existing searchable content. The sections below discuss additional ways to tune your configuration for the best results. ### Use stopwords The quality of your cluster labels depends on the quality of your [stopwords](/docs/lucidworks-search/07-improve-your-queries/stopwords-file/overview). Lucidworks Search comes with a very basic stopword list, but there may be other words in your corpus that you want to remove. If you see terms in your cluster labels that your do not want, add them to your stopwords and then re-run the job. ### Scale your Spark resources Make sure that you have enough resources in Spark to run the job, especially if you have many documents or your documents are long. If Spark doesn’t have enough memory, you may see out-of-memory errors in the Job History tab. If there are obstacles to scalability, you can index a subset of the text from each document then run the job on this downsized version of the documents. The algorithm does not need all of the text to generate meaningful clusters. Stopword removal also decreases the document size significantly. ### Select the clustering method The job provides these clustering methods: * Hierarchical Bisecting Kmeans (“hierarchical”)\ The default choice is “hierarchical,” which is a mixed method between Kmeans and hierarchical clustering. It can tackle the problem of uneven cluster sizes produced by standard Kmeans, and is more robust regarding initialization. In addition, it runs much faster than the standard hierarchical-clustering method, and has fewer problems dealing with overlapping topic documents. * Standard Kmeans (“kmeans”)\ For use cases such as novel and review clustering, several words can express similar meanings. In that case, Kmeans can perform well in combination with the Word2Vec featurization method described below. This method is also helpful when you have a corpus with a large vocabulary. Kmeans also works well when clusters are "convex", meaning that they are regularly shaped. There are two ways to configure the number of clusters: * Setting **Number Of Clusters** speeds up the processing time, but finding the best single value can be difficult unless you know exactly how many clusters are in the dataset. * Setting **Minimum Possible Number Of Clusters** and **Maximum Possible Number Of Clusters**) allows Lucidworks Search to test up to 20 different values within the configured range to find the best number of clusters for your dataset based on metrics like how far each datapoint is from its associated cluster center. This optimizes the algorithm to detect true groups of similar documents and thus create better-quality clusters.\ For example, if `kMin=2` and `kMax=100`, then the job searches through 2, 7, 12, …, 100 with a step size of 5. A large `kMax` can increase the running time. The algorithm incurs a penalty if k is unnecessarily large. You can use the parameter `kDiscount` to reduce this penalty and use a larger k-chosen. However, if `kMax` is small (for example, 10 or less), then not using a discount (`kDiscount=1`) is recommended. ### Select the featurization method The job provides two text-vectorization methods: * TFIDF You can trim out noisy terms for TFIDF by specifying the **Min Doc Support** and **Max Doc Support** parameters (minimum and maximum number of documents that contain the term).\ If you are using the hierarchical clustering method, then you should apply the TFIDF featurization method; it can provide better-detailed clusters for use cases like clustering email or product descriptions. * Word2Vec\ Word2Vec can reduce dimensions and extract contextual information by putting co-occurring words in the same subspace. However, it can also lose some detailed information by abstraction. If you assign **Word2Vec Dimension** an integer greater than 0, then Lucidworks Search chooses the Word2Vec method over TFIDF.\ If you are using the standard kmeans clustering method, then you should enable the Word2Vec featurization method.\ For a large corpus dataset with a big vocabulary, Word2Vec is preferred to help deal with the dimensionality. ### Configure de-noise parameters The job provides three layers of protection from the impact of noisy documents: * In the `analyzerConfig` parameter, you can specify stopword deletion, stemming, short token treatment, and regular expressions. The `analyzerConfig` is used in the featurization step. * You can add an optional phase to separate out documents that are extremely long or short (as measured by the number of tokens). Extremely short or long documents can contaminate the clustering process. Documents with a length between **Length Threshold for Short Doc** and **Length Threshold for Long Doc** are kept for clustering. * The job performs outlier detection using the Kmeans method. Lucidworks Search groups documents into **Number of Outlier Groups**, then trims out clusters with a size less than **Outlier Cutoff** as outliers.\ Outliers are documents that are too distant from the nearest centroid or nearest neighbors. These distant outliers are grouped into outlier clusters and labeled as `outlier_group0`, `outlier_group1` …​ `outlier_groupN`, where N is the value of **Number of Outlier Groups**. ### Configure cluster labelling The configuration parameter **Number Of Keywords For Each Cluster** determines the number of keywords to pick to describe each cluster. Test different values until you find that the cluster labels give an accurate depiction of the data contained within a given cluster. ### Evaluating and tuning the results When the job has finished: 1. Navigate to the output collection. 2. Open the Query Workbench. 3. Click **Add a Field Facet** and select `cluster_label`.\ The `cluster_label` field contains the five terms that best define the center of each cluster. 4. Click **Add a Field Facet** again and select `freq_terms`.\ The `freq_terms` field contains the five terms that appear most often in each cluster.\ In many cases, the frequent terms in a cluster are also among those that best define it. The `dist_to_center` field can be used to sort documents by similarity to their cluster labels. 5. Explore the facets to determine whether the clusters are useful. * Be sure to examine the `long_doc` cluster and any `outlier_group` clusters. If no outliers are detected, consider increasing `outlierK` or `outlierThreshold`. * If you see terms that you do not want in your cluster labels, add them to your stopwords. * To tune the granularity of your clusters, adjust the values of **Min Possible Number of Clusters** and **Max Possible Number of Clusters**. Increasing these values will break up some of the bigger groups into smaller ones. Decreasing the values can consolidate smaller groups into larger ones. Experiment until you find the level of granularity that produces the most meaningful clusters. * If the clusters are very uneven, such as when most documents are in one large cluster: * Try increasing the outlier cutoff (some outlier groups are being labeled as clusters). * Try increasing k (you have many clusters combined into one). * If you have many outlier groups, but only a few docs per outlier group, try increasing the outlier cutoff to avoid saturating your number of outlier groups before all outliers have been removed. * If the corpus is large and the clustering job is taking too long: * Use Kmeans and Word2Vec (Kmeans does not usually do well with TF\*IDF). * Use an exact k value (**Number Of Clusters**) instead of a range (**Min Possible Number of Clusters** and **Max Possible Number of Clusters**). * If the clustering job completes successfully according to logs, but is not writing data to Solr, check the schema specs on output fields. Solr strings have a maximum length that Spark strings do not. Use text type if your outputs are very long. this problem will only appear if reading from and writing to different collections. 6. Re-run the job and examine the facets again to see whether the results are more useful. ### Back up your primary collection Before running this job over your primary collection, make sure you have a backup of the original content. This can come in handy if you change your mind about the results and want to overwrite the document clustering fields. ## Configuration properties