> ## Documentation Index > Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt > Use this file to discover all available pages before exploring further. # Outlier Detection Jobs export const schema = { "type": "object", "title": "Outlier Detection", "description": "Detects outliers in a document collection by clustering documents and flagging those far from any cluster center.", "required": ["id", "trainingCollection", "fieldToVectorize", "dataFormat", "uidField", "outputCollection", "type"], "properties": { "id": { "type": "string", "title": "Spark Job ID", "description": "The ID for this Spark job. Used in the API to reference this job. Allowed characters: a-z, A-Z, dash (-) and underscore (_).", "maxLength": 63, "pattern": "[a-zA-Z][_\\-a-zA-Z0-9]*[a-zA-Z0-9]?" }, "sparkConfig": { "type": "array", "title": "Spark Settings", "description": "Spark configuration settings.", "hints": ["advanced"], "items": { "type": "object", "required": ["key"], "properties": { "key": { "type": "string", "title": "Parameter Name" }, "value": { "type": "string", "title": "Parameter Value" } } } }, "trainingCollection": { "type": "string", "title": "Training Collection", "description": "Specifies the Solr collection or cloud storage path containing training data.", "minLength": 1 }, "fieldToVectorize": { "type": "string", "title": "Field to Vectorize", "description": "Specifies the Solr field containing text to vectorize. Combine multiple fields with weights using the format `field1:weight1,field2:weight2`.", "minLength": 1 }, "dataFormat": { "type": "string", "title": "Data format", "description": "Specifies the Spark-compatible input data format, such as `solr`, `parquet`, or `orc`.", "default": "solr", "minLength": 1 }, "trainingDataFrameConfigOptions": { "type": "object", "title": "Dataframe Config Options", "description": "Sets additional key-value configuration options passed to the Spark DataFrame reader at load time.", "properties": {}, "additionalProperties": { "type": "string" }, "hints": ["advanced"] }, "trainingDataFilterQuery": { "type": "string", "title": "Training data filter query", "description": "Filters training data using a Solr query when reading from Solr, or a SQL expression for other data sources.", "default": "*:*", "hints": ["advanced"] }, "sparkSQL": { "type": "string", "title": "Spark SQL filter query", "description": "Filters the loaded DataFrame using a Spark SQL query before processing. The input data is registered as a temporary table named `spark_input`.", "default": "SELECT * from spark_input", "hints": ["code/sql", "advanced"] }, "trainingDataSamplingFraction": { "type": "number", "title": "Training data sampling fraction", "description": "Sets the fraction of training data to sample before processing. Use a value between `0` and `1`. `1.0` uses all available data.", "default": 1, "hints": ["advanced"], "maximum": 1, "exclusiveMaximum": false }, "randomSeed": { "type": "integer", "title": "Random seed", "description": "Sets the random seed for deterministic operations including sampling and initialization. Fix this value to reproduce identical results across runs.", "default": 1234, "hints": ["advanced"] }, "outputCollection": { "type": "string", "title": "Output Collection", "description": "Specifies the Solr collection where output documents are written.", "minLength": 1 }, "overwriteOutput": { "type": "boolean", "title": "Overwrite Output", "description": "When enabled, overwrites the output collection before writing new results.", "default": true, "hints": ["hidden", "advanced"] }, "dataOutputFormat": { "type": "string", "title": "Data output format", "description": "Specifies the Spark-compatible output format, such as `solr`, `parquet`, or `orc`.", "default": "solr", "hints": ["advanced"], "minLength": 1 }, "sourceFields": { "type": "string", "title": "Fields to Load", "description": "Specifies the document fields to load from the input source.", "hints": ["advanced"] }, "partitionCols": { "type": "string", "title": "Partition fields", "description": "Specifies a comma-delimited list of column names used to partition output when writing to non-Solr sinks.", "hints": ["advanced"] }, "writeOptions": { "type": "array", "title": "Write Options", "description": "Sets additional key-value options passed to the Spark writer when writing output to Solr or other sinks.", "hints": ["advanced"], "items": { "type": "object", "required": ["key"], "properties": { "key": { "type": "string", "title": "Parameter Name" }, "value": { "type": "string", "title": "Parameter Value" } } } }, "readOptions": { "type": "array", "title": "Read Options", "description": "Sets additional key-value options passed to the Spark reader when loading input from Solr or other sources.", "hints": ["advanced"], "items": { "type": "object", "required": ["key"], "properties": { "key": { "type": "string", "title": "Parameter Name" }, "value": { "type": "string", "title": "Parameter Value" } } } }, "modelId": { "type": "string", "title": "Model ID", "description": "Specifies the identifier for the trained model. Defaults to the Spark job ID if not provided.", "hints": ["advanced"], "minLength": 1 }, "outlierGroupIdField": { "type": "string", "title": "Output Field Name for Outlier Group Id", "description": "Specifies the output field name where the outlier group ID is stored on each document.", "default": "outlier_group_id" }, "outlierGroupLabelField": { "type": "string", "title": "Top Unique Terms Field Name", "description": "Specifies the output field name where the top unique terms describing the outlier group are stored.", "default": "outlier_group_label" }, "outputOutliersOnly": { "type": "boolean", "title": "Only save outliers?", "description": "When `true`, writes only outlier documents to the output collection. When `false`, writes the entire dataset.", "default": false }, "uidField": { "type": "string", "title": "ID Field Name", "description": "Specifies the document field used as the unique identifier. Default is `id`.", "default": "id", "minLength": 1 }, "analyzerConfig": { "type": "string", "title": "Lucene Analyzer Schema", "description": "Specifies the Lucene analyzer configuration in JSON format used for text tokenization and processing.", "default": "{ \"analyzers\": [{ \"name\": \"StdTokLowerStop\",\"charFilters\": [ { \"type\": \"htmlstrip\" } ],\"tokenizer\": { \"type\": \"standard\" },\"filters\": [{ \"type\": \"lowercase\" },{ \"type\": \"KStem\" },{ \"type\": \"length\", \"min\": \"2\", \"max\": \"32767\" },{ \"type\": \"fusionstop\", \"ignoreCase\": \"true\", \"format\": \"snowball\", \"words\": \"org/apache/lucene/analysis/snowball/english_stop.txt\" }] }],\"fields\": [{ \"regex\": \".+\", \"analyzer\": \"StdTokLowerStop\" } ]}", "hints": ["lengthy", "code/json"], "minLength": 1 }, "freqTermField": { "type": "string", "title": "Top Frequent Terms Field Name", "description": "Specifies the output field name where the top frequent terms for each cluster are stored.", "default": "freq_terms" }, "distToCenterField": { "type": "string", "title": "Output Field Name for doc distance to its cluster center", "description": "Specifies the output field name where each document's distance to its cluster center is stored.", "default": "dist_to_center" }, "norm": { "type": "integer", "title": "Vector normalization", "description": "Sets the p-norm used to normalize feature vectors. Use `-1` to disable normalization.", "enum": [-1, 0, 1, 2], "default": 2, "hints": ["advanced"] }, "minDF": { "type": "number", "title": "Min Doc Support", "description": "Sets the minimum document frequency for terms to be included. Values less than `1.0` are treated as a fraction. `1.0` or above as an absolute count.", "default": 5 }, "maxDF": { "type": "number", "title": "Max Doc Support", "description": "Sets the maximum document frequency for terms to be included. Values less than `1.0` are treated as a fraction. `1.0` or above as an absolute count.", "default": 0.75 }, "numKeywordsPerLabel": { "type": "integer", "title": "Number of Keywords for Each Cluster", "description": "Sets the number of keywords used to label each cluster.", "default": 5 }, "outlierK": { "type": "integer", "title": "Number of outlier groups", "description": "Sets the number of clusters used to identify outliers.", "default": 10, "hints": ["advanced"] }, "outlierThreshold": { "type": "number", "title": "Outlier cutoff", "description": "Sets the distance threshold above which a document is classified as an outlier.", "default": 0.01, "hints": ["advanced"] }, "stopwordsList": { "type": "array", "title": "List of stopwords", "description": "Specifies stop words defined in the Lucene analyzer configuration to exclude from processing.", "hints": ["readonly", "hidden"], "items": { "type": "string", "minLength": 1, "reference": "blob", "blobType": "file:spark" } }, "type": { "type": "string", "title": "Spark Job Type", "enum": ["outlier_detection"], "default": "outlier_detection", "hints": ["readonly"] } }, "additionalProperties": true, "category": "Other", "categoryPriority": 1, "propertyGroups": [{ "label": "Input/Output Parameters", "properties": ["trainingCollection", "outputCollection", "dataFormat", "trainingDataFilterQuery", "readOptions", "writeOptions", "trainingDataFrameConfigOptions", "trainingDataSamplingFraction", "randomSeed", "outputOutliersOnly"] }, { "label": "Field Parameters", "properties": ["fieldToVectorize", "sourceFields", "uidField", "outlierGroupIdField", "outlierGroupLabelField", "freqTermField", "distToCenterField"] }, { "label": "Model Tuning Parameters", "properties": ["outlierK", "outlierThreshold", "maxDF", "minDF", "norm", "numKeywordsPerLabel"] }, { "label": "Featurization Parameters", "properties": ["analyzerConfig"] }, { "label": "Misc. Parameters", "properties": ["modelId"] }] }; export const SchemaParamFields = ({schema}) => { const sanitize = str => { if (typeof str !== "string") return str; return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'"); }; const renderMd = str => { const s = sanitize(str); const text = (/[.!?]\)*$/).test(s) ? s : `${s}.`; return text.split(/(\*\*[^*]+\*\*|_[^_]+_|`[^`]+`)/g).map((part, i) => { if (part.startsWith("**")) return {part.slice(2, -2)}; if (part.startsWith("_")) return {part.slice(1, -1)}; if (part.startsWith("`")) return {part.slice(1, -1)}; return part; }); }; const {description, properties = {}, required: requiredProps = []} = schema; const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]); const renderProp = ([name, prop]) => { const isRequired = requiredProps.includes(name); const hasDefault = prop.default !== undefined; const rawDefault = prop.default; const hints = prop.hints || []; const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"'))); const postBadges = []; if (prop.title) { postBadges.push(<>API property: {name}); } const constraints = []; if (prop.minimum !== undefined && prop.maximum !== undefined) { constraints.push(`Range: ${prop.minimum} – ${prop.maximum}`); } else if (prop.minimum !== undefined) { constraints.push(`Min: ${prop.minimum}`); } else if (prop.maximum !== undefined) { constraints.push(`Max: ${prop.maximum}`); } if (prop.minLength !== undefined && prop.maxLength !== undefined) { constraints.push(`Length: ${prop.minLength} – ${prop.maxLength}`); } else if (prop.minLength !== undefined) { constraints.push(`Min length: ${prop.minLength}`); } else if (prop.maxLength !== undefined) { constraints.push(`Max length: ${prop.maxLength}`); } const fieldProps = { key: name, body: prop.title || name, type: prop.type, ...postBadges.length > 0 && ({ post: postBadges }), ...isRequired && ({ required: true }), ...!isComplexDefault && hasDefault ? { default: sanitize(String(rawDefault)) } : {} }; const isObject = prop.type === "object" && prop.properties; const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties; return {prop.description &&

{renderMd(prop.description)}

} {prop.enum &&

Allowed values: {prop.enum.map((v, i) => <>{i > 0 && ", "}{String(v)})}

} {constraints.length > 0 &&

{constraints.join(" · ")}

} {isComplexDefault &&

Default: 

              
                {JSON.stringify(rawDefault, null, 2)}
} {isArrayOfObjects && } {isObject && } ; }; return
{description &&

{renderMd(description)}

} {visibleProps.map(renderProp)}
; }; export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => { const [isLoaded, setIsLoaded] = useState(false); useEffect(() => { const timer = setTimeout(() => { setIsLoaded(true); }, 500); return () => clearTimeout(timer); }, []); return
{isLoaded && ` }} />} Powered by Lucidworks Agent Studio
; }; [localhost link]: http://localhost:3000/docs/5/fusion/reference/config-ref/jobs/outlier-detection [mintlify link]: https://doc.lucidworks.com/docs/5/fusion/reference/config-ref/jobs/outlier-detection [old doc.lw link]: https://doc.lucidworks.com/fusion/5.9/578 Outlier detection jobs are run against your data collections, and also perform the following actions: * Identify information that significantly differs from other data in the collection * Attach labels to designate each outlier group To create an Outlier Detection job, sign in to Fusion and click **Collections > Jobs**. Then click **Add+** and in the Clustering and Outlier Analysis Jobs section, select **Outlier Detection**. You can enter basic and advanced parameters to configure the job. If the field has a default value, it is populated when you click to add the job. ## Basic parameters To enter advanced parameters in the UI, click **Advanced**. Those parameters are described in [the advanced parameters section](#advanced-parameters). * **Spark job ID.** The unique ID for the Spark job that references this job in the API. This is the `id` field in the configuration file. Required field. * **Input/Output Parameters.** This section includes these parameters: * **Training collection.** The Solr collection that contains documents that will be clustered. The job will be run against this information. This is the `trainingCollection` field in the configuration file. Required field. * **Output collection.** The Solr collection where the job output is stored. The job will write the output to this collection. This is the `outputCollection` field in the configuration file. Required field. * **Data format.** The format that contains training data. The format must be compatible with Spark and options include `solr`, `parquet`, and `orc`. This is the `dataFormat` field in the configuration file. Required field. * **Only save outliers?** If this checkbox is selected (set to `true`), only outliers are saved in the job’s output collection. If not selected (set to `false`), the entire dataset is saved in the job’s output collection. This is the `outputOutliersOnly` field in the configuration file. Optional field. * **Field Parameters.** This section includes these parameters: * **Field to vectorize.** The Solr field that contains text training data. To combine data from multiple fields with different weights, enter `field1:weight1`,`field2:weight2`, etc. This is the `fieldToVectorize` field in the configuration file. Required field. * **ID field name.** The unique ID for each document. This is the `uidField` field in the configuration file. Required field. * **Output field name for outlier group ID.** The field that contains the ID for the outlier group. This is the `outlierGroupIdField` field in the configuration file. Optional field. * **Top unique terms field name.** The field where the job output stores the top frequent terms that, for the most part, are unique for each outlier group. The information is computed based on term frequency-inverse document frequency (TF-IDF) and group ID. This is the `outlierGroupLabelField` field in the configuration file. Optional field. * **Top frequent terms field name.** The field where the job output stores top frequent terms in each cluster. Terms may overlap with other clusters. This is the `freqTermField` field in the configuration file. Optional field. * **Output field name for doc distance to its corresponding cluster center.** The field that contains the document’s distance from the center of its cluster. This is based on the arithmetic mean of all of the documents in the cluster. This denotes how representative the document is in the cluster. This is the `distToCenterField` field in the configuration file. Optional field. * **Model Tuning Parameters.** This section includes these parameters: * **Max doc support.** The maximum number of documents that can contain the term. Values that are `<1.0` indicate a percentage, `1.0` is 100 percent, and `>1.0` indicates the exact number. This is the `maxDF` field in the configuration file. Optional field. * **Min doc support.** The minimum number of documents that must contain the term. Values that are `<1.0` indicate a percentage, `1.0` is 100 percent, and `>1.0` indicates the exact number. This is the `minDF` field in the configuration file. Optional field. * **Number of keywords for each cluster.** The number of keywords required to label each cluster. This is the `numKeywordsPerLabel` field in the configuration file. Optional field. * **Featurization Parameters.** This section includes the following parameter: * **Lucene analyzer schema.** This is the JSON-encoded Lucene text analyzer schema used for tokenization. This is the `analyzerConfig` field in the configuration file. Optional field. ## Advanced parameters If you click the **Advanced** toggle, the following optional fields are displayed in the UI. * **Spark Settings.** The Spark configuration settings include the following: * **Spark SQL filter query.** This field contains the Spark SQL query that filters your input data. For example, `SELECT * from spark_input` registers the input data as `spark_input`. This is the `sparkSQL` field in the configuration file. * **Data output format.** The format for the job output. The format must be compatible with Spark and options include `solr` and `parquet`. This is the `dataOutputFormat` field in the configuration file. * **Partition fields.** If the job output is written to non-Solr sources, this field contains a comma-delimited list of column names that partition the dataframe before the external output is written. This is the `partitionCols` field in the configuration file. * **Input/Output Parameters.** This advanced option adds these parameters: * **Training data filter query.** If Solr is used, this field contains the Solr query executed to load training data. This is the `trainingDataFilterQuery` field in the configuration file. * **Read Options.** This section lets you enter `parameter name:parameter value` options to use when reading input from Solr or other sources. This is the `readOptions` field in the configuration file. * **Write Options.** This section lets you enter `parameter name:parameter value` options to use when writing output to Solr or other sources. This is the `writeOptions` field in the configuration file. * **Dataframe config options.** This section includes these parameters: * **Property name:property value.** Each entry defines an additional Spark dataframe loading configuration option. This is the `trainingDataFrameConfigOptions` field in the configuration file. * **Training data sampling fraction.** This is the fractional amount of the training data the job will use. This is the `trainingDataSamplingFraction` field in the configuration file. * **Random seed.** This value is used in any deterministic pseudorandom number generation to group documents into clusters based on similarities in their content. This is the `randomSeed` field in the configuration file. * **Field Parameters.** The advanced option adds this parameter: * **Fields to load.** This field contains a comma-delimited list of Solr fields to load. If blank, the job selects the required fields to load at runtime. This is the `sourceFields` field in the configuration file. * **Model Tuning Parameters.** The advanced option adds these parameters: * **Number of outlier groups.** The number of clusters to help find outliers. This is the `outlierK` field in the configuration file. * **Outlier cutoff.** The fraction out of the total documents to designate as an outlier group. Values that are `<1.0` indicate a percentage, `1.0` is 100 percent, and `>1.0` indicates the exact number. This is the `outlierThreshold` field in the configuration file. * **Vector normalization.** The p-norm value used to normalize vectors. A value of `-1` turns off normalization. This is the `norm` field in the configuration file. * **Miscellaneous Parameters.** This section includes this parameter: * **Model ID.** The unique identifier for the model to be trained. If no value is entered, the `Spark Job ID` is used. This is the `modelId` field in the configuration file. ## Configuration properties