> ## Documentation Index
> Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt
> Use this file to discover all available pages before exploring further.
# Classification
> Job configuration specifications
export const schema = {
"type": "object",
"title": "Classification",
"description": "Trains a text classification model that assigns category labels to documents or queries. Supports Logistic Regression and Starspace methods. Configure the training collection, text field, and label field to get started. The trained model is deployed automatically using the name specified in `deployModelName`.",
"required": ["id", "trainingCollection", "trainingFormat", "textField", "labelField", "deployModelName", "workflowType", "type"],
"properties": {
"id": {
"type": "string",
"title": "Job ID",
"description": "Sets the unique identifier for this job, used to reference it in the API. Allowed characters: a-z, A-Z, dash (`-`), and underscore (`_`). Must start with a letter.",
"maxLength": 63,
"pattern": "[a-zA-Z][_\\-a-zA-Z0-9]*[a-zA-Z0-9]?"
},
"sparkConfig": {
"type": "array",
"title": "Additional parameters",
"description": "Injects additional key-value pairs into the training JSON map at runtime. String values must be wrapped in quotes. Use this to pass custom parameters not exposed in the UI.",
"hints": ["advanced"],
"items": {
"type": "object",
"required": ["key"],
"properties": {
"key": {
"type": "string",
"title": "Parameter Name"
},
"value": {
"type": "string",
"title": "Parameter Value"
}
}
}
},
"writeOptions": {
"type": "array",
"title": "Write Options",
"description": "Sets additional key-value options passed to the Spark writer when writing output to Solr or other sinks. Use entries like `commitWithin=10000` to control Solr commit behavior.",
"hints": ["advanced"],
"items": {
"type": "object",
"required": ["key"],
"properties": {
"key": {
"type": "string",
"title": "Parameter Name"
},
"value": {
"type": "string",
"title": "Parameter Value"
}
}
}
},
"readOptions": {
"type": "array",
"title": "Read Options",
"description": "Sets additional key-value options passed to the Spark reader when loading input from Solr or other sources. Use these to configure connector-specific behavior such as `rows` or `zkHost`.",
"hints": ["advanced"],
"items": {
"type": "object",
"required": ["key"],
"properties": {
"key": {
"type": "string",
"title": "Parameter Name"
},
"value": {
"type": "string",
"title": "Parameter Value"
}
}
}
},
"stopwordsBlobName": {
"type": "string",
"title": "Stopwords Blob Store",
"description": "Specifies the blob store resource path to a plain-text stopword file with one word per line. The default `stopwords/stopwords_en.txt` is used unless a custom file is uploaded and referenced here.",
"default": "stopwords/stopwords_en.txt",
"reference": "blob",
"blobType": "file:spark"
},
"trainingCollection": {
"type": "string",
"title": "Training data path",
"description": "Specifies the Solr collection or cloud storage path containing labeled training data. This collection must include both the text field and the label field configured below.",
"minLength": 1
},
"trainingFormat": {
"type": "string",
"title": "Training data format",
"description": "Specifies the format of the training data, such as `solr` or `parquet`. This controls which Spark connector loads the training data from `trainingCollection`.",
"default": "solr",
"minLength": 1
},
"secretName": {
"type": "string",
"title": "Cloud storage secret name",
"description": "Specifies the Kubernetes secret used to authenticate access to cloud storage. Define this secret in the K8s namespace before referencing it here. Omit if training data is in Solr.",
"hints": ["advanced"],
"minLength": 1
},
"textField": {
"type": "string",
"title": "Training collection content field",
"description": "Specifies the field in the training collection containing the text content to classify. This field is vectorized and used as the model input during both training and inference.",
"minLength": 1
},
"labelField": {
"type": "string",
"title": "Training collection class field",
"description": "Specifies the field in the training collection containing the ground-truth class labels. Each unique value in this field becomes a classification target during training.",
"minLength": 1
},
"trainingDataFilterQuery": {
"type": "string",
"title": "Training Data Filter Query",
"description": "Filters training data using a Solr query when reading from a Solr collection, or a SQL expression for cloud storage. Use this to restrict training to a specific subset of labeled documents.",
"hints": ["code/sql", "advanced"]
},
"randomSeed": {
"type": "integer",
"title": "Random Seed",
"description": "Sets the random seed used during training and data splitting to produce deterministic results. Fix this value to reproduce identical model outputs across runs.",
"default": 12345,
"hints": ["advanced"]
},
"trainingSampleFraction": {
"type": "number",
"title": "Training Data Sampling Fraction",
"description": "Sets the fraction of training data sampled before model training. Values less than `1.0` reduce training time on very large datasets at the cost of model coverage.",
"default": 1,
"hints": ["advanced"],
"maximum": 1,
"exclusiveMaximum": false
},
"deployModelName": {
"type": "string",
"title": "Model Deployment Name",
"description": "Sets the DNS-compatible name used to identify and deploy the trained model. Must be lowercase, contain no underscores, and be at most 30 characters. This name is used in the Seldon Core deployment.",
"maxLength": 30,
"pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$"
},
"workflowType": {
"type": "string",
"title": "Method",
"description": "Selects the classification algorithm to use for training. Choose `Logistic Regression` for fast, interpretable models on structured text. Choose `Starspace` for embedding-based classification on short or sparse text.",
"enum": ["Logistic Regression", "Starspace"],
"default": "Logistic Regression"
},
"minCharLen": {
"type": "integer",
"title": "Minimum No. of Characters",
"description": "Sets the minimum character length for text samples included in training. Samples shorter than this threshold are excluded. Raising this value removes very short or single-character entries.",
"default": 2,
"minimum": 1,
"exclusiveMinimum": false
},
"maxCharLen": {
"type": "integer",
"title": "Maximum No. of Characters",
"description": "Sets the maximum character length for text samples included in training. Samples longer than this are truncated to this length to avoid out-of-memory issues during vectorization.",
"default": 100000,
"minimum": 1,
"exclusiveMinimum": false
},
"lowercaseTexts": {
"type": "boolean",
"title": "Lowercase Text",
"description": "Controls whether training text is lowercased before featurization. Enabling this normalizes casing and typically improves classification accuracy for case-insensitive domains.",
"default": true
},
"unidecodeTexts": {
"type": "boolean",
"title": "Unidecode Text",
"description": "Controls whether training text is transliterated to ASCII using unidecode before featurization. Enabling this normalizes accented characters and improves consistency across multilingual datasets.",
"default": true
},
"minClassSize": {
"type": "integer",
"title": "Minimum no. of examples per class",
"description": "Sets the minimum number of training examples required for a class to be included in training. Classes with fewer examples than this threshold are dropped to avoid training on unreliable labels.",
"default": 5,
"minimum": 2,
"exclusiveMinimum": false
},
"valSize": {
"type": "number",
"title": "Validation set size",
"description": "Sets the size of the validation set used to evaluate model performance during training. Provide a float between 0 and 1 to sample a fraction, or an integer >= 1 to specify an exact number of records.",
"default": 0.1
},
"topK": {
"type": "integer",
"title": "Number of Output classes",
"description": "Sets the number of most-probable output classes returned per sample along with their confidence scores. Increase this value to return multiple classification candidates for downstream re-ranking.",
"default": 1,
"minimum": 1,
"exclusiveMinimum": false
},
"featurizerType": {
"type": "string",
"title": "Featurizer",
"description": "Selects the text vectorization method used to generate features for Logistic Regression classification. Choose `tfidf` for frequency-weighted token features. Choose `count` for raw token occurrence counts.",
"enum": ["tfidf", "count"],
"default": "tfidf",
"hints": ["advanced"]
},
"useCharacters": {
"type": "boolean",
"title": "Use Characters",
"description": "Controls whether character n-grams or word n-grams are used for vectorization. Use character n-grams for short text such as queries. Word n-grams are preferable for longer documents to avoid excessive memory use.",
"default": true
},
"tokenPattern": {
"type": "string",
"title": "Token filtering pattern",
"description": "Sets the regex pattern used to filter tokens during vectorization. This hidden field defaults to the standard word-boundary pattern. Change it only if your text requires custom tokenization logic.",
"default": "(?u)\\b\\w\\w+\\b",
"hints": ["hidden"]
},
"minDf": {
"type": "number",
"title": "Min Document Frequency",
"description": "Sets the minimum document frequency required for a token to be included in the vocabulary. Provide a float between 0 and 1 for a fraction, or an integer >= 1 for an exact document count.",
"default": 1,
"hints": ["advanced"]
},
"maxDf": {
"type": "number",
"title": "Max Document Frequency",
"description": "Sets the maximum document frequency allowed for a token to be included in the vocabulary. Tokens appearing in more documents than this threshold are treated as stop words and excluded.",
"default": 0.8,
"hints": ["advanced"]
},
"minNgram": {
"type": "integer",
"title": "Min Ngram size",
"description": "Sets the minimum word or character n-gram size generated during vectorization. Use this alongside `maxNgram` to control the range of n-gram features included in the model vocabulary.",
"minimum": 1,
"exclusiveMinimum": false
},
"maxNgram": {
"type": "integer",
"title": "Max Ngram size",
"description": "Sets the maximum word or character n-gram size generated during vectorization. Larger values capture more context but increase vocabulary size and memory requirements.",
"minimum": 1,
"exclusiveMinimum": false
},
"maxFeatures": {
"type": "integer",
"title": "Maximum Vocab Size",
"description": "Sets the maximum number of tokens (including n-grams) included in the vocabulary. Tokens that fall below the frequency cutoff are omitted to bound memory use and reduce noise.",
"default": 250000,
"minimum": 1,
"exclusiveMinimum": false
},
"norm": {
"type": "string",
"title": "Use Norm",
"description": "Selects the vector normalization method applied to feature vectors after TF-IDF weighting. `L2` normalizes by Euclidean length. `L1` by sum of absolute values. `None` applies no normalization.",
"enum": ["None", "L1", "L2"],
"default": "None",
"hints": ["advanced"]
},
"smoothIdf": {
"type": "boolean",
"title": "Smooth IDF",
"description": "Controls whether IDF weights are smoothed by adding 1 to document frequencies before computing log-IDF. Enabling this prevents division-by-zero errors for terms that appear in every document.",
"default": true,
"hints": ["advanced"]
},
"sublinearTf": {
"type": "boolean",
"title": "Sublinear TF",
"description": "Controls whether sublinear TF scaling (replacing TF with `1 + log(TF)`) is applied during vectorization. Enabling this typically improves classification when character n-grams are used on short text.",
"default": true,
"hints": ["advanced"]
},
"scaling": {
"type": "boolean",
"title": "Scale Features",
"description": "Controls whether standard scaling is applied to feature vectors before classification. For sparse vectors, only standard deviation normalization is applied. Enabling this often improves Logistic Regression convergence.",
"default": true
},
"dimReduction": {
"type": "boolean",
"title": "Perform Dimensionality Reduction",
"description": "Controls whether Truncated SVD dimensionality reduction is applied to feature vectors before classification. Enabling this reduces overfitting and speeds up training on high-dimensional TF-IDF vectors.",
"default": false
},
"dimReductionSize": {
"type": "integer",
"title": "Reduced Dimension Size",
"description": "Sets the target number of dimensions after applying Truncated SVD dimensionality reduction. Larger values preserve more information but provide less compression. Set this when `dimReduction` is `true`.",
"default": 256,
"minimum": 1,
"exclusiveMinimum": false
},
"penalty": {
"type": "string",
"title": "Penalty",
"description": "Selects the regularization norm applied to the Logistic Regression objective. `l2` works with most solvers. `elasticnet` requires the `saga` solver. `none` disables regularization, which is not supported by `liblinear`.",
"enum": ["l1", "l2", "elsaticnet", "none"],
"default": "l2",
"hints": ["advanced"]
},
"l1Ratio": {
"type": "number",
"title": "L1 penalty ratio",
"description": "Sets the L1 penalty ratio for `elasticnet` regularization, controlling the mix between L1 and L2 penalties. A value of `0` applies pure L2 regularization. A value of `1` applies pure L1 regularization.",
"default": 0.5,
"hints": ["advanced"],
"maximum": 1,
"exclusiveMaximum": false
},
"tol": {
"type": "number",
"title": "Stopping tolerance",
"description": "Sets the convergence tolerance for the Logistic Regression optimization algorithm. The solver stops when the improvement between iterations falls below this threshold. Smaller values allow more precise convergence.",
"default": 0.0001
},
"reg": {
"type": "number",
"title": "Regularization term",
"description": "Sets the inverse regularization strength for Logistic Regression. Smaller values apply stronger regularization to reduce overfitting. Larger values allow the model to fit training data more closely.",
"default": 1
},
"useClassWeights": {
"type": "boolean",
"title": "Use class weights",
"description": "Controls whether class weights are applied during Logistic Regression training to handle class imbalance. When `true`, each class is weighted inversely proportional to its frequency in the training set.",
"default": false
},
"solver": {
"type": "string",
"title": "Optimization Algorithm",
"description": "Selects the optimization algorithm used to fit the Logistic Regression model. `lbfgs` and `saga` are good defaults. `liblinear` does not support `l2` penalty without limitations.",
"enum": ["lbfgs", "newton-cg", "liblinear", "sag", "saga"],
"default": "lbfgs",
"hints": ["advanced"]
},
"multiClass": {
"type": "string",
"title": "Loss Method",
"description": "Selects the multi-class classification strategy for Logistic Regression. `auto` selects `ovr` for binary data or `liblinear`. `multinomial` uses a joint loss across all classes for better calibration.",
"enum": ["auto", "ovr", "multinomial"],
"default": "auto",
"hints": ["advanced"]
},
"maxIter": {
"type": "integer",
"title": "Maximum iterations for algorithm",
"description": "Sets the maximum number of iterations allowed for the Logistic Regression solver to converge. Increase this value if the solver reports convergence warnings on complex or large datasets.",
"default": 200,
"minimum": 1,
"exclusiveMinimum": false
},
"textLayersSizes": {
"type": "string",
"title": "Hidden sizes before text embedding",
"description": "Specifies the hidden layer sizes before the text embedding layer in the Starspace model as a list, such as `[256, 128]`. Leave blank if no hidden layers are needed before the embedding.",
"default": "[256, 128]",
"pattern": "^(\\[(((\\d)*,\\s*)*(\\d+)+)?\\])?$"
},
"labelLayersSizes": {
"type": "string",
"title": "Hidden sizes before class embedding",
"description": "Specifies the hidden layer sizes before the class embedding layer in the Starspace model as a list, such as `[128]`. Leave blank if no hidden layers are needed before the class embedding.",
"default": "[]",
"pattern": "^(\\[(((\\d)*,\\s*)*(\\d+)+)?\\])?$"
},
"embeddingsSize": {
"type": "integer",
"title": "Embedding size",
"description": "Sets the dimension of the final embedding vectors for both text and class representations in the Starspace model. Larger values capture more semantic information but require more memory and training time.",
"default": 100,
"minimum": 1,
"exclusiveMinimum": false
},
"regTerm": {
"type": "number",
"title": "Regularization Term",
"description": "Sets the L2 regularization scale applied during Starspace training to penalize large embedding weights. Increasing this value reduces overfitting at the cost of model expressiveness.",
"default": 0.002
},
"dropout": {
"type": "number",
"title": "Dropout",
"description": "Sets the dropout probability applied during Starspace training to regularize the embedding layers. Values between `0.1` and `0.5` are typical. Set to `0` to disable dropout.",
"default": 0.2
},
"embeddingReg": {
"type": "number",
"title": "Embedding regularization",
"description": "Sets the scale of the penalty applied to minimize maximum similarity between different class embeddings. Higher values push class embeddings further apart, improving inter-class separation.",
"default": 0.8,
"hints": ["advanced"]
},
"minBatchSize": {
"type": "integer",
"title": "Minimum Batch Size",
"description": "Sets the smallest mini-batch size used at the start of Starspace training. Batch size increases linearly each epoch up to `maxBatchSize`. Starting small helps stabilize early training.",
"default": 64,
"minimum": 1,
"exclusiveMinimum": false
},
"maxBatchSize": {
"type": "integer",
"title": "Maximum Batch Size",
"description": "Sets the largest mini-batch size reached at the end of Starspace training. Batch size grows linearly from `minBatchSize` each epoch. Larger batches improve throughput but increase memory use.",
"default": 128,
"minimum": 1,
"exclusiveMinimum": false
},
"numEpochs": {
"type": "integer",
"title": "Number of training epochs",
"description": "Sets the number of training epochs for the Starspace model. More epochs allow the model to converge more fully but increase training time. Monitor validation metrics to avoid overfitting.",
"default": 40,
"minimum": 1,
"exclusiveMinimum": false
},
"muPos": {
"type": "number",
"title": "Maximum correct class similarity",
"description": "Sets the target similarity threshold that the Starspace model tries to exceed for correct class embeddings. The model maximizes similarity between text and correct class beyond this value.",
"default": 0.8,
"hints": ["advanced"],
"maximum": 1,
"exclusiveMaximum": false
},
"muNeg": {
"type": "number",
"title": "Maximum negative class similarity",
"description": "Sets the similarity threshold that the Starspace model tries to stay below for incorrect class embeddings. The model minimizes similarity between text and negative classes below this value.",
"default": -0.4,
"hints": ["advanced"],
"maximum": 1,
"exclusiveMaximum": false
},
"similarityType": {
"type": "string",
"title": "Similarity type",
"description": "Selects the similarity function used to compare text and class embedding vectors during Starspace training. `cosine` normalizes by vector magnitude. `inner` uses raw dot product similarity.",
"enum": ["cosine", "inner"],
"default": "cosine",
"hints": ["advanced"]
},
"numNeg": {
"type": "integer",
"title": "Number of negative classes for training",
"description": "Sets the number of negative class examples used per training step in the Starspace model. Must be less than the total number of classes. More negatives improve discrimination but slow training.",
"hints": ["advanced"],
"minimum": 1,
"exclusiveMinimum": false
},
"useMaxNegSim": {
"type": "boolean",
"title": "Only minimize max. negative similarity",
"description": "Controls whether only the maximum similarity among negative classes is minimized during Starspace training. When `true`, the model focuses on the hardest negative. When `false`, all negatives are penalized.",
"default": true,
"hints": ["advanced"]
},
"modelReplicas": {
"type": "integer",
"title": "Model replicas",
"description": "Sets the number of Seldon Core replicas deployed for this classification model. Increase this value to handle higher inference throughput or improve availability.",
"default": 1,
"minimum": 1,
"exclusiveMinimum": false
},
"type": {
"type": "string",
"title": "Spark Job Type",
"enum": ["argo-classification"],
"default": "argo-classification",
"hints": ["readonly"]
}
},
"additionalProperties": true,
"category": "Other",
"categoryPriority": 1,
"propertyGroups": [{
"label": "Input/Output Parameters",
"properties": ["deployModelName", "trainingCollection", "trainingFormat", "modelReplicas", "secretName"]
}, {
"label": "Training Data Settings",
"properties": ["trainingDataFilterQuery", "trainingSampleFraction", "randomSeed", "textField", "labelField"]
}, {
"label": "Preprocessing Parameters",
"properties": ["minCharLen", "maxCharLen", "minClassSize", "lowercaseTexts", "unidecodeTexts"]
}, {
"label": "Eval and Output Parameters",
"properties": ["valSize", "topK"]
}, {
"label": "Vectorization Parameters",
"properties": ["featurizerType", "useCharacters", "stopwordsBlobName", "minDf", "maxDf", "minNgram", "maxNgram", "maxFeatures", "norm", "smoothIdf", "sublinearTf", "scaling", "dimReduction", "dimReductionSize"]
}, {
"label": "Logistic Regression Parameters",
"properties": ["penalty", "l1Ratio", "tol", "reg", "useClassWeights", "solver", "multiClass", "maxIter"]
}, {
"label": "Starspace Parameters",
"properties": ["textLayersSizes", "labelLayersSizes", "embeddingsSize", "regTerm", "dropout", "embeddingReg", "minBatchSize", "maxBatchSize", "numEpochs", "muPos", "muNeg", "similarityType", "numNeg", "useMaxNegSim"]
}]
};
export const SchemaParamFields = ({schema}) => {
const sanitize = str => {
if (typeof str !== "string") return str;
return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'");
};
const renderMd = str => {
const s = sanitize(str);
const text = (/[.!?]\)*$/).test(s) ? s : `${s}.`;
return text.split(/(\*\*[^*]+\*\*|_[^_]+_|`[^`]+`)/g).map((part, i) => {
if (part.startsWith("**")) return {part.slice(2, -2)};
if (part.startsWith("_")) return {part.slice(1, -1)};
if (part.startsWith("`")) return {part.slice(1, -1)};
return part;
});
};
const {description, properties = {}, required: requiredProps = []} = schema;
const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]);
const renderProp = ([name, prop]) => {
const isRequired = requiredProps.includes(name);
const hasDefault = prop.default !== undefined;
const rawDefault = prop.default;
const hints = prop.hints || [];
const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"')));
const postBadges = [];
if (prop.title) {
postBadges.push(<>API property: {name});
}
const constraints = [];
if (prop.minimum !== undefined && prop.maximum !== undefined) {
constraints.push(`Range: ${prop.minimum} – ${prop.maximum}`);
} else if (prop.minimum !== undefined) {
constraints.push(`Min: ${prop.minimum}`);
} else if (prop.maximum !== undefined) {
constraints.push(`Max: ${prop.maximum}`);
}
if (prop.minLength !== undefined && prop.maxLength !== undefined) {
constraints.push(`Length: ${prop.minLength} – ${prop.maxLength}`);
} else if (prop.minLength !== undefined) {
constraints.push(`Min length: ${prop.minLength}`);
} else if (prop.maxLength !== undefined) {
constraints.push(`Max length: ${prop.maxLength}`);
}
const fieldProps = {
key: name,
body: prop.title || name,
type: prop.type,
...postBadges.length > 0 && ({
post: postBadges
}),
...isRequired && ({
required: true
}),
...!isComplexDefault && hasDefault ? {
default: sanitize(String(rawDefault))
} : {}
};
const isObject = prop.type === "object" && prop.properties;
const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties;
return
{prop.description &&
{isLoaded && `
}} />}
Powered by Lucidworks Agent Studio
;
};
[localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/classification
[mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/classification
[old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/seo2hq
This job analyzes how your existing documents are categorized and produces a classification model that can be used to predict the categories of new documents at index time.
For detailed configuration instructions and examples, see **Automatically classify new documents at index time** or **Automatically classify new queries**.
You can predict the categories of new documents at index time by using [Classification](/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/classification) to analyze previously-classified documents in your index and produce a training model, then referencing the model in [Machine learning index stage](/docs/lucidworks-search/09-developer-documentation/config-specs/index-pipeline-stages/machine-learning).
**Classification job dataflow (documents)**
```mermaid placement="bottom-right" actions={true} theme={"dark"}
flowchart LR
accTitle: Classification job dataflow
accDescr: A flowchart showing how a data source feeds into an index pipeline with a machine learning stage, and how a classification training job trains a Seldon model used by the ML stage
DS(["Data Source"])
subgraph IP["Index Pipeline"]
ML["Machine Learning Stage"]
end
Docs[("documents\ncollection")]
DocsIn[("documents\ncollection")]
Job["Classification training job\n(docid-description-\nlabel/category)"]
Model[("Deployed Seldon\nmodel")]
DS --> IP --> Docs
DocsIn --> Job --> Model
Model <-->|"document classifier"| IP
```
{/*  */}
## How to configure new document classification
1. Sign in to Lucidworks Search and click your application.
2. Click **Collections > Jobs > Add+ > Classification** to create a new Classification job.
3. In the **Model Deployment Name** field, enter an ID for the new classification model.
4. In the **Training Data Path** field, enter the collection name or cloud storage path where your main content is stored.
5. In the **Training Data Format** field, leave the default `solr` value if the **Training Data Path** is a collection. Otherwise, specify the format of your data in cloud storage.
6. In the **Training collection content field**, enter the name of the field that contains the content to analyze.
The content field you choose depends on your use case and the typical user query types.
For example, you could choose the description field if users tend to make descriptive queries like "4k TV" or "soft waterproof jacket".
But if users are more likely to search for specific brands or products, such as "LG TV" or "North Face jacket", then the product name field might be more suitable.
7. In the **Training collection class field**, enter the name of the field that contains the category data.
For additional configuration details, see [Best practices](#best-practices-for-configuring-the-classification-job) below.
8. Save the job.
9. Specify the model’s name in the [Machine learning stage](/docs/lucidworks-search/09-developer-documentation/config-specs/index-pipeline-stages/machine-learning) of your index pipeline.
10. In the **Model input transformation script** field, enter the following:
```js theme={"dark"}
/*
Name of the document field to feed into the model.
*/
var documentFeatureField = "body_t"
/*
Model input construction.
*/
var modelInput = new java.util.HashMap()
modelInput.put("text", doc.getFirstFieldValue(documentFeatureField))
modelInput
```
11. In the **Model output transformation script** field, enter the following:
```js theme={"dark"}
{/* // In case if top_k_predictions are needed */}
var top1ClassField = "top_1_class_s"
var top1ScoreField = "top_1_score_d"
var topKClassesField = "top_k_classes_ss"
var topKScoresField = "top_k_scores_ds"
var jsonOutput = JSON.parse(modelOutput.get("_rawJsonResponse"))
var parsedOutput = {};
for (var i=0; i
You can predict the categories most likely to satisfy a new query using this workflow:
1. Use the [Build Training Data job](/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/build-training-data) to join your signals data with your catalog data and produce training data in the form of query/class pairs.
2. Use the [Classification job](/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/classification) to train a classification model using the output collection of the Build Training Data job as the training collection.
**Query-time classification workflow**
```mermaid placement="bottom-right" actions={true} theme={"dark"}
flowchart LR
accTitle: Query-time classification workflow
accDescr: A flowchart showing how documents and signals feed into a Build Training Data job, which trains a Classification Job model deployed with Seldon for use in the query pipeline
DocCol[("Documents Collection")]
SigCol[("Signals Collection")]
BTD["Build Training Data job"]
IntCol[("query-docId-category")]
ClassJob["Classification Job: query-docID-department"]
Seldon["Model Deploy with Seldon"]
UI["UI"]
QProfile["Query Profile"]
subgraph qpipe["Query Pipeline"]
ML["ML Stage"]
end
DocCol --> BTD
SigCol --> BTD
BTD --> IntCol
IntCol --> ClassJob
ClassJob --> Seldon
Seldon --> ML
qpipe <--> QProfile
QProfile --> UI
```
{/*  */}
See the detailed steps below.
## To predict the categories of new queries
1. Navigate to **Collections** > **Jobs** > **Add+** > **Build Training Data** to create a new Build Training Data job.
2. Configure the job as follows:
1. In the **Catalog Path** field, enter the collection name or cloud storage path where your main content is stored.
2. In the **Catalog Format** field, enter `solr` if you are analyzing a Solr collection, or another format if your content is in the cloud.
3. In the **Signals Path** field, enter the collection name or cloud storage path where your signals data is stored.
4. In the **Output Path** field, enter the collection name or cloud storage path where you want to store the training data.
5. In the **Category Field in Catalog** field, enter the field name for the category data in your main content.
6. In the **Item ID Field in Catalog** field, enter the field name for the item IDs in your main content.
7. Check that the values of **Item ID Field in Signals** and **Count Field in Signals** match the field names in your signals data.
3. Save the job.
4. Click **Run** > **Start** to run the job.
5. Navigate to **Collections** > **Jobs** > **Add+** > **Classification** to create a new Classification job.
6. Configure the job as follows:
1. In the **Model Deployment Name** field, enter an ID for the new classification model.
2. In the **Training Data Path** field, enter the collection name or cloud storage path from the Build Training Data job’s **Output Path** field.
3. In the **Training Data Format** field, leave the default `solr` value if the **Training Data Path** is a collection or if you used the default format in your Build Training Data job configuration. If you configured the Build Training Data job to output a different format, enter it here.
4. In the **Training collection content field**, enter `query_s`, the default content field name in the Build Training Data job’s output.
5. In the **Training collection class field**, enter `category_s`, the default category field name in the Build Training Data job’s output.
For additional configuration details, see [Best practices](#best-practices-for-configuring-the-classification-job) below.
7. Save the job.
8. Verify that the Build Training Data job has finished successfully.
9. Click **Run** > **Start** to run the job.
10. Navigate to **Indexing** > **Query Workbench** > **Load** and select your query pipeline.
11. Configure the query pipeline as follows:
1. Add a new Machine Learning stage.
2. In the **Model ID** field, enter the name from the Classification job’s **Model Deployment Name** field.
3. In the **Model input transformation script** field, enter the following:
```js theme={"dark"}
var modelInput = new java.util.HashMap()
modelInput.put("text", request.getFirstParam("q"))
modelInput
```
4. In the **Model output transformation script** field, enter the following:
```js theme={"dark"}
{/* // In case if top_k_predictions are needed */}
{/* // To put into response documents (can be done only after Solr Query stage) */}
var jsonOutput = JSON.parse(modelOutput.get("_rawJsonResponse"))
var parsedOutput = {};
for (var i=0; i
This job takes raw text and an associated single class as input. Although it trains on single classes, there is an option to predict the top several classes with their scores.
At a minimum, you must configure these:
* An ID for this job
* A **Method**; Logistic Regression is the default
* A **Model Deployment Name**
* The **Training Collection**
* The **Training collection content field**, the document field containing the raw text
* The **Training collection class field** containing the classes, labels, or other category data for the text
The course for **Classification** focuses on understanding the different classifier models in Fusion.
## Classification at index time
Used at index time, a classification model can be applied to predict the categories of new, incoming documents. To train a model for this use case, use your main content collection as the training collection. The model requires at least 100 examples in the training data for each category predicted.
```mermaid theme={"dark"}
flowchart LR
accTitle: Classification job dataflow
accDescr: A flowchart showing how a data source feeds into an index pipeline with a machine learning stage, and how a classification training job trains a Seldon model used by the ML stage
DS(["Data Source"])
subgraph IP["Index Pipeline"]
ML["Machine Learning Stage"]
end
Docs[("documents\ncollection")]
DocsIn[("documents\ncollection")]
Job["Classification training job\n(docid-description-\nlabel/category)"]
Model[("Deployed Seldon\nmodel")]
DS --> IP --> Docs
DocsIn --> Job --> Model
Model <-->|"document classifier"| IP
```
Once you have run the job, you can specify the model name in the [Machine learning](/docs/lucidworks-search/09-developer-documentation/config-specs/index-pipeline-stages/machine-learning).
## Job flow
The first part of the job is *vectorization* which is the same for all available classification algorithms. Mainly it supports two types of featurization:
* **Character-based** - for queries or short texts, like document titles, sentences, and so on.
* **Word-based** - for long texts like paragraphs, documents, and so on.
The second part is *classification algorithms*:
* **Logistic Regression.** A classical algorithm with a good trade-off between training speed and results quality. It provides a robust baseline out of the box. Consider using it as a first choice.
* **StarSpace.** A deep learning algorithm that jointly trains to maximize similarity between text and correct class and minimize similarity between text and incorrect classes. This usually requires more tuning and time for training, but with potentially more accurate results. Consider using it and then tuning it if better results are needed.
The third part of the job deploys the new classification model to Lucidworks Search using Seldon Core.
## Best practices
These tips describe how to tune the options under **Vectorization Parameters** for best results with different use cases.
### Query intent / short texts
If you want to train a model to predict query intents or to do short text classification, then enable **Use Characters**.
Another vectorization parameter that can improve model quality is **Max Ngram size**, with reasonable defaults between 3 and 5.
The more character ngrams are used the bigger the vocabulary, so it is worthwhile to tune the **Maximum Vocab Size** parameter that controls how many unique tokens will be used. Lower values will make training faster and will prevent overfitting but might provide lower quality too. It’s important to find a good balance.
Activating the advanced **Sublinear TF** option usually helps if characters are used.
### Documents / long texts
If you want to train a model to predict classes for documents or long texts like one or more paragraphs, then uncheck **Use Characters**.
The reasonable values for word-based **Max Ngram size** are 2-3. Be sure to tune **Maximum Vocab Size** parameter too. Usually it’s better to leave the advanced **Sublinear TF** option deactivated.
### Performance tuning
If the text is very long and **Use Characters** is checked, the job may take a lot of memory and possibly fail if the amount of memory requested by the job is not available. This may result in pods being evicted or failing with OOM errors. If you see this happening, try the following:
* Uncheck **Use Characters**.
* Reduce the vocabulary size and ngram range of the documents.
* Allocate more memory to the pod.
### Algorithm-specific
If you are going to train a model via LogisticRegression algorithm, dimensionality reduction usually doesn’t help so it makes sense to leave **Reduce Dimensionality** unchecked. But scaling seems to improve results, so it’s suggested to activate **Scale Features**.
For models trained by StarSpace algorithm it’s vice-versa. Dimensionality reduction usually helps to get better results as well as much faster model training. But scaling usually doesn’t help or might make results a little bit worse.
## Index pipeline configuration
**Model input transformation script**
```js wrap theme={"dark"}
/*
Name of the document field to feed into the model.
*/
var documentFeatureField = "body_t"
/*
Model input construction.
*/
var modelInput = new java.util.HashMap()
modelInput.put("text", doc.getFirstFieldValue(documentFeatureField))
modelInput
```
**Model output transformation script**
```js wrap theme={"dark"}
var top1ClassField = "top_1_class_s"
var top1ScoreField = "top_1_score_d"
doc.addField(top1ClassField, modelOutput.get("top_1_class")[0])
doc.addField(top1ScoreField, modelOutput.get("top_1_score")[0])
```
```js wrap theme={"dark"}
// In case if top_k_predictions are needed
var top1ClassField = "top_1_class_s"
var top1ScoreField = "top_1_score_d"
var topKClassesField = "top_k_classes_ss"
var topKScoresField = "top_k_scores_ds"
var jsonOutput = JSON.parse(modelOutput.get("_rawJsonResponse"))
var parsedOutput = {};
for (var i=0; i