> ## Documentation Index
> Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Build Training Data

> Job configuration specifications

export const schema = {
  "type": "object",
  "title": "Build Training Data",
  "description": "Builds training data for query classification by joining click signals with a product catalog. Use this job to generate labeled query-category pairs needed to train a classification model. The output can be consumed directly by the Classification job.",
  "required": ["id", "fieldToVectorize", "catalogPath", "catalogFormat", "signalsPath", "outputPath", "categoryField", "catalogIdField", "itemIdField", "countField", "analyzerConfig", "type"],
  "properties": {
    "id": {
      "type": "string",
      "title": "Spark Job ID",
      "description": "Sets the unique identifier for this Spark job, used to reference it in the API. Allowed characters: a-z, A-Z, dash (`-`), and underscore (`_`). Must start with a letter.",
      "maxLength": 63,
      "pattern": "[a-zA-Z][_\\-a-zA-Z0-9]*[a-zA-Z0-9]?"
    },
    "sparkConfig": {
      "type": "array",
      "title": "Spark Settings",
      "description": "Sets Spark configuration key-value pairs applied to this job's execution context. Use entries like `spark.executor.memory=4g` to tune resource allocation beyond cluster defaults.",
      "hints": ["advanced"],
      "items": {
        "type": "object",
        "required": ["key"],
        "properties": {
          "key": {
            "type": "string",
            "title": "Parameter Name"
          },
          "value": {
            "type": "string",
            "title": "Parameter Value"
          }
        }
      }
    },
    "trainingCollection": {
      "type": "string",
      "title": "Training Collection",
      "description": "Specifies the Solr collection containing labeled training data. This field is hidden. Use `signalsPath` and `catalogPath` to configure the actual data sources.",
      "hints": ["dummy", "hidden"],
      "minLength": 1
    },
    "fieldToVectorize": {
      "type": "string",
      "title": "Query Field",
      "description": "Specifies the field containing query strings used to generate training features. Maps the `query_s` field by default. Change this to match the query field in your signals collection.",
      "default": "query_s",
      "minLength": 1
    },
    "dataFormat": {
      "type": "string",
      "title": "Signals Format",
      "description": "Specifies the Spark-compatible format of the signals training data, such as `solr`, `parquet`, or `orc`. This controls which Spark connector loads the signals DataFrame.",
      "default": "solr",
      "hints": ["dummy"],
      "minLength": 1
    },
    "trainingDataFrameConfigOptions": {
      "type": "object",
      "title": "Dataframe Config Options",
      "description": "Sets additional Spark DataFrame loading options as key-value pairs applied when reading training data. Use connector-specific options such as `zkHost` or `collection` for non-default sources.",
      "properties": {},
      "additionalProperties": {
        "type": "string"
      },
      "hints": ["advanced"]
    },
    "trainingDataFilterQuery": {
      "type": "string",
      "title": "Signal Data Filter Query",
      "description": "Filters signals data using a Solr query before joining with the catalog. For non-Solr data sources, use the Spark SQL filter query field in the Advanced section instead.",
      "default": "*:*",
      "hints": ["dummy"]
    },
    "sparkSQL": {
      "type": "string",
      "title": "Spark SQL filter query",
      "description": "Filters input signals data using a Spark SQL expression before processing. The input data is registered as `spark_input`, allowing standard SQL `WHERE` clauses and joins.",
      "default": "SELECT * from spark_input",
      "hints": ["code/sql", "advanced"]
    },
    "trainingDataSamplingFraction": {
      "type": "number",
      "title": "Training data sampling fraction",
      "description": "Sets the fraction of training data sampled before building the training dataset. Values less than `1.0` reduce dataset size to speed up iteration. Use on very large signals collections.",
      "default": 1,
      "hints": ["advanced"],
      "maximum": 1,
      "exclusiveMaximum": false
    },
    "randomSeed": {
      "type": "integer",
      "title": "Random seed",
      "description": "Sets the random seed used during data sampling to produce deterministic results. Fix this value to reproduce the same training dataset across multiple runs.",
      "default": 1234,
      "hints": ["advanced"]
    },
    "outputCollection": {
      "type": "string",
      "title": "Output Collection",
      "description": "Specifies the Solr collection where model-labeled training output is stored. This field is hidden. Use `outputPath` to configure the actual output destination.",
      "hints": ["dummy", "hidden"]
    },
    "overwriteOutput": {
      "type": "boolean",
      "title": "Overwrite Output",
      "description": "Controls whether the output collection is overwritten when the job runs. When `true`, existing output data is replaced. When `false`, the job may fail if the output already exists.",
      "default": true,
      "hints": ["hidden", "advanced"]
    },
    "dataOutputFormat": {
      "type": "string",
      "title": "Data output format",
      "description": "Specifies the Spark-compatible format for writing the output training data, such as `solr` or `parquet`. Must match the target storage system specified in `outputPath`.",
      "default": "solr",
      "hints": ["dummy"],
      "minLength": 1
    },
    "sourceFields": {
      "type": "string",
      "title": "Fields to Load",
      "description": "Specifies the field names to load from the input source, limiting which columns are read into the DataFrame. This field is hidden. Leave unset to load all available fields.",
      "hints": ["dummy", "hidden"]
    },
    "partitionCols": {
      "type": "string",
      "title": "Partition fields",
      "description": "Specifies a comma-delimited list of column names used to partition the output DataFrame before writing to non-Solr storage. Partition columns improve downstream query performance by enabling file pruning.",
      "hints": ["advanced"]
    },
    "writeOptions": {
      "type": "array",
      "title": "Write Options",
      "description": "Sets additional key-value options passed to the Spark writer when writing output to Solr or other sinks. Use entries like `commitWithin=10000` to control Solr commit behavior.",
      "hints": ["advanced"],
      "items": {
        "type": "object",
        "required": ["key"],
        "properties": {
          "key": {
            "type": "string",
            "title": "Parameter Name"
          },
          "value": {
            "type": "string",
            "title": "Parameter Value"
          }
        }
      }
    },
    "readOptions": {
      "type": "array",
      "title": "Read Options",
      "description": "Sets additional key-value options passed to the Spark reader when loading input data from Solr or other sources. Use these to configure connector-specific behavior such as `rows` or `zkHost`.",
      "hints": ["advanced"],
      "items": {
        "type": "object",
        "required": ["key"],
        "properties": {
          "key": {
            "type": "string",
            "title": "Parameter Name"
          },
          "value": {
            "type": "string",
            "title": "Parameter Value"
          }
        }
      }
    },
    "catalogPath": {
      "type": "string",
      "title": "Catalog Path",
      "description": "Specifies the file system or URL path to the product catalog data. This path is joined with signals data to associate items with their categories for training label generation."
    },
    "catalogFormat": {
      "type": "string",
      "title": "Catalog Format",
      "description": "Specifies the Spark-compatible format of the catalog data source, such as `solr`, `parquet`, or `orc`. This controls which Spark connector loads the catalog DataFrame."
    },
    "signalsPath": {
      "type": "string",
      "title": "Signals Path",
      "description": "Specifies the file system or URL path to the click signals data. This path is joined with catalog data using item ID fields to produce labeled query-category training pairs."
    },
    "outputPath": {
      "type": "string",
      "title": "Output Path",
      "description": "Specifies the file system or URL path where the labeled training output is written. This output is typically consumed by a Classification job to train a query classifier."
    },
    "categoryField": {
      "type": "string",
      "title": "Category Field in Catalog",
      "description": "Specifies the field in the catalog that contains item category labels. This field is used as the classification target when joining signals with catalog data."
    },
    "catalogIdField": {
      "type": "string",
      "title": "Item Id Field in Catalog",
      "description": "Specifies the field in the catalog used as the item ID for joining with signals data. This must match the field referenced by `itemIdField` in the signals collection."
    },
    "itemIdField": {
      "type": "string",
      "title": "Item Id Field in Signals",
      "description": "Specifies the field in the signals data containing item identifiers used to join with the catalog. Maps the `doc_id_s` field by default. Adjust to match your signals collection schema.",
      "default": "doc_id_s"
    },
    "countField": {
      "type": "string",
      "title": "Count Field in Signals",
      "description": "Specifies the field in the signals data containing interaction counts used to weight training examples. Maps the `aggr_count_i` field by default. Adjust to match your aggregation collection schema.",
      "default": "aggr_count_i"
    },
    "topCategoryProportion": {
      "type": "number",
      "title": "Top Category Proportion",
      "description": "Sets the minimum proportion that a query's top category must represent among all associated categories for the query to be included in training. Queries where no category dominates are excluded to reduce label noise.",
      "default": 0.5
    },
    "topCategoryThreshold": {
      "type": "integer",
      "title": "Minimum Count",
      "description": "Sets the minimum count a query must have to be included in the training dataset. Queries with fewer occurrences than this threshold are excluded to remove rare or low-signal training examples.",
      "default": 1,
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "analyzerConfig": {
      "type": "string",
      "title": "Lucene Text Analyzer",
      "description": "Sets the Lucene text analyzer configuration applied to query strings during featurization. The JSON defines tokenizer, char filters, and token filters. The default uses standard tokenization with lowercasing.",
      "default": "{ \"analyzers\": [{ \"name\": \"StdTokLowerStop\",\"charFilters\": [ { \"type\": \"htmlstrip\" } ],\"tokenizer\": { \"type\": \"standard\" },\"filters\": [{ \"type\": \"lowercase\" }] }],\"fields\": [{ \"regex\": \".+\", \"analyzer\": \"StdTokLowerStop\" } ]}",
      "hints": ["lengthy", "code/json"]
    },
    "type": {
      "type": "string",
      "title": "Spark Job Type",
      "enum": ["build-training"],
      "default": "build-training",
      "hints": ["readonly"]
    }
  },
  "additionalProperties": true,
  "category": "Other",
  "categoryPriority": 1,
  "propertyGroups": [{
    "label": "Input/Output Parameters",
    "properties": ["trainingCollection", "outputCollection", "dataFormat", "trainingDataFilterQuery", "readOptions", "writeOptions", "trainingDataFrameConfigOptions", "trainingDataSamplingFraction", "randomSeed", "catalogPath", "catalogFormat", "signalsPath", "outputPath", "dataOutputFormat", "partitionCols", "sparkSQL"]
  }, {
    "label": "Field Parameters",
    "properties": ["fieldToVectorize", "sourceFields", "categoryField", "catalogIdField", "itemIdField", "countField"]
  }, {
    "label": "Training Parameters",
    "properties": ["topCategoryProportion", "topCategoryThreshold"]
  }, {
    "label": "Featurization Parameters",
    "properties": ["analyzerConfig"]
  }]
};

export const SchemaParamFields = ({schema}) => {
  const sanitize = str => {
    if (typeof str !== "string") return str;
    return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'");
  };
  const renderMd = str => {
    const s = sanitize(str);
    const text = (/[.!?]\)*$/).test(s) ? s : `${s}.`;
    return text.split(/(\*\*[^*]+\*\*|_[^_]+_|`[^`]+`)/g).map((part, i) => {
      if (part.startsWith("**")) return <strong key={i}>{part.slice(2, -2)}</strong>;
      if (part.startsWith("_")) return <em key={i}>{part.slice(1, -1)}</em>;
      if (part.startsWith("`")) return <code key={i}>{part.slice(1, -1)}</code>;
      return part;
    });
  };
  const {description, properties = {}, required: requiredProps = []} = schema;
  const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]);
  const renderProp = ([name, prop]) => {
    const isRequired = requiredProps.includes(name);
    const hasDefault = prop.default !== undefined;
    const rawDefault = prop.default;
    const hints = prop.hints || [];
    const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"')));
    const postBadges = [];
    if (prop.title) {
      postBadges.push(<><span className="text-stone-400 dark:text-stone-500">API property: </span>{name}</>);
    }
    const constraints = [];
    if (prop.minimum !== undefined && prop.maximum !== undefined) {
      constraints.push(`Range: ${prop.minimum} – ${prop.maximum}`);
    } else if (prop.minimum !== undefined) {
      constraints.push(`Min: ${prop.minimum}`);
    } else if (prop.maximum !== undefined) {
      constraints.push(`Max: ${prop.maximum}`);
    }
    if (prop.minLength !== undefined && prop.maxLength !== undefined) {
      constraints.push(`Length: ${prop.minLength} – ${prop.maxLength}`);
    } else if (prop.minLength !== undefined) {
      constraints.push(`Min length: ${prop.minLength}`);
    } else if (prop.maxLength !== undefined) {
      constraints.push(`Max length: ${prop.maxLength}`);
    }
    const fieldProps = {
      key: name,
      body: prop.title || name,
      type: prop.type,
      ...postBadges.length > 0 && ({
        post: postBadges
      }),
      ...isRequired && ({
        required: true
      }),
      ...!isComplexDefault && hasDefault ? {
        default: sanitize(String(rawDefault))
      } : {}
    };
    const isObject = prop.type === "object" && prop.properties;
    const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties;
    return <ParamField {...fieldProps}>
        {prop.description && <p>{renderMd(prop.description)}</p>}

        {prop.enum && <p>
            Allowed values: 
            {prop.enum.map((v, i) => <>{i > 0 && ", "}<code key={i}>{String(v)}</code></>)}
          </p>}

        {constraints.length > 0 && <p className="text-stone-500 dark:text-stone-400 text-sm">
            {constraints.join(" · ")}
          </p>}

        {isComplexDefault && <div className="flex">
            <p>
              <strong>Default:</strong>
            </p>
            <pre className="!my-0">
              <code>
                {JSON.stringify(rawDefault, null, 2)}
              </code>
            </pre>
          </div>}

        {isArrayOfObjects && <Expandable title="item properties">
            <SchemaParamFields schema={{
      properties: prop.items.properties,
      required: prop.items.required
    }} />
          </Expandable>}

        {isObject && <Expandable title="properties">
            <SchemaParamFields schema={{
      properties: prop.properties,
      required: prop.required
    }} />
          </Expandable>}
      </ParamField>;
  };
  return <div>
      {description && <p>{renderMd(description)}</p>}

      {visibleProps.map(renderProp)}
    </div>;
};

export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => {
  const [isLoaded, setIsLoaded] = useState(false);
  useEffect(() => {
    const timer = setTimeout(() => {
      setIsLoaded(true);
    }, 500);
    return () => clearTimeout(timer);
  }, []);
  return <div className="lw-template-container">
      <Card title={title} icon={icon}>
        {isLoaded && <span dangerouslySetInnerHTML={{
    __html: `<lw-template id="a029c1a9-28be-427e-b0e1-5d918920246a"></lw-template
            >`
  }} />}
        <Link href={linkHref} className="agent-studio-link text-left text-gray-600 gap-2 dark:text-gray-400 text-sm font-medium flex flex-row items-center hover:text-primary dark:hover:text-primary-light group-hover:text-primary group-hover:dark:text-primary-light">Powered by Lucidworks Agent Studio</Link>
      </Card>
    </div>;
};

[localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/build-training-data

[mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/build-training-data

[old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/dwgzki

Use this job to build training data for query classification by joining signals data with catalog data.

The output of this job can be used as input for the [Classification job](/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/classification), which analyzes how documents are categorized and generates a model. That model can then be used to predict categories of new documents when they are indexed.

The Build Training Data job can be configured in **Collections > Jobs** in your Lucidworks Search UI instance.

Enter the following information:

* Spark Job ID used by the API to reference the job.
* Location where your signals are stored, the Spark-compatible format for the signals, and filters for the query.
* Location where your content catalog is stored and the Spark-compatible format of the catalog data.
* Location and Spark-compatible format of the job output.
* Field names for the query string, the category and item from the catalog, the signals item ID, and the signals count field.
* Style of the text analyzer you want to use for the job.
* Top category proportion in relation to all the categories, and the minimum number of the query category pair counts.

For detailed configuration steps, see **Automatically classify new queries**. That process lets you predict the categories that are most likely to be returned successfully in a query.

<Accordion title="Automatically classify new queries">
  You can predict the categories most likely to satisfy a new query using this workflow:

  1. Use the [Build Training Data job](/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/build-training-data) to join your signals data with your catalog data and produce training data in the form of query/class pairs.
  2. Use the [Classification job](/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/classification) to train a classification model using the output collection of the Build Training Data job as the training collection.

  **Query-time classification workflow**

  ```mermaid placement="bottom-right" actions={true} theme={"dark"}
  flowchart LR
  accTitle: Query-time classification workflow
  accDescr: A flowchart showing how documents and signals feed into a Build Training Data job, which trains a Classification Job model deployed with Seldon for use in the query pipeline
      DocCol[("Documents Collection")]
      SigCol[("Signals Collection")]
      BTD["Build Training Data job"]
      IntCol[("query-docId-category")]
      ClassJob["Classification Job: query-docID-department"]
      Seldon["Model Deploy with Seldon"]
      UI["UI"]

      QProfile["Query Profile"]

      subgraph qpipe["Query Pipeline"]
          ML["ML Stage"]
      end

      DocCol --> BTD
      SigCol --> BTD
      BTD --> IntCol
      IntCol --> ClassJob
      ClassJob --> Seldon
      Seldon --> ML
      qpipe <--> QProfile
      QProfile --> UI
  ```

  {/* ![Query-time classification workflow](/assets/images/5.3/diagrams/AI_Diagrams_DC-QueryClassification.png) */}

  See the detailed steps below.

  <LwTemplate />

  ## To predict the categories of new queries

  1. Navigate to **Collections** > **Jobs** > **Add+** > **Build Training Data** to create a new Build Training Data job.
  2. Configure the job as follows:
     1. In the **Catalog Path** field, enter the collection name or cloud storage path where your main content is stored.
     2. In the **Catalog Format** field, enter `solr` if you are analyzing a Solr collection, or another format if your content is in the cloud.
     3. In the **Signals Path** field, enter the collection name or cloud storage path where your signals data is stored.
     4. In the **Output Path** field, enter the collection name or cloud storage path where you want to store the training data.
     5. In the **Category Field in Catalog** field, enter the field name for the category data in your main content.
     6. In the **Item ID Field in Catalog** field, enter the field name for the item IDs in your main content.
     7. Check that the values of **Item ID Field in Signals** and **Count Field in Signals** match the field names in your signals data.
  3. Save the job.
  4. Click **Run** > **Start** to run the job.
  5. Navigate to **Collections** > **Jobs** > **Add+** > **Classification** to create a new Classification job.
  6. Configure the job as follows:
     1. In the **Model Deployment Name** field, enter an ID for the new classification model.
     2. In the **Training Data Path** field, enter the collection name or cloud storage path from the Build Training Data job’s **Output Path** field.
     3. In the **Training Data Format** field, leave the default `solr` value if the **Training Data Path** is a collection or if you used the default format in your Build Training Data job configuration. If you configured the Build Training Data job to output a different format, enter it here.
     4. In the **Training collection content field**, enter `query_s`, the default content field name in the Build Training Data job’s output.
     5. In the **Training collection class field**, enter `category_s`, the default category field name in the Build Training Data job’s output.

        <Tip>      For additional configuration details, see [Best practices](#best-practices-for-configuring-the-classification-job) below.</Tip>
  7. Save the job.
  8. Verify that the Build Training Data job has finished successfully.
  9. Click **Run** > **Start** to run the job.
  10. Navigate to **Indexing** > **Query Workbench** > **Load** and select your query pipeline.
  11. Configure the query pipeline as follows:

      1. Add a new Machine Learning stage.
      2. In the **Model ID** field, enter the name from the Classification job’s **Model Deployment Name** field.
      3. In the **Model input transformation script** field, enter the following:

      ```js theme={"dark"}
      var modelInput = new java.util.HashMap()
      modelInput.put("text", request.getFirstParam("q"))
      modelInput
      ```

      4. In the **Model output transformation script** field, enter the following:

      ```js theme={"dark"}
      {/* // In case if top_k_predictions are needed */}
      {/* // To put into response documents (can be done only after Solr Query stage) */}
      var jsonOutput = JSON.parse(modelOutput.get("_rawJsonResponse"))
      var parsedOutput = {};
      for (var i=0; i<jsonOutput["names"].length;i++){
        parsedOutput[jsonOutput["names"][i]] = jsonOutput["ndarray"][i]
      }

      var docs = response.get().getInnerResponse().getDocuments();
      var ndocs = new java.util.ArrayList();
      for (var i=0; i<docs.length;i++){
        var doc = docs[i];
        doc.putField("top_1_class", parsedOutput["top_1_class"][0])
        doc.putField("top_1_score", parsedOutput["top_1_score"][0])
        if ("top_k_classes" in parsedOutput) {
          doc.putField("top_k_classes", new java.util.ArrayList(parsedOutput["top_k_classes"][0]))
          doc.putField("top_k_scores", new java.util.ArrayList(parsedOutput["top_k_scores"][0]))
        }
        ndocs.add(doc);
      }
      response.get().getInnerResponse().updateDocuments(ndocs);
      ```

      5. Click **Apply**.

  {/* // Should the user see something different in the preview at this point?  What can they look for to confirm that the config is working? */}

  12. Save the query pipeline.

  ## Custom output transformation script examples

  ```js theme={"dark"}
  {/* // To put into request */}
  request.putSingleParam("class", modelOutput.get("top_1_class")[0])
  request.putSingleParam("score", modelOutput.get("top_1_score")[0])

  {/* // Or for example to apply Filter Query */}
  request.putSingleParam("fq", "class:" + modelOutput.get("top_1_class")[0])
  ```

  ```js theme={"dark"}
  {/* // To put into query context */}
  context.put("class", modelOutput.get("top_1_class")[0])
  context.put("score", modelOutput.get("top_1_score")[0])
  ```

  ```js theme={"dark"}
  {/* // To put into response documents (can be done only after Solr Query stage) */}
  var docs = response.get().getInnerResponse().getDocuments();
  var ndocs = new java.util.ArrayList();

  for (var i=0; i<docs.length;i++){
    var doc = docs[i];
    doc.putField("query_class", modelOutput.get("top_1_class")[0])
    doc.putField("query_score", modelOutput.get("top_1_score")[0])
    ndocs.add(doc);
  }

  response.get().getInnerResponse().updateDocuments(ndocs);
  ```

  ## Best practices for configuring the Classification job

  This job analyzes how your existing documents are categorized and produces a classification model that can be used to predict the categories of new documents at index time.

  In addition to the information in this topic, see [Classify new documents at index time](/docs/5/fusion/reference/config-ref/jobs/classification#classification-at-index-time) for configuration and examples.

  This job takes raw text and an associated single class as input. Although it trains on single classes, there is an option to predict the top several classes with their scores.

  At a minimum, you must configure these:

  * An ID for this job
  * A **Method**; Logistic Regression is the default
  * A **Model Deployment Name**
  * The **Training Collection**
  * The **Training collection content field**, the document field containing the raw text
  * The **Training collection class field** containing the classes, labels, or other category data for the text
</Accordion>

## Configuration properties

<SchemaParamFields schema={schema} />