> ## Documentation Index
> Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Content-Based Recommender

> Job configuration specifications

export const schema = {
  "type": "object",
  "title": "Content based Recommender",
  "description": "Computes item-to-item similarities based on content fields such as product descriptions. Configure training and output paths to generate batch-predicted similarity recommendations stored in Solr or cloud storage. Supports deep learning, Word2Vec, and TF-IDF vectorization methods, which can be combined into an ensemble.",
  "required": ["id", "trainingCollection", "trainingFormat", "outputCollection", "outputFormat", "itemIdField", "contentField", "type"],
  "properties": {
    "id": {
      "type": "string",
      "title": "Job ID",
      "description": "Sets the unique identifier for this job, used to reference it in the API. Allowed characters: a-z, A-Z, dash (`-`), and underscore (`_`). Must start with a letter.",
      "maxLength": 63,
      "pattern": "[a-zA-Z][_\\-a-zA-Z0-9]*[a-zA-Z0-9]?"
    },
    "sparkConfig": {
      "type": "array",
      "title": "Additional parameters",
      "description": "Injects additional key-value pairs into the training JSON map at runtime. String values must be wrapped in quotes. Use this to pass custom parameters not exposed in the UI.",
      "hints": ["advanced"],
      "items": {
        "type": "object",
        "required": ["key"],
        "properties": {
          "key": {
            "type": "string",
            "title": "Parameter Name"
          },
          "value": {
            "type": "string",
            "title": "Parameter Value"
          }
        }
      }
    },
    "writeOptions": {
      "type": "array",
      "title": "Write Options",
      "description": "Sets additional key-value options passed to the Spark writer when writing output to Solr or other sinks. Use entries like `commitWithin=10000` to control Solr commit behavior.",
      "hints": ["advanced"],
      "items": {
        "type": "object",
        "required": ["key"],
        "properties": {
          "key": {
            "type": "string",
            "title": "Parameter Name"
          },
          "value": {
            "type": "string",
            "title": "Parameter Value"
          }
        }
      }
    },
    "readOptions": {
      "type": "array",
      "title": "Read Options",
      "description": "Sets additional key-value options passed to the Spark reader when loading input from Solr or other sources. Use these to configure connector-specific behavior such as `rows` or `zkHost`.",
      "hints": ["advanced"],
      "items": {
        "type": "object",
        "required": ["key"],
        "properties": {
          "key": {
            "type": "string",
            "title": "Parameter Name"
          },
          "value": {
            "type": "string",
            "title": "Parameter Value"
          }
        }
      }
    },
    "outputBatchSize": {
      "type": "string",
      "title": "Output Batch Size",
      "description": "Controls how many output records are written per batch operation when saving recommendations. Larger batch sizes improve throughput but increase memory usage. Reduce if out-of-memory errors occur.",
      "default": "15000",
      "hints": ["advanced"]
    },
    "unidecodeText": {
      "type": "boolean",
      "title": "Unidecode Text",
      "description": "Controls whether item content text is transliterated to ASCII using unidecode before vectorization. Enabling this normalizes accented characters and improves consistency across multilingual catalogs.",
      "default": true
    },
    "lowercaseText": {
      "type": "boolean",
      "title": "Lowercase Text",
      "description": "Controls whether item content text is lowercased before vectorization. Enabling this normalizes casing and typically improves similarity quality for case-insensitive content.",
      "default": true
    },
    "vectorizationUseDl": {
      "type": "boolean",
      "title": "Use Deep Learning for vectorization",
      "description": "Enables deep learning as a vectorization method for content-based similarity computation. When combined with other methods, the outputs are combined into an ensemble using `vectorizationDlEnsembleWeight`.",
      "default": true
    },
    "vectorizationUseFasttext": {
      "type": "boolean",
      "title": "Use Word2Vec for vectorization",
      "description": "Enables Word2Vec (via FastText) as a vectorization method for content-based similarity computation. Custom word embeddings are learned from the item content. Can be combined with other methods into an ensemble."
    },
    "vectorizationUseTfidf": {
      "type": "boolean",
      "title": "Use Tf-Idf for vectorization",
      "description": "Enables TF-IDF as a vectorization method for content-based similarity computation. Can be combined with deep learning or Word2Vec into an ensemble weighted by `vectorizationTfidfEnsembleWeight`."
    },
    "vectorizationDlEnsembleWeight": {
      "type": "number",
      "title": "Deep learning vectorization ensemble weight",
      "description": "Sets the ensemble weight applied to the deep learning vectorization output when multiple methods are combined. Higher values give the deep learning embeddings more influence on the final similarity scores.",
      "default": 1
    },
    "vectorizationFasttextVectorsSize": {
      "type": "integer",
      "title": "Size of word vectors",
      "description": "Sets the number of dimensions in the Word2Vec word vectors learned from item content. Larger values capture more semantic information but require more memory and training time.",
      "default": 150,
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "vectorizationFasttextWindowSize": {
      "type": "integer",
      "title": "Word2Vec window size",
      "description": "Sets the context window size used during Word2Vec training, controlling how many neighboring words are considered. Larger windows capture broader context. Smaller windows focus on local co-occurrence patterns.",
      "default": 5,
      "hints": ["advanced"],
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "vectorizationFasttextEpochs": {
      "type": "integer",
      "title": "Word2Vec training epochs",
      "description": "Sets the number of training epochs for the Word2Vec model learned from item content. More epochs improve embedding quality at the cost of training time. Typical values range from 5 to 30.",
      "default": 15,
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "vectorizationFasttextSkipGram": {
      "type": "boolean",
      "title": "Use SkipGram model",
      "description": "Controls whether the skip-gram or CBOW architecture is used for Word2Vec training. Skip-gram generally produces better embeddings for infrequent words. CBOW is faster for large vocabularies. This field is hidden.",
      "default": true,
      "hints": ["hidden"]
    },
    "vectorizationFasttextMinCount": {
      "type": "integer",
      "title": "Min count of words",
      "description": "Sets the minimum number of times a token must appear in the corpus to be included in the Word2Vec vocabulary. Tokens below this threshold are discarded as too rare to learn reliable embeddings. This field is hidden.",
      "default": 1,
      "hints": ["hidden"],
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "vectorizationFasttextMaxVocabSize": {
      "type": "integer",
      "title": "Max vocab size",
      "description": "Sets the maximum number of tokens included in the Word2Vec vocabulary. Less frequent tokens beyond this limit are excluded to bound memory usage during embedding training.",
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "vectorizationFasttextUseSubwordNgram": {
      "type": "boolean",
      "title": "Use subword ngrams",
      "description": "Controls whether subword character n-grams are used during Word2Vec training. Enabling this improves handling of rare or morphologically complex words at the cost of training time. This field is hidden.",
      "default": true,
      "hints": ["hidden"]
    },
    "vectorizationFasttextMinNgram": {
      "type": "integer",
      "title": "Min Ngram size",
      "description": "Sets the minimum character n-gram size used when subword n-grams are enabled. Smaller values capture more subword patterns but increase vocabulary size. This field is hidden.",
      "default": 3,
      "hints": ["hidden"],
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "vectorizationFasttextMaxNgram": {
      "type": "integer",
      "title": "Max Ngram size",
      "description": "Sets the maximum character n-gram size used when subword n-grams are enabled. Larger values capture longer subword sequences but increase vocabulary size and memory usage. This field is hidden.",
      "default": 6,
      "hints": ["hidden"],
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "vectorizationFasttextEnsembleWeight": {
      "type": "number",
      "title": "Word2Vec vectorization ensemble weight",
      "description": "Sets the ensemble weight applied to the Word2Vec vectorization output when multiple methods are combined. Higher values give Word2Vec embeddings more influence on the final similarity scores.",
      "default": 1
    },
    "vectorizationTfidfUseCharacters": {
      "type": "boolean",
      "title": "Use characters ngrams",
      "description": "Controls whether character n-grams rather than word tokens are used for TF-IDF vectorization. Character n-grams can improve similarity for short or noisy text such as product SKUs."
    },
    "vectorizationTfidfFilterStopwords": {
      "type": "boolean",
      "title": "Filter stopwords",
      "description": "Controls whether stop words are filtered from the vocabulary before computing TF-IDF weights. Enabling this removes high-frequency words that carry little semantic meaning, improving similarity quality.",
      "default": true
    },
    "vectorizationTfidfMinDf": {
      "type": "number",
      "title": "Min Document Frequency",
      "description": "Sets the minimum document frequency required for a token to be included in the TF-IDF vocabulary. This hidden field filters out very rare terms that contribute noise rather than signal.",
      "hints": ["hidden"]
    },
    "vectorizationTfidfMaxDf": {
      "type": "number",
      "title": "Max Document Frequency",
      "description": "Sets the maximum document frequency allowed for a token in the TF-IDF vocabulary. This hidden field filters out tokens that appear too frequently to distinguish items from one another.",
      "default": 1,
      "hints": ["hidden"]
    },
    "vectorizationTfidfMinNgram": {
      "type": "integer",
      "title": "Min Ngram size",
      "description": "Sets the minimum n-gram size used when building the TF-IDF vocabulary. Use `1` for unigrams. Increase to capture multi-word phrases or character sequences in content fields.",
      "default": 1,
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "vectorizationTfidfMaxNgram": {
      "type": "integer",
      "title": "Max Ngram size",
      "description": "Sets the maximum n-gram size used when building the TF-IDF vocabulary. Larger values capture longer phrases but increase vocabulary size and memory requirements.",
      "default": 3,
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "vectorizationTfIdfMaxVocabSize": {
      "type": "integer",
      "title": "Max vocab size",
      "description": "Sets the maximum number of tokens included in the TF-IDF vocabulary. Less frequent tokens beyond this limit are excluded to bound memory usage during vectorization.",
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "vectorizationTfidfEnsembleWeight": {
      "type": "number",
      "title": "Tf-Idf vectorization ensemble weight",
      "description": "Sets the ensemble weight applied to the TF-IDF vectorization output when multiple methods are combined. Higher values give TF-IDF features more influence on the final similarity scores.",
      "default": 1
    },
    "topKAnn": {
      "type": "integer",
      "title": "No. of Item Recs to  compute for ensemble",
      "description": "Sets the number of candidate items fetched via ANN before filtering to produce the final `numSimsPerItem` recommendations. Set this to approximately 10x `numSimsPerItem` to ensure the target count is satisfied.",
      "default": 100,
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "jobRunName": {
      "type": "string",
      "title": "Job Run Name",
      "description": "Sets an identifier attached to this job run's output records. Use this to filter recommendations produced by a specific run when multiple runs write to the same output collection.",
      "hints": ["advanced"]
    },
    "trainingCollection": {
      "type": "string",
      "title": "Training data path",
      "description": "Specifies the Solr collection or cloud storage path containing item content data used for similarity computation. This is the primary data source for vectorizing item descriptions.",
      "minLength": 1
    },
    "trainingFormat": {
      "type": "string",
      "title": "Training data format",
      "description": "Specifies the format of the training data, such as `solr` or `parquet`. This controls which Spark connector loads the item content data from `trainingCollection`.",
      "default": "solr",
      "minLength": 1
    },
    "secretName": {
      "type": "string",
      "title": "Cloud storage secret name",
      "description": "Specifies the Kubernetes secret used to authenticate access to cloud storage. Define this secret in the K8s namespace before referencing it here. Omit if training data is in Solr.",
      "hints": ["advanced"],
      "minLength": 1
    },
    "outputCollection": {
      "type": "string",
      "title": "Output data path",
      "description": "Specifies the Solr collection or cloud storage path where item similarity recommendations are written. This path receives the output produced by the vectorization and ANN similarity computation."
    },
    "outputFormat": {
      "type": "string",
      "title": "Output data format",
      "description": "Specifies the format used to write output recommendations, such as `solr` or `parquet`. Must match the target storage system specified in `outputCollection`.",
      "default": "solr",
      "minLength": 1
    },
    "partitionFields": {
      "type": "string",
      "title": "Partition fields",
      "description": "Comma-delimited list of field names to partition by when writing output.",
      "hints": ["advanced"]
    },
    "numSimsPerItem": {
      "type": "integer",
      "title": "No. of Item Recs to Compute",
      "description": "Sets the number of similar items saved per item in the output collection. Increase to surface more recommendations. Higher values require more storage and compute time.",
      "default": 10,
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "deleteOldRecs": {
      "type": "boolean",
      "title": "Delete Old Recommendations",
      "description": "Controls whether existing recommendations are deleted before writing new ones. When `false`, new recommendations are appended with a different job run ID, and both old and new records coexist in the collection.",
      "default": true
    },
    "excludeFromDeleteFilter": {
      "type": "string",
      "title": "Exclude from Delete Filter",
      "description": "Specifies a Solr filter query identifying recommendation documents to retain when `deleteOldRecs` is `true`. Use this to protect specific documents, such as manually curated recommendations, from deletion.",
      "hints": ["advanced"]
    },
    "metadataCategoryFields": {
      "type": "array",
      "title": "Metadata fields for item-item evaluation",
      "description": "Specifies metadata fields used to evaluate whether recommended item pairs belong to the same category. These fields are used during the item-to-item evaluation phase to assess recommendation quality.",
      "hints": ["advanced"],
      "items": {
        "type": "string"
      }
    },
    "trainingDataFilterQuery": {
      "type": "string",
      "title": "Training Data Filter Query",
      "description": "Filters training data using a Solr query when reading from a Solr collection, or a SQL expression for cloud storage. Use this to restrict training to a specific subset of items.",
      "hints": ["code/sql", "advanced"]
    },
    "trainingSampleFraction": {
      "type": "number",
      "title": "Training Data Sampling Fraction",
      "description": "Sets the fraction of training data sampled before model training. Values less than `1.0` reduce training time on very large item catalogs at the cost of coverage.",
      "default": 1,
      "hints": ["advanced"],
      "maximum": 1,
      "exclusiveMaximum": false
    },
    "itemIdField": {
      "type": "string",
      "title": "Training Item Id Field",
      "description": "Specifies the field in the training collection containing item identifiers. Maps the `item_id_s` field by default. Adjust this to match the actual item ID field in your collection.",
      "default": "item_id_s",
      "minLength": 1
    },
    "contentField": {
      "type": "array",
      "title": "Training Content Field",
      "description": "Specifies the field(s) in the training collection containing item content, such as product descriptions. Multiple fields can be provided. All are concatenated and vectorized to compute item similarities.",
      "items": {
        "type": "string"
      }
    },
    "randomSeed": {
      "type": "integer",
      "title": "Random Seed",
      "description": "Sets the random seed used during training to produce deterministic results. Fix this value to reproduce identical similarity outputs across runs.",
      "default": 12345,
      "hints": ["advanced"]
    },
    "itemMetadataFields": {
      "type": "array",
      "title": "Item Metadata Fields",
      "description": "Specifies item metadata fields to include in recommendation output documents, such as `title` or `price`. Adding many fields can significantly increase output size.",
      "hints": ["advanced"],
      "items": {
        "type": "string"
      }
    },
    "vectorizationDlBatchSize": {
      "type": "integer",
      "title": "Batch size to compute encodings",
      "description": "Sets the number of items encoded per batch during deep learning vectorization. Reduce this value if GPU or CPU memory is exhausted during the encoding phase.",
      "hints": ["advanced"],
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "performANN": {
      "type": "boolean",
      "title": "Perform approximate nearest neighbor search",
      "description": "Controls whether approximate nearest neighbor (ANN) search is used to speed up similarity computation. Disable only for very small datasets. ANN significantly reduces training time with minimal accuracy loss.",
      "default": true
    },
    "maxNeighbors": {
      "type": "integer",
      "title": "Max neighbors for indexing",
      "description": "Sets the number of candidate neighbors evaluated during ANN index construction. Higher values improve recall at the cost of longer indexing time. The recommended range is 5–100.",
      "hints": ["advanced"],
      "maximum": 100,
      "exclusiveMaximum": false,
      "minimum": 5,
      "exclusiveMinimum": false
    },
    "searchNN": {
      "type": "integer",
      "title": "Search Depth",
      "description": "Sets the search depth used to find neighbors during ANN retrieval. Higher values improve recall but increase retrieval time. The recommended range is 100–2000.",
      "hints": ["advanced"],
      "maximum": 2000,
      "exclusiveMaximum": false,
      "minimum": 100,
      "exclusiveMinimum": false
    },
    "indexNN": {
      "type": "integer",
      "title": "Indexing Depth",
      "description": "Sets the depth of the ANN index structure built during indexing. Higher values improve recall at the cost of longer indexing time. The recommended range is 100–2000.",
      "hints": ["advanced"],
      "maximum": 2000,
      "exclusiveMaximum": false,
      "minimum": 100,
      "exclusiveMinimum": false
    },
    "type": {
      "type": "string",
      "title": "Spark Job Type",
      "enum": ["argo-item-recommender-content"],
      "default": "argo-item-recommender-content",
      "hints": ["readonly"]
    }
  },
  "additionalProperties": true,
  "category": "Other",
  "categoryPriority": 1,
  "propertyGroups": [{
    "label": "Input/Output Parameters",
    "properties": ["trainingCollection", "trainingFormat", "outputCollection", "outputFormat", "outputBatchSize", "secretName", "partitionFields"]
  }, {
    "label": "Training Data Settings",
    "properties": ["trainingDataFilterQuery", "trainingSampleFraction", "randomSeed", "itemIdField", "contentField"]
  }, {
    "label": "Model Tuning Parameters",
    "properties": ["numSimsPerItem", "topKAnn", "performANN", "maxNeighbors", "searchNN", "indexNN", "unidecodeText", "lowercaseText", "deleteOldRecs", "excludeFromDeleteFilter"]
  }, {
    "label": "Vectorization Parameters",
    "properties": ["vectorizationUseDl", "vectorizationUseFasttext", "vectorizationUseTfidf"]
  }, {
    "label": "Deep Learning Vectorization Parameters",
    "properties": ["vectorizationDlBatchSize", "vectorizationDlEnsembleWeight"]
  }, {
    "label": "Word2Vec Vectorization Parameters",
    "properties": ["vectorizationFasttextVectorsSize", "vectorizationFasttextWindowSize", "vectorizationFasttextEpochs", "vectorizationFasttextMinNgram", "vectorizationFasttextEnsembleWeight", "vectorizationFasttextMaxVocabSize"]
  }, {
    "label": "Tf-Idf Vectorization Parameters",
    "properties": ["vectorizationTfidfUseCharacters", "vectorizationTfidfFilterStopwords", "vectorizationTfidfMinNgram", "vectorizationTfidfMaxNgram", "vectorizationTfIdfMaxVocabSize", "vectorizationTfidfEnsembleWeight"]
  }, {
    "label": "Item Metadata Settings",
    "properties": ["itemMetadataFields", "metadataCategoryFields"]
  }]
};

export const SchemaParamFields = ({schema}) => {
  const sanitize = str => {
    if (typeof str !== "string") return str;
    return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'");
  };
  const renderMd = str => {
    const s = sanitize(str);
    const text = (/[.!?]\)*$/).test(s) ? s : `${s}.`;
    return text.split(/(\*\*[^*]+\*\*|_[^_]+_|`[^`]+`)/g).map((part, i) => {
      if (part.startsWith("**")) return <strong key={i}>{part.slice(2, -2)}</strong>;
      if (part.startsWith("_")) return <em key={i}>{part.slice(1, -1)}</em>;
      if (part.startsWith("`")) return <code key={i}>{part.slice(1, -1)}</code>;
      return part;
    });
  };
  const {description, properties = {}, required: requiredProps = []} = schema;
  const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]);
  const renderProp = ([name, prop]) => {
    const isRequired = requiredProps.includes(name);
    const hasDefault = prop.default !== undefined;
    const rawDefault = prop.default;
    const hints = prop.hints || [];
    const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"')));
    const postBadges = [];
    if (prop.title) {
      postBadges.push(<><span className="text-stone-400 dark:text-stone-500">API property: </span>{name}</>);
    }
    const constraints = [];
    if (prop.minimum !== undefined && prop.maximum !== undefined) {
      constraints.push(`Range: ${prop.minimum} – ${prop.maximum}`);
    } else if (prop.minimum !== undefined) {
      constraints.push(`Min: ${prop.minimum}`);
    } else if (prop.maximum !== undefined) {
      constraints.push(`Max: ${prop.maximum}`);
    }
    if (prop.minLength !== undefined && prop.maxLength !== undefined) {
      constraints.push(`Length: ${prop.minLength} – ${prop.maxLength}`);
    } else if (prop.minLength !== undefined) {
      constraints.push(`Min length: ${prop.minLength}`);
    } else if (prop.maxLength !== undefined) {
      constraints.push(`Max length: ${prop.maxLength}`);
    }
    const fieldProps = {
      key: name,
      body: prop.title || name,
      type: prop.type,
      ...postBadges.length > 0 && ({
        post: postBadges
      }),
      ...isRequired && ({
        required: true
      }),
      ...!isComplexDefault && hasDefault ? {
        default: sanitize(String(rawDefault))
      } : {}
    };
    const isObject = prop.type === "object" && prop.properties;
    const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties;
    return <ParamField {...fieldProps}>
        {prop.description && <p>{renderMd(prop.description)}</p>}

        {prop.enum && <p>
            Allowed values: 
            {prop.enum.map((v, i) => <>{i > 0 && ", "}<code key={i}>{String(v)}</code></>)}
          </p>}

        {constraints.length > 0 && <p className="text-stone-500 dark:text-stone-400 text-sm">
            {constraints.join(" · ")}
          </p>}

        {isComplexDefault && <div className="flex">
            <p>
              <strong>Default:</strong>
            </p>
            <pre className="!my-0">
              <code>
                {JSON.stringify(rawDefault, null, 2)}
              </code>
            </pre>
          </div>}

        {isArrayOfObjects && <Expandable title="item properties">
            <SchemaParamFields schema={{
      properties: prop.items.properties,
      required: prop.items.required
    }} />
          </Expandable>}

        {isObject && <Expandable title="properties">
            <SchemaParamFields schema={{
      properties: prop.properties,
      required: prop.required
    }} />
          </Expandable>}
      </ParamField>;
  };
  return <div>
      {description && <p>{renderMd(description)}</p>}

      {visibleProps.map(renderProp)}
    </div>;
};

export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => {
  const [isLoaded, setIsLoaded] = useState(false);
  useEffect(() => {
    const timer = setTimeout(() => {
      setIsLoaded(true);
    }, 500);
    return () => clearTimeout(timer);
  }, []);
  return <div className="lw-template-container">
      <Card title={title} icon={icon}>
        {isLoaded && <span dangerouslySetInnerHTML={{
    __html: `<lw-template id="a029c1a9-28be-427e-b0e1-5d918920246a"></lw-template
            >`
  }} />}
        <Link href={linkHref} className="agent-studio-link text-left text-gray-600 gap-2 dark:text-gray-400 text-sm font-medium flex flex-row items-center hover:text-primary dark:hover:text-primary-light group-hover:text-primary group-hover:dark:text-primary-light">Powered by Lucidworks Agent Studio</Link>
      </Card>
    </div>;
};

[localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/content-recommend

[mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/content-recommend

[old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/g02utf

Use this job when you want to compute item similarities based on their content, such as product descriptions.

|                      |                                                                                                                                                                             |
| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| **Default job name** | `COLLECTION_NAME_content_recs`                                                                                                                                              |
| **Input**            | Searchable content from the primary collection.                                                                                                                             |
| **Output**           | [Items-for-item recommendations](/docs/lucidworks-search/07-improve-your-queries/recommendations/items-for-item) (the `COLLECTION_NAME_content_recs` collection by default) |

First, item content is vectorized; different vectorization methods are available. Then, similar items are selected based on cosine similarity ("nearest neighbor") between their vectors.

At a minimum, you must specify these:

* An ID for this job
* The name of the training collection, that is, the collection with your content
* An output collection; create a separate collection for this
* The name of the ID field for documents in the training collection, such as `item_id_s`
* The names of one or more content fields in the training collection

```mermaid theme={"dark"}
flowchart LR
accTitle: Content-based recommender job dataflow
accDescr: A flowchart showing the content_recs query pipeline connecting to the content_recs collection, fed by the Content-Based Recommender Job processing the documents collection
    UI["UI"]

    subgraph profile["Query Profile"]
        Pipe["content_recs Query Pipeline"]
    end

    Col[("content_recs collection")]
    Job["Content-Based Recommender Job (Experimental)"]
    Docs[("documents collection")]

    UI <--> profile
    Pipe <--> Col
    Docs --> Job <--> Col
```

<LwTemplate />

## Tuning tips

* Configure **Metadata fields for item-item evaluation** to use those fields during evaluation to determine whether pairs belong to the same category.
* **Perform approximate nearest neighbor search** is enabled by default to significantly reduce the job’s running time, with a small decrease in accuracy. If your training dataset is very small, then you can disable this option.
* If your content contains a lot of domain-specific jargon, enable **Use Word2Vec for vectorization**.
* If your documents are too short or too long, enable **Use TF-IDF for vectorization**.

## Query pipeline setup

<a href="/assets/attachments/pipelines/APPName_item_item_rec_pipelines_content.json" download>Download the APPName\_item\_item\_rec\_pipelines\_content.json file</a> and **Migrate Lucidworks Search Objects** to create the query pipeline that consumes this job’s output. See **Fetch Items-for-Item Recommendations (Content-Based Method)** for details.

<AccordionGroup>
  <Accordion title="Migrate Lucidworks Search Objects">
    You can use the Fusion UI and the [Objects API](/api-reference/objects/get-objects-service-status) to migrate collections and related objects, including your searchable data, configuration data, pipelines, aggregations, and other objects on which your collections depend. You can also migrate entire apps.

    <Check>To upgrade from one Fusion version to a 5.x version, see Fusion 5 Upgrades to migrate objects.</Check>

    You might need to migrate objects in the following circumstances:

    * When migrating data from development environments into testing and production environments
    * To back up data, so you can restore it after an incident of data loss
    * When the migrator script was not able to migrate an object automatically

    <Warning>When you export and import objects to migrate them, make sure you that any data gets to where it is going before deleting the sources.</Warning>

    ## Migration approaches

    Several approaches are available for migrating Fusion objects. This table summarizes the approaches.

    |                                   | Export an app              | Import an app                                                      | Export an object           | Import an object          | Add an object to an app |
    | --------------------------------- | -------------------------- | ------------------------------------------------------------------ | -------------------------- | ------------------------- | ----------------------- |
    | **Fusion UI**                     | App configuration          | Launcher<br /> (entire app) App configuration<br /> (combine apps) | -                          | -                         | Object Explorer         |
    | **Objects API**<br /> (endpoints) | GET from `export` endpoint | POST to `import` endpoint                                          | GET from `export` endpoint | POST to `import` endpoint | -                       |

    For more information about using the Objects API to export and import objects, see [Objects API](/api-reference/objects/get-objects-service-status).

    The remainder of this topic describes approaches in the Fusion UI.

    Use the parts of the Fusion UI indicated in the table to export and import apps and specific objects. Exporting creates a zip file. To import, you select a data file and possibly a variable file.

    The approach with Object Explorer differs. With Object Explorer, you can add objects from other apps (or that are not linked to any apps) to the currently open app.

    ## Export an app with the Fusion UI

    **How to export an app with the Fusion UI**

    1. Navigate to the launcher.
    2. Hover over the app you want to export and click the Configure icon:

           <img src="https://mintcdn.com/lucidworks/L5PMnIeZ03zhv8Ti/assets/images/5.6/launcher-app-config-hover-56.png?fit=max&auto=format&n=L5PMnIeZ03zhv8Ti&q=85&s=8ae921d898899048020c96681719b4ee" alt="App config button" width="3138" height="1134" data-path="assets/images/5.6/launcher-app-config-hover-56.png" />
    3. In the app config window, click **Export app to zip**:

           <img style={{ width: "300px" }} src="https://mintcdn.com/lucidworks/L5PMnIeZ03zhv8Ti/assets/images/5.6/launcher-export-app-56.png?fit=max&auto=format&n=L5PMnIeZ03zhv8Ti&q=85&s=198aa7f16fddd9adc758115e7ec132c2" width="869" height="989" data-path="assets/images/5.6/launcher-export-app-56.png" />

    See import for information to import the downloaded zip file into other instances of Fusion 5.x.

    ## Import an app with the Fusion UI

    **How to import an app with the Fusion UI**

    1. Navigate to the launcher.
    2. Click **Import app**.

           <img style={{ width: "300px" }} src="https://mintcdn.com/lucidworks/zH_ln2rWO5G9pvTA/assets/images/5.0/launcher-import-app.png?fit=max&auto=format&n=zH_ln2rWO5G9pvTA&q=85&s=c093d9e28cc5ab76f2c4a7323d741afa" width="591" height="578" data-path="assets/images/5.0/launcher-import-app.png" />
    3. Under **Data File**, click **Choose File** and select the zip file containing the app you want to import.
    4. If your app has [usernames and passwords](/api-reference/objects/list-imported-variables) in a separate file, select it under **Variables File**.
       If the Variables File is needed, it *must* be a separate file that is not in a .zip file. It is a .json map of variables to values. The following is an example:
       ```json theme={"dark"}
       {
       "secret.dataSources.Inventory_BR_S3_DS.password":"SOMETHING",
       "secret.dataSources.LLM_A_BR_S3_DS.password":"FmJSaDE9Tj5REDACTED",
       "secret.dataSources.LLM_BR_Inventory_S3_DS.password":"FmJSaDE9Tj5GzIVvethAC4Huh",
       "secret.dataSources.LLM_BR_Load_S3_DS.password":"FmJSaDE9Tj5GzIVvethAC4"
       }
       ```
    5. You can sometimes edit parameter values to use the new values in the imported app. If this is the case, Fusion displays a dialog box that lets you edit the parameter values.

           <img src="https://mintcdn.com/lucidworks/de_1M1m_4TTyJqw0/assets/images/5.0/import-app-edit-parameters.png?fit=max&auto=format&n=de_1M1m_4TTyJqw0&q=85&s=2211f3a81b3394a7959b582d59afa2b9" alt="Edit parameter values" width="1658" height="1152" data-path="assets/images/5.0/import-app-edit-parameters.png" />

       Make desired changes, and then click **Import**.

    ## Copy an app

    To copy an app from one deployment to a different one, [export the app](#export-an-app-with-the-fusion-ui) on the source deployment, and then [import the app](#import-an-app-with-the-fusion-ui) on the target deployment.

    ## Import objects into an app

    You can import objects into the currently open app.

    **How to import objects into an open app**

    1. In the Fusion launcher, click the app into which you want to import objects.
       The Fusion workspace appears.
    2. Click **System  > Import Fusion Objects**.
       The Import Fusion Objects window opens.
           <img src="https://mintcdn.com/lucidworks/NgNm7Bp5nEBDIA7H/assets/images/4.0/import-objects.png?fit=max&auto=format&n=NgNm7Bp5nEBDIA7H&q=85&s=04697d36b39bd2229b87bedfee422a09" alt="Import Objects Window" width="2880" height="1508" data-path="assets/images/4.0/import-objects.png" />
    3. Select the data file from your local filesystem.
       If you are importing [usernames and passwords](/api-reference/objects/import-objects) in a separate file, select it under **Variables File**.
    4. Click **Import**.
       If there are conflicts, Fusion prompts you to specify an import policy:
           <img src="https://mintcdn.com/lucidworks/NgNm7Bp5nEBDIA7H/assets/images/4.0/import-conflicts.png?fit=max&auto=format&n=NgNm7Bp5nEBDIA7H&q=85&s=53b711be62ea740fb033b6dbc96a29a6" alt="Import Conflicts" width="2880" height="1508" data-path="assets/images/4.0/import-conflicts.png" />
       * Click **Overwrite** to overwrite the objects on the target system with the ones in the import file.
       * Click **Merge** to skip all conflicting objects and import only the non-conflicting objects.
       * Click **Start Over** to abort the import.
         Fusion confirms that the import was successful:
             <img src="https://mintcdn.com/lucidworks/NgNm7Bp5nEBDIA7H/assets/images/4.0/import-objects-success.png?fit=max&auto=format&n=NgNm7Bp5nEBDIA7H&q=85&s=e2416d872ac6a74f2d40eaa4adbde56f" alt="Successful Import" width="2880" height="1508" data-path="assets/images/4.0/import-objects-success.png" />
    5. Click **Close** to close the Import Fusion Objects window.

    ## Add an object to an app

    You can add objects present in other apps (or in no apps) to the open app. Some objects are linked to other apps. You can also add those directly to an app.

    * **Add an object to an app** – While in the Fusion workspace for the app to which you want to add an object, open Object Explorer and click **In Any App**. Search for or browse to the object you want to add. Hover over the object, click the App <img className="inline-image" alt="App" src="https://mintcdn.com/lucidworks/5yWZ-KtZuBe4Y_Fg/assets/images/4.0/icons/oe-app-menu.png?fit=max&auto=format&n=5yWZ-KtZuBe4Y_Fg&q=85&s=1e9322222ea76f70c278fb9c2a04ce9f" width="40" height="42" data-path="assets/images/4.0/icons/oe-app-menu.png" /> icon, and then click **Add to this app**.

          <img src="https://mintcdn.com/lucidworks/l9y7VqRhZkN9hmR0/assets/images/4.0/object-explorer-add-to-app.png?fit=max&auto=format&n=l9y7VqRhZkN9hmR0&q=85&s=472088e9361aa1bc278ad7da50e9bbdf" alt="Add to this app" width="2560" height="938" data-path="assets/images/4.0/object-explorer-add-to-app.png" />
    * **Add an object to an app directly** – In cases when an object is linked to an app, but is not linked directly to the app (it is linked via some dependency), you can add the object to an app directly.

      While in the Fusion workspace for the app to which you want to add an object directly, open Object Explorer and click **In Any App**. Search for or browse to the object you want to add. Hover over the object, click the App <img className="inline-image" alt="App" src="https://mintcdn.com/lucidworks/5yWZ-KtZuBe4Y_Fg/assets/images/4.0/icons/oe-app-menu.png?fit=max&auto=format&n=5yWZ-KtZuBe4Y_Fg&q=85&s=1e9322222ea76f70c278fb9c2a04ce9f" width="40" height="42" data-path="assets/images/4.0/icons/oe-app-menu.png" /> icon, and then click **Add to this app directly**.

          <img src="https://mintcdn.com/lucidworks/l9y7VqRhZkN9hmR0/assets/images/4.0/object-explorer-add-directly-to-app.png?fit=max&auto=format&n=l9y7VqRhZkN9hmR0&q=85&s=af54840e096bbb4e6814c7eb5960d492" alt="Add to this app directly" width="2560" height="861" data-path="assets/images/4.0/object-explorer-add-directly-to-app.png" />
  </Accordion>

  <Accordion title="Fetch Items-for-Item Recommendations (Content-Based Method)">
    You can download and import a query pipeline that works out of the box to fetch the items-for-item recommendations generated by the default [Content-Based Recommender](/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/content-recommend) job.

    There are two separate pipelines attached below which work in different ways to query and return recommendations.

    **Query for recommendations only**

    1. <a href="/assets/attachments/pipelines/APPName_item_item_rec_pipelines_content.json" download>Download the `APPName_item_item_rec_pipelines_content.json` file</a>.
    2. Rename the file to replace `APPName` with the name of your Lucidworks Search app, such as `ProductCatalog_item_item_rec_pipelines_bpr.json`.
    3. Open the JSON file, replace all instances of `APPName` with the name of your Lucidworks Search app, such as `ProductCatalog`, and save it.
    4. Import the JSON file into your Lucidworks Search instance using the Query Pipelines REST API:
       ```
       curl -u USERNAME:PASSWORD https://EXAMPLE_COMPANY.b.lucidworks.cloud/api/query-pipelines -XPOST -H 'content-type:application/json' -d@<path/to/filename.json>
       ```
    5. In the Lucidworks Search UI, navigate to **Query** > **Query Pipelines** to verify that the new pipeline is available.

    <Note>This pipeline should be used to query the collection where the recommendations are stored. It makes a query against the `itemId` field and returns the recommended `itemId` values. To get the actual items, you need to make a second query to the respective catalog collection with the returned `itemId` values.</Note>

    No additional configuration is needed to use this pipeline with the default BPR job configuration.

    **Boost recommended products from catalog**

    1. <a href="/assets/attachments/pipelines/APPName_content_boost.json" download>Download the `APPName_content_boost.json` file</a>
    2. Rename the file to replace `APPName` with the name of your Lucidworks Search app, such as `ProductCatalog_item_item_rec_pipelines_bpr.json`.
    3. Open the JSON file, replace all instances of `APPName` with the name of your Lucidworks Search app, such as `ProductCatalog`.
    4. Fill in the `collection` name field in the first `Recommend Items for Item` stage and save the file.
    5. Import the JSON file into your Lucidworks Search instance using the Query Pipelines REST API:
       ```
       curl -u USERNAME:PASSWORD https://EXAMPLE_COMPANY.b.lucidworks.cloud/api/query-pipelines -XPOST -H 'content-type:application/json' -d@<path/to/filename.json>
       ```
    6. In the Lucidworks Search UI, navigate to **Query** > **Query Pipelines** to verify that the new pipeline is available.

    <Note>This pipeline queries the recommendations collection and then makes a subsequent query to the actual catalog collection boosting the recommended items and returning the actual items from the catalog. This pipeline will therefore also return recommendations even if none were generated/available.</Note>

    <Note>This pipeline expects a request parameter called `id=itemId` to be appended to the request. An example query URL to this pipeline would look like `https://EXAMPLE_COMPANY.b.lucidworks.cloud/api/query-pipelines/APPName_items_for_user_bpr_boost/collections/catalog/select?q=**:**&id=SomeItemId`</Note>

    <Tip>If the pipeline does not appear in the **Query Pipelines** panel, you may need to attach it to your app like this:  Go to **System** > **Object Explorer**, click the **In No Apps** filter, hover over the pipeline, click the <img className="inline-image" alt="oe-app-menu" src="https://mintcdn.com/lucidworks/5yWZ-KtZuBe4Y_Fg/assets/images/4.0/icons/oe-app-menu.png?fit=max&auto=format&n=5yWZ-KtZuBe4Y_Fg&q=85&s=1e9322222ea76f70c278fb9c2a04ce9f" width="40" height="42" data-path="assets/images/4.0/icons/oe-app-menu.png" /> icon, and select **Add to this app**.</Tip>

    **See also**

    * [Items-For-Item Recommendations](/docs/lucidworks-search/07-improve-your-queries/recommendations/items-for-item)
    * [Content-Based Recommender job](/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/content-recommend)
    * [Get Started with Recommendations and Boosting](/docs/lucidworks-search/07-improve-your-queries/recommendations/getting-started)
  </Accordion>
</AccordionGroup>

## Configuration properties

<SchemaParamFields schema={schema} />
