> ## Documentation Index
> Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Phrase Extraction

> Job configuration specifications

export const schema = {
  "type": "object",
  "title": "Phrase Extraction",
  "description": "Identifies statistically significant phrases in document content and writes them to the output collection for use in query rewrites.",
  "required": ["id", "trainingCollection", "fieldToVectorize", "dataFormat", "analyzerConfig", "type"],
  "properties": {
    "id": {
      "type": "string",
      "title": "Spark Job ID",
      "description": "The ID for this Spark job. Used in the API to reference this job. Allowed characters: a-z, A-Z, dash (-) and underscore (_).",
      "maxLength": 63,
      "pattern": "[a-zA-Z][_\\-a-zA-Z0-9]*[a-zA-Z0-9]?"
    },
    "sparkConfig": {
      "type": "array",
      "title": "Spark Settings",
      "description": "Spark configuration settings.",
      "hints": ["advanced"],
      "items": {
        "type": "object",
        "required": ["key"],
        "properties": {
          "key": {
            "type": "string",
            "title": "Parameter Name"
          },
          "value": {
            "type": "string",
            "title": "Parameter Value"
          }
        }
      }
    },
    "trainingCollection": {
      "type": "string",
      "title": "Training Collection",
      "description": "Specifies the Solr collection or cloud storage path containing training data.",
      "minLength": 1
    },
    "fieldToVectorize": {
      "type": "string",
      "title": "Field to Vectorize",
      "description": "Specifies the Solr field containing text to vectorize. Combine multiple fields with weights using the format `field1:weight1,field2:weight2`.",
      "minLength": 1
    },
    "dataFormat": {
      "type": "string",
      "title": "Data format",
      "description": "Specifies the Spark-compatible input data format, such as `solr`, `parquet`, or `orc`.",
      "default": "solr",
      "minLength": 1
    },
    "trainingDataFrameConfigOptions": {
      "type": "object",
      "title": "Dataframe Config Options",
      "description": "Sets additional key-value configuration options passed to the Spark DataFrame reader at load time.",
      "properties": {},
      "additionalProperties": {
        "type": "string"
      },
      "hints": ["advanced"]
    },
    "trainingDataFilterQuery": {
      "type": "string",
      "title": "Training data filter query",
      "description": "Filters training data using a Solr query when reading from Solr, or a SQL expression for other data sources.",
      "default": "*:*",
      "hints": ["advanced"]
    },
    "sparkSQL": {
      "type": "string",
      "title": "Spark SQL filter query",
      "description": "Filters the loaded DataFrame using a Spark SQL query before processing. The input data is registered as a temporary table named `spark_input`.",
      "default": "SELECT * from spark_input",
      "hints": ["code/sql", "advanced"]
    },
    "trainingDataSamplingFraction": {
      "type": "number",
      "title": "Training data sampling fraction",
      "description": "Sets the fraction of training data to sample before processing. Use a value between `0` and `1`. `1.0` uses all available data.",
      "default": 1,
      "hints": ["advanced"],
      "maximum": 1,
      "exclusiveMaximum": false
    },
    "randomSeed": {
      "type": "integer",
      "title": "Random seed",
      "description": "Sets the random seed for deterministic operations including sampling and initialization. Fix this value to reproduce identical results across runs.",
      "default": 8180,
      "hints": ["advanced"]
    },
    "outputCollection": {
      "type": "string",
      "title": "Output Collection",
      "description": "Specifies the Solr collection where extracted phrases are written. Defaults to the `query_rewrite_staging` collection."
    },
    "overwriteOutput": {
      "type": "boolean",
      "title": "Overwrite Output",
      "description": "When enabled, overwrites the output collection before writing new results.",
      "default": true,
      "hints": ["hidden", "advanced"]
    },
    "dataOutputFormat": {
      "type": "string",
      "title": "Data output format",
      "description": "Specifies the Spark-compatible output format, such as `solr`, `parquet`, or `orc`.",
      "default": "solr",
      "hints": ["advanced"],
      "minLength": 1
    },
    "sourceFields": {
      "type": "string",
      "title": "Fields to Load",
      "description": "Specifies the document fields to load from the input source.",
      "hints": ["advanced"]
    },
    "partitionCols": {
      "type": "string",
      "title": "Partition fields",
      "description": "Specifies a comma-delimited list of column names used to partition output when writing to non-Solr sinks.",
      "hints": ["advanced"]
    },
    "writeOptions": {
      "type": "array",
      "title": "Write Options",
      "description": "Sets additional key-value options passed to the Spark writer when writing output to Solr or other sinks.",
      "hints": ["advanced"],
      "items": {
        "type": "object",
        "required": ["key"],
        "properties": {
          "key": {
            "type": "string",
            "title": "Parameter Name"
          },
          "value": {
            "type": "string",
            "title": "Parameter Value"
          }
        }
      }
    },
    "readOptions": {
      "type": "array",
      "title": "Read Options",
      "description": "Sets additional key-value options passed to the Spark reader when loading input from Solr or other sources.",
      "hints": ["advanced"],
      "items": {
        "type": "object",
        "required": ["key"],
        "properties": {
          "key": {
            "type": "string",
            "title": "Parameter Name"
          },
          "value": {
            "type": "string",
            "title": "Parameter Value"
          }
        }
      }
    },
    "ngramSize": {
      "type": "integer",
      "title": "Ngram Size",
      "description": "Sets the number of words in each n-gram candidate considered for phrase extraction.",
      "default": 3,
      "maximum": 5,
      "exclusiveMaximum": false,
      "minimum": 2,
      "exclusiveMinimum": false
    },
    "minmatch": {
      "type": "integer",
      "title": "Minimum Count",
      "description": "Sets the minimum number of occurrences a phrase must have to be considered significant.",
      "default": 100,
      "minimum": 1,
      "exclusiveMinimum": false
    },
    "analyzerConfig": {
      "type": "string",
      "title": "Lucene Text Analyzer",
      "description": "Specifies the Lucene analyzer configuration in JSON format used for text tokenization and processing.",
      "default": "{ \"analyzers\": [{ \"name\": \"StdTokLowerStop\",\"charFilters\": [ { \"type\": \"htmlstrip\" } ],\"tokenizer\": { \"type\": \"standard\" },\"filters\": [{ \"type\": \"lowercase\" }] }],\"fields\": [{ \"regex\": \".+\", \"analyzer\": \"StdTokLowerStop\" } ]}",
      "hints": ["lengthy", "code/json"]
    },
    "attachPhrases": {
      "type": "boolean",
      "title": "Extract Key Phrases from Input Text",
      "description": "When enabled, associates extracted phrases with their source documents and writes the associations to the output collection.",
      "default": false,
      "hints": ["advanced"]
    },
    "stopwordsList": {
      "type": "array",
      "title": "List of stopwords",
      "description": "Specifies stop words defined in the Lucene analyzer configuration to exclude from processing.",
      "hints": ["readonly", "hidden"],
      "items": {
        "type": "string",
        "minLength": 1,
        "reference": "blob",
        "blobType": "file:spark"
      }
    },
    "minLikelihood": {
      "type": "number",
      "title": "Minimum Likelihood Score",
      "description": "Sets the minimum likelihood score for a phrase to be included in the output. Phrases below this threshold are excluded.",
      "hints": ["advanced"]
    },
    "enableAutoPublish": {
      "type": "boolean",
      "title": "Enable auto-publishing",
      "description": "When enabled, automatically publishes rewrites for rules without requiring manual review.",
      "default": false,
      "hints": ["advanced"]
    },
    "sparkPartitions": {
      "type": "integer",
      "title": "Set minimum Spark partitions for input",
      "description": "Sets the number of Spark partitions to repartition input into before processing. Increase for greater parallelism on large datasets.",
      "default": 200,
      "hints": ["advanced"]
    },
    "type": {
      "type": "string",
      "title": "Spark Job Type",
      "enum": ["sip"],
      "default": "sip",
      "hints": ["readonly"]
    }
  },
  "additionalProperties": true,
  "category": "Other",
  "categoryPriority": 1,
  "propertyGroups": [{
    "label": "Input/Output Parameters",
    "properties": ["trainingCollection", "outputCollection", "dataFormat", "trainingDataFilterQuery", "readOptions", "writeOptions", "trainingDataFrameConfigOptions", "trainingDataSamplingFraction", "randomSeed"]
  }, {
    "label": "Field Parameters",
    "properties": ["fieldToVectorize", "sourceFields"]
  }, {
    "label": "Model Tuning Parameters",
    "properties": ["minmatch", "ngramSize"]
  }, {
    "label": "Featurization Parameters",
    "properties": ["analyzerConfig"]
  }]
};

export const SchemaParamFields = ({schema}) => {
  const sanitize = str => {
    if (typeof str !== "string") return str;
    return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'");
  };
  const renderMd = str => {
    const s = sanitize(str);
    const text = (/[.!?]\)*$/).test(s) ? s : `${s}.`;
    return text.split(/(\*\*[^*]+\*\*|_[^_]+_|`[^`]+`)/g).map((part, i) => {
      if (part.startsWith("**")) return <strong key={i}>{part.slice(2, -2)}</strong>;
      if (part.startsWith("_")) return <em key={i}>{part.slice(1, -1)}</em>;
      if (part.startsWith("`")) return <code key={i}>{part.slice(1, -1)}</code>;
      return part;
    });
  };
  const {description, properties = {}, required: requiredProps = []} = schema;
  const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]);
  const renderProp = ([name, prop]) => {
    const isRequired = requiredProps.includes(name);
    const hasDefault = prop.default !== undefined;
    const rawDefault = prop.default;
    const hints = prop.hints || [];
    const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"')));
    const postBadges = [];
    if (prop.title) {
      postBadges.push(<><span className="text-stone-400 dark:text-stone-500">API property: </span>{name}</>);
    }
    const constraints = [];
    if (prop.minimum !== undefined && prop.maximum !== undefined) {
      constraints.push(`Range: ${prop.minimum} – ${prop.maximum}`);
    } else if (prop.minimum !== undefined) {
      constraints.push(`Min: ${prop.minimum}`);
    } else if (prop.maximum !== undefined) {
      constraints.push(`Max: ${prop.maximum}`);
    }
    if (prop.minLength !== undefined && prop.maxLength !== undefined) {
      constraints.push(`Length: ${prop.minLength} – ${prop.maxLength}`);
    } else if (prop.minLength !== undefined) {
      constraints.push(`Min length: ${prop.minLength}`);
    } else if (prop.maxLength !== undefined) {
      constraints.push(`Max length: ${prop.maxLength}`);
    }
    const fieldProps = {
      key: name,
      body: prop.title || name,
      type: prop.type,
      ...postBadges.length > 0 && ({
        post: postBadges
      }),
      ...isRequired && ({
        required: true
      }),
      ...!isComplexDefault && hasDefault ? {
        default: sanitize(String(rawDefault))
      } : {}
    };
    const isObject = prop.type === "object" && prop.properties;
    const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties;
    return <ParamField {...fieldProps}>
        {prop.description && <p>{renderMd(prop.description)}</p>}

        {prop.enum && <p>
            Allowed values: 
            {prop.enum.map((v, i) => <>{i > 0 && ", "}<code key={i}>{String(v)}</code></>)}
          </p>}

        {constraints.length > 0 && <p className="text-stone-500 dark:text-stone-400 text-sm">
            {constraints.join(" · ")}
          </p>}

        {isComplexDefault && <div className="flex">
            <p>
              <strong>Default:</strong>
            </p>
            <pre className="!my-0">
              <code>
                {JSON.stringify(rawDefault, null, 2)}
              </code>
            </pre>
          </div>}

        {isArrayOfObjects && <Expandable title="item properties">
            <SchemaParamFields schema={{
      properties: prop.items.properties,
      required: prop.items.required
    }} />
          </Expandable>}

        {isObject && <Expandable title="properties">
            <SchemaParamFields schema={{
      properties: prop.properties,
      required: prop.required
    }} />
          </Expandable>}
      </ParamField>;
  };
  return <div>
      {description && <p>{renderMd(description)}</p>}

      {visibleProps.map(renderProp)}
    </div>;
};

export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => {
  const [isLoaded, setIsLoaded] = useState(false);
  useEffect(() => {
    const timer = setTimeout(() => {
      setIsLoaded(true);
    }, 500);
    return () => clearTimeout(timer);
  }, []);
  return <div className="lw-template-container">
      <Card title={title} icon={icon}>
        {isLoaded && <span dangerouslySetInnerHTML={{
    __html: `<lw-template id="a029c1a9-28be-427e-b0e1-5d918920246a"></lw-template
            >`
  }} />}
        <Link href={linkHref} className="agent-studio-link text-left text-gray-600 gap-2 dark:text-gray-400 text-sm font-medium flex flex-row items-center hover:text-primary dark:hover:text-primary-light group-hover:text-primary group-hover:dark:text-primary-light">Powered by Lucidworks Agent Studio</Link>
      </Card>
    </div>;
};

[localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/phrase-extraction

[mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/phrase-extraction

[old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/xzyz7w

Identify multi-word phrases in signals.

<Note>
  This job is deprecated in Lucidworks Search 5.9.15 and will be removed in a future release.
  Lucidworks recommends migrating to [Neural Hybrid Search](/docs/lucidworks-search/11-vector-search/overview), which achieves superior relevance compared to legacy machine learning methods.
</Note>

<Card title="Resolving Underperforming Queries" class="note-image" href="https://academy.lucidworks.com/resolving-underperforming-queries-with-query-rewriting-jobs" cta="Take this course on the LucidAcademy." icon="graduation-cap" iconType="duotone">
  The course for **Resolving Underperforming Queries** focuses on tips for tuning, running, and cleaning up Fusion’s query rewrite jobs.
</Card>

|                      |                                                                                       |
| -------------------- | ------------------------------------------------------------------------------------- |
| **Default job name** | `COLLECTION_NAME_phrase_extraction`                                                   |
| **Input**            | Raw signals (the `COLLECTION_NAME_signals` collection by default)                     |
| **Output**           | Extracted phrases (the `COLLECTION_NAME_query_rewrite_staging` collection by default) |

This job writes to the `COLLECTION_NAME_query_rewrite_staging` collection. It also uses reviewed documents from that collection to improve the accuracy of the job. You can review, edit, deploy, or delete output from this job using the [Query Rewriting](/docs/lucidworks-search/07-improve-your-queries/query-rewriting/overview).

Lucidworks Search ships with the [OpenNLP Maxent model](http://opennlp.sourceforge.net/models-1.5/en-pos-maxent.bin) already loaded in the [blob store](/docs/lucidworks-search/04-move-data-in/blob-storage/overview).

This job’s output, and output from the [Token and Phrase Spell Correction job](/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/token-phrase-spell-correction), can be used as input for the [Synonym Detection job](/docs/lucidworks-search/09-developer-documentation/config-specs/jobs/synonym-detection).

<LwTemplate />

## Minimum configuration

For most use cases, the minimum configuration for this job consists of these fields:

* `id`/**Spark Job ID**
  Give this job an arbitrary ID string.
* `trainingCollection`/**Training Collection**
  Specify the input collection.
* `fieldToVectorize`/**Field to Vectorize**
  Specify the field in the input collection where phrases can be found.
* `outputCollection`/**Output Collection**
  Specify the collection in which the output documents should be indexed.

<Tip>
  When running this job over a content document collection, be sure to set `attachPhrases`/**Extract Key Phrases from Input Text** to "true". The default is "false", which works well when running the job over a signals collection.
</Tip>

## Output documents

By default, the job only outputs the phrases found from the original document. In each row of the phrases output, these fields are most useful:

* The phrase itself is in the `phrases_s` field, which can be used for faceting.
* The `likelihood_d` field gives the likelihood that the phrase is legitimate, from 0 to infinity.\
  Low-probability phrases are automatically trimmed from the results.
* When a phrase’s likelihood value is ambiguous, the `review` field is set to "true" to indicate that the phrase should be reviewed.
* A `phrase_count` field indicates the number of instances of the phrase in the input collection.

The complete list of output fields is shown below.

## Output fields

|                    |                                                                                  |
| ------------------ | -------------------------------------------------------------------------------- |
| `aggr_id_s`        | The name of the Phrase Extraction job that generated this document.              |
| `doc_type_s`       | This is always `key_phrases` for documents generated by a Phrase Extraction job. |
| `id`               | A unique ID for this document.                                                   |
| `input_collection` | The collection used for this job’s input.                                        |
| `likelihood_d`     | The likelihood that this `phrases_s` is a phrase, from 0 to infinity.            |
| `phrase_count`     | The number of occurrences of this phrase in the input collection.                |
| `phrases_s`        | The phrase detected by the job.                                                  |
| `review`           | "True" indicates that this may not be a valid phrase and should be reviewed.     |
| `score`            | This is always "1".                                                              |
| `timestamp`        | The date and time when the document was generated.                               |
| `word_num_i`       | The number of words in this phrase.                                              |
| `_version_`        | An internal Solr field used for partial updates.                                 |

If the `attachPhrases`/**Extract Key Phrases from Input Text** parameter is set to "true", then the job also outputs the original documents from the input collection with an appended field, `phrases_extracted_tt`, that lists the extracted phrases from this document.

The way to distinguish the phrases output from the original document output is by the field `doc_type_s`, with one of these values:

* `key_phrases` denotes phrases output.
* `original_doc_with_phrases` denotes the original documents.

## Configuration properties

<SchemaParamFields schema={schema} />