> ## Documentation Index
> Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Include Documents

> Index pipeline stage configuration specifications

export const schema = {
  "type": "object",
  "title": "Include Documents",
  "description": "Passes a document through the pipeline if any of the specified rules match. Drops the document if no rules match. Use this stage to allow only documents meeting specific criteria to continue for indexing.",
  "required": ["matchRules"],
  "properties": {
    "skip": {
      "type": "boolean",
      "title": "Skip This Stage",
      "description": "Controls whether this stage executes during pipeline processing at runtime. When set to `true`, the stage is completely bypassed and documents pass through unchanged to the next stage. Useful for A/B testing, gradual rollouts, or temporarily disabling a stage without removing it from the pipeline.",
      "default": false,
      "hints": ["advanced"]
    },
    "label": {
      "type": "string",
      "title": "Label",
      "description": "Human-readable identifier displayed in the Fusion Admin UI, monitoring dashboards, and log messages. Use descriptive labels like `Parse Product PDFs` to aid debugging and team collaboration. Labels appear in performance metrics and error reports, making it easier to identify which stage failed.",
      "hints": ["advanced"],
      "maxLength": 255
    },
    "condition": {
      "type": "string",
      "title": "Condition",
      "description": "JavaScript expression evaluated at runtime on each document to conditionally execute this stage. The expression must return `true` to execute or `false` to skip. Access document fields using `doc.getFieldValue('fieldName')` and request parameters via `request.getFirstParam('paramName')`. For example, `doc.getFieldValue('type') === 'premium'` executes this stage only for premium content.",
      "hints": ["code", "code/javascript", "advanced"]
    },
    "matchRules": {
      "type": "array",
      "title": "Fields and Patterns",
      "minItems": 1,
      "items": {
        "type": "object",
        "required": ["field", "pattern"],
        "properties": {
          "field": {
            "type": "string",
            "title": "Field",
            "description": "Specifies the field name(s) for field operations in the document processing pipeline."
          },
          "pattern": {
            "type": "string",
            "title": "Regex Pattern",
            "description": "Pattern to match the field value against. The value may be a regex pattern.",
            "format": "regex"
          }
        }
      }
    }
  },
  "category": "Document Filtering and Enrichment",
  "categoryPriority": 8,
  "unsafe": false
};

export const SchemaParamFields = ({schema}) => {
  const sanitize = str => {
    if (typeof str !== "string") return str;
    return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'");
  };
  const renderMd = str => {
    const s = sanitize(str);
    const text = (/[.!?]\)*$/).test(s) ? s : `${s}.`;
    return text.split(/(\*\*[^*]+\*\*|_[^_]+_|`[^`]+`)/g).map((part, i) => {
      if (part.startsWith("**")) return <strong key={i}>{part.slice(2, -2)}</strong>;
      if (part.startsWith("_")) return <em key={i}>{part.slice(1, -1)}</em>;
      if (part.startsWith("`")) return <code key={i}>{part.slice(1, -1)}</code>;
      return part;
    });
  };
  const {description, properties = {}, required: requiredProps = []} = schema;
  const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]);
  const renderProp = ([name, prop]) => {
    const isRequired = requiredProps.includes(name);
    const hasDefault = prop.default !== undefined;
    const rawDefault = prop.default;
    const hints = prop.hints || [];
    const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"')));
    const postBadges = [];
    if (prop.title) {
      postBadges.push(<><span className="text-stone-400 dark:text-stone-500">API property: </span>{name}</>);
    }
    const constraints = [];
    if (prop.minimum !== undefined && prop.maximum !== undefined) {
      constraints.push(`Range: ${prop.minimum} – ${prop.maximum}`);
    } else if (prop.minimum !== undefined) {
      constraints.push(`Min: ${prop.minimum}`);
    } else if (prop.maximum !== undefined) {
      constraints.push(`Max: ${prop.maximum}`);
    }
    if (prop.minLength !== undefined && prop.maxLength !== undefined) {
      constraints.push(`Length: ${prop.minLength} – ${prop.maxLength}`);
    } else if (prop.minLength !== undefined) {
      constraints.push(`Min length: ${prop.minLength}`);
    } else if (prop.maxLength !== undefined) {
      constraints.push(`Max length: ${prop.maxLength}`);
    }
    const fieldProps = {
      key: name,
      body: prop.title || name,
      type: prop.type,
      ...postBadges.length > 0 && ({
        post: postBadges
      }),
      ...isRequired && ({
        required: true
      }),
      ...!isComplexDefault && hasDefault ? {
        default: sanitize(String(rawDefault))
      } : {}
    };
    const isObject = prop.type === "object" && prop.properties;
    const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties;
    return <ParamField {...fieldProps}>
        {prop.description && <p>{renderMd(prop.description)}</p>}

        {prop.enum && <p>
            Allowed values: 
            {prop.enum.map((v, i) => <>{i > 0 && ", "}<code key={i}>{String(v)}</code></>)}
          </p>}

        {constraints.length > 0 && <p className="text-stone-500 dark:text-stone-400 text-sm">
            {constraints.join(" · ")}
          </p>}

        {isComplexDefault && <div className="flex">
            <p>
              <strong>Default:</strong>
            </p>
            <pre className="!my-0">
              <code>
                {JSON.stringify(rawDefault, null, 2)}
              </code>
            </pre>
          </div>}

        {isArrayOfObjects && <Expandable title="item properties">
            <SchemaParamFields schema={{
      properties: prop.items.properties,
      required: prop.items.required
    }} />
          </Expandable>}

        {isObject && <Expandable title="properties">
            <SchemaParamFields schema={{
      properties: prop.properties,
      required: prop.required
    }} />
          </Expandable>}
      </ParamField>;
  };
  return <div>
      {description && <p>{renderMd(description)}</p>}

      {visibleProps.map(renderProp)}
    </div>;
};

export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => {
  const [isLoaded, setIsLoaded] = useState(false);
  useEffect(() => {
    const timer = setTimeout(() => {
      setIsLoaded(true);
    }, 500);
    return () => clearTimeout(timer);
  }, []);
  return <div className="lw-template-container">
      <Card title={title} icon={icon}>
        {isLoaded && <span dangerouslySetInnerHTML={{
    __html: `<lw-template id="a029c1a9-28be-427e-b0e1-5d918920246a"></lw-template
            >`
  }} />}
        <Link href={linkHref} className="agent-studio-link text-left text-gray-600 gap-2 dark:text-gray-400 text-sm font-medium flex flex-row items-center hover:text-primary dark:hover:text-primary-light group-hover:text-primary group-hover:dark:text-primary-light">Powered by Lucidworks Agent Studio</Link>
      </Card>
    </div>;
};

[localhost link]: http://localhost:3000/docs/lucidworks-search/09-developer-documentation/config-specs/index-pipeline-stages/include-documents

[mintlify link]: https://doc.lucidworks.com/docs/lucidworks-search/09-developer-documentation/config-specs/index-pipeline-stages/include-documents

[old doc.lw link]: https://doc.lucidworks.com/managed-fusion/5.9/lzzy5s

This stage passes documents to the next stage in the pipeline if they match one or more of the specified rules (Boolean OR). If some field has multiple values then at least one value must match against specified pattern. All non-matching documents are dropped. Rules are defined using regular expression field matching.

<LwTemplate />

## Examples

**Give the "simple-include" pipeline a stage that includes only certain document types**

```bash wrap  theme={"dark"}
curl -u USERNAME:PASSWORD -X POST -H "Content-type: application/json" 'https://EXAMPLE_COMPANY.lucidworks.cloud/api/index-pipelines' -d '
{
  "id" : "simple-include",
  "stages" : [ {
    "type" : "include-doc",
    "matchRules" : [ {
        "field" : "document_type",
        "pattern" : "(xls|xlsx|xlst|doc|docx)"
    }]
  }]
}'
```

<Note>
  Replace `EXAMPLE_COMPANY` with the name provided by your Lucidworks representative.
</Note>

Response:

```json wrap  theme={"dark"}
{
  "id" : "simple-include",
  "stages" : [ {
    "type" : "include-doc",
    "id" : "f701f96b-780e-4355-9dd3-6e53a89afe3e",
    "matchRules" : [ {
      "field" : "document_type",
      "pattern" : "(xls|xlsx|xlst|doc|docx)"
    } ],
    "type" : "include-doc",
    "skip" : false,
    "label" : "include-doc"
  } ],
  "properties" : { }
}
```

**Send a text document through the "simple-include" pipeline**

```bash wrap  theme={"dark"}
curl -u USERNAME:PASSWORD 'http://EXAMPLE_COMPANY.lucidworks.cloud/api/index-pipelines/simple-include/collections/logs/index?simulate=true&echo=true' -H 'Content-type: application/json' -d '
{
  "document_type": "txt"
}'
```

The empty response indicates the document was dropped:

```json wrap  theme={"dark"}
[ ]
```

**Send an XLS document through the pipeline**

```bash wrap  theme={"dark"}
curl -u USERNAME:PASSWORD 'http://EXAMPLE_COMPANY.lucidworks.cloud/api/index-pipelines/simple-include/collections/logs/index?simulate=true&echo=true' -H 'Content-type: application/json' -d '
{
  "document_type": "xls"
}'
```

The response is document metadata, indicating the document passed the stage:

```json wrap  theme={"dark"}
 {
  "id" : "9e7d1c2e-343a-49de-bc6a-1d1fc25fa93f",
  "fields" : [ {
    "name" : "document_type",
    "value" : "xls",
    "metadata" : { },
    "annotations" : [ ]
  } ]
} ]
```

## Configuration

<Tip>
  When entering configuration values in the UI, use *unescaped* characters, such as `\t` for the tab character. When entering configuration values in the API, use *escaped* characters, such as `\\t` for the tab character.
</Tip>

<SchemaParamFields schema={schema} />