> ## Documentation Index > Fetch the complete documentation index at: https://doc.lucidworks.com/llms.txt > Use this file to discover all available pages before exploring further. # Web V2 > The Web V2 connector retrieves data from a website using HTTP and starting from a specified URL. export const schema = { "type": "object", "title": "Web (v2)", "description": "Connector for websites and web-based content resources.", "required": ["id", "properties", "pipeline"], "properties": { "id": { "type": "string", "title": "Configuration ID", "description": "A unique identifier for this Configuration.", "minLength": 1, "pattern": "^[a-zA-Z0-9_-]+$" }, "pipeline": { "type": "string", "title": "Pipeline ID", "description": "Name of the IndexPipeline used for processing output.", "minLength": 1, "pattern": "^[a-zA-Z0-9_-]+$" }, "parserId": { "type": "string", "title": "Parser ID", "description": "The Parser to use in the associated IndexPipeline.", "pattern": "^[a-zA-Z0-9_-]+$" }, "description": { "type": "string", "title": "Description", "description": "Optional description", "hints": ["lengthy"], "maxLength": 125 }, "modified": { "type": "string", "title": "Date Modified", "description": "The date at which this Configuration was last modified.", "hints": ["readonly", "hidden"] }, "created": { "type": "string", "title": "Date Created", "description": "The date at which this Configuration was created.", "hints": ["readonly", "hidden"] }, "type": { "type": "string", "title": "Type", "description": "A type ID for this connector.", "hints": ["readonly", "hidden"] }, "diagnosticLogging": { "type": "boolean", "title": "Diagnostic Logging", "description": "[Deprecated] Enables verbose diagnostic logging for troubleshooting. May increase log volume. Disabled by default.", "default": false }, "collection": { "type": "string", "title": "Collection ID", "description": "The associated content Collection.", "hints": ["readonly", "hidden"], "minLength": 1, "pattern": "^[a-zA-Z0-9_-]+$" }, "type_description": { "type": "string", "title": "Type Description", "default": "Connector for websites and web-based content resources.", "hints": ["hidden", "readonly"] }, "category": { "type": "string", "title": "Category", "default": "Connector for websites and web-based content resources.", "hints": ["hidden", "readonly"] }, "connector": { "title": "Connector Type", "description": "Connector type.", "minLength": 1, "type": "string", "hints": ["hidden"] }, "properties": { "type": "object", "title": "Properties", "description": "Plugin specific properties.", "required": [], "properties": { "startLinks": { "type": "array", "title": "Start Links", "description": "The URL(s) that the crawler will start crawling from, for example: https://en.wikipedia.org/wiki/Main_Page", "items": { "type": "string" } }, "bulkStartLinks": { "type": "string", "title": "Bulk start links", "description": "If a large number of start links must be defined, you can provide them here. One link per line.", "hints": ["lengthy"] }, "limitDocumentsConfig": { "type": "object", "title": "Limit Documents Properties", "required": [], "properties": { "depth": { "type": "number", "title": "Max crawling depth", "description": "Maximum number of link levels to explore within a site.", "default": -1, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "maxItems": { "type": "number", "title": "Max items", "description": "Maximum number of documents to fetch. The default (-1) means no limit.", "default": -1, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "excludeExtensions": { "type": "array", "title": "Excluded file extensions", "description": "File extensions that should not to be fetched. This will limit this datasource to all extensions except this list.", "items": { "type": "string" } }, "excludeRegexes": { "type": "array", "title": "Exclusive regexes", "description": "Regular expressions for URI patterns to exclude. This will limit this datasource to only URIs that do not match the regular expression.", "items": { "type": "string" } }, "includeExtensions": { "type": "array", "title": "Included file extensions", "description": "File extensions to be fetched. This will limit this datasource to only these file extensions.", "items": { "type": "string" } }, "includeRegexes": { "type": "array", "title": "Inclusive regexes", "description": "Regular expressions for URI patterns to include. This will limit this datasource to only URIs that match the regular expression.", "items": { "type": "string" } }, "maxSizeBytes": { "type": "number", "title": "Max file size (bytes)", "description": "Maximum size, in bytes, of a document to fetch.", "default": 20000000, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "indexItemsDiscarded": { "type": "boolean", "title": "Index discarded document metadata", "description": "Enable to index discarded document metadata", "default": false, "hints": ["hidden"] } } }, "crawlAuthenticationConfig": { "type": "object", "title": "Crawl Authentication Properties", "required": [], "properties": { "maintainCookies": { "type": "boolean", "title": "Maintain Cookies for No-Auth Crawls", "description": "If you are not using authentication, then by default cookies are not stored in between web requests (stateless). If checked, cookies will be maintained between requests during the web crawl even when you are not using authentication. If you are using authentication, this checkbox has no effect on the crawl and can be ignored.", "default": false, "hints": ["advanced"] }, "basicAuth": { "type": "array", "title": "Basic Authentication", "description": "Settings for Basic authentication", "items": { "type": "object", "required": [], "properties": { "host": { "type": "string", "title": "Host", "description": "The host of the site. You can specify * to authenticate against any host. (Warning authentication cookies will be send to any host)" }, "port": { "type": "number", "title": "Port", "description": "The port. You can specify -1 to authenticate against any port", "maximum": 65535, "exclusiveMaximum": false, "minimum": -1, "exclusiveMinimum": false, "multipleOf": 1 }, "realm": { "type": "string", "title": "Realm", "description": "The realm for the site, if applicable" }, "username": { "type": "string", "title": "Username", "description": "The username to use for authentication." }, "password": { "type": "string", "title": "Password", "description": "The password to use for authentication.", "hints": ["secret"] }, "passwordVaultLabel": { "type": "string", "title": "Password from Vault", "description": "The Fusion managed secret to use for Password.", "hints": ["vaultvaluetarget:password", "enumUrl:/apps-manager/vault/labels?type=secret", "vaultref"] } } } }, "digestAuth": { "type": "array", "title": "Digest Authentication", "description": "Settings for Digest authentication", "items": { "type": "object", "required": [], "properties": { "host": { "type": "string", "title": "Host", "description": "The host of the site. You can specify * to authenticate against any host. (Warning authentication cookies will be send to any host)" }, "port": { "type": "number", "title": "Port", "description": "The port. You can specify -1 to authenticate against any port", "maximum": 65535, "exclusiveMaximum": false, "minimum": -1, "exclusiveMinimum": false, "multipleOf": 1 }, "realm": { "type": "string", "title": "Realm", "description": "The realm for the site, if applicable" }, "username": { "type": "string", "title": "Username", "description": "The username to use for authentication." }, "password": { "type": "string", "title": "Password", "description": "The password to use for authentication.", "hints": ["secret"] }, "passwordVaultLabel": { "type": "string", "title": "Password from Vault", "description": "The Fusion managed secret to use for Password.", "hints": ["vaultvaluetarget:password", "enumUrl:/apps-manager/vault/labels?type=secret", "vaultref"] } } } }, "ntlmAuth": { "type": "array", "title": "NTLM Authentication", "description": "Settings for NTLM authentication", "items": { "type": "object", "required": [], "properties": { "domain": { "type": "string", "title": "Domain", "description": "The NTLM Domain" }, "workstation": { "type": "string", "title": "Workstation", "description": "The NTLM Workstation name" }, "host": { "type": "string", "title": "Host", "description": "The host of the site. You can specify * to authenticate against any host. (Warning authentication cookies will be send to any host)" }, "port": { "type": "number", "title": "Port", "description": "The port. You can specify -1 to authenticate against any port", "maximum": 65535, "exclusiveMaximum": false, "minimum": -1, "exclusiveMinimum": false, "multipleOf": 1 }, "realm": { "type": "string", "title": "Realm", "description": "The realm for the site, if applicable" }, "username": { "type": "string", "title": "Username", "description": "The username to use for authentication." }, "password": { "type": "string", "title": "Password", "description": "The password to use for authentication.", "hints": ["secret"] }, "passwordVaultLabel": { "type": "string", "title": "Password from Vault", "description": "The Fusion managed secret to use for Password.", "hints": ["vaultvaluetarget:password", "enumUrl:/apps-manager/vault/labels?type=secret", "vaultref"] } } } }, "formAuth": { "type": "array", "title": "Form Authentication", "description": "Settings for Form based authentication", "items": { "type": "object", "required": [], "properties": { "action": { "type": "string", "title": "URL", "description": "The URL of the authentication endpoint" }, "ttl": { "type": "number", "title": "TTL (ms)", "description": "The \"time to live\" in milliseconds for the session that will be created after authentication.", "default": 300000, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "passwordParamName": { "type": "string", "title": "Password Parameter", "description": "Name of the parameter containing the password" }, "password": { "type": "string", "title": "Password", "description": "The password to use for the authentication request. This will be copied into the \"Parameters\" using the \"Password Parameter\" name as the key", "hints": ["secret"] }, "passwordVaultLabel": { "type": "string", "title": "Password from Vault", "description": "The Fusion managed secret to use for Password.", "hints": ["vaultvaluetarget:password", "enumUrl:/apps-manager/vault/labels?type=secret", "vaultref"] }, "paramsList": { "type": "array", "title": "Parameters", "description": "Parameters sent to the authentication endpoint", "items": { "type": "object", "required": [], "properties": { "key": { "type": "string", "title": "Parameters Key" }, "value": { "type": "string", "title": "Parameters value" } } } } } } }, "samlAuth": { "type": "array", "title": "SAML/Smart Form Authentication", "description": "Settings for SAML/Smart Form based authentication allows you to visit one or more web pages that contain form inputs such as username, password, security questions, etc., submitting each one in turn in order to become authenticated.", "items": { "type": "object", "required": [], "properties": { "action": { "type": "string", "title": "URL", "description": "The URL of the authentication endpoint" }, "ttl": { "type": "number", "title": "TTL (ms)", "description": "The \"time to live\" in milliseconds for the session that will be created after authentication.", "default": 300000, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "passwordParamName": { "type": "string", "title": "Password Parameter", "description": "Name of the parameter containing the password" }, "password": { "type": "string", "title": "Password", "description": "The password to use for the authentication request. This will be copied into the \"Parameters\" using the \"Password Parameter\" name as the key", "hints": ["secret"] }, "passwordVaultLabel": { "type": "string", "title": "Password from Vault", "description": "The Fusion managed secret to use for Password.", "hints": ["vaultvaluetarget:password", "enumUrl:/apps-manager/vault/labels?type=secret", "vaultref"] }, "paramsList": { "type": "array", "title": "Parameters", "description": "Parameters sent to the authentication endpoint", "items": { "type": "object", "required": [], "properties": { "key": { "type": "string", "title": "Parameters Key" }, "value": { "type": "string", "title": "Parameters value" } } } } } } }, "credentialsFile": { "type": "string", "title": "credentialsFile", "description": "This property is no longer in use, and is only in place due to backwards compatible configuration validation purposes." }, "kerberosEnabled": { "type": "boolean", "title": "kerberosEnabled", "description": "This property is no longer in use, and is only in place due to backwards compatible configuration validation purposes.", "default": false }, "kerberosLoginContextName": { "type": "string", "title": "kerberosLoginContextName", "description": "This property is no longer in use, and is only in place due to backwards compatible configuration validation purposes." }, "kerberosSpn": { "type": "string", "title": "kerberosSpn", "description": "This property is no longer in use, and is only in place due to backwards compatible configuration validation purposes." }, "kerberosPrincipal": { "type": "string", "title": "kerberosPrincipal", "description": "This property is no longer in use, and is only in place due to backwards compatible configuration validation purposes." }, "kerberosKeytabFile": { "type": "string", "title": "kerberosKeytabFile", "description": "This property is no longer in use, and is only in place due to backwards compatible configuration validation purposes." }, "kerberosKeytabBase64": { "type": "string", "title": "kerberosKeytabBase64", "description": "This property is no longer in use, and is only in place due to backwards compatible configuration validation purposes." }, "kerberosPassword": { "type": "string", "title": "kerberosPassword", "description": "This property is no longer in use, and is only in place due to backwards compatible configuration validation purposes.", "hints": ["secret"] }, "kerberosPasswordVaultLabel": { "type": "string", "title": "kerberosPassword from Vault", "description": "The Fusion managed secret to use for kerberosPassword.", "hints": ["vaultvaluetarget:kerberosPassword", "enumUrl:/apps-manager/vault/labels?type=secret", "vaultref"] }, "obeyRobots": { "type": "boolean", "title": "Obey robots.txt", "description": "If true, Allow, Disallow and other rules found in a robots.txt file will be obeyed.", "default": false }, "obeyRobotsMeta": { "type": "boolean", "title": "Obey robots meta tags and headers", "description": "If true, rules like 'noindex', 'nofollow' and others found in a robots meta tag on a page or in the headers of the HTTP response are obeyed.", "default": false }, "obeyLinkNofollow": { "type": "boolean", "title": "Obey link nofollow attributes", "description": "If true, rel='nofollow' on links are obeyed", "default": false }, "proxy": { "type": "string", "title": "HTTP proxy address", "description": "Address of the HTTP proxy, if required. This should be entered in the format host:port." }, "allowAllCertificates": { "type": "boolean", "title": "Allow all HTTPS certificates", "description": "If false, security checks will be performed on all SSL/TLS certificate signers and origins. This means self-signed certificates would not be supported.", "default": false }, "useIpAddressForSslConnections": { "type": "boolean", "title": "Use IP address for SSL connections", "description": "Use IP address instead of host name for SSL connections. This is used to work around mis-configured HTTP server throwing 'unrecognized name' error whenSNI is enabled. (This only works if 'Allow all certificates' setting is also enabled)", "default": false }, "accessTokenConfig": { "type": "object", "title": "Access Token", "description": "Access Token will be automatically requested using this request. Secret params that are set up below will be added automatically.", "required": ["requestUrl"], "properties": { "requestUrl": { "type": "string", "title": "Access Token Request URL" }, "params": { "type": "array", "items": { "type": "object", "required": [], "properties": { "name": { "type": "string", "title": "Name", "description": "Parameter Name" }, "value": { "type": "string", "title": "Value", "description": "Parameter Value" } } } }, "secretParams": { "type": "array", "items": { "type": "object", "required": [], "properties": { "name": { "type": "string", "title": "Name", "description": "Parameter Name" }, "value": { "type": "string", "title": "Value", "description": "Parameter Value", "hints": ["secret"] }, "valueVaultLabel": { "type": "string", "title": "Value from Vault", "description": "The Fusion managed secret to use for Value.", "hints": ["enumUrl:/apps-manager/vault/labels?type=secret", "vaultref", "vaultvaluetarget:value"] } } } } } } } }, "crawlHistoryConfig": { "type": "object", "title": "Crawl History Properties", "required": [], "properties": { "crawlDBType": { "type": "string", "title": "Crawl database type", "description": "The type of crawl database to use, in-memory or on-disk.", "default": "on-disk", "hints": ["hidden"] }, "commitAfterItems": { "type": "number", "title": "Commit After This Many Items", "description": "Commit the crawlDB to disk after this many items have been received. A smaller number here will result in a slower crawl because of commits to disk being more frequent; conversely, a larger number here will cause a resumed job after a crash to need to recrawl more records.", "default": 10000, "hints": ["hidden"], "maximum": 9999999, "exclusiveMaximum": false, "minimum": 1, "exclusiveMinimum": false, "multipleOf": 1 }, "retainOutlinks": { "type": "boolean", "title": "Retain links in the crawldb", "description": "Set to true for links found during fetching to be stored in the crawldb. This increases precision in certain recrawl scenarios, but requires more memory and disk space.", "default": false, "hints": ["hidden"] }, "aliasExpiration": { "type": "number", "title": "Alias expiration", "description": "The number of crawls after which an alias will expire. The default is 1 crawl.", "default": 1, "hints": ["hidden"], "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "discardLinkURLQueries": { "type": "boolean", "title": "Discard queries in link URLs", "description": "If true, query parameters found in URLs will be removed before being added to the discovery queue.", "default": false, "hints": ["advanced"] }, "discardLinkURLAnchors": { "type": "boolean", "title": "Discard anchors in link URLs", "description": "If true, anchors found in URLs will be removed before being added to the discovery queue.", "default": false, "hints": ["advanced"] } }, "hints": ["advanced"] }, "crawlIdConfig": { "type": "object", "title": "Crawl Id Properties", "required": [], "properties": { "userAgentName": { "type": "string", "title": "HTTP user-agent name", "description": "Name the connector should use when identifying itself to a website in order to crawl it.", "default": "Lucidworks-Anda/2.0", "hints": ["advanced"] }, "userAgentEmail": { "type": "string", "title": "HTTP user-agent email address", "description": "Email address to use as part of connector identification.", "hints": ["advanced"] }, "userAgentWebAddr": { "type": "string", "title": "HTTP user-agent web address", "description": "Web address to use as part of connector identification.", "hints": ["advanced"] } }, "hints": ["advanced"] }, "crawlPerformanceConfig": { "type": "object", "title": "Crawl Performance Properties", "required": [], "properties": { "fetchDelayMSPerHost": { "type": "boolean", "title": "Fetch delay per host", "description": "If true, the 'Fetch delay (ms)' property will be applied for each host.", "default": false, "hints": ["advanced"] }, "fetchThreads": { "type": "number", "title": "Fetch threads", "description": "The number of threads to use during fetching. The default is 5.", "default": 5, "hints": ["hidden"], "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "emitThreads": { "type": "number", "title": "Emit threads", "description": "The number of threads used to send documents from the connector to the index pipeline. The default is 5.", "default": 5, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "chunkSize": { "type": "number", "title": "Fetch batch size", "description": "The number of items to batch for each round of fetching. A higher value can make crawling faster, but memory usage is also increased. The default is 1.", "default": 1, "hints": ["advanced"], "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "fetchDelayMS": { "type": "number", "title": "Fetch delay", "description": "Number of milliseconds to wait between fetch requests. The default is 0. This property can be used to throttle a crawl if necessary.", "default": 0, "hints": ["advanced"], "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "retryEmit": { "type": "boolean", "title": "Retry emits", "description": "Set to true for emit batch failures to be retried on a document-by-document basis.", "default": true, "hints": ["hidden"] }, "failFastOnStartLinkFailure": { "type": "boolean", "title": "Fail crawl if start links are invalid", "description": "If true, when Fusion cannot connect to any of the provided start links, the crawl is stopped and an exception logged.", "default": true, "hints": ["hidden"] }, "requestRetryCount": { "type": "number", "title": "Request Retry Count", "description": "If an http request fails, retry up to this many times before giving up. If set to 0, requests will not be retried. This is useful in situations where your crawls are failing with errors like \"The target server failed to respond\".", "default": 0, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "obeyRobotsDelay": { "type": "boolean", "title": "Obey robots.txt Crawl-Delay", "description": "If true, Crawl-Delay rules in robots.txt will be obeyed. Disabling this option will speed up crawling, but is considered negative behavior for sites you do not control.", "default": true, "hints": ["advanced"] }, "parserRetryCount": { "type": "number", "title": "Max Parser Retries", "description": "The maximum number of times the configured parser will try getting content before giving up", "default": 0, "hints": ["hidden"], "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "timeoutMS": { "type": "number", "title": "Connection timeout (ms)", "description": "Time in milliseconds to wait for server response. This setting is only used when 'Use infinite timeout' is disabled.", "default": 10000, "hints": ["advanced"], "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "infiniteTimeout": { "type": "boolean", "title": "Enable Infinite timeout", "description": "When unchecked infinite timeout is disabled. if the property is checked the timeoutMS value is ignored", "default": false, "hints": ["advanced"] } } }, "dedupeConfig": { "type": "object", "title": "Dedupe Properties", "required": [], "properties": { "dedupe": { "type": "boolean", "title": "Dedupe documents", "description": "If true, documents will be deduplicated. Deduplication can be done based on an analysis of the content, on the content of a specific field, or by a JavaScript function. If neither a field nor a script are defined, content analysis will be used.", "default": false, "hints": ["hidden"] }, "dedupeField": { "type": "string", "title": "Dedupe field", "description": "Field to be used for dedupe. Define either a field or a dedupe script, otherwise the full raw content of each document will be used.", "hints": ["hidden"] }, "dedupeScript": { "type": "string", "title": "Dedupe script", "description": "Custom javascript to dedupe documents. The script must define a 'genSignature(content){}' function, but can use any combination of document fields. The function must return a string.", "hints": ["code", "hidden", "javascript"] }, "dedupeSaveSignature": { "type": "boolean", "title": "Save dedupe signature", "description": "If true,the signature used for dedupe will be stored in a 'dedupeSignature_s' field. Note this may cause errors about 'immense terms' in that field.", "default": false, "hints": ["hidden"] }, "followCanonicalTags": { "type": "boolean", "title": "Deduplication via canonical tag", "description": "Deduplicate, by only indexing the document at the URL specified in the canonical tag. https://en.wikipedia.org/wiki/Canonical_link_element", "default": false, "hints": ["advanced"] }, "canonicalTagsRedirectLimit": { "type": "number", "title": "The maximum number of requests to perform while resolving the canonical URL for a page.", "description": "Because canonical tag resolution may be cyclical, a limit must be applied to the total number of requests. This value ensures that the resolution finishes in a reasonable amount of time.", "default": 4, "hints": ["advanced"], "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 } }, "hints": ["advanced"] }, "documentParsingConfig": { "type": "object", "title": "Document Parsing Properties", "required": [], "properties": { "defaultCharSet": { "type": "string", "title": "Default character set", "description": "Default character set to use when one is not declared in the HTTP headers.", "default": "UTF-8", "hints": ["advanced"] }, "obeyCharSet": { "type": "boolean", "title": "Obey server-supplied charset", "description": "Use the encoding sent by the web server (if any) when parsing content. If unset, Fusion will try to guess the character set when parsing.", "default": true, "hints": ["hidden"] }, "defaultMIMEType": { "type": "string", "title": "Default MIME type", "description": "Default MIME type to use when one is not declared in the HTTP headers.", "default": "application/octet-stream", "hints": ["advanced"] }, "appendTrailingSlashToLinks": { "type": "boolean", "title": "Add trailing slash to link URLs", "description": "If true, a trailing '/' will be added to link URLs when the URL does not end in a dot ('.').", "default": false, "hints": ["advanced"] }, "scrapeLinksBeforeFiltering": { "type": "boolean", "title": "Scrape links before filtering", "description": "If true, links will be extracted from documents before any other document processing has ocurred. By default, links are extracted after all other document processing.", "default": false, "hints": ["advanced"] }, "tagFields": { "type": "array", "title": "Tag fields", "description": "HTML tags of elements to put into their own field in the index. The field will have the same name as the tag.", "hints": ["advanced"], "items": { "type": "string" } }, "tagIDFields": { "type": "array", "title": "Tag-ID fields", "description": "HTML tag IDs of elements to put into their own field in the index. The field will have the same name as the tag ID.", "hints": ["advanced"], "items": { "type": "string" } }, "tagClassFields": { "type": "array", "title": "Tag-class fields", "description": "HTML tag classes of elements to put into their own field in the index. The field will have the same name as the tag class.", "hints": ["advanced"], "items": { "type": "string" } }, "selectorFields": { "type": "array", "title": "Jsoup selector fields", "description": "List of Jsoup selectors for elements to put into their separate field in the index. The field will have the same name as the element. Syntax for jsoup selectors is available at http://jsoup.org/apidocs/org/jsoup/select/Selector.html.", "hints": ["advanced"], "items": { "type": "string" } }, "filteringRootTags": { "type": "array", "title": "Root elements to filter", "description": "Root HTML elements whose child elements will be used to extract content. By default 'body' and 'head' elements are already included.", "hints": ["advanced"], "items": { "type": "string" } }, "includeSelectors": { "type": "array", "title": "Jsoup inclusive selectors", "description": "Jsoup-formatted selectors for elements to include in the crawled content.", "hints": ["advanced"], "items": { "type": "string" } }, "includeTags": { "type": "array", "title": "Included tags", "description": "HTML tag names of elements to include in the crawled content.", "hints": ["advanced"], "items": { "type": "string" } }, "includeTagClasses": { "type": "array", "title": "Included tag classes", "description": "HTML tag classes of elements to include in the crawled content.", "hints": ["advanced"], "items": { "type": "string" } }, "includeTagIDs": { "type": "array", "title": "Included tag IDs", "description": "HTML tag IDs of elements to include in the crawled content.", "hints": ["advanced"], "items": { "type": "string" } }, "excludeSelectors": { "type": "array", "title": "Jsoup exclusive selectors", "description": "Jsoup-formatted selectors for elements to exclude from the crawled content. Syntax for jsoup selectors is available at http://jsoup.org/apidocs/org/jsoup/select/Selector.html.", "hints": ["advanced"], "items": { "type": "string" } }, "excludeTags": { "type": "array", "title": "Excluded tags", "description": "HTML tag names of elements to exclude from the crawled content.", "hints": ["advanced"], "items": { "type": "string" } }, "excludeTagClasses": { "type": "array", "title": "Excluded tag classes", "description": "HTML tag classes of elements to exclude from the crawled content.", "hints": ["advanced"], "items": { "type": "string" } }, "excludeTagIDs": { "type": "array", "title": "Excluded tag IDs", "description": "HTML tag IDs of elements to exclude from the crawled content.", "hints": ["advanced"], "items": { "type": "string" } }, "customLinkSelectors": { "type": "array", "title": "Custom XPath Link Selectors", "description": "By default, only standard anchor tags, iframe tags, frame tags, and link tags are fetched. This allows you to use one or more XPath expressions to parse links from custom places. Such as //option/@value", "items": { "type": "string" } } } }, "javascriptEvaluationConfig": { "type": "object", "title": "Javascript Evaluation Properties", "required": [], "properties": { "crawlJS": { "type": "boolean", "title": "Evaluate Javascript", "description": "Evaluate JavaScript on web pages when crawling. This makes it possible for the Web fetcher to extract content from pages that is only available after JavaScript has prepared the document, but it may make the crawl slower because JavaScript loading can be time consuming.", "default": false }, "jsEnabledAuth": { "type": "boolean", "title": "Evaluate JavaScript during SmartForms/SAML Login", "description": "Evaluate JavaScript when doing SAML/SmartForm authentication. This is only applicable if you have specified a SmartForms/SAML Authentication element in the \"Crawl Authentication\" area.", "default": false }, "jsPageLoadTimeout": { "type": "number", "title": "Timeout", "description": "The time to wait in milliseconds for a page load to complete. If the timeout is -1, page loads can be indefinite. Maximum: 180,000ms i.e. 3 minutes", "default": 15000, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "jsScriptTimeout": { "type": "number", "title": "Script Timeout", "description": "The time to wait in milliseconds for JavaScript code execution to complete. Controls how long the connector waits for complex scripts to finish running. Suggested timeouts: Standard / Basic websites (30000ms), Complex JS applications / SPA (60000ms), Very slow and Legacy sites (up to 180000ms), Fast/simple sites (15000ms)", "default": 30000, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "jsImplicitWaitTimeout": { "type": "number", "title": "Implicit Wait Timeout", "description": "The time to wait in milliseconds for web elements to become available. This timeout is used when searching for elements on the page. For customers with slow-loading JavaScript content, consider increasing from the default.", "default": 5000, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "jsAjaxTimeout": { "type": "number", "title": "AJAX Timeout", "description": "The time in milliseconds after which an AJAX request will be ignored when considering whether all AJAX requests have completed. Maximum: 180,000ms i.e. 3 minutes", "default": 20000, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "extraLoadTimeMs": { "type": "number", "title": "Extra time to wait for content after page load (ms)", "description": "The JavaScript evaluation process will first wait for the DOM 'document.readyState' to be set to 'complete'; then it will wait until there are no more pending Ajax before emitting the page’s contents. Use this property to wait an additional number of milliseconds before emitting the contents. This gives background JavaScript routines a chance to finish rendering the page before the contents is emitted.", "default": 250, "hints": ["hidden"], "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "extraPageLoadDeltaChars": { "type": "number", "title": "Stop waiting for extraLoadTimeMs if page size increases by this many bytes.", "description": "This parameter is used when the \"Extra time to wait for content after page load (ms)\" parameter is > 0. It will stop the additional wait time if it sees the web page's content grows by at least this many characters. If set to 0 (the default) any increase in character count indicates the page load is finished.", "default": 0, "hints": ["hidden"], "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "quitTimeoutMs": { "type": "number", "title": "Web Driver Quit Timeout (milliseconds)", "description": "The amount of time to wait for a web browser to quit before killing the browser process.", "default": 5000, "hints": ["hidden"], "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "useRequestCounter": { "type": "boolean", "title": "Use Request Counter", "description": "Use the request counter plugin to wait for all pending ajax requests to be complete before loading the page contents.", "default": true, "hints": ["hidden"] }, "waitForLateAjaxRequests": { "type": "boolean", "title": "Wait for late AJAX requests", "description": "Wait for late AJAX requests to be completed before loading the page contents.", "default": false }, "requestCounterMinWaitMs": { "type": "number", "title": "Request counter min wait (ms)", "description": "When the requestcounter is enabled, often early on the requestcount may say there are 0 pending requests... but there may still be ajax requests that haven't run yet. This parameter provides a certain time in milliseconds to wait for a non-zero count to be returned. If a requestcount is non-zero at any point, then the next requestcount = 0 is assumed to signify this page is done loading.", "default": 500, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "requestCounterMaxWaitMs": { "type": "number", "title": "Request counter max wait (ms)", "description": "The request counter plugin counts active ajax requests after a page was loaded until there are no more pending ajax requests. This parameter says how long to wait in milliseconds for the requestcount to go to 0 before giving up.", "default": 3000, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "useHighPerfJsEval": { "type": "boolean", "title": "High Performance Mode", "description": "This property is no longer in use, and is only in place due to backwards compatible configuration validation purposes.", "default": false, "hints": ["hidden"] }, "headlessBrowser": { "type": "boolean", "title": "Headless Browser", "description": "Applicable only when \"Evaluate JavaScript\" is selected, deselect this checkbox if you want to actually see browser windows display while fetchers process web pages. Otherwise, if selected, browsers will run in \"headless\" mode which means they will run in the background. If running on a server with no desktop interface, this must stay selected.", "default": true }, "takeScreenshot": { "type": "boolean", "title": "Index a screenshot of rendered page", "description": "Applicable only when \"Evaluate JavaScript\" is selected, take a screenshot of the fully rendered page and index it. Screenshots will be indexed in a field called \"screenshot_bin\". You must make sure your schema specifies this field as a binary field or indexing will fail. To add this, go to System -> Solr Config -> Managed Schema then add ", "default": false, "hints": ["hidden"] }, "screenshotFullscreen": { "type": "boolean", "title": "Screenshots Full Screen", "description": "When taking a screenshot, capture the full screen.", "default": false, "hints": ["hidden"] }, "viewportWidth": { "type": "number", "title": "Viewport Width", "description": "Set an optional browser viewport width. If not specified, will default to 800.", "default": 800, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "viewportHeight": { "type": "number", "title": "Viewport Height", "description": "Set an optional browser viewport height. If not specified, will default to 600.", "default": 600, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "deviceScreenFactor": { "type": "number", "title": "Device Screen Factor", "description": "Set an optional browser device screen factor. If not specified, will default to 1 (no scaling).", "default": 1, "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "simulateMobile": { "type": "boolean", "title": "Simulate mobile", "description": "Simulate a mobile device", "default": false, "hints": ["hidden"] }, "mobileScreenWidth": { "type": "number", "title": "Mobile screen width (Only used for simulate mobile)", "description": "If simulate mobile is checked, this species the device's emulated screen width.", "hints": ["hidden"], "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "mobileScreenHeight": { "type": "number", "title": "Mobile screen height (Only used for simulate mobile)", "description": "If simulate mobile is checked, this species the device's emulated screen height.", "hints": ["hidden"], "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "chromeBinaryPath": { "type": "string", "title": "chromeBinaryPath", "description": "This property is no longer in use, and is only in place due to backwards compatible configuration validation purposes.", "hints": ["hidden"] }, "chromeExtraCommandLineArgs": { "type": "string", "title": "Google Chrome Extra Command Line Options", "description": "Specify additional command line arguments to add to the chromium executable when it is run.", "hints": ["advanced"] } } }, "linkDiscoveryConfig": { "type": "object", "title": "Link Discovery", "required": [], "properties": { "restrictToTreeIgnoredHostPrefixes": { "type": "array", "title": "Restrict crawl host prefix exemptions", "description": "Modifies the behavior of 'Restrict crawl to start-link tree' to ignore the configured list of prefixes when restricting the crawl. Commonly, 'www.' is ignored so links with the same domain are allowed, whether of the form 'http://host.com' or 'http://www.host.com'. This option requires 'Restrict to start-link tree' to be enabled to have any effect.", "items": { "type": "string" } }, "restrictToTree": { "type": "boolean", "title": "Restrict crawl to start-link tree", "description": "If true, only URLs that match the startLinks URL domain will be followed", "default": true }, "restrictToTreeAllowSubdomains": { "type": "boolean", "title": "Ignore sub-domains when restricting crawl", "description": "Modifies the behavior of 'Restrict crawl to start-link tree' so that a link to any sub-domain of the start links is allowed. For example, if the start link is 'http://host.com', this option ensures that links to 'http://news.host.com' are also followed. This option requires 'Restrict to start-link tree' to be enabled to have any effect.", "default": false }, "restrictToTreeUseHostAndPath": { "type": "boolean", "title": "Restrict crawl to start-link path", "description": "Modifies the behavior of 'Restrict crawl to start-link tree' to include the 'path' of the start link in the restriction logic. For example, if the start link is 'http://host.com/US', this option will limit all followed URLs to ones starting with the '/US/' path. This option requires 'Restrict to start-link tree' to be enabled to have any effect.", "default": false }, "sitemapURLs": { "type": "array", "title": "Sitemap URLs", "description": "URLs for sitemaps, to be used a basis for link discovery. Rules found in sitemaps will not be processed.", "items": { "type": "string" } }, "respectMetaEquivRedirects": { "type": "boolean", "title": "Respect refresh redirects", "description": "If true, the connector will follow metatags with refresh redirects such as .", "default": false }, "allowCircularRedirects": { "type": "boolean", "title": "Allow circular redirects", "description": "If true, a request can be redirected to the same URL multiple times", "default": false }, "addedHeaders": { "type": "string", "title": "Headers to add to HTTP requests", "description": "Add these headers to http requests. This is useful for web sites that require certain headers to let you visit them. Write each header on its own line in the format HeaderName: HeaderValue", "hints": ["lengthy"] } } }, "recrawlRulesConfig": { "type": "object", "title": "Recrawl Rules", "required": [], "properties": { "delete": { "type": "boolean", "title": "Delete dead URIs", "description": "Set to true to remove documents from the index when they can no longer be accessed as unique documents.", "default": true, "hints": ["hidden"] }, "deleteErrorsAfter": { "type": "number", "title": "Fetch failure allowance", "description": "Number of times a website can error out, for example with a 500 error or a connection timeout, before a document is removed from the index. The default of -1 means such documents are never removed. Note that pages that return a 404 status code can be configured to be removed immediately regardless of this setting.", "default": -1, "hints": ["advanced"], "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "refreshAll": { "type": "boolean", "title": "Recrawl all items", "description": "Set to true to always recrawl all items found in the crawldb.", "default": true, "hints": ["advanced"] }, "refreshStartLinks": { "type": "boolean", "title": "Recrawl start links", "description": "Set to true to recrawl items specified in the list of start links.", "default": false, "hints": ["advanced"] }, "refreshErrors": { "type": "boolean", "title": "Recrawl errors", "description": "Set to true to recrawl items that failed during the last crawl.", "default": false, "hints": ["advanced"] }, "refreshOlderThan": { "type": "number", "title": "Recrawl age", "description": "Number of seconds to recrawl items whose last fetched date is longer ago than this value.", "default": -1, "hints": ["advanced"], "maximum": 2147483647, "exclusiveMaximum": false, "minimum": -2147483648, "exclusiveMinimum": false, "multipleOf": 1 }, "refreshIDPrefixes": { "type": "array", "title": "Recrawl ID prefixes", "description": "A prefix to recrawl all items whose IDs begin with this value.", "hints": ["advanced"], "items": { "type": "string" } }, "refreshIDRegexes": { "type": "array", "title": "Recrawl ID regexes", "description": "A regular expression to recrawl all items whose IDs match this pattern.", "hints": ["advanced"], "items": { "type": "string" } }, "refreshScript": { "type": "string", "title": "Recrawl script", "description": "A JavaScript function ('shouldRefresh()') to customize the items re-crawled. Example: function shouldRefresh(id, lastModified, lastFetched, lastEmitted, signature, error) {return true;} ", "hints": ["code", "advanced", "javascript"] }, "forceRefresh": { "type": "boolean", "title": "Force recrawl", "description": "Set to true to recrawl all items even if they have not changed since the last crawl.", "default": false, "hints": ["advanced"] }, "forceRefreshClearSignatures": { "type": "boolean", "title": "Clear signatures", "description": "If true, signatures will be cleared if force recrawl is enabled.", "default": false, "hints": ["advanced"] }, "delete404": { "type": "boolean", "title": "Remove 404/410 pages", "description": "Select this option to delete indexed pages that return a 404 or 410 error.", "default": true, "hints": ["advanced"] }, "sitemapIncrementalCrawling": { "type": "boolean", "title": "Process Sitemap URLs", "description": "When enabled, only URLs found in the sitemap will be processed and crawled.", "default": false, "hints": ["advanced"] } }, "hints": ["advanced"] }, "cookieSpec": { "type": "string", "title": "Cookie spec", "default": "browser-compatibility" }, "rewriteLinkScript": { "type": "string", "title": "URI rewrite script", "description": "A Javascript function 'rewriteLink(link) { }' to modify links to documents before they are fetched.", "hints": ["code", "advanced", "javascript"] } } }, "coreProperties": { "type": "object", "title": "Core Properties", "description": "Common behavior and performance settings.", "required": [], "properties": { "fetchSettings": { "type": "object", "title": "Fetch Settings", "description": "System level settings for controlling fetch behavior and performance.", "required": [], "properties": { "numFetchThreads": { "type": "number", "title": "Fetch Threads", "description": "Maximum number of fetch threads; defaults to 5.This setting controls the number of threads that call the Connectors fetch method.Higher values can, but not always, help with overall fetch performance.", "default": 5, "maximum": 500, "exclusiveMaximum": false, "minimum": 1, "exclusiveMinimum": false, "multipleOf": 1 }, "indexingThreads": { "type": "number", "title": "Index Subscription Threads", "description": "Maximum number of indexing threads; defaults to 4.This setting controls the number of threads in the indexing service used for processing content documents emitted by this datasource.Higher values can sometimes help with overall fetch performance.", "default": 4, "maximum": 10, "exclusiveMaximum": false, "minimum": 1, "exclusiveMinimum": false, "multipleOf": 1 }, "pluginInstances": { "type": "number", "title": "Number of plugin instances for distributed fetching", "description": "Maximum number of plugin instances for distributed fetching. Only specified number of plugin instanceswill do fetching. This is useful for distributing load between different instances.", "default": 0, "maximum": 500, "exclusiveMaximum": false, "minimum": 0, "exclusiveMinimum": false, "multipleOf": 1 }, "fetchItemQueueSize": { "type": "number", "title": "Fetch Item Queue Size", "description": "Size of the fetch item queue.Larger values result in increased memory usage, but potentially higher performance.Default is 10k.", "default": 10000, "hints": ["hidden"], "maximum": 500000, "exclusiveMaximum": false, "minimum": 1, "exclusiveMinimum": false, "multipleOf": 1 }, "fetchRequestCheckInterval": { "type": "number", "title": "Fetch request check interval(ms)", "description": "The amount of time to wait before check if a request is done", "default": 15000, "hints": ["hidden"], "maximum": 500000, "exclusiveMaximum": false, "minimum": 1000, "exclusiveMinimum": false, "multipleOf": 1 }, "fetchResponseScheduledTimeout": { "type": "number", "title": "Fetch response scheduled timeout(ms)", "description": "The maximum amount of time for a response to be scheduled. The task will be canceled if this setting is exceeded.", "default": 300000, "maximum": 500000, "exclusiveMaximum": false, "minimum": 1000, "exclusiveMinimum": false, "multipleOf": 1 }, "fetchResponseCompletedTimeout": { "type": "number", "title": "Fetch response completion timeout(ms)", "description": "The maximum amount of time for a response to be completed. If exceeded, the task will be retried if the job is still running", "default": 300000, "hints": ["hidden"], "maximum": 600000, "exclusiveMaximum": false, "minimum": 1, "exclusiveMinimum": false, "multipleOf": 1 }, "indexingInactivityTimeout": { "type": "number", "title": "Indexing inactivity timeout(seconds)", "description": "The maximum amount of time to wait for indexing results (in seconds). If exceeded, the job will fail with an indexing inactivity timeout.", "default": 86400, "maximum": 691200, "exclusiveMaximum": false, "minimum": 60, "exclusiveMinimum": false, "multipleOf": 1 }, "pluginInactivityTimeout": { "type": "number", "title": "Plugin inactivity timeout(seconds)", "description": "The maximum amount of time to wait for plugin activity (in seconds). If exceeded, the job will fail with a plugin inactivity timeout.", "default": 600, "maximum": 691200, "exclusiveMaximum": false, "minimum": 60, "exclusiveMinimum": false, "multipleOf": 1 }, "indexMetadata": { "type": "boolean", "title": "Index metadata", "description": "When enabled the metadata of skipped items will be indexed to the content collection.", "default": false }, "indexContentFields": { "type": "boolean", "title": "Index content fields", "description": "When enabled, content fields will be indexed to the crawl-db collection.", "default": false }, "asyncParsing": { "type": "boolean", "title": "Async Parsing", "description": "When enabled, content will be indexed asynchronously.", "default": false } } }, "strayDeletionSettings": { "type": "object", "title": "Stray Content Deletion Settings", "description": "System level settings for controlling stray content deletion behavior", "required": [], "properties": { "enableStrayDeletion": { "type": "boolean", "title": "Enable Stray Deletion", "description": "When enabled, items not re-encountered in the current crawl are deleted from the content collection. Defaults to true.", "default": true }, "circuitBreakerSettings": { "type": "object", "title": "Circuit Breaker Settings", "description": "Settings that control the circuit breaker, which prevents mass deletion when an unusually high percentage of items are identified as stray in a crawl.", "required": [], "properties": { "percentageThreshold": { "type": "number", "title": "Stray Deletion Threshold (%)", "description": "Maximum percentage of indexed items that may be deleted as stray in a crawl. If the total stray item exceeds this percentage, the circuit breaker blocks the deletion to prevent accidental data loss. Accepts values from 0 to 100. Defaults to 80.", "default": 80, "maximum": 100, "exclusiveMaximum": false, "minimum": 0, "exclusiveMinimum": false, "multipleOf": 1 }, "enableCircuitBreaker": { "type": "boolean", "title": "Enable Circuit Breaker", "description": "When enabled, stray deletion is blocked if the percentage of items to be deleted exceeds the configured threshold. Disable only if unconditional stray deletion is required regardless of volume. Defaults to true.", "default": true } } } } }, "skipConfigValidation": { "type": "boolean", "title": "Skip Validation", "description": "Enable to skip configuration validation when it takes too long and causes timeout issue", "default": false } }, "hints": ["advanced"] } }, "category": "Web", "categoryPriority": 1 }; export const SchemaParamFields = ({schema}) => { const sanitize = str => { if (typeof str !== "string") return str; return str.replace(/^"(.*)"$/s, "$1").replace(/\\/g, "").replace(/"/g, "'"); }; const formatDescription = str => { const s = sanitize(str); return (/[.!?]\)*$/).test(s) ? s : `${s}.`; }; const {description, properties = {}, required: requiredProps = []} = schema; const visibleProps = useMemo(() => Object.entries(properties).filter(([, prop]) => !prop.hints?.includes("hidden")), [properties]); return
{description &&

{formatDescription(description)}

} {visibleProps.map(([name, prop]) => { const isRequired = requiredProps.includes(name); const hasDefault = prop.default !== undefined; const rawDefault = prop.default; const isComplexDefault = hasDefault && (typeof rawDefault === "object" || typeof rawDefault === "string" && (rawDefault.length > 20 || rawDefault.includes('"'))); const fieldProps = { key: name, body: prop.title || name, type: prop.type, ...prop.title && ({ post: [<>API property: {name}] }), ...isRequired && ({ required: true }), ...!isComplexDefault && hasDefault ? { default: sanitize(String(rawDefault)) } : {} }; const isObject = prop.type === "object" && prop.properties; const isArrayOfObjects = prop.type === "array" && prop.items?.type === "object" && prop.items.properties; return {prop.description &&

{formatDescription(prop.description)}

} {isComplexDefault &&

Default: 

                  
                    {JSON.stringify(rawDefault, null, 2)}
} {isArrayOfObjects &&

Object attributes: 

                
                  {'{\n'}
                  {Object.entries(prop.items.properties).map(([iname, iprop]) => <>
                      {`  ${iname}`}
                      {prop.items?.required?.includes(iname) &&  required}
                      {`: {\n    display name: ${sanitize(iprop.title || '')}\n    type: ${iprop.type}\n  }\n`}
                    )}
                  {'}'}
} {isObject && } ; })}
; }; export const LwTemplate = ({title = "Key questions to get you started", icon = "sparkles", cta = "Powered by Agent Studio", linkHref = "https://lucidworks.com/demo/?utm_source=docs&utm_medium=referral&utm_campaign=docs_cta_ai"}) => { const [isLoaded, setIsLoaded] = useState(false); useEffect(() => { const timer = setTimeout(() => { setIsLoaded(true); }, 500); return () => clearTimeout(timer); }, []); return
{isLoaded && ` }} />} Powered by Lucidworks Agent Studio
; }; [localhost link]: http://localhost:3000/docs/fusion-connectors/connectors/web-v2 [mintlify link]: https://doc.lucidworks.com/docs/fusion-connectors/connectors/web-v2 [old doc.lw link]: https://doc.lucidworks.com/fusion-connectors/tq6l3z * **Latest version:** v2.2.2 * **Compatible with Fusion version:** 5.9.0 and later The Web V2 connector for Fusion crawls and ingests content from websites over HTTP. It works best for crawling static content by retrieving data from a website starting at one or more seed URLs (the `startLinks`), following links up to a configurable depth, and indexing the fetched pages. Starting with Fusion 5.9.11, users of the Web V2 connector must upgrade to version 2.0.0 or later. Previous versions are incompatible due to changes introduced by the upgraded JDK in Fusion 5.9.11. For Web V2 v2.1.0 and later, up to three Web V2 connectors can run simultaneously in a single cluster. This prevents reaching a max concurrency limit per Web V2 connector, which affects how much data can be sent to Selenium Grid at one time. Fusion 5.6 and later uses the [Open Graph Protocol](https://ogp.me/) as the default configuration for fields. Deviation from that standard configuration may exclude information from indexing during the crawl. If crawls fail with a corrupted CrawlDB error, reinstall the connector. ## Prerequisites Perform these prerequisites to ensure the connector can reliably access, crawl, and index your data. Proper setup helps avoid configuration or permission errors, so use the following guidelines to keep your content available for discovery and search in Fusion. Confirm network and security configurations: * Egress must allow HTTP/2 traffic from your on-prem host into the Fusion gRPC endpoint. * You need a Fusion user account with the `remote-connectors` or `admin` role to authenticate the connector. * If the standalone host doesn't trust Fusion's TLS cert, point to your truststore file path. ## Authentication Setting up the correct authentication according to your organization's data governance policies helps keep sensitive data secure while allowing authorized indexing. The Web V2 connector supports many crawl-time authentication methods, including OAuth token retrieval. You configure these under Crawl Authentication Properties in your datasource. The supported authentication schemes include the following: * Basic HTTP Authentication: Provide host, port, realm (if any), username, and password. * Digest HTTP Authentication: Same parameters as Basic, but using the Digest challenge/response scheme. * NTLM Authentication For Windows-style authentication: Provide domain, workstation, host, port, realm (if any), username, and password. * Form Authentication: Post to a login form URL with whatever name/value pairs the site expects, plus a TTL for the session. * SAML/Smart Form Authentication: For multi-step or SAML-backed form logins, submit a sequence of forms until you're authenticated. * OAuth 2.0: Fetch a bearer token using client-credentials before crawling. You supply the token-endpoint URL and parameter names/values (for example, `client_id`, `client_secret`, `scope`, or `grant_type`) in the Access Token section of Crawl Authentication Properties. Once you've picked your scheme, Fusion will maintain cookies and session state automatically for the duration of the crawl. ### Authentication resources The Web V1 connector retrieves data from a Web site using HTTP and starting from a specified URL. The Web connector supports Basic, Digest, Form, and NTLM authentication to websites. The credentials for a crawl are stored in a credentials file in `https://FUSION_HOST:FUSION_PORT/data/connectors/container/lucid.web/datasourceName`, where `datasourceName` is the name of the datasource. After you create a datasource, Fusion creates this directory for you. The file should be a JSON formatted file, ending with the `.json` file extension. When defining the datasource, use the name of the file in the **Authentication credentials filename** field in the UI (or for the `f.credentialsFile` property if using the REST API). All authentication types require the credentials file to include a property called `type` that defines the type of authentication to use. The other required properties vary depending on the type of authentication chosen. ## Basic form-based Authentication To use basic form-based authentication, use `form` for the type. The other properties are: * ttl. The "time to live" for the session created after authentication. After the specified time, the crawler logs in again to keep the crawl activity from failing due to an expired session. This value is defined in seconds. * action. The action to take to log in. That is, the URL for the login form. * params. The parameters for the form, likely the username and password, but could be other required properties. In the example below, we pass two parameters, `os_username` and `os_password`, which are expected by the system we crawl. Here is an example using form-based authentication: ```json theme={"dark"} [ { "credential" : { "type" : "form", "ttl" : 300000, "action" : "http://some.server.com/login.action?os_destination=%2Fpages%2Fviewpage.action%3Ftitle%3DAcme%2B5%2BDocumentation%26spaceKey%3DAcme5", "params" : { "os_username" : "username", "os_password" : "password" } } } ] ``` ## Complex Form-based Authentication Some websites do not manage their own authentication, but rather trust a third-party authority to authenticate the user. An example of this is websites that use SAML to log in a user via a central single-signon authority. To configure fusion to log in to a website like this, use `smartForm` for the type. The other properties are: * ttl. The "time to live" for the session created after authentication. After the specified time, the crawler logs in again to keep the crawl activity from failing due to an expired session. This value is defined in seconds. * loginUrl. The URL on which the first page that initializes the login chain is located * params. A list of parameters to use for the form logins, likely the username and password, but could be other required properties. In the example below, we pass two parameters, `os_username` and `os_password`, which are expected by the system we crawl. Additionally we expect that once that login has happened, a new form is presented to the user which then posts back to where we came from. No data need to be entered in this form, which is why we include an empty `{ }` in the params list. Here is an example using form-based authentication: ```json theme={"dark"} [ { "credential" : { "type" : "smartForm", "ttl" : 300000, "loginUrl" : "http://some.example.com/login", "params" : [{ "os_username" : "username", "os_password" : "password" }, { } ] } } ] ``` To figure out what parameters you need to specify, turn off JavaScript in your browser and go through the login work flow. Though you normally see only a single login form on your screen, you might find many more forms you need to submit before you get logged in when JavaScript is not available to perform those form submissions automatically. Each form in that login chain needs to be represented in the list of `params`. If no user input is required, simply include an empty `{ }`. ## Basic and Digest Authentication Basic and Digest authentication are simple HTTP authentication methods still in use in some places. To use either of these types, in the credentials file, for the `type` property use "basic" or "digest". The other properties are: * host. The host of the site. * port. The port, if any. * userName. The username to use for authentication. * password. The password for the userName. * realm. The [security realm](/docs/5/fusion/operations/security/access-control/security-realms) for the site, if any. Example basic auth configuration: ``` [ { "credential" : { "type" : "basic", "ttl" : 300000, "userName" : "usr", "password" : "pswd", "host":"hostname.exampledomain.com” "port": 443 } } ] ``` ## NTLM Authentication To use NTLM authentication, in the credentials file, for the `type` property, use `ntlm`. The other properties available are: * host. The host of the site. * port. The port, if any. * userName. The username to use for authentication. * password. The password for the userName. * realm. The [security realm](/docs/5/fusion/operations/security/access-control/security-realms) for the site, if any. * domain. The domain. * workstation. The workstation, as needed. Example NTLM credential configuration: ```json theme={"dark"} [ {"credential" : { "type" : "ntlm", "ttl" : 300000, "port" : 80, "host" : "someHost", "domain" : "someDomain", "userName" : "someUser", "password" : "XXXXXXXX" } } ] ``` ## Crawl a Web site protected by Kerberos In Fusion 4.1 and later, the Web connector can crawl Web sites protected by Kerberos using SPNEGO. This is a way to access Web sites without requiring a user’s login credentials. Kerberos support requires Fusion 5.9.5. The Fusion Web connector can optionally use Kerberos with SAML/Smart Form authentication. To crawl Kerberos-protected Web sites, first create the necessary configuration files, then configure Fusion to use them. {/* // tag::kerberos[] */} ### Create standard Java configuration files to connect to Kerberos Fusion uses the JDK standard JAAS Kerberos implementation, which is based on three system properties that reference three separate files. The files are as follows: * On the Kerberos-protected server, a keytab file, named `kerberuser.keytab` in our examples. * On the Fusion system, a configuration file named `login.conf`. * On the Fusion system, an initialization file named `krb5.ini`. #### Create a Kerberos keytab Create and validate the keytab file for the Kerberos client principal you want to use to authenticate to the website. If you do not specify the `kerberosPrincipalName` and `kerberosKeytabFilePath` or `kerberosKeytabBase64` when [creating the Fusion datasource](#configure-fusion-to-use-kerberos), Fusion uses the default login principal and ticket cache. You can see the default values by logging into the Fusion server as the user who runs Fusion and running `klist`. If you do not want to use the default account and credentials, specify these configuration properties when creating a keytab as well as in the [Web datasource setup](#configure-fusion-to-use-kerberos). Use the Kerberos user principal name (UPN), not the service principal name (SPN, which is used with the Kerberos [security realm](/docs/5/fusion/operations/security/access-control/security-realms)). In some cases the UPN can be a service. In our examples, the Fusion Web crawler authenticates to the Web sites using the user [kerbuser@win.lab.lucidworks.com](mailto:kerbuser@win.lab.lucidworks.com). We create a keytab file `kerbuser.keytab` for the user principal [kerbuser@WIN.LAB.LUCIDWORKS.COM](mailto:kerbuser@WIN.LAB.LUCIDWORKS.COM). ##### Create a Kerberos keytab on Windows Example: ``` ktpass -out kerbuser.keytab -princ kerbuser@WIN.LAB.LUCIDWORKS.COM -mapUser kerbuser -mapOp set -pass YOUR_PASSWORD -crypto AES256-SHA1 -pType KRB5_NT_PRINCIPAL ``` The following weak encryption types are not supported by Fusion: * DES-CBC-CRC \* DES\_CBC\_MD5 * Microsoft Windows 2000 RC4-HMAC ##### Create a Kerberos keytab on Ubuntu Linux Prerequisite: Install the `krb5-user` package: `sudo apt-get install krb5-user` Example: ``` ktutil addent -password -p HTTP/kerbuser@WIN.LAB.LUCIDWORKS.COM -k 1 -e aes128-cts-hmac-sha1-96 - it will ask you for password of kerbuser - wkt kerbuser.keytab q ``` ##### Test the keytab Once you create a keytab, verify that it works. Prerequisite: You need a version of curl installed that allows SPNEGO. To test whether your version of curl does this, run `curl --version` and make sure SPNEGO is in the output. Run the following curl command (replace the keytab path and site): ``` export KRB5CCNAME=FILE:/path/to/kerbuser.keytab curl -vvv --negotiate -u : http://your-site.com ``` Note that the first request is a 401 status code for the negotiate request followed by a second request, which is a status of 200. #### Create a login.conf and krb5.ini On the Fusion server, create `login.conf` and `krb5.ini` files as follows. ##### Create a login.conf on Windows For hosted connectors, Selenium Grid support is available through Kubernetes. For remote connectors, Selenium Grid support is available through Docker Compose. See [the Web V2 remote support repository](https://github.com/lucidworks/web-v2-remote-support/) for full setup instructions and YAML files. In this example, the keytab is stored at `C:\\kerb\\kerbuser.keytab` ``` KrbLogin { com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab="C:\\kerb\\kerbuser.keytab" useTicketCache=true principal="kerbuser@WIN.LAB.LUCIDWORKS.COM" debug=true; }; ``` ##### Create a login.conf on Linux In this example, the keytab is stored at `/home/lucidworks/kb.keytab` ``` com.sun.security.jgss.initiate { com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab="/home/lucidworks/kerbuser.keytab" useTicketCache=true principal="kerbuser@WIN.LAB.LUCIDWORKS.COM" debug=true; }; ``` The format of the login.conf is described on [the Oracle Web site](https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/tutorials/LoginConfigFile.html). ##### Create a krb5.ini When you install krb5 on Linux, you can find a Kerberos configuration file in `/etc/krb5.conf`. You can optionally create a custom one instead. Creating a `krb5.conf` is the same for Linux and Windows. On Windows the file is `krb5.ini`. In this example the domain is WIN.LAB.LUCIDWORKS.COM, the Kerberos kdc host is `my.kdc-dns.com`, and the Kerberos admin server is `my-admin-server-dns.com`. Example: ``` default_realm = WIN.LAB.LUCIDWORKS.COM default_tkt_enctypes = aes128-cts-hmac-sha1-96 default_tgs_enctypes = aes128-cts-hmac-sha1-96 permitted_enctypes = aes128-cts-hmac-sha1-96 dns_lookup_realm = false dns_lookup_kdc = false ticket_lifetime = 24h renew_lifetime = 7d forwardable = true udp_preference_limit = 1 WIN.LAB.LUCIDWORKS.COM = { kdc = my.kdc-dns.com admin_server = my.admin-server-dns.com } **WIN.LAB.LUCIDWORKS.COM = WIN.LAB.LUCIDWORKS.COM** WIN.LAB.LUCIDWORKS.COM = WIN.LAB.LUCIDWORKS.COM ``` The format of the `krb5.ini` file is described in [the MIT Kerberos documentation](https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html). You can change the encryption algorithms by changing the properties `default_tkt_enctypes`, `default_tgs_enctypes`, and `permitted_enctypes` as needed. For example: ``` default_tkt_enctypes = RC4-HMAC default_tgs_enctypes = RC4-HMAC Permitted_enctypes = RC4-HMAC ``` {/* // end::kerberos[] */} ### Configure Fusion to use Kerberos Once you have the keytab, `login.conf`, and `krb5.ini` files, configure Fusion to use Kerberos. You must set a property in a Fusion configuration file in addition to defining the datasource in the Fusion UI. At the command line on every machine in your Fusion cluster: 1. In `$FUSION_HOME/conf/fusion.cors` (`fusion.properties` in Fusion 4.x), add the following property to the `connectors-classic.jvmOptions` setting: `-Djavax.security.auth.useSubjectCredsOnly=false` 2. Restart the `connectors-classic` service using `./bin/connectors-classic restart` on Linux or `bin\connectors-classic.cmd restart` on Windows. In the Fusion UI: 1. Click **Indexing > Datasources**. 2. Click **Add+**, then **Web**. 3. Enter a datasource ID and a start link. 4. Click **Crawl authorization**. 5. At the bottom of the section, check **Enable SPNEGO/Kerberos Authentication**. 6. You can either use the default principal name or specify a principal name to use. * If you do not specify the principal name, then Fusion uses the default login principal and ticket cache. You can see those default values by logging into the Fusion server as the user who runs Fusion and running `klist`. 7. If you specify a principal name, you must provide a keytab, either in Base64 or as a file path. * If you specify a keytab file path, the file must be on the machine running the Fusion connector, for each connector’s node in the cluster. * The Base64 option lets you supply the keytab in one place, in the UI. 8. Fill in any remaining options to configure the datasource. 9. Click **Save**. {/* // tag::troubleshoot-kerberos[] */} ### Troubleshoot Kerberos authentication #### javax.security.auth.login.LoginException: No key to store Problem: When trying to crawl a Kerberos-authenticated Web site, you get an error like this: ```bash wrap expandable theme={"dark"} crawler.common.ComponentInitException: Could not initialize Spnego/Kerberos. at crawler.fetch.impl.http.WebFetcher.init(WebFetcher.java:282) ~[lucid.web-4.0.2.jar:?] at crawler.common.ComponentFactory.initComponent(ComponentFactory.java:37) ~[lucid.anda-4.0.2.jar:?] at crawler.Crawler.initComponents(Crawler.java:125) ~[lucid.anda-4.0.2.jar:?] at crawler.Crawler.init(Crawler.java:108) ~[lucid.anda-4.0.2.jar:?] at crawler.common.ComponentFactory.initComponent(ComponentFactory.java:37) ~[lucid.anda-4.0.2.jar:?] at crawler.common.config.CrawlConfig.buildCrawler(CrawlConfig.java:212) ~[lucid.anda-4.0.2.jar:?] at com.lucidworks.connectors.anda.AndaFetcher.start(AndaFetcher.java:139) [lucid.anda-4.0.2.jar:?] at com.lucidworks.connectors.ConnectorJob.start(ConnectorJob.java:200) [lucid.shared-4.0.2.jar:?] at com.lucidworks.connectors.Connector$RunnableJob.run(Connector.java:319) [lucid.shared-4.0.2.jar:?] Caused by: java.lang.Exception: Could not perform spnego/kerberos login. java.security.krb5.conf = /etc/krb5.conf,, Keytab file = /home/ndipiazza/Downloads/kerbuser.keytab, login config = {principal=HTTP/kerbuser@WIN.LAB.LUCIDWORKS.COM, debug=false, storeKey=true, keyTab=/home/ndipiazza/Downloads/kerbuser.keytab, useKeyTab=true, useTicketCache=true, refreshKrb5Config=true} at com.lucidworks.connectors.spnego.SpnegoAuth.<init>(SpnegoAuth.java:83) ~[?:?] at crawler.fetch.impl.http.WebFetcher.init(WebFetcher.java:279) ~[?:?] ... 8 more Caused by: javax.security.auth.login.LoginException: No key to store at com.sun.security.auth.module.Krb5LoginModule.commit(Krb5LoginModule.java:1119) ~[?:1.8.0_161] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.invoke(LoginContext.java:755) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680) ~[?:1.8.0_161] at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.login(LoginContext.java:588) ~[?:1.8.0_161] at com.lucidworks.connectors.spnego.SpnegoAuth.<init>(SpnegoAuth.java:76) ~[?:?] at crawler.fetch.impl.http.WebFetcher.init(WebFetcher.java:279) ~[?:?] ... 8 more ``` Resolution: First test your keytab as described in [test the keytab](#test-the-keytab) above. If your keytab passes validation, another possibility is that the `/tmp/krb*` cache file got corrupted or is not compatible after you went through other troubleshooting steps. To rule that out, remove the `/tmp/krb*` cache file on all hosts, restart your connectors-classic, and try the crawl again. That is, on each host: ``` rm -f /tmp/krb* $FUSION_HOME/bin/connectors-classic restart ``` #### 401 error Problem: Crawling using the Web connector with Kerberos results in a 401 error, but curl with Kerberos works fine. Resolution: Make sure you have this system property set in `connectors-classic jvmOptions` **on all nodes**: ``` -Djavax.security.auth.useSubjectCredsOnly=false ``` You must restart `connectors-classic` after making that change. If that doesn’t work, make sure the user you are authenticating with from Curl matches the user you are trying to authenticate with from the Web connector. To see your Kerberos principal user name, run `klist`. #### Error: “Pre-authentication information was invalid - Identifier doesn’t match expected value” Problem: When crawling using the Web connector with Kerberos enabled, you get an error like this: ```bash wrap expandable theme={"dark"} crawler.common.ComponentInitException: Could not initialize Spnego/Kerberos. at crawler.fetch.impl.http.WebFetcher.init(WebFetcher.java:282) ~[lucid.web-4.0.2.jar:?] at crawler.common.ComponentFactory.initComponent(ComponentFactory.java:37) ~[lucid.anda-4.0.2.jar:?] at crawler.Crawler.initComponents(Crawler.java:125) ~[lucid.anda-4.0.2.jar:?] at crawler.Crawler.init(Crawler.java:108) ~[lucid.anda-4.0.2.jar:?] at crawler.common.ComponentFactory.initComponent(ComponentFactory.java:37) ~[lucid.anda-4.0.2.jar:?] at crawler.common.config.CrawlConfig.buildCrawler(CrawlConfig.java:212) ~[lucid.anda-4.0.2.jar:?] at com.lucidworks.connectors.anda.AndaFetcher.start(AndaFetcher.java:139) [lucid.anda-4.0.2.jar:?] at com.lucidworks.connectors.ConnectorJob.start(ConnectorJob.java:200) [lucid.shared-4.0.2.jar:?] at com.lucidworks.connectors.Connector$RunnableJob.run(Connector.java:319) [lucid.shared-4.0.2.jar:?] Caused by: java.lang.Exception: Could not perform spnego/kerberos login. java.security.krb5.conf = /etc/krb5.conf,, Keytab file = /home/ndipiazza/Downloads/kerbuser.keytab, login config = {principal=kerbuser@WIN.LAB.LUCIDWORKS.COM, debug=false, storeKey=true, keyTab=/home/ndipiazza/Downloads/kerbuser.keytab, useKeyTab=true, useTicketCache=true, refreshKrb5Config=true} at com.lucidworks.connectors.spnego.SpnegoAuth.<init>(SpnegoAuth.java:83) ~[?:?] at crawler.fetch.impl.http.WebFetcher.init(WebFetcher.java:279) ~[?:?] ... 8 more Caused by: javax.security.auth.login.LoginException: Pre-authentication information was invalid (24) at com.sun.security.auth.module.Krb5LoginModule.attemptAuthentication(Krb5LoginModule.java:804) ~[?:1.8.0_161] at com.sun.security.auth.module.Krb5LoginModule.login(Krb5LoginModule.java:617) ~[?:1.8.0_161] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.invoke(LoginContext.java:755) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680) ~[?:1.8.0_161] at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.login(LoginContext.java:587) ~[?:1.8.0_161] at com.lucidworks.connectors.spnego.SpnegoAuth.<init>(SpnegoAuth.java:76) ~[?:?] at crawler.fetch.impl.http.WebFetcher.init(WebFetcher.java:279) ~[?:?] ... 8 more Caused by: sun.security.krb5.KrbException: Pre-authentication information was invalid (24) at sun.security.krb5.KrbAsRep.<init>(KrbAsRep.java:76) ~[?:1.8.0_161] at sun.security.krb5.KrbAsReqBuilder.send(KrbAsReqBuilder.java:316) ~[?:1.8.0_161] at sun.security.krb5.KrbAsReqBuilder.action(KrbAsReqBuilder.java:361) ~[?:1.8.0_161] at com.sun.security.auth.module.Krb5LoginModule.attemptAuthentication(Krb5LoginModule.java:776) ~[?:1.8.0_161] at com.sun.security.auth.module.Krb5LoginModule.login(Krb5LoginModule.java:617) ~[?:1.8.0_161] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.invoke(LoginContext.java:755) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680) ~[?:1.8.0_161] at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.login(LoginContext.java:587) ~[?:1.8.0_161] at com.lucidworks.connectors.spnego.SpnegoAuth.<init>(SpnegoAuth.java:76) ~[?:?] at crawler.fetch.impl.http.WebFetcher.init(WebFetcher.java:279) ~[?:?] ... 8 more Caused by: sun.security.krb5.Asn1Exception: Identifier does not match expected value (906) at sun.security.krb5.internal.KDCRep.init(KDCRep.java:140) ~[?:1.8.0_161] at sun.security.krb5.internal.ASRep.init(ASRep.java:64) ~[?:1.8.0_161] at sun.security.krb5.internal.ASRep.<init>(ASRep.java:59) ~[?:1.8.0_161] at sun.security.krb5.KrbAsRep.<init>(KrbAsRep.java:60) ~[?:1.8.0_161] at sun.security.krb5.KrbAsReqBuilder.send(KrbAsReqBuilder.java:316) ~[?:1.8.0_161] at sun.security.krb5.KrbAsReqBuilder.action(KrbAsReqBuilder.java:361) ~[?:1.8.0_161] at com.sun.security.auth.module.Krb5LoginModule.attemptAuthentication(Krb5LoginModule.java:776) ~[?:1.8.0_161] at com.sun.security.auth.module.Krb5LoginModule.login(Krb5LoginModule.java:617) ~[?:1.8.0_161] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.invoke(LoginContext.java:755) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.access$000(LoginContext.java:195) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext$4.run(LoginContext.java:682) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext$4.run(LoginContext.java:680) ~[?:1.8.0_161] at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:680) ~[?:1.8.0_161] at javax.security.auth.login.LoginContext.login(LoginContext.java:587) ~[?:1.8.0_161] at com.lucidworks.connectors.spnego.SpnegoAuth.<init>(SpnegoAuth.java:76) ~[?:?] at crawler.fetch.impl.http.WebFetcher.init(WebFetcher.java:279) ~[?:?] ... 8 more ``` Resolution: Your keytab’s principal name doesn’t match the value on the ticket server. Check the principal name for your user. {/* // end::troubleshoot-kerberos[] */} The Web V2 connector v1.4.0 added the ability to authenticate using OAuth. ## Example with PingFederate This example uses Client Credentials Grant Type and a basic configuration. 1. Create new app in PingIdentity. 2. Edit the app configuration: 1. Check **Client Credentials** Grant Type. 2. Set **Token Endpoint Authentication Method** to `Client Secret Post`. 3. Enable the app by clicking on the toggle button to the right of the app name. 4. Go to the **Configuration** tab and click **Get Access Token**. At this point it will probably show an error about scope. Fix it by adding a scope, such as `testScope`. 5. Click the **Get Access Token** button again under the **Configuration** tab. It should generate and display your access token. 6. Go to the **Configuration** tab and click **URLs**. Copy the Token Endpoint URL for further use. 7. You can view a sample of Client Credentials Grant Type access token request in the [PingIdentity documentation](https://docs.pingidentity.com/developer-resources/oauth_20_developer_guide/client-credentials-grant-type.html). In this example the access token request is: ```curl theme={"dark"} $ curl -X POST "https://auth.pingone.com/ENV_ID/as/token" -d "grant_type=client_credentials&client_id=CLIENT_ID&client_secret=CLIENT_SECRET&scope=testScope" --header "Content-Type:application/x-www-form-urlencoded" { "access_token" : "ACCESS_TOKEN", "token_type" : "Bearer", "expires_in" : 3600, "scope" : "testScope" } ``` 8. In Fusion, go to **CRAWL AUTHENTICATION PROPERTIES** in the datasource configuration and fill in the **ACCESS TOKEN** section by adding the URL and parameters of the request. Replace CLIENT\_ID with your own client ID. * ACCESS TOKEN REQUEST URL: `https://auth.pingone.com/ENV_ID/as/token` * PARAMETER NAME | PARAMETER VALUE: `grant_type` | `client_credentials` * PARAMETER NAME | PARAMETER VALUE: `client_id` | CLIENT\_ID * PARAMETER NAME | PARAMETER VALUE: `scope` | `testScope` ## Example with Microsoft Azure AD 1. In Microsoft Entra ID, go to **App Registrations** and register a new app. 2. Go to your app and click **Certificates & secrets** and add a client secret. 3. Copy the client secret **Value** for further use. 4. [View the Microsoft documentation](https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#first-case-access-token-request-with-a-shared-secret) for a request to get access token. In this example, we use the client credentials grant type: ```curl theme={"dark"} $ curl -X POST "https://login.microsoftonline.com/TENANT_ID/oauth2/v2.0/token" -d "client_id=CLIENT_ID&scope=https%3A%2F%2Fgraph.microsoft.com%2F.default&client_secret=CLIENT_SECRET&grant_type=client_credentials" -H "Content-Type: application/x-www-form-urlencoded" {"token_type":"Bearer","expires_in":3599,"ext_expires_in":3599,"access_token":"ACCESS_TOKEN"} ``` 5. Copy the endpoint URL and the parameters to the **CRAWL AUTHENTICATION PROPERTIES > ACCESS TOKEN** section in the datasource configuration in Fusion. Replace CLIENT\_ID with your own client ID. * ACCESS TOKEN REQUEST URL: `https://login.microsoftonline.com/TENANT_ID/oauth2/v2.0/token` * PARAMETER NAME | PARAMETER VALUE: `grant_type` | `client_credentials` * PARAMETER NAME | PARAMETER VALUE: `client_id` | CLIENT\_ID * PARAMETER NAME | PARAMETER VALUE: `scope` | `https://graph.microsoft.com/.default` ## Remote connectors V2 connectors support [running remotely](/docs/fusion-connectors/developers/remote-v2-connectors) in Fusion versions 5.7.1 and later. If you need to index data from behind a firewall, you can configure a V2 connector to run remotely on-premises using TLS-enabled gRPC. ## Prerequisites Before you can set up an on-prem V2 connector, you must configure the egress from your network to allow HTTP/2 communication into the Fusion cloud. You can use a [forward proxy server](#egress-and-proxy-server-configuration) to act as an intermediary between the connector and Fusion. The following is required to run V2 connectors remotely: * The [plugin zip file and the connector-plugin-standalone JAR](https://plugins.lucidworks.com/). * A configured connector backend gRPC endpoint. * Username and password of a user with a `remote-connectors` or `admin` role. * If the host where the remote connector is running is not configured to trust the server’s TLS certificate, you must configure the file path of the trust certificate collection. If your version of Fusion doesn’t have the `remote-connectors` role by default, you can create one. No API or UI permissions are required for the role. ## Connector compatibility Only V2 connectors are able to run remotely on-premises. You also need the remote connector client JAR file that matches your Fusion version. You can download the latest files at [V2 Connectors Downloads](/docs/fusion-connectors/downloads/v2-connectors-downloads). Whenever you upgrade Fusion, you must also update your remote connectors to match the new version of Fusion. The gRPC connector backend is not supported in Fusion environments deployed on AWS. ## System requirements The following is required for the on-prem host of the remote connector: * (Fusion 5.9.0-5.9.10) JVM version 11 * (Fusion 5.9.11) JVM version 17 * Minimum of 2 CPUs * 4GB Memory Note that memory requirements depend on the number and size of ingested documents. ## Enable backend ingress In your `values.yaml` file, configure this section as needed: ```yaml theme={"dark"} ingress: enabled: false pathtype: "Prefix" path: "/" #host: "ingress.example.com" ingressClassName: "nginx" # Fusion 5.9.6 only tls: enabled: false certificateArn: "" # Enable the annotations field to override the default annotations #annotations: "" ``` * Set `enabled` to `true` to enable the backend ingress. * Set `pathtype` to `Prefix` or `Exact`. * Set `path` to the path where the backend will be available. * Set `host` to the host where the backend will be available. * In Fusion 5.9.6 *only*, you can set `ingressClassName` to one of the following: * `nginx` for Nginx Ingress Controller * `alb` for AWS Application Load Balancer (ALB) * Configure TLS and certificates according to your CA’s procedures and policies. TLS must be enabled in order to use AWS ALB for ingress. ## Connector configuration example ```yaml wrap expandable theme={"dark"} kafka-bridge: target: mynamespace-connectors-backend.lucidworkstest.com:443 # mandatory plain-text: false # optional, false by default. proxy-server: # optional - needed when a forward proxy server is used to provide outbound access to the standalone connector host: host port: some-port user: user # optional password: password # optional trust: # optional - needed when the client's system doesn't trust the server's certificate cert-collection-filepath: path1 proxy: # mandatory fusion-proxy user: admin password: password123 url: https://fusiontest.com/ # needed only when the connector plugin requires blob store access plugin: # mandatory path: ./fs.zip type: #optional - the suffix is added to the connector id suffix: remote ``` ### Minimal example ```yaml theme={"dark"} kafka-bridge: target: mynamespace-connectors-backend.lucidworkstest.com:443 proxy: user: admin password: "password123" plugin: path: ./testplugin.zip ``` ### Logback XML configuration file example ```xml wrap expandable theme={"dark"} %d - %-5p [%t:%C{3.}@%L] - %m{nolookups}%n utf8 ${LOGDIR:-.}/connector.log ${LOGDIR:-.}/connector-%d{yyyy-MM-dd}.%i.log.gz 50MB 10GB %d - %-5p [%t:%C{3.}@%L] - %m{nolookups}%n utf8 ``` ## Run the remote connector ```java theme={"dark"} java [-Dlogging.config=[LOGBACK_XML_FILE]] \ -jar connector-plugin-client-standalone.jar [YAML_CONFIG_FILE] ``` The `logging.config` property is optional. If not set, logging messages are sent to the console. ## Test communication You can run the connector in communication testing mode. This mode tests the communication with the backend without running the plugin, reports the result, and exits. ```java theme={"dark"} java -Dstandalone.connector.connectivity.test=true -jar connector-plugin-client-standalone.jar [YAML_CONFIG_FILE] ``` ## Encryption In a deployment, communication to the connector’s backend server is encrypted using TLS. You should only run this configuration without TLS in a testing scenario. To disable TLS, set `plain-text` to `true`. ## Egress and proxy server configuration One of the methods you can use to allow outbound communication from behind a firewall is a proxy server. You can configure a proxy server to allow certain communication traffic while blocking unauthorized communication. If you use a proxy server at the site where the connector is running, you must configure the following properties: * **Host.** The hosts where the proxy server is running. * **Port.** The port the proxy server is listening to for communication requests. * **Credentials.** Optional proxy server user and password. When you configure egress, it is important to disable any connection or activity timeouts because the connector uses long running gRPC calls. ## Password encryption If you use a login name and password in your configuration, run the following utility to encrypt the password: 1. Enter a user name and password in the [connector configuration YAML](#configuration). 2. Run the standalone JAR with this property: ```java theme={"dark"} -Dstandalone.connector.encrypt.password=true ``` 3. Retrieve the encrypted passwords from the log that is created. 4. Replace the clear password in the configuration YAML with the encrypted password. ## Connector restart (5.7 and earlier) The connector will shut down automatically whenever the connection to the server is disrupted, to prevent it from getting into a bad state. Communication disruption can happen, for example, when the server running in the `connectors-backend` pod shuts down and is replaced by a new pod. Once the connector shuts down, connector configuration and job execution are disabled. To prevent that from happening, you should restart the connector as soon as possible. You can use Linux scripts and utilities to restart the connector automatically, such as [Monit](https://mmonit.com/monit/). ## Recoverable bridge (5.8 and later) If communication to the remote connector is disrupted, the connector will try to recover communication and gRPC calls. By default, six attempts will be made to recover each gRPC call. The number of attempts can be configured with the `max-grpc-retries` bridge parameters. ## Job expiration duration (5.9.5 only) The timeout value for irresponsive backend jobs can be configured with the `job-expiration-duration-seconds` parameter. The default value is `120` seconds. ## Use the remote connector Once the connector is running, it is available in the Datasources dropdown. If the standalone connector terminates, it disappears from the list of available connectors. Once it is re-run, it is available again and configured connector instances will not get lost. ## Enable asynchronous parsing (5.9 and later) To separate document crawling from document parsing, enable Tika Asynchronous Parsing on remote V2 connectors. Below is an example configuration showing how to specify the file system to index under the `connector-plugins` entry in your `values.yaml` file: ```yaml theme={"dark"} additionalVolumes: - name: fusion-data1-pvc persistentVolumeClaim: claimName: fusion-data1-pvc - name: fusion-data2-pvc persistentVolumeClaim: claimName: fusion-data2-pvc additionalVolumeMounts: - name: fusion-data1-pvc mountPath: "/connector/data1" - name: fusion-data2-pvc mountPath: "/connector/data2" ``` You may also need to specify the user that is authorized to access the file system, as in this example: ```yaml theme={"dark"} securityContext: fsGroup: 1002100000 runAsUser: 1002100000 ``` ## Selenium Grid setup If you are using Web V2 v2.1.0 or later, you must also use Selenium Grid as part of your Web V2 connector setup. When using Web V2 v2.2.1 or later on Fusion 5.9.15 or later, you do not need to perform any Selenium setup. Selenium is installed automatically with the connector and removed automatically when the connector is removed. Manual installation of Selenium is still required in Fusion 5.9.14 or earlier, even if you use Web V2 v2.2.1. For hosted connectors, Selenium Grid support is available through Kubernetes. For remote connectors, Selenium Grid support is avialable through Docker Compose. See [the Web V2 remote support repository](https://github.com/lucidworks/web-v2-remote-support/) for full setup instructions and YAML files. Before you set up Selenium Grid, install the connector standalone plugin file for the version of Fusion that you are using and the most recent version of the Web V2 connector. Verify that you have the correct files at [the Lucidworks plugins site](https://plugins.lucidworks.com/). The Selenium services require an x86 architecture to run properly. Running the Selenium services on an ARM-based system such as Apple Silicon is not supported. ### Set up Selenium Grid in Kubernetes If you are using a hosted connector, use Kubernetes to set up Selenium Grid. These steps explain how to deploy the Selenium Hub component and the two Chrome browser nodes that connect to Selenium Hub. The referenced YAML files are available in the `k8s` directory of [the Web V2 remote support repository](https://github.com/lucidworks/web-v2-remote-support/). To set up Selenium Grid: 1. In a terminal, apply the Kubernetes YAML configurations: ```bash theme={"dark"} kubectl apply -f deployment.yaml -n NAMESPACE kubectl apply -f chrome-deployment.yaml -n NAMESPACE kubectl apply -f service.yaml -n NAMESPACE ``` 2. Verify that the deployments are successful. Replace `NAMESPACE` with your namespace. ```bash theme={"dark"} kubectl get pods -n NAMESPACE kubectl get services -n NAMESPACE ``` 3. Adjust the network policy to allow port 4444. Enter the following command in a terminal: ```bash theme={"dark"} kubectl edit networkpolicy NAMESPACE-connector-plugin -n NAMESPACE ``` 4. Add the following snippet to the file: ```yaml theme={"dark"} - ports: - port: 4444 protocol: TCP - port: 4444 protocol: UDP ``` 5. Save the file. ### Set up Selenium Grid in Docker Compose If you are using a remote connector, use Docker Compose to set up Selenium Grid. The referenced YAML files are available in [the Web V2 remote support repository](https://github.com/lucidworks/web-v2-remote-support/). Before setting up Selenium Grid in Docker Compose, you must know what version of JDK your Fusion connectors are using. If you are using Fusion 5.9.10 or earlier, you are using JDK 11. If you are using Fusion 5.9.11 or later, you are using JDK 17. To set up Selenium Grid in Docker Compose: 1. Visit the [the Web V2 remote support repository](https://github.com/lucidworks/web-v2-remote-support/) and select the folder corresponding to your JDK version. Download the contents of that folder. 2. Edit the `bin/conf/connector-config.yaml` file to configure the Kafka bridge settings, the proxy settings, and the plugin path. The following snippet shows an example configuration. Quotation marks are required around the password. ```yaml theme={"dark"} kafka-bridge: target: EXAMPLE_CONNECTORS_BACKEND.example.com:443 # Uncomment proxy-server section if needed proxy: user: EXAMPLE_USERNAME password: "EXAMPLE_PASSWORD" url: https://FUSION_HOST:FUSION_PORT/ plugin: path: /app/connector-plugin.zip type: suffix: remote- ``` 3. Save the configuration file. Now you can start the Docker Compose environment, which uses standard Docker Compose commands. You can start the environment in background mode or with live logs. To start Docker Compose in background mode, navigate to the directory for your environment and enter the following command in a terminal: ```bash theme={"dark"} docker-compose up -d ``` To start Docker Compose in live mode, navigate to the directory for your environment and enter the following command in a terminal: ```bash theme={"dark"} docker-compose up ``` When you’ve started Docker Compose, verify that the services are running. You can access the Selenium Grid console at `http://localhost:4444/ui`. Verify that the Selenium Hub is running and the Chrome nodes are connected. The Lucidworks connector is available on port 8764. Run `docker-compose logs lucidworks-connector` in a terminal to verify that the service is up. To check the container status, run `docker-compose ps` in a terminal and verify that all containers are up. Press `Ctrl-C` to stop the services when viewing logs in real-time. To stop all services, run `docker-compose down` in a terminal. If you want to remove all volumes when stopping all services, run `docker-compose down -v`. ## Control the web connector's crawl depth The Max Crawling Depth setting controls how many levels the Web V2 connector follows from your starting URL. You can think of these crawl levels as link hops or navigation levels from a starting URL. Setting the crawl depth ensures you are crawling the appropriate pages for your search experience. As you increase the Max Crawling Depth value, the Web V2 connector may crawl exponentially more pages. For example, if the start link contains links to ten pages, and each of those pages links to ten different pages, then a crawl depth of `2` can grow to 100 pages. As the number of pages crawled increases, the time to complete the crawl also increases. Deeper crawl depths consume more memory, use more HTTP requests and bandwidth, and may cause your website to limit the number of pages the web connector can crawl or time out before completing the crawl. Without strict limits, it's possible for the Web V2 connector to crawl extremely large websites indefinitely if the website contains dynamic content or circular links. ### Use a start link A start link can be a page of your website or a webpage with a list of HTML links. The Web V2 connector starts the crawl from this page and crawls the page to the specified depth. You may specify multiple start URLs in the **Bulk Start Links** field. A simple example is starting a crawl from a company's home page with the following layout: ``` Company Home Page (Start page - Depth 0) ├── About Us (Depth 1) │ ├── Our History (Depth 2) │ └── Leadership Team (Depth 2) ├── Products (Depth 1) │ ├── Product A (Depth 2) │ │ └── Product A Details (Depth 3) │ └── Product B (Depth 2) └── Contact (Depth 1) ``` If you set the **Max Crawling Depth** field to a different value, the Web V2 connector crawls a different set of pages. For example: | Depth | Crawled pages | Example crawl results | | :---- | :---------------------------------- | :----------------------------------------------------------- | | 0 | Only start URLs | Just the home page | | 1 | Start URLs and their direct links | Home page + About, Products, Contact | | 2 | Previous + links from depth 1 pages | Depth 1 links plus History, Leadership, Product A, Product B | | 3 | Previous + links from depth 2 pages | Depth 2 links plus Product A Details | | -1 | No limit | Everything the crawler can find | If you set the max crawling depth to `0`, the web connector crawls only your start page if you are using a start URL. This setting is recommended for testing basic connectivity or troubleshooting. A max crawling depth of `2` or `3` is sufficient for most business needs, especially when combined with URL filters and page limits. Very large websites may need a max crawling depth of `4`. If you need comprehensive coverage, set the max crawling depth to `-1`. If you set the max crawling depth to `-1`, set strict document limits and URL filters so your crawls complete successfully. ### Use a sitemap file The Web V2 connector supports XML sitemap files. These files must be in XML format and are typically named `sitemap.xml` or similar. To use a sitemap instead of a start link, select **Link Discovery**, enter a sitemap URL in the **Sitemap URLs** field, and click **Add**. You may add multiple sitemap URLs. Unlike using a start link, setting the max crawling depth to `0` when using a sitemap crawls the URLs listed in the sitemap. Setting the max crawling depth to `1` crawls the URLs listed in the sitemap and the links included in those pages. ## Configure the timeout The connector timeout can be configured as infinite to handle large files. Configure the timeout by enabling **Crawl performance properties** and setting the **Connection timeout (ms)**. Connector flow The **Connection timeout (ms)** option becomes visible only after selecting the **Advanced** checkbox. Use the following guidelines when setting the timeout: * To enable an infinite timeout for large file downloads, set the timeout configuration to `0`. * For standard timeout behavior, use any positive value. For example, a timeout of `30000` gives a 30-second timeout with clean connection termination. ## Enable Javascript Evaluation in Fusion JavaScript evaluation allows the Web V2 connector to extract content from a website that is only available after JavaScript has rendered the document. It is available for Web V2 v2.1.0 and later on hosted and remote connectors. To enable JavaScript evaluation in the Web V2 connector: 1. Navigate to your Web V2 datasource in Fusion. 2. Select **Javascript Evaluation Properties**. A variety of settings displays. In this section you can customize your JavaScript evaluation settings. 3. Select **Evaluate JavaScript**. This is required for using JavaScript evaluation. 4. If you specified a SmartForms or SAML element in the Crawl Authentication Properties area, select **Evaluate JavaScript during SmartForms/SAML Login**. 5. Headless browser mode is selected by default, which runs the browser performing the website crawl in the background without being visible. If your website renders pages on the server side, the **Headless browser** field must be unchecked for the crawl to work correctly and retrieve links. If your website renders pages on the client side, the **Headless browser** field should be checked. 6. Click **Apply**. For the full JavaScript evaluation settings, see `javascriptEvaluationConfig` under `Properties` in the [configuration specifications](#configuration). ## Configuration When entering configuration values in the UI, use *unescaped* characters, such as `\t` for the tab character. When entering configuration values in the API, use *escaped* characters, such as `\\t` for the tab character.