CrawlConfig

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "#/components/schemas/CrawlConfig",
  "title": "CrawlConfig",
  "properties": {
    "max_scan_time_in_minutes": {
      "type": "integer",
      "format": "int64"
    },
    "max_domain": {
      "type": "integer",
      "format": "int64"
    },
    "max_crawl_results": {
      "type": "integer",
      "format": "int64"
    },
    "max_per_web_site_crawl_results": {
      "type": "integer",
      "format": "int64"
    },
    "max_per_dir_crawl_results": {
      "type": "integer",
      "format": "int64"
    },
    "max_per_link_crawl_results": {
      "type": "integer",
      "format": "int64"
    },
    "max_per_normalized_link_crawl_result": {
      "type": "integer",
      "format": "int64"
    },
    "max_per_dir_child_nodes": {
      "type": "integer",
      "format": "int64"
    },
    "max_deny_list_ext_crawl_results": {
      "type": "integer",
      "format": "int64"
    },
    "max_attack_feedback_links_count": {
      "type": "integer",
      "format": "int64"
    },
    "max_per_file_name_crawl_results": {
      "type": "integer",
      "format": "int64"
    },
    "max_per_query_crawl_results": {
      "type": "integer",
      "format": "int64"
    },
    "recursion_depth": {
      "type": "integer",
      "format": "int64"
    },
    "max_dir_depth": {
      "type": "integer",
      "format": "int64"
    },
    "discovery_depth": {
      "type": "integer",
      "format": "int64"
    },
    "url_repetition_tolerance": {
      "type": "integer",
      "format": "int64"
    },
    "sequence_repetition_tolerance": {
      "type": "integer",
      "format": "int64"
    },
    "max_reported_images": {
      "type": "integer",
      "format": "int64"
    },
    "max_reported_links": {
      "type": "integer",
      "format": "int64"
    },
    "max_reported_comments": {
      "type": "integer",
      "format": "int64"
    },
    "max_reported_scripts": {
      "type": "integer",
      "format": "int64"
    },
    "max_reported_emails": {
      "type": "integer",
      "format": "int64"
    },
    "max_reported_forms": {
      "type": "integer",
      "format": "int64"
    },
    "extract_crawl_browser_storage": {
      "type": "boolean"
    },
    "max_browser_page_wait_timeout": {
      "type": "integer",
      "format": "int64"
    },
    "max_browser_wait_till_request_timeout": {
      "type": "integer",
      "format": "int64"
    },
    "max_browser_dom_depth": {
      "type": "integer",
      "format": "int64"
    },
    "max_browser_events_per_link": {
      "type": "integer",
      "format": "int64"
    },
    "max_browser_events_per_crawl_result": {
      "type": "integer",
      "format": "int64"
    },
    "max_browser_events_per_dom": {
      "type": "integer",
      "format": "int64"
    },
    "max_browser_events_per_browse_dom": {
      "type": "integer",
      "format": "int64"
    },
    "max_browser_no_new_resource_dom_count": {
      "type": "integer",
      "format": "int64"
    },
    "not_inserted_link_count_threshold": {
      "type": "integer",
      "format": "int64"
    },
    "max_cookies_from_javascript": {
      "type": "integer",
      "format": "int64"
    },
    "max_cookies_same_name_from_javascript": {
      "type": "integer",
      "format": "int64"
    },
    "max_days_threshold_good_traffic": {
      "type": "integer",
      "format": "int64"
    },
    "crawl_prioritization": {
      "type": "string",
      "enum": [
        "FIRST_IN_FIRST_OUT",
        "SMART",
        "DIRECTORY_BREADTH_FIRST",
        "FOUND_BREADTH_FIRST",
        "FOUND_DEPTH_FIRST",
        "JUICY",
        "LOGIN_FORM_DISCOVERY",
        "LOGIN"
      ]
    },
    "file_not_found_regex": {
      "type": "string"
    },
    "server_error_regex": {
      "type": "string"
    },
    "invalid_url_regex_attack": {
      "type": "string"
    },
    "invalid_url_regex_crawl": {
      "type": "string"
    },
    "nonce_regex": {
      "type": "string"
    },
    "priority_links_regex": {
      "type": "string"
    },
    "lock_cookies": {
      "type": "boolean"
    },
    "max_cookie_header_size": {
      "type": "integer",
      "format": "int64"
    },
    "case_sensitivity": {
      "type": "string",
      "enum": [
        "AUTO_DETECT",
        "CASE_SENSITIVE",
        "CASE_INSENSITIVE"
      ]
    },
    "stay_on_port": {
      "type": "boolean"
    },
    "restrict_to_macro": {
      "type": "boolean"
    },
    "restrict_to_manual_crawling": {
      "type": "boolean"
    },
    "restrict_to_seed_list": {
      "type": "boolean"
    },
    "restrict_to_web_service": {
      "type": "boolean"
    },
    "restrict_to_selenium": {
      "type": "boolean"
    },
    "restrict_to_swagger": {
      "type": "boolean"
    },
    "restrict_to_agent_routes": {
      "type": "boolean"
    },
    "import_cookies_from_traffic": {
      "type": "boolean"
    },
    "page_equal_threshhold": {
      "type": "number",
      "format": "double"
    },
    "page_similar_threshhold": {
      "type": "number",
      "format": "double"
    },
    "experimental_crawling": {
      "type": "string",
      "enum": [
        "DISABLED",
        "ENABLED",
        "FRAMEWORK_MATCH"
      ]
    },
    "flash": {
      "type": "boolean"
    },
    "enable_advanced_parsers": {
      "type": "boolean"
    },
    "search_for_urls": {
      "type": "boolean"
    },
    "cookie_comma_separator": {
      "type": "boolean"
    },
    "denylist_contact_forms": {
      "type": "boolean"
    },
    "denylist_all_forms": {
      "type": "boolean"
    },
    "enable_robotstxt_sitemap_crawling": {
      "type": "boolean"
    },
    "log_dom_contents": {
      "type": "boolean"
    },
    "use_browser_response_caching": {
      "type": "boolean"
    },
    "use_browser_response_reference_caching": {
      "type": "boolean"
    },
    "remove_content_security_policy": {
      "type": "boolean"
    },
    "page_loaded": {
      "type": "string",
      "enum": [
        "NO_NETWORK_CONNECTION_FOR_OVER_500_MS",
        "DOM_CONTENT_LOADED_EVENT_FIRED",
        "LOAD_EVENT_FIRED",
        "DOCUMENT_STARTS_LOADING"
      ]
    },
    "page_delay_milliseconds": {
      "type": "integer",
      "format": "int64"
    },
    "page_timeout_milliseconds": {
      "type": "integer",
      "format": "int64"
    },
    "mock_images": {
      "type": "boolean"
    },
    "mock_videos": {
      "type": "boolean"
    },
    "screen_width": {
      "type": "integer",
      "format": "int64"
    },
    "screen_height": {
      "type": "integer",
      "format": "int64"
    },
    "screen_responsive": {
      "type": "boolean"
    },
    "inject_initial_delay_milliseconds": {
      "type": "integer",
      "format": "int64"
    },
    "inject_combo_delay_milliseconds": {
      "type": "integer",
      "format": "int64"
    },
    "error_limit_percentage": {
      "type": "integer",
      "format": "int64"
    },
    "error_limit_lower_absolute": {
      "type": "integer",
      "format": "int64"
    },
    "events_full_selectors_only": {
      "type": "boolean"
    },
    "events_add_iframe_events": {
      "type": "boolean"
    },
    "dom_element_selector_type": {
      "type": "string",
      "enum": [
        "X_PATH",
        "CSS",
        "TEXT"
      ]
    },
    "disable_before_unload": {
      "type": "boolean"
    },
    "seed_url_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/SeedUrl"
      }
    },
    "scope_constraint_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/ScopeConstraint"
      }
    },
    "deny_list_extension_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/DenyListExtension"
      }
    },
    "gray_list_extension_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/GrayListExtension"
      }
    },
    "binary_extension_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/BinaryExtension"
      }
    },
    "text_extension_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/TextExtension"
      }
    },
    "binary_content_type_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/BinaryContentType"
      }
    },
    "html_content_type_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/HtmlContentType"
      }
    },
    "text_content_type_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/TextContentType"
      }
    },
    "xml_content_type_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/XmlContentType"
      }
    },
    "dom_restrictions_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/DomRestrictions"
      }
    },
    "browser_download_allowlist_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/BrowserDownloadAllowlist"
      }
    },
    "browser_do_not_download_extension_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/BrowserDoNotDownloadExtension"
      }
    },
    "browser_do_not_download_content_type_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/BrowserDoNotDownloadContentType"
      }
    },
    "locked_cookie_list": {
      "type": "array",
      "items": {
        "$ref": "#/components/schemas/LockedCookie"
      }
    },
    "frameworks_crawl_config": {
      "$ref": "#/components/schemas/FrameworksCrawlConfig"
    },
    "crawler_initialization_config": {
      "$ref": "#/components/schemas/CrawlerInitializationConfig"
    },
    "crawler_monitoring_config": {
      "$ref": "#/components/schemas/CrawlerMonitoringConfig"
    }
  }
}