逐页抓取整站干净内容的网站爬虫

{
  "type": "object",
  "required": [
    "body"
  ],
  "properties": {
    "body": {
      "type": "object",
      "$schema": "https://json-schema.org/draft/2020-12/schema",
      "required": [
        "url"
      ],
      "~standard": {
        "vendor": "zod",
        "version": 1,
        "jsonSchema": {}
      },
      "properties": {
        "url": {
          "type": "string",
          "format": "uri",
          "description": "开始抓取的网址。"
        },
        "delay": {
          "type": "number",
          "maximum": 30,
          "minimum": 0,
          "description": "两次页面请求之间的间隔，单位为秒，用于控制访问频率。"
        },
        "limit": {
          "type": "integer",
          "default": 25,
          "maximum": 10000,
          "minimum": 1,
          "description": "最多抓取的页面数。调用前必须有足够额度支付整个上限，请按实际需要填写。"
        },
        "prompt": {
          "type": "string",
          "maxLength": 2000,
          "minLength": 1,
          "description": "用自然语言描述要抓取的内容，Firecrawl 会据此生成路径筛选条件。"
        },
        "sitemap": {
          "enum": [
            "skip",
            "include",
            "only"
          ],
          "type": "string"
        },
        "excludePaths": {
          "type": "array",
          "items": {
            "type": "string",
            "minLength": 1
          },
          "maxItems": 100
        },
        "includePaths": {
          "type": "array",
          "items": {
            "type": "string",
            "minLength": 1
          },
          "maxItems": 100
        },
        "scrapeOptions": {
          "type": "object",
          "properties": {
            "json": {
              "type": "boolean",
              "description": "通过大语言模型从页面提取结构化 JSON，费用为普通采集的五倍，具体见价格卡中的 JSON 档位。"
            },
            "proxy": {
              "enum": [
                "basic",
                "enhanced",
                "auto"
              ],
              "type": "string",
              "description": "auto 为默认值：普通请求被拦截时，使用增强的反爬代理重试，两种情况均不额外收费。"
            },
            "maxAge": {
              "type": "integer",
              "maximum": 604800000,
              "minimum": 0,
              "description": "可复用缓存的时间范围，单位为毫秒。使用足够新的缓存可更快返回；设为 0 时强制重新获取页面。"
            },
            "mobile": {
              "type": "boolean"
            },
            "formats": {
              "type": "array",
              "items": {
                "enum": [
                  "markdown",
                  "html",
                  "rawHtml",
                  "links",
                  "images",
                  "summary"
                ],
                "type": "string"
              },
              "maxItems": 6,
              "minItems": 1,
              "description": "输出格式，默认 markdown。使用大语言模型提取内容需另行设置 json 选项。"
            },
            "timeout": {
              "type": "integer",
              "maximum": 120000,
              "minimum": 1000,
              "description": "每个页面的超时时间，单位为毫秒；等待浏览器的排队时间也计入其中。"
            },
            "waitFor": {
              "type": "integer",
              "maximum": 30000,
              "minimum": 0,
              "description": "页面加载后额外等待的毫秒数，适合依赖大量 JavaScript 的页面。"
            },
            "blockAds": {
              "type": "boolean"
            },
            "location": {
              "type": "object",
              "properties": {
                "country": {
                  "type": "string",
                  "maxLength": 2,
                  "minLength": 2,
                  "description": "要模拟的访问国家，使用 ISO 3166-1 alpha-2 代码，例如 'US'。"
                },
                "languages": {
                  "type": "array",
                  "items": {
                    "type": "string",
                    "maxLength": 10,
                    "minLength": 2
                  },
                  "maxItems": 5,
                  "description": "优先使用的语言，服务会尽量满足，例如 ['en']。"
                }
              },
              "additionalProperties": false
            },
            "jsonPrompt": {
              "type": "string",
              "maxLength": 2000,
              "minLength": 1,
              "description": "要从页面提取的内容，在 json: true 时使用。"
            },
            "jsonSchema": {
              "type": "object",
              "description": "提取结果必须符合的 JSON Schema，在 json: true 时使用。",
              "propertyNames": {
                "type": "string"
              },
              "additionalProperties": {}
            },
            "excludeTags": {
              "type": "array",
              "items": {
                "type": "string",
                "minLength": 1
              },
              "maxItems": 50
            },
            "includeTags": {
              "type": "array",
              "items": {
                "type": "string",
                "minLength": 1
              },
              "maxItems": 50
            },
            "onlyMainContent": {
              "type": "boolean",
              "description": "是否去除页头、导航和页脚，默认 true。"
            },
            "removeBase64Images": {
              "type": "boolean"
            }
          },
          "additionalProperties": false
        },
        "allowSubdomains": {
          "type": "boolean"
        },
        "crawlEntireDomain": {
          "type": "boolean"
        },
        "maxDiscoveryDepth": {
          "type": "integer",
          "maximum": 20,
          "minimum": 0
        }
      },
      "description": "逐页抓取整个网站，提取整理后的内容。",
      "additionalProperties": false
    }
  },
  "additionalProperties": false
}