Firecrawl 中文文档

Scrape multiple URLs and optionally extract information using an LLM

curl --request POST \
  --url https://api.firecrawl.dev/v1/batch/scrape \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "urls": [
    "<string>"
  ],
  "ignoreInvalidURLs": false,
  "formats": [
    "markdown"
  ],
  "onlyMainContent": true,
  "includeTags": [
    "<string>"
  ],
  "excludeTags": [
    "<string>"
  ],
  "headers": {},
  "waitFor": 0,
  "mobile": false,
  "skipTlsVerification": false,
  "timeout": 30000,
  "jsonOptions": {
    "schema": {},
    "systemPrompt": "<string>",
    "prompt": "<string>"
  },
  "actions": [
    {
      "type": "wait",
      "milliseconds": 2,
      "selector": "#my-element"
    }
  ],
  "location": {
    "country": "US",
    "languages": [
      "en-US"
    ]
  },
  "removeBase64Images": true,
  "blockAds": true
}
'

import requests

url = "https://api.firecrawl.dev/v1/batch/scrape"

payload = {
    "urls": ["<string>"],
    "ignoreInvalidURLs": False,
    "formats": ["markdown"],
    "onlyMainContent": True,
    "includeTags": ["<string>"],
    "excludeTags": ["<string>"],
    "headers": {},
    "waitFor": 0,
    "mobile": False,
    "skipTlsVerification": False,
    "timeout": 30000,
    "jsonOptions": {
        "schema": {},
        "systemPrompt": "<string>",
        "prompt": "<string>"
    },
    "actions": [
        {
            "type": "wait",
            "milliseconds": 2,
            "selector": "#my-element"
        }
    ],
    "location": {
        "country": "US",
        "languages": ["en-US"]
    },
    "removeBase64Images": True,
    "blockAds": True
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    urls: ['<string>'],
    ignoreInvalidURLs: false,
    formats: ['markdown'],
    onlyMainContent: true,
    includeTags: ['<string>'],
    excludeTags: ['<string>'],
    headers: {},
    waitFor: 0,
    mobile: false,
    skipTlsVerification: false,
    timeout: 30000,
    jsonOptions: {schema: {}, systemPrompt: '<string>', prompt: '<string>'},
    actions: [{type: 'wait', milliseconds: 2, selector: '#my-element'}],
    location: {country: 'US', languages: ['en-US']},
    removeBase64Images: true,
    blockAds: true
  })
};

fetch('https://api.firecrawl.dev/v1/batch/scrape', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.firecrawl.dev/v1/batch/scrape",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'urls' => [
        '<string>'
    ],
    'ignoreInvalidURLs' => false,
    'formats' => [
        'markdown'
    ],
    'onlyMainContent' => true,
    'includeTags' => [
        '<string>'
    ],
    'excludeTags' => [
        '<string>'
    ],
    'headers' => [
        
    ],
    'waitFor' => 0,
    'mobile' => false,
    'skipTlsVerification' => false,
    'timeout' => 30000,
    'jsonOptions' => [
        'schema' => [
                
        ],
        'systemPrompt' => '<string>',
        'prompt' => '<string>'
    ],
    'actions' => [
        [
                'type' => 'wait',
                'milliseconds' => 2,
                'selector' => '#my-element'
        ]
    ],
    'location' => [
        'country' => 'US',
        'languages' => [
                'en-US'
        ]
    ],
    'removeBase64Images' => true,
    'blockAds' => true
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.firecrawl.dev/v1/batch/scrape"

	payload := strings.NewReader("{\n  \"urls\": [\n    \"<string>\"\n  ],\n  \"ignoreInvalidURLs\": false,\n  \"formats\": [\n    \"markdown\"\n  ],\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"headers\": {},\n  \"waitFor\": 0,\n  \"mobile\": false,\n  \"skipTlsVerification\": false,\n  \"timeout\": 30000,\n  \"jsonOptions\": {\n    \"schema\": {},\n    \"systemPrompt\": \"<string>\",\n    \"prompt\": \"<string>\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"wait\",\n      \"milliseconds\": 2,\n      \"selector\": \"#my-element\"\n    }\n  ],\n  \"location\": {\n    \"country\": \"US\",\n    \"languages\": [\n      \"en-US\"\n    ]\n  },\n  \"removeBase64Images\": true,\n  \"blockAds\": true\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.firecrawl.dev/v1/batch/scrape")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"urls\": [\n    \"<string>\"\n  ],\n  \"ignoreInvalidURLs\": false,\n  \"formats\": [\n    \"markdown\"\n  ],\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"headers\": {},\n  \"waitFor\": 0,\n  \"mobile\": false,\n  \"skipTlsVerification\": false,\n  \"timeout\": 30000,\n  \"jsonOptions\": {\n    \"schema\": {},\n    \"systemPrompt\": \"<string>\",\n    \"prompt\": \"<string>\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"wait\",\n      \"milliseconds\": 2,\n      \"selector\": \"#my-element\"\n    }\n  ],\n  \"location\": {\n    \"country\": \"US\",\n    \"languages\": [\n      \"en-US\"\n    ]\n  },\n  \"removeBase64Images\": true,\n  \"blockAds\": true\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.firecrawl.dev/v1/batch/scrape")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"urls\": [\n    \"<string>\"\n  ],\n  \"ignoreInvalidURLs\": false,\n  \"formats\": [\n    \"markdown\"\n  ],\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"headers\": {},\n  \"waitFor\": 0,\n  \"mobile\": false,\n  \"skipTlsVerification\": false,\n  \"timeout\": 30000,\n  \"jsonOptions\": {\n    \"schema\": {},\n    \"systemPrompt\": \"<string>\",\n    \"prompt\": \"<string>\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"wait\",\n      \"milliseconds\": 2,\n      \"selector\": \"#my-element\"\n    }\n  ],\n  \"location\": {\n    \"country\": \"US\",\n    \"languages\": [\n      \"en-US\"\n    ]\n  },\n  \"removeBase64Images\": true,\n  \"blockAds\": true\n}"

response = http.request(request)
puts response.read_body

{
  "success": true,
  "id": "<string>",
  "url": "<string>",
  "invalidURLs": [
    "<string>"
  ]
}

{
  "error": "Payment required to access this resource."
}

{
  "error": "Request rate limit exceeded. Please wait and try again later."
}

{
  "error": "An unexpected error occurred on the server."
}

POST

batch

scrape

Scrape multiple URLs and optionally extract information using an LLM

curl --request POST \
  --url https://api.firecrawl.dev/v1/batch/scrape \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "urls": [
    "<string>"
  ],
  "ignoreInvalidURLs": false,
  "formats": [
    "markdown"
  ],
  "onlyMainContent": true,
  "includeTags": [
    "<string>"
  ],
  "excludeTags": [
    "<string>"
  ],
  "headers": {},
  "waitFor": 0,
  "mobile": false,
  "skipTlsVerification": false,
  "timeout": 30000,
  "jsonOptions": {
    "schema": {},
    "systemPrompt": "<string>",
    "prompt": "<string>"
  },
  "actions": [
    {
      "type": "wait",
      "milliseconds": 2,
      "selector": "#my-element"
    }
  ],
  "location": {
    "country": "US",
    "languages": [
      "en-US"
    ]
  },
  "removeBase64Images": true,
  "blockAds": true
}
'

import requests

url = "https://api.firecrawl.dev/v1/batch/scrape"

payload = {
    "urls": ["<string>"],
    "ignoreInvalidURLs": False,
    "formats": ["markdown"],
    "onlyMainContent": True,
    "includeTags": ["<string>"],
    "excludeTags": ["<string>"],
    "headers": {},
    "waitFor": 0,
    "mobile": False,
    "skipTlsVerification": False,
    "timeout": 30000,
    "jsonOptions": {
        "schema": {},
        "systemPrompt": "<string>",
        "prompt": "<string>"
    },
    "actions": [
        {
            "type": "wait",
            "milliseconds": 2,
            "selector": "#my-element"
        }
    ],
    "location": {
        "country": "US",
        "languages": ["en-US"]
    },
    "removeBase64Images": True,
    "blockAds": True
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    urls: ['<string>'],
    ignoreInvalidURLs: false,
    formats: ['markdown'],
    onlyMainContent: true,
    includeTags: ['<string>'],
    excludeTags: ['<string>'],
    headers: {},
    waitFor: 0,
    mobile: false,
    skipTlsVerification: false,
    timeout: 30000,
    jsonOptions: {schema: {}, systemPrompt: '<string>', prompt: '<string>'},
    actions: [{type: 'wait', milliseconds: 2, selector: '#my-element'}],
    location: {country: 'US', languages: ['en-US']},
    removeBase64Images: true,
    blockAds: true
  })
};

fetch('https://api.firecrawl.dev/v1/batch/scrape', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.firecrawl.dev/v1/batch/scrape",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'urls' => [
        '<string>'
    ],
    'ignoreInvalidURLs' => false,
    'formats' => [
        'markdown'
    ],
    'onlyMainContent' => true,
    'includeTags' => [
        '<string>'
    ],
    'excludeTags' => [
        '<string>'
    ],
    'headers' => [
        
    ],
    'waitFor' => 0,
    'mobile' => false,
    'skipTlsVerification' => false,
    'timeout' => 30000,
    'jsonOptions' => [
        'schema' => [
                
        ],
        'systemPrompt' => '<string>',
        'prompt' => '<string>'
    ],
    'actions' => [
        [
                'type' => 'wait',
                'milliseconds' => 2,
                'selector' => '#my-element'
        ]
    ],
    'location' => [
        'country' => 'US',
        'languages' => [
                'en-US'
        ]
    ],
    'removeBase64Images' => true,
    'blockAds' => true
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.firecrawl.dev/v1/batch/scrape"

	payload := strings.NewReader("{\n  \"urls\": [\n    \"<string>\"\n  ],\n  \"ignoreInvalidURLs\": false,\n  \"formats\": [\n    \"markdown\"\n  ],\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"headers\": {},\n  \"waitFor\": 0,\n  \"mobile\": false,\n  \"skipTlsVerification\": false,\n  \"timeout\": 30000,\n  \"jsonOptions\": {\n    \"schema\": {},\n    \"systemPrompt\": \"<string>\",\n    \"prompt\": \"<string>\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"wait\",\n      \"milliseconds\": 2,\n      \"selector\": \"#my-element\"\n    }\n  ],\n  \"location\": {\n    \"country\": \"US\",\n    \"languages\": [\n      \"en-US\"\n    ]\n  },\n  \"removeBase64Images\": true,\n  \"blockAds\": true\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.firecrawl.dev/v1/batch/scrape")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"urls\": [\n    \"<string>\"\n  ],\n  \"ignoreInvalidURLs\": false,\n  \"formats\": [\n    \"markdown\"\n  ],\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"headers\": {},\n  \"waitFor\": 0,\n  \"mobile\": false,\n  \"skipTlsVerification\": false,\n  \"timeout\": 30000,\n  \"jsonOptions\": {\n    \"schema\": {},\n    \"systemPrompt\": \"<string>\",\n    \"prompt\": \"<string>\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"wait\",\n      \"milliseconds\": 2,\n      \"selector\": \"#my-element\"\n    }\n  ],\n  \"location\": {\n    \"country\": \"US\",\n    \"languages\": [\n      \"en-US\"\n    ]\n  },\n  \"removeBase64Images\": true,\n  \"blockAds\": true\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.firecrawl.dev/v1/batch/scrape")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"urls\": [\n    \"<string>\"\n  ],\n  \"ignoreInvalidURLs\": false,\n  \"formats\": [\n    \"markdown\"\n  ],\n  \"onlyMainContent\": true,\n  \"includeTags\": [\n    \"<string>\"\n  ],\n  \"excludeTags\": [\n    \"<string>\"\n  ],\n  \"headers\": {},\n  \"waitFor\": 0,\n  \"mobile\": false,\n  \"skipTlsVerification\": false,\n  \"timeout\": 30000,\n  \"jsonOptions\": {\n    \"schema\": {},\n    \"systemPrompt\": \"<string>\",\n    \"prompt\": \"<string>\"\n  },\n  \"actions\": [\n    {\n      \"type\": \"wait\",\n      \"milliseconds\": 2,\n      \"selector\": \"#my-element\"\n    }\n  ],\n  \"location\": {\n    \"country\": \"US\",\n    \"languages\": [\n      \"en-US\"\n    ]\n  },\n  \"removeBase64Images\": true,\n  \"blockAds\": true\n}"

response = http.request(request)
puts response.read_body

{
  "success": true,
  "id": "<string>",
  "url": "<string>",
  "invalidURLs": [
    "<string>"
  ]
}

{
  "error": "Payment required to access this resource."
}

{
  "error": "Request rate limit exceeded. Please wait and try again later."
}

{
  "error": "An unexpected error occurred on the server."
}

Authorizations

Authorization

string

header

required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Body

application/json

urls

string<uri>[]

required

The URL to scrape

webhook

object

A webhook specification object.

Show child attributes

ignoreInvalidURLs

boolean

default:false

If invalid URLs are specified in the urls array, they will be ignored. Instead of them failing the entire request, a batch scrape using the remaining valid URLs will be created, and the invalid URLs will be returned in the invalidURLs field of the response.

formats

enum<string>[]

Formats to include in the output.

Available options:

markdown,

html,

rawHtml,

links,

screenshot,

screenshot@fullPage,

json

onlyMainContent

boolean

default:true

Only return the main content of the page excluding headers, navs, footers, etc.

includeTags

string[]

Tags to include in the output.

excludeTags

string[]

Tags to exclude from the output.

headers

object

Headers to send with the request. Can be used to send cookies, user-agent, etc.

waitFor

integer

default:0

Specify a delay in milliseconds before fetching the content, allowing the page sufficient time to load.

mobile

boolean

default:false

Set to true if you want to emulate scraping from a mobile device. Useful for testing responsive pages and taking mobile screenshots.

skipTlsVerification

boolean

default:false

Skip TLS certificate verification when making requests

timeout

integer

default:30000

Timeout in milliseconds for the request

jsonOptions

object

Extract object

Show child attributes

actions

Actions to perform on the page before grabbing the content

Show child attributes

location

object

Location settings for the request. When specified, this will use an appropriate proxy if available and emulate the corresponding language and timezone settings. Defaults to 'US' if not specified.

Show child attributes

removeBase64Images

boolean

Removes all base 64 images from the output, which may be overwhelmingly long. The image's alt text remains in the output, but the URL is replaced with a placeholder.

blockAds

boolean

default:true

Enables ad-blocking and cookie popup blocking.

proxy

enum<string>

Specifies the type of proxy to use.

basic: Proxies for scraping sites with none to basic anti-bot solutions. Fast and usually works.
stealth: Stealth proxies for scraping sites with advanced anti-bot solutions. Slower, but more reliable on certain sites.

If you do not specify a proxy, Firecrawl will automatically attempt to determine which one you need based on the target site.

Available options:

basic,

stealth

Response

Successful response

success

boolean

string

url

string<uri>

invalidURLs

string[] | null

If ignoreInvalidURLs is true, this is an array containing the invalid URLs that were specified in the request. If there were no invalid URLs, this will be an empty array. If ignoreInvalidURLs is false, this field will be undefined.

使用 API

抓取接口

爬取接口

映射接口

提取接口

搜索接口

账户接口

批量抓取

Authorizations

Body

Response