Crawl multiple URLs based on options
curl --request POST \
--url https://api.firecrawl.dev/v1/crawl \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "<string>",
"excludePaths": [
"<string>"
],
"includePaths": [
"<string>"
],
"maxDepth": 10,
"maxDiscoveryDepth": 123,
"ignoreSitemap": false,
"ignoreQueryParameters": false,
"limit": 10000,
"allowBackwardLinks": false,
"allowExternalLinks": false,
"scrapeOptions": {
"formats": [
"markdown"
],
"onlyMainContent": true,
"includeTags": [
"<string>"
],
"excludeTags": [
"<string>"
],
"headers": {},
"waitFor": 0,
"mobile": false,
"skipTlsVerification": false,
"timeout": 30000,
"jsonOptions": {
"schema": {},
"systemPrompt": "<string>",
"prompt": "<string>"
},
"actions": [
{
"type": "wait",
"milliseconds": 2,
"selector": "#my-element"
}
],
"location": {
"country": "US",
"languages": [
"en-US"
]
},
"removeBase64Images": true,
"blockAds": true
}
}
'import requests
url = "https://api.firecrawl.dev/v1/crawl"
payload = {
"url": "<string>",
"excludePaths": ["<string>"],
"includePaths": ["<string>"],
"maxDepth": 10,
"maxDiscoveryDepth": 123,
"ignoreSitemap": False,
"ignoreQueryParameters": False,
"limit": 10000,
"allowBackwardLinks": False,
"allowExternalLinks": False,
"scrapeOptions": {
"formats": ["markdown"],
"onlyMainContent": True,
"includeTags": ["<string>"],
"excludeTags": ["<string>"],
"headers": {},
"waitFor": 0,
"mobile": False,
"skipTlsVerification": False,
"timeout": 30000,
"jsonOptions": {
"schema": {},
"systemPrompt": "<string>",
"prompt": "<string>"
},
"actions": [
{
"type": "wait",
"milliseconds": 2,
"selector": "#my-element"
}
],
"location": {
"country": "US",
"languages": ["en-US"]
},
"removeBase64Images": True,
"blockAds": True
}
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
excludePaths: ['<string>'],
includePaths: ['<string>'],
maxDepth: 10,
maxDiscoveryDepth: 123,
ignoreSitemap: false,
ignoreQueryParameters: false,
limit: 10000,
allowBackwardLinks: false,
allowExternalLinks: false,
scrapeOptions: {
formats: ['markdown'],
onlyMainContent: true,
includeTags: ['<string>'],
excludeTags: ['<string>'],
headers: {},
waitFor: 0,
mobile: false,
skipTlsVerification: false,
timeout: 30000,
jsonOptions: {schema: {}, systemPrompt: '<string>', prompt: '<string>'},
actions: [{type: 'wait', milliseconds: 2, selector: '#my-element'}],
location: {country: 'US', languages: ['en-US']},
removeBase64Images: true,
blockAds: true
}
})
};
fetch('https://api.firecrawl.dev/v1/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.firecrawl.dev/v1/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'excludePaths' => [
'<string>'
],
'includePaths' => [
'<string>'
],
'maxDepth' => 10,
'maxDiscoveryDepth' => 123,
'ignoreSitemap' => false,
'ignoreQueryParameters' => false,
'limit' => 10000,
'allowBackwardLinks' => false,
'allowExternalLinks' => false,
'scrapeOptions' => [
'formats' => [
'markdown'
],
'onlyMainContent' => true,
'includeTags' => [
'<string>'
],
'excludeTags' => [
'<string>'
],
'headers' => [
],
'waitFor' => 0,
'mobile' => false,
'skipTlsVerification' => false,
'timeout' => 30000,
'jsonOptions' => [
'schema' => [
],
'systemPrompt' => '<string>',
'prompt' => '<string>'
],
'actions' => [
[
'type' => 'wait',
'milliseconds' => 2,
'selector' => '#my-element'
]
],
'location' => [
'country' => 'US',
'languages' => [
'en-US'
]
],
'removeBase64Images' => true,
'blockAds' => true
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.firecrawl.dev/v1/crawl"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"excludePaths\": [\n \"<string>\"\n ],\n \"includePaths\": [\n \"<string>\"\n ],\n \"maxDepth\": 10,\n \"maxDiscoveryDepth\": 123,\n \"ignoreSitemap\": false,\n \"ignoreQueryParameters\": false,\n \"limit\": 10000,\n \"allowBackwardLinks\": false,\n \"allowExternalLinks\": false,\n \"scrapeOptions\": {\n \"formats\": [\n \"markdown\"\n ],\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"headers\": {},\n \"waitFor\": 0,\n \"mobile\": false,\n \"skipTlsVerification\": false,\n \"timeout\": 30000,\n \"jsonOptions\": {\n \"schema\": {},\n \"systemPrompt\": \"<string>\",\n \"prompt\": \"<string>\"\n },\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 2,\n \"selector\": \"#my-element\"\n }\n ],\n \"location\": {\n \"country\": \"US\",\n \"languages\": [\n \"en-US\"\n ]\n },\n \"removeBase64Images\": true,\n \"blockAds\": true\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.firecrawl.dev/v1/crawl")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"excludePaths\": [\n \"<string>\"\n ],\n \"includePaths\": [\n \"<string>\"\n ],\n \"maxDepth\": 10,\n \"maxDiscoveryDepth\": 123,\n \"ignoreSitemap\": false,\n \"ignoreQueryParameters\": false,\n \"limit\": 10000,\n \"allowBackwardLinks\": false,\n \"allowExternalLinks\": false,\n \"scrapeOptions\": {\n \"formats\": [\n \"markdown\"\n ],\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"headers\": {},\n \"waitFor\": 0,\n \"mobile\": false,\n \"skipTlsVerification\": false,\n \"timeout\": 30000,\n \"jsonOptions\": {\n \"schema\": {},\n \"systemPrompt\": \"<string>\",\n \"prompt\": \"<string>\"\n },\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 2,\n \"selector\": \"#my-element\"\n }\n ],\n \"location\": {\n \"country\": \"US\",\n \"languages\": [\n \"en-US\"\n ]\n },\n \"removeBase64Images\": true,\n \"blockAds\": true\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.firecrawl.dev/v1/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"excludePaths\": [\n \"<string>\"\n ],\n \"includePaths\": [\n \"<string>\"\n ],\n \"maxDepth\": 10,\n \"maxDiscoveryDepth\": 123,\n \"ignoreSitemap\": false,\n \"ignoreQueryParameters\": false,\n \"limit\": 10000,\n \"allowBackwardLinks\": false,\n \"allowExternalLinks\": false,\n \"scrapeOptions\": {\n \"formats\": [\n \"markdown\"\n ],\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"headers\": {},\n \"waitFor\": 0,\n \"mobile\": false,\n \"skipTlsVerification\": false,\n \"timeout\": 30000,\n \"jsonOptions\": {\n \"schema\": {},\n \"systemPrompt\": \"<string>\",\n \"prompt\": \"<string>\"\n },\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 2,\n \"selector\": \"#my-element\"\n }\n ],\n \"location\": {\n \"country\": \"US\",\n \"languages\": [\n \"en-US\"\n ]\n },\n \"removeBase64Images\": true,\n \"blockAds\": true\n }\n}"
response = http.request(request)
puts response.read_body{
"success": true,
"id": "<string>",
"url": "<string>"
}{
"error": "Payment required to access this resource."
}{
"error": "Request rate limit exceeded. Please wait and try again later."
}{
"error": "An unexpected error occurred on the server."
}爬取接口
抓取
POST
/
crawl
Crawl multiple URLs based on options
curl --request POST \
--url https://api.firecrawl.dev/v1/crawl \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "<string>",
"excludePaths": [
"<string>"
],
"includePaths": [
"<string>"
],
"maxDepth": 10,
"maxDiscoveryDepth": 123,
"ignoreSitemap": false,
"ignoreQueryParameters": false,
"limit": 10000,
"allowBackwardLinks": false,
"allowExternalLinks": false,
"scrapeOptions": {
"formats": [
"markdown"
],
"onlyMainContent": true,
"includeTags": [
"<string>"
],
"excludeTags": [
"<string>"
],
"headers": {},
"waitFor": 0,
"mobile": false,
"skipTlsVerification": false,
"timeout": 30000,
"jsonOptions": {
"schema": {},
"systemPrompt": "<string>",
"prompt": "<string>"
},
"actions": [
{
"type": "wait",
"milliseconds": 2,
"selector": "#my-element"
}
],
"location": {
"country": "US",
"languages": [
"en-US"
]
},
"removeBase64Images": true,
"blockAds": true
}
}
'import requests
url = "https://api.firecrawl.dev/v1/crawl"
payload = {
"url": "<string>",
"excludePaths": ["<string>"],
"includePaths": ["<string>"],
"maxDepth": 10,
"maxDiscoveryDepth": 123,
"ignoreSitemap": False,
"ignoreQueryParameters": False,
"limit": 10000,
"allowBackwardLinks": False,
"allowExternalLinks": False,
"scrapeOptions": {
"formats": ["markdown"],
"onlyMainContent": True,
"includeTags": ["<string>"],
"excludeTags": ["<string>"],
"headers": {},
"waitFor": 0,
"mobile": False,
"skipTlsVerification": False,
"timeout": 30000,
"jsonOptions": {
"schema": {},
"systemPrompt": "<string>",
"prompt": "<string>"
},
"actions": [
{
"type": "wait",
"milliseconds": 2,
"selector": "#my-element"
}
],
"location": {
"country": "US",
"languages": ["en-US"]
},
"removeBase64Images": True,
"blockAds": True
}
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
excludePaths: ['<string>'],
includePaths: ['<string>'],
maxDepth: 10,
maxDiscoveryDepth: 123,
ignoreSitemap: false,
ignoreQueryParameters: false,
limit: 10000,
allowBackwardLinks: false,
allowExternalLinks: false,
scrapeOptions: {
formats: ['markdown'],
onlyMainContent: true,
includeTags: ['<string>'],
excludeTags: ['<string>'],
headers: {},
waitFor: 0,
mobile: false,
skipTlsVerification: false,
timeout: 30000,
jsonOptions: {schema: {}, systemPrompt: '<string>', prompt: '<string>'},
actions: [{type: 'wait', milliseconds: 2, selector: '#my-element'}],
location: {country: 'US', languages: ['en-US']},
removeBase64Images: true,
blockAds: true
}
})
};
fetch('https://api.firecrawl.dev/v1/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.firecrawl.dev/v1/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'excludePaths' => [
'<string>'
],
'includePaths' => [
'<string>'
],
'maxDepth' => 10,
'maxDiscoveryDepth' => 123,
'ignoreSitemap' => false,
'ignoreQueryParameters' => false,
'limit' => 10000,
'allowBackwardLinks' => false,
'allowExternalLinks' => false,
'scrapeOptions' => [
'formats' => [
'markdown'
],
'onlyMainContent' => true,
'includeTags' => [
'<string>'
],
'excludeTags' => [
'<string>'
],
'headers' => [
],
'waitFor' => 0,
'mobile' => false,
'skipTlsVerification' => false,
'timeout' => 30000,
'jsonOptions' => [
'schema' => [
],
'systemPrompt' => '<string>',
'prompt' => '<string>'
],
'actions' => [
[
'type' => 'wait',
'milliseconds' => 2,
'selector' => '#my-element'
]
],
'location' => [
'country' => 'US',
'languages' => [
'en-US'
]
],
'removeBase64Images' => true,
'blockAds' => true
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.firecrawl.dev/v1/crawl"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"excludePaths\": [\n \"<string>\"\n ],\n \"includePaths\": [\n \"<string>\"\n ],\n \"maxDepth\": 10,\n \"maxDiscoveryDepth\": 123,\n \"ignoreSitemap\": false,\n \"ignoreQueryParameters\": false,\n \"limit\": 10000,\n \"allowBackwardLinks\": false,\n \"allowExternalLinks\": false,\n \"scrapeOptions\": {\n \"formats\": [\n \"markdown\"\n ],\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"headers\": {},\n \"waitFor\": 0,\n \"mobile\": false,\n \"skipTlsVerification\": false,\n \"timeout\": 30000,\n \"jsonOptions\": {\n \"schema\": {},\n \"systemPrompt\": \"<string>\",\n \"prompt\": \"<string>\"\n },\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 2,\n \"selector\": \"#my-element\"\n }\n ],\n \"location\": {\n \"country\": \"US\",\n \"languages\": [\n \"en-US\"\n ]\n },\n \"removeBase64Images\": true,\n \"blockAds\": true\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.firecrawl.dev/v1/crawl")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"excludePaths\": [\n \"<string>\"\n ],\n \"includePaths\": [\n \"<string>\"\n ],\n \"maxDepth\": 10,\n \"maxDiscoveryDepth\": 123,\n \"ignoreSitemap\": false,\n \"ignoreQueryParameters\": false,\n \"limit\": 10000,\n \"allowBackwardLinks\": false,\n \"allowExternalLinks\": false,\n \"scrapeOptions\": {\n \"formats\": [\n \"markdown\"\n ],\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"headers\": {},\n \"waitFor\": 0,\n \"mobile\": false,\n \"skipTlsVerification\": false,\n \"timeout\": 30000,\n \"jsonOptions\": {\n \"schema\": {},\n \"systemPrompt\": \"<string>\",\n \"prompt\": \"<string>\"\n },\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 2,\n \"selector\": \"#my-element\"\n }\n ],\n \"location\": {\n \"country\": \"US\",\n \"languages\": [\n \"en-US\"\n ]\n },\n \"removeBase64Images\": true,\n \"blockAds\": true\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.firecrawl.dev/v1/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"excludePaths\": [\n \"<string>\"\n ],\n \"includePaths\": [\n \"<string>\"\n ],\n \"maxDepth\": 10,\n \"maxDiscoveryDepth\": 123,\n \"ignoreSitemap\": false,\n \"ignoreQueryParameters\": false,\n \"limit\": 10000,\n \"allowBackwardLinks\": false,\n \"allowExternalLinks\": false,\n \"scrapeOptions\": {\n \"formats\": [\n \"markdown\"\n ],\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"headers\": {},\n \"waitFor\": 0,\n \"mobile\": false,\n \"skipTlsVerification\": false,\n \"timeout\": 30000,\n \"jsonOptions\": {\n \"schema\": {},\n \"systemPrompt\": \"<string>\",\n \"prompt\": \"<string>\"\n },\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 2,\n \"selector\": \"#my-element\"\n }\n ],\n \"location\": {\n \"country\": \"US\",\n \"languages\": [\n \"en-US\"\n ]\n },\n \"removeBase64Images\": true,\n \"blockAds\": true\n }\n}"
response = http.request(request)
puts response.read_body{
"success": true,
"id": "<string>",
"url": "<string>"
}{
"error": "Payment required to access this resource."
}{
"error": "Request rate limit exceeded. Please wait and try again later."
}{
"error": "An unexpected error occurred on the server."
}Authorizations
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Body
application/json
The base URL to start crawling from
URL pathname regex patterns that exclude matching URLs from the crawl. For example, if you set "excludePaths": ["blog/.*"] for the base URL firecrawl.dev, any results matching that pattern will be excluded, such as https://www.firecrawl.dev/blog/firecrawl-launch-week-1-recap.
URL pathname regex patterns that include matching URLs in the crawl. Only the paths that match the specified patterns will be included in the response. For example, if you set "includePaths": ["blog/.*"] for the base URL firecrawl.dev, only results matching that pattern will be included, such as https://www.firecrawl.dev/blog/firecrawl-launch-week-1-recap.
Maximum depth to crawl relative to the base URL. Basically, the max number of slashes the pathname of a scraped URL may contain.
Maximum depth to crawl based on discovery order. The root site and sitemapped pages has a discovery depth of 0. For example, if you set it to 1, and you set ignoreSitemap, you will only crawl the entered URL and all URLs that are linked on that page.
Ignore the website sitemap when crawling
Do not re-scrape the same path with different (or none) query parameters
Maximum number of pages to crawl. Default limit is 10000.
Enables the crawler to navigate from a specific URL to previously linked pages.
Allows the crawler to follow links to external websites.
A webhook specification object.
Show child attributes
Show child attributes
Show child attributes
Show child attributes