Scrape a single URL and optionally extract information using an LLM
curl --request POST \
--url https://api.firecrawl.dev/v1/scrape \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "<string>",
"formats": [
"markdown"
],
"onlyMainContent": true,
"includeTags": [
"<string>"
],
"excludeTags": [
"<string>"
],
"headers": {},
"waitFor": 0,
"mobile": false,
"skipTlsVerification": false,
"timeout": 30000,
"jsonOptions": {
"schema": {},
"systemPrompt": "<string>",
"prompt": "<string>"
},
"actions": [
{
"type": "wait",
"milliseconds": 2,
"selector": "#my-element"
}
],
"location": {
"country": "US",
"languages": [
"en-US"
]
},
"removeBase64Images": true,
"blockAds": true
}
'import requests
url = "https://api.firecrawl.dev/v1/scrape"
payload = {
"url": "<string>",
"formats": ["markdown"],
"onlyMainContent": True,
"includeTags": ["<string>"],
"excludeTags": ["<string>"],
"headers": {},
"waitFor": 0,
"mobile": False,
"skipTlsVerification": False,
"timeout": 30000,
"jsonOptions": {
"schema": {},
"systemPrompt": "<string>",
"prompt": "<string>"
},
"actions": [
{
"type": "wait",
"milliseconds": 2,
"selector": "#my-element"
}
],
"location": {
"country": "US",
"languages": ["en-US"]
},
"removeBase64Images": True,
"blockAds": True
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
formats: ['markdown'],
onlyMainContent: true,
includeTags: ['<string>'],
excludeTags: ['<string>'],
headers: {},
waitFor: 0,
mobile: false,
skipTlsVerification: false,
timeout: 30000,
jsonOptions: {schema: {}, systemPrompt: '<string>', prompt: '<string>'},
actions: [{type: 'wait', milliseconds: 2, selector: '#my-element'}],
location: {country: 'US', languages: ['en-US']},
removeBase64Images: true,
blockAds: true
})
};
fetch('https://api.firecrawl.dev/v1/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.firecrawl.dev/v1/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'formats' => [
'markdown'
],
'onlyMainContent' => true,
'includeTags' => [
'<string>'
],
'excludeTags' => [
'<string>'
],
'headers' => [
],
'waitFor' => 0,
'mobile' => false,
'skipTlsVerification' => false,
'timeout' => 30000,
'jsonOptions' => [
'schema' => [
],
'systemPrompt' => '<string>',
'prompt' => '<string>'
],
'actions' => [
[
'type' => 'wait',
'milliseconds' => 2,
'selector' => '#my-element'
]
],
'location' => [
'country' => 'US',
'languages' => [
'en-US'
]
],
'removeBase64Images' => true,
'blockAds' => true
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.firecrawl.dev/v1/scrape"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"formats\": [\n \"markdown\"\n ],\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"headers\": {},\n \"waitFor\": 0,\n \"mobile\": false,\n \"skipTlsVerification\": false,\n \"timeout\": 30000,\n \"jsonOptions\": {\n \"schema\": {},\n \"systemPrompt\": \"<string>\",\n \"prompt\": \"<string>\"\n },\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 2,\n \"selector\": \"#my-element\"\n }\n ],\n \"location\": {\n \"country\": \"US\",\n \"languages\": [\n \"en-US\"\n ]\n },\n \"removeBase64Images\": true,\n \"blockAds\": true\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.firecrawl.dev/v1/scrape")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"formats\": [\n \"markdown\"\n ],\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"headers\": {},\n \"waitFor\": 0,\n \"mobile\": false,\n \"skipTlsVerification\": false,\n \"timeout\": 30000,\n \"jsonOptions\": {\n \"schema\": {},\n \"systemPrompt\": \"<string>\",\n \"prompt\": \"<string>\"\n },\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 2,\n \"selector\": \"#my-element\"\n }\n ],\n \"location\": {\n \"country\": \"US\",\n \"languages\": [\n \"en-US\"\n ]\n },\n \"removeBase64Images\": true,\n \"blockAds\": true\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.firecrawl.dev/v1/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"formats\": [\n \"markdown\"\n ],\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"headers\": {},\n \"waitFor\": 0,\n \"mobile\": false,\n \"skipTlsVerification\": false,\n \"timeout\": 30000,\n \"jsonOptions\": {\n \"schema\": {},\n \"systemPrompt\": \"<string>\",\n \"prompt\": \"<string>\"\n },\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 2,\n \"selector\": \"#my-element\"\n }\n ],\n \"location\": {\n \"country\": \"US\",\n \"languages\": [\n \"en-US\"\n ]\n },\n \"removeBase64Images\": true,\n \"blockAds\": true\n}"
response = http.request(request)
puts response.read_body{
"success": true,
"data": {
"markdown": "<string>",
"html": "<string>",
"rawHtml": "<string>",
"screenshot": "<string>",
"links": [
"<string>"
],
"actions": {
"screenshots": [
"<string>"
]
},
"metadata": {
"title": "<string>",
"description": "<string>",
"language": "<string>",
"sourceURL": "<string>",
"<any other metadata> ": "<string>",
"statusCode": 123,
"error": "<string>"
},
"llm_extraction": {},
"warning": "<string>"
}
}{
"error": "Payment required to access this resource."
}{
"error": "Request rate limit exceeded. Please wait and try again later."
}{
"error": "An unexpected error occurred on the server."
}抓取接口
抓取
POST
/
scrape
Scrape a single URL and optionally extract information using an LLM
curl --request POST \
--url https://api.firecrawl.dev/v1/scrape \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "<string>",
"formats": [
"markdown"
],
"onlyMainContent": true,
"includeTags": [
"<string>"
],
"excludeTags": [
"<string>"
],
"headers": {},
"waitFor": 0,
"mobile": false,
"skipTlsVerification": false,
"timeout": 30000,
"jsonOptions": {
"schema": {},
"systemPrompt": "<string>",
"prompt": "<string>"
},
"actions": [
{
"type": "wait",
"milliseconds": 2,
"selector": "#my-element"
}
],
"location": {
"country": "US",
"languages": [
"en-US"
]
},
"removeBase64Images": true,
"blockAds": true
}
'import requests
url = "https://api.firecrawl.dev/v1/scrape"
payload = {
"url": "<string>",
"formats": ["markdown"],
"onlyMainContent": True,
"includeTags": ["<string>"],
"excludeTags": ["<string>"],
"headers": {},
"waitFor": 0,
"mobile": False,
"skipTlsVerification": False,
"timeout": 30000,
"jsonOptions": {
"schema": {},
"systemPrompt": "<string>",
"prompt": "<string>"
},
"actions": [
{
"type": "wait",
"milliseconds": 2,
"selector": "#my-element"
}
],
"location": {
"country": "US",
"languages": ["en-US"]
},
"removeBase64Images": True,
"blockAds": True
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
formats: ['markdown'],
onlyMainContent: true,
includeTags: ['<string>'],
excludeTags: ['<string>'],
headers: {},
waitFor: 0,
mobile: false,
skipTlsVerification: false,
timeout: 30000,
jsonOptions: {schema: {}, systemPrompt: '<string>', prompt: '<string>'},
actions: [{type: 'wait', milliseconds: 2, selector: '#my-element'}],
location: {country: 'US', languages: ['en-US']},
removeBase64Images: true,
blockAds: true
})
};
fetch('https://api.firecrawl.dev/v1/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.firecrawl.dev/v1/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'formats' => [
'markdown'
],
'onlyMainContent' => true,
'includeTags' => [
'<string>'
],
'excludeTags' => [
'<string>'
],
'headers' => [
],
'waitFor' => 0,
'mobile' => false,
'skipTlsVerification' => false,
'timeout' => 30000,
'jsonOptions' => [
'schema' => [
],
'systemPrompt' => '<string>',
'prompt' => '<string>'
],
'actions' => [
[
'type' => 'wait',
'milliseconds' => 2,
'selector' => '#my-element'
]
],
'location' => [
'country' => 'US',
'languages' => [
'en-US'
]
],
'removeBase64Images' => true,
'blockAds' => true
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.firecrawl.dev/v1/scrape"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"formats\": [\n \"markdown\"\n ],\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"headers\": {},\n \"waitFor\": 0,\n \"mobile\": false,\n \"skipTlsVerification\": false,\n \"timeout\": 30000,\n \"jsonOptions\": {\n \"schema\": {},\n \"systemPrompt\": \"<string>\",\n \"prompt\": \"<string>\"\n },\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 2,\n \"selector\": \"#my-element\"\n }\n ],\n \"location\": {\n \"country\": \"US\",\n \"languages\": [\n \"en-US\"\n ]\n },\n \"removeBase64Images\": true,\n \"blockAds\": true\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.firecrawl.dev/v1/scrape")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"formats\": [\n \"markdown\"\n ],\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"headers\": {},\n \"waitFor\": 0,\n \"mobile\": false,\n \"skipTlsVerification\": false,\n \"timeout\": 30000,\n \"jsonOptions\": {\n \"schema\": {},\n \"systemPrompt\": \"<string>\",\n \"prompt\": \"<string>\"\n },\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 2,\n \"selector\": \"#my-element\"\n }\n ],\n \"location\": {\n \"country\": \"US\",\n \"languages\": [\n \"en-US\"\n ]\n },\n \"removeBase64Images\": true,\n \"blockAds\": true\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.firecrawl.dev/v1/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"formats\": [\n \"markdown\"\n ],\n \"onlyMainContent\": true,\n \"includeTags\": [\n \"<string>\"\n ],\n \"excludeTags\": [\n \"<string>\"\n ],\n \"headers\": {},\n \"waitFor\": 0,\n \"mobile\": false,\n \"skipTlsVerification\": false,\n \"timeout\": 30000,\n \"jsonOptions\": {\n \"schema\": {},\n \"systemPrompt\": \"<string>\",\n \"prompt\": \"<string>\"\n },\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 2,\n \"selector\": \"#my-element\"\n }\n ],\n \"location\": {\n \"country\": \"US\",\n \"languages\": [\n \"en-US\"\n ]\n },\n \"removeBase64Images\": true,\n \"blockAds\": true\n}"
response = http.request(request)
puts response.read_body{
"success": true,
"data": {
"markdown": "<string>",
"html": "<string>",
"rawHtml": "<string>",
"screenshot": "<string>",
"links": [
"<string>"
],
"actions": {
"screenshots": [
"<string>"
]
},
"metadata": {
"title": "<string>",
"description": "<string>",
"language": "<string>",
"sourceURL": "<string>",
"<any other metadata> ": "<string>",
"statusCode": 123,
"error": "<string>"
},
"llm_extraction": {},
"warning": "<string>"
}
}{
"error": "Payment required to access this resource."
}{
"error": "Request rate limit exceeded. Please wait and try again later."
}{
"error": "An unexpected error occurred on the server."
}Authorizations
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Body
application/json
The URL to scrape
Formats to include in the output.
Available options:
markdown, html, rawHtml, links, screenshot, screenshot@fullPage, json Only return the main content of the page excluding headers, navs, footers, etc.
Tags to include in the output.
Tags to exclude from the output.
Headers to send with the request. Can be used to send cookies, user-agent, etc.
Specify a delay in milliseconds before fetching the content, allowing the page sufficient time to load.
Set to true if you want to emulate scraping from a mobile device. Useful for testing responsive pages and taking mobile screenshots.
Skip TLS certificate verification when making requests
Timeout in milliseconds for the request
Extract object
Show child attributes
Show child attributes
actions
(Wait · object | Screenshot · object | Click · object | Write text · object | Press a key · object | Scroll · object | Scrape · object | Execute JavaScript · object)[]
Actions to perform on the page before grabbing the content
- Wait
- Screenshot
- Click
- Write text
- Press a key
- Scroll
- Scrape
- Execute JavaScript
Show child attributes
Show child attributes
Location settings for the request. When specified, this will use an appropriate proxy if available and emulate the corresponding language and timezone settings. Defaults to 'US' if not specified.
Show child attributes
Show child attributes
Removes all base 64 images from the output, which may be overwhelmingly long. The image's alt text remains in the output, but the URL is replaced with a placeholder.
Enables ad-blocking and cookie popup blocking.
Specifies the type of proxy to use.
- basic: Proxies for scraping sites with none to basic anti-bot solutions. Fast and usually works.
- stealth: Stealth proxies for scraping sites with advanced anti-bot solutions. Slower, but more reliable on certain sites.
If you do not specify a proxy, Firecrawl will automatically attempt to determine which one you need based on the target site.
Available options:
basic, stealth ⌘I