Scrape a single URL and optionally extract information using an LLM
curl --request POST \
--url https://api.firecrawl.dev/v0/scrape \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "<string>",
"pageOptions": {
"headers": {},
"includeHtml": false,
"includeRawHtml": false,
"onlyIncludeTags": [
"<string>"
],
"onlyMainContent": false,
"removeTags": [
"<string>"
],
"replaceAllPathsWithAbsolutePaths": false,
"screenshot": false,
"fullPageScreenshot": false,
"waitFor": 0
},
"extractorOptions": {},
"timeout": 30000
}
'import requests
url = "https://api.firecrawl.dev/v0/scrape"
payload = {
"url": "<string>",
"pageOptions": {
"headers": {},
"includeHtml": False,
"includeRawHtml": False,
"onlyIncludeTags": ["<string>"],
"onlyMainContent": False,
"removeTags": ["<string>"],
"replaceAllPathsWithAbsolutePaths": False,
"screenshot": False,
"fullPageScreenshot": False,
"waitFor": 0
},
"extractorOptions": {},
"timeout": 30000
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
pageOptions: {
headers: {},
includeHtml: false,
includeRawHtml: false,
onlyIncludeTags: ['<string>'],
onlyMainContent: false,
removeTags: ['<string>'],
replaceAllPathsWithAbsolutePaths: false,
screenshot: false,
fullPageScreenshot: false,
waitFor: 0
},
extractorOptions: {},
timeout: 30000
})
};
fetch('https://api.firecrawl.dev/v0/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.firecrawl.dev/v0/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'pageOptions' => [
'headers' => [
],
'includeHtml' => false,
'includeRawHtml' => false,
'onlyIncludeTags' => [
'<string>'
],
'onlyMainContent' => false,
'removeTags' => [
'<string>'
],
'replaceAllPathsWithAbsolutePaths' => false,
'screenshot' => false,
'fullPageScreenshot' => false,
'waitFor' => 0
],
'extractorOptions' => [
],
'timeout' => 30000
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.firecrawl.dev/v0/scrape"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"pageOptions\": {\n \"headers\": {},\n \"includeHtml\": false,\n \"includeRawHtml\": false,\n \"onlyIncludeTags\": [\n \"<string>\"\n ],\n \"onlyMainContent\": false,\n \"removeTags\": [\n \"<string>\"\n ],\n \"replaceAllPathsWithAbsolutePaths\": false,\n \"screenshot\": false,\n \"fullPageScreenshot\": false,\n \"waitFor\": 0\n },\n \"extractorOptions\": {},\n \"timeout\": 30000\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.firecrawl.dev/v0/scrape")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"pageOptions\": {\n \"headers\": {},\n \"includeHtml\": false,\n \"includeRawHtml\": false,\n \"onlyIncludeTags\": [\n \"<string>\"\n ],\n \"onlyMainContent\": false,\n \"removeTags\": [\n \"<string>\"\n ],\n \"replaceAllPathsWithAbsolutePaths\": false,\n \"screenshot\": false,\n \"fullPageScreenshot\": false,\n \"waitFor\": 0\n },\n \"extractorOptions\": {},\n \"timeout\": 30000\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.firecrawl.dev/v0/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"pageOptions\": {\n \"headers\": {},\n \"includeHtml\": false,\n \"includeRawHtml\": false,\n \"onlyIncludeTags\": [\n \"<string>\"\n ],\n \"onlyMainContent\": false,\n \"removeTags\": [\n \"<string>\"\n ],\n \"replaceAllPathsWithAbsolutePaths\": false,\n \"screenshot\": false,\n \"fullPageScreenshot\": false,\n \"waitFor\": 0\n },\n \"extractorOptions\": {},\n \"timeout\": 30000\n}"
response = http.request(request)
puts response.read_body{
"success": true,
"data": {
"markdown": "<string>",
"content": "<string>",
"html": "<string>",
"rawHtml": "<string>",
"metadata": {
"title": "<string>",
"description": "<string>",
"language": "<string>",
"sourceURL": "<string>",
"<any other metadata> ": "<string>",
"pageStatusCode": 123,
"pageError": "<string>"
},
"llm_extraction": {},
"warning": "<string>"
}
}{
"error": "Payment required to access this resource."
}{
"error": "Request rate limit exceeded. Please wait and try again later."
}{
"error": "An unexpected error occurred on the server."
}接口
抓取
POST
/
scrape
Scrape a single URL and optionally extract information using an LLM
curl --request POST \
--url https://api.firecrawl.dev/v0/scrape \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "<string>",
"pageOptions": {
"headers": {},
"includeHtml": false,
"includeRawHtml": false,
"onlyIncludeTags": [
"<string>"
],
"onlyMainContent": false,
"removeTags": [
"<string>"
],
"replaceAllPathsWithAbsolutePaths": false,
"screenshot": false,
"fullPageScreenshot": false,
"waitFor": 0
},
"extractorOptions": {},
"timeout": 30000
}
'import requests
url = "https://api.firecrawl.dev/v0/scrape"
payload = {
"url": "<string>",
"pageOptions": {
"headers": {},
"includeHtml": False,
"includeRawHtml": False,
"onlyIncludeTags": ["<string>"],
"onlyMainContent": False,
"removeTags": ["<string>"],
"replaceAllPathsWithAbsolutePaths": False,
"screenshot": False,
"fullPageScreenshot": False,
"waitFor": 0
},
"extractorOptions": {},
"timeout": 30000
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url: '<string>',
pageOptions: {
headers: {},
includeHtml: false,
includeRawHtml: false,
onlyIncludeTags: ['<string>'],
onlyMainContent: false,
removeTags: ['<string>'],
replaceAllPathsWithAbsolutePaths: false,
screenshot: false,
fullPageScreenshot: false,
waitFor: 0
},
extractorOptions: {},
timeout: 30000
})
};
fetch('https://api.firecrawl.dev/v0/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.firecrawl.dev/v0/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => '<string>',
'pageOptions' => [
'headers' => [
],
'includeHtml' => false,
'includeRawHtml' => false,
'onlyIncludeTags' => [
'<string>'
],
'onlyMainContent' => false,
'removeTags' => [
'<string>'
],
'replaceAllPathsWithAbsolutePaths' => false,
'screenshot' => false,
'fullPageScreenshot' => false,
'waitFor' => 0
],
'extractorOptions' => [
],
'timeout' => 30000
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.firecrawl.dev/v0/scrape"
payload := strings.NewReader("{\n \"url\": \"<string>\",\n \"pageOptions\": {\n \"headers\": {},\n \"includeHtml\": false,\n \"includeRawHtml\": false,\n \"onlyIncludeTags\": [\n \"<string>\"\n ],\n \"onlyMainContent\": false,\n \"removeTags\": [\n \"<string>\"\n ],\n \"replaceAllPathsWithAbsolutePaths\": false,\n \"screenshot\": false,\n \"fullPageScreenshot\": false,\n \"waitFor\": 0\n },\n \"extractorOptions\": {},\n \"timeout\": 30000\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.firecrawl.dev/v0/scrape")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"<string>\",\n \"pageOptions\": {\n \"headers\": {},\n \"includeHtml\": false,\n \"includeRawHtml\": false,\n \"onlyIncludeTags\": [\n \"<string>\"\n ],\n \"onlyMainContent\": false,\n \"removeTags\": [\n \"<string>\"\n ],\n \"replaceAllPathsWithAbsolutePaths\": false,\n \"screenshot\": false,\n \"fullPageScreenshot\": false,\n \"waitFor\": 0\n },\n \"extractorOptions\": {},\n \"timeout\": 30000\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.firecrawl.dev/v0/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"<string>\",\n \"pageOptions\": {\n \"headers\": {},\n \"includeHtml\": false,\n \"includeRawHtml\": false,\n \"onlyIncludeTags\": [\n \"<string>\"\n ],\n \"onlyMainContent\": false,\n \"removeTags\": [\n \"<string>\"\n ],\n \"replaceAllPathsWithAbsolutePaths\": false,\n \"screenshot\": false,\n \"fullPageScreenshot\": false,\n \"waitFor\": 0\n },\n \"extractorOptions\": {},\n \"timeout\": 30000\n}"
response = http.request(request)
puts response.read_body{
"success": true,
"data": {
"markdown": "<string>",
"content": "<string>",
"html": "<string>",
"rawHtml": "<string>",
"metadata": {
"title": "<string>",
"description": "<string>",
"language": "<string>",
"sourceURL": "<string>",
"<any other metadata> ": "<string>",
"pageStatusCode": 123,
"pageError": "<string>"
},
"llm_extraction": {},
"warning": "<string>"
}
}{
"error": "Payment required to access this resource."
}{
"error": "Request rate limit exceeded. Please wait and try again later."
}{
"error": "An unexpected error occurred on the server."
}Authorizations
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Body
application/json
The URL to scrape
Show child attributes
Show child attributes
Options for extraction of structured information from the page content. Note: LLM-based extraction is not performed by default and only occurs when explicitly configured. The 'markdown' mode simply returns the scraped markdown and is the default mode for scraping.
Show child attributes
Show child attributes
Timeout in milliseconds for the request
⌘I