Crawl

pip install firecrawl-py

npm install @mendable/firecrawl-js

go get github.com/mendableai/firecrawl-go

# add the following to your Cargo.toml

[dependencies]
firecrawl = "^0.1"
tokio = { version = "^1", features = ["full"] }
serde = { version = "^1.0", features = ["derive"] }
serde_json = "^1.0"
uuid = { version = "^1.10", features = ["v4"] }

[build-dependencies]
tokio = { version = "1", features = ["full"] }

from firecrawl import FirecrawlApp

app = FirecrawlApp(api_key="YOUR_API_KEY")

crawl_result = app.crawl_url('mendable.ai', {'crawlerOptions': {'excludes': ['blog/*']}})

# Get the markdown
for result in crawl_result:
    print(result['markdown'])

import FirecrawlApp from '@mendable/firecrawl-js';

// Initialize the FirecrawlApp with your API key
const app = new FirecrawlApp({ apiKey: 'YOUR_API_KEY' });

// Crawl a website
const crawlResult = await app.crawlUrl('mendable.ai', { crawlerOptions: { excludes: ['blog/*'] } });

// Log the markdown
console.log(crawlResult.map(result => result.markdown));

import (
  "fmt"
  "log"

  "github.com/mendableai/firecrawl-go"
)

func main() {
  app, err := firecrawl.NewFirecrawlApp("YOUR_API_KEY")
  if err != nil {
    log.Fatalf("Failed to initialize FirecrawlApp: %v", err)
  }

  crawlResult, err := app.CrawlURL("mendable.ai", nil)
  if err != nil {
    log.Fatalf("Failed to crawl URL: %v", err)
  }

  fmt.Println(crawlResult)
}

use firecrawl::FirecrawlApp;

#[tokio::main]
async fn main() {
  // Initialize the FirecrawlApp with the API key
  let api_key = "YOUR_API_KEY";
  let api_url = "https://api.firecrawl.dev";
  let app = FirecrawlApp::new(api_key, api_url).expect("Failed to initialize FirecrawlApp");

  // Crawl the URL
  let crawl_params = json!({
    "crawlerOptions": {
        "excludes": ["blog/*"]
    }
  });

  let crawl_result = app
      .crawl_url("https://example.com", Some(crawl_params), true, 2, None)
      .await;

  // Print the crawl result
  match crawl_result {
      Ok(data) => println!("Crawl Result:
{}", data),
      Err(e) => eprintln!("Crawl failed: {}", e),
  }
}

curl -X POST https://api.firecrawl.dev/v0/crawl \
    -H 'Content-Type: application/json' \
    -H 'Authorization: Bearer YOUR_API_KEY' \
    -d '{
      "url": "https://mendable.ai"
    }'

status = app.check_crawl_status(job_id)

const status = await app.checkCrawlStatus(jobId);

status, err := app.CheckCrawlStatus(jobId)
if err != nil {
  log.Fatalf("Failed to check crawl status: %v", err)
}

let status = match app.check_crawl_status(jobId).await {
    Ok(status) => status,
    Err(e) => panic!("Failed to check crawl status: {:?}", e),
};

println!("Crawl Status: {:?}", status);

curl -X GET https://api.firecrawl.dev/v0/crawl/status/1234-5678-9101 \
  -H 'Content-Type: application/json' \
  -H 'Authorization: Bearer YOUR_API_KEY'