Bright Data Debug Bundle

Overview

Collect all diagnostic information needed for Bright Data support tickets: proxy connectivity, zone status, response headers, and error logs.

Prerequisites

• Bright Data zone credentials configured
• curl available
• Permission to collect environment info

Instructions

Step 1: Create Debug Bundle Script

#!/bin/bash
# brightdata-debug-bundle.sh
set -euo pipefail

BUNDLE_DIR="brightdata-debug-$(date +%Y%m%d-%H%M%S)"
mkdir -p "$BUNDLE_DIR"

echo "=== Bright Data Debug Bundle ===" | tee "$BUNDLE_DIR/summary.txt"
echo "Generated: $(date -u)" | tee -a "$BUNDLE_DIR/summary.txt"
echo "" >> "$BUNDLE_DIR/summary.txt"

Step 2: Collect Environment and Connectivity

# Runtime versions
echo "--- Runtime ---" >> "$BUNDLE_DIR/summary.txt"
node --version >> "$BUNDLE_DIR/summary.txt" 2>&1 || echo "Node.js: not found" >> "$BUNDLE_DIR/summary.txt"
python3 --version >> "$BUNDLE_DIR/summary.txt" 2>&1 || echo "Python: not found" >> "$BUNDLE_DIR/summary.txt"

# Credential check (presence only, never log values)
echo "--- Credentials ---" >> "$BUNDLE_DIR/summary.txt"
echo "BRIGHTDATA_CUSTOMER_ID: ${BRIGHTDATA_CUSTOMER_ID:+[SET]}" >> "$BUNDLE_DIR/summary.txt"
echo "BRIGHTDATA_ZONE: ${BRIGHTDATA_ZONE:-[NOT SET]}" >> "$BUNDLE_DIR/summary.txt"
echo "BRIGHTDATA_ZONE_PASSWORD: ${BRIGHTDATA_ZONE_PASSWORD:+[SET]}" >> "$BUNDLE_DIR/summary.txt"
echo "BRIGHTDATA_API_TOKEN: ${BRIGHTDATA_API_TOKEN:+[SET]}" >> "$BUNDLE_DIR/summary.txt"

# SSL cert check
echo "--- SSL Certificate ---" >> "$BUNDLE_DIR/summary.txt"
if [ -f "./brd-ca.crt" ]; then
  openssl x509 -in ./brd-ca.crt -noout -subject -enddate >> "$BUNDLE_DIR/summary.txt" 2>&1
else
  echo "brd-ca.crt: NOT FOUND" >> "$BUNDLE_DIR/summary.txt"
fi

Step 3: Test Proxy Connectivity with Verbose Headers

# Proxy connectivity test with full response headers
echo "--- Proxy Test ---" >> "$BUNDLE_DIR/summary.txt"
PROXY_USER="brd-customer-${BRIGHTDATA_CUSTOMER_ID}-zone-${BRIGHTDATA_ZONE}"
curl -x "http://${PROXY_USER}:${BRIGHTDATA_ZONE_PASSWORD}@brd.superproxy.io:33335" \
  -s -D "$BUNDLE_DIR/proxy-headers.txt" \
  -o "$BUNDLE_DIR/proxy-response.txt" \
  -w "HTTP %{http_code} in %{time_total}s\n" \
  https://lumtest.com/myip.json 2>> "$BUNDLE_DIR/summary.txt" || echo "Proxy FAILED" 

Bright Data Deploy Integration

Overview

Deploy Bright Data scraping applications to cloud platforms with proper secrets management. Key consideration: Bright Data proxy connections require outbound TCP to brd.superproxy.io on ports 33335 (proxy) and 9222 (Scraping Browser).

Prerequisites

• Bright Data production zone credentials
• Platform CLI installed (vercel, fly, or gcloud)
• Application tested in staging

Instructions

Step 1: Vercel Deployment (Serverless)

# Add secrets
vercel env add BRIGHTDATA_CUSTOMER_ID production
vercel env add BRIGHTDATA_ZONE production
vercel env add BRIGHTDATA_ZONE_PASSWORD production
vercel env add BRIGHTDATA_API_TOKEN production

// vercel.json
{
  "functions": {
    "api/scrape.ts": {
      "maxDuration": 60
    }
  }
}

// api/scrape.ts — Vercel serverless function
import type { VercelRequest, VercelResponse } from '@vercel/node';
import axios from 'axios';
import https from 'https';

export default async function handler(req: VercelRequest, res: VercelResponse) {
  const { url } = req.body;
  if (!url) return res.status(400).json({ error: 'url required' });

  const proxy = {
    host: 'brd.superproxy.io',
    port: 33335,
    auth: {
      username: `brd-customer-${process.env.BRIGHTDATA_CUSTOMER_ID}-zone-${process.env.BRIGHTDATA_ZONE}`,
      password: process.env.BRIGHTDATA_ZONE_PASSWORD!,
    },
  };

  try {
    const response = await axios.get(url, {
      proxy,
      httpsAgent: new https.Agent({ rejectUnauthorized: false }),
      timeout: 55000, // Leave 5s buffer for Vercel's 60s limit
    });
    res.json({ status: response.status, length: response.data.length });
  } catch (error: any) {
    res.status(502).json({ error: error.message });
  }
}

Step 2: Fly.io Deployment (Long-Running)

# fly.toml — better for Scraping Browser (needs WebSocket)
app = "my-scraper"
primary_region = "iad"

[env]
  NODE_ENV = "production"

[http_service]
  internal_port = 3000
  force_https = true
  auto_stop_machines = true
  auto_start_machines = true
  min_machines_running = 0

# Set secrets
fly secrets set BRIGHTDATA_CUSTOMER_ID=c_abc123
fly secrets set BRIGHTDATA_ZONE=web_unlocker_prod
fly secrets set BRIGHTDATA_ZONE_PASSWORD=z_prod_pass
fly secrets set BRIGHTDATA_API_TOKEN=prod_token

# Deploy
fly deploy

Step 3: Google Cloud Run

# Store secrets in Secret Manager
echo -n "c_abc123" | gcloud secrets create brightdata-customer-id --data-file=-
echo -n "web_unlocker_prod" | gcloud secrets create brightdata-zone --data-file=-
echo -n "z_pro

Bright Data Hello World

Overview

Scrape a real webpage through Bright Data's Web Unlocker proxy. Web Unlocker handles CAPTCHAs, fingerprinting, and retries automatically — you send a normal HTTP request through the proxy endpoint at brd.superproxy.io:33335.

Prerequisites

• Completed brightdata-install-auth setup
• Web Unlocker zone active in Bright Data control panel
• brd-ca.crt SSL certificate downloaded

Instructions

Step 1: Scrape via Web Unlocker Proxy (Node.js)

// hello-brightdata.ts
import axios from 'axios';
import https from 'https';
import 'dotenv/config';

const { BRIGHTDATA_CUSTOMER_ID, BRIGHTDATA_ZONE, BRIGHTDATA_ZONE_PASSWORD } = process.env;

const proxy = {
  host: 'brd.superproxy.io',
  port: 33335,
  auth: {
    username: `brd-customer-${BRIGHTDATA_CUSTOMER_ID}-zone-${BRIGHTDATA_ZONE}`,
    password: BRIGHTDATA_ZONE_PASSWORD!,
  },
};

async function scrape(url: string) {
  const response = await axios.get(url, {
    proxy,
    httpsAgent: new https.Agent({ rejectUnauthorized: false }),
    timeout: 60000,
  });
  console.log(`Status: ${response.status}`);
  console.log(`Content length: ${response.data.length} chars`);
  console.log(response.data.substring(0, 500));
  return response.data;
}

scrape('https://example.com').catch(console.error);

Step 2: Scrape via REST API

// hello-brightdata-api.ts
import 'dotenv/config';

async function scrapeViaAPI(url: string) {
  const response = await fetch('https://api.brightdata.com/request', {
    method: 'POST',
    headers: {
      'Authorization': `Bearer ${process.env.BRIGHTDATA_API_TOKEN}`,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify({
      zone: process.env.BRIGHTDATA_ZONE,
      url,
      format: 'raw',
    }),
  });
  const html = await response.text();
  console.log(`Status: ${response.status}, Length: ${html.length}`);
  return html;
}

scrapeViaAPI('https://example.com').catch(console.error);

Step 3: Python Version

# hello_brightdata.py
import os, requests
from dotenv import load_dotenv

load_dotenv()
proxy_url = (
    f"http://brd-customer-{os.environ['BRIGHTDATA_CUSTOMER_ID']}"
    f"-zone-{os.environ['BRIGHTDATA_ZONE']}"
    f":{os.environ['BRIGHTDATA_ZONE_PASSWORD']}"
    f"@brd.superproxy.io:33335"
)
response = requests.get(
    'https://example.com',
    proxies={'http': proxy_url, 'https': proxy_url},
    verify='./brd-ca.crt',
    timeout=60,
)
print(f"Status: {response.status_code}, Length: {len(response.text)}")

Geo-Targeting

Add country or

Bright Data Local Dev Loop

Overview

Set up a fast, reproducible local development workflow for Bright Data scraping projects with mocked proxy responses, cached results, and vitest integration.

Prerequisites

• Completed brightdata-install-auth setup
• Node.js 18+ with npm/pnpm
• brd-ca.crt SSL certificate downloaded

Instructions

Step 1: Create Project Structure

my-scraper/
├── src/
│   ├── brightdata/
│   │   ├── proxy.ts         # Proxy configuration helper
│   │   ├── scraper.ts       # Scraping functions
│   │   └── cache.ts         # Response caching for dev
│   └── index.ts
├── tests/
│   ├── fixtures/            # Cached HTML responses
│   │   └── example.html
│   └── scraper.test.ts
├── .env.local               # Local credentials (git-ignored)
├── .env.example             # Template for team
├── brd-ca.crt               # Bright Data SSL cert (git-ignored)
└── package.json

Step 2: Build Proxy Configuration Module

// src/brightdata/proxy.ts
import 'dotenv/config';

export interface BrightDataProxy {
  host: string;
  port: number;
  auth: { username: string; password: string };
}

export function getProxy(options?: {
  country?: string;
  city?: string;
  session?: string;
}): BrightDataProxy {
  const { BRIGHTDATA_CUSTOMER_ID, BRIGHTDATA_ZONE, BRIGHTDATA_ZONE_PASSWORD } = process.env;
  if (!BRIGHTDATA_CUSTOMER_ID || !BRIGHTDATA_ZONE || !BRIGHTDATA_ZONE_PASSWORD) {
    throw new Error('Missing BRIGHTDATA_* environment variables');
  }

  let username = `brd-customer-${BRIGHTDATA_CUSTOMER_ID}-zone-${BRIGHTDATA_ZONE}`;
  if (options?.country) username += `-country-${options.country}`;
  if (options?.city) username += `-city-${options.city}`;
  if (options?.session) username += `-session-${options.session}`;

  return {
    host: 'brd.superproxy.io',
    port: 33335,
    auth: { username, password: BRIGHTDATA_ZONE_PASSWORD },
  };
}

Step 3: Add Response Cache for Development

// src/brightdata/cache.ts — cache scraped pages to avoid burning proxy credits
import { createHash } from 'crypto';
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
import { join } from 'path';

const CACHE_DIR = join(process.cwd(), '.scrape-cache');

export function getCachedResponse(url: string): string | null {
  const key = createHash('md5').update(url).digest('hex');
  const path = join(CACHE_DIR, `${key}.html`);
  return existsSync(path) ? readFileSync(path, 'utf-8') : null;
}

export function setCachedResponse(url: string, html: string): void {
  mkdirSync(CACHE_DIR, { recursive: true });
  const key = createHash('md5').update(url).digest('hex');
  writeFileSync(join(CACHE_DIR, `${key}.html`), html);
}

Step

Bright Data Production Checklist

Overview

Complete checklist for deploying Bright Data scraping integrations to production with zone verification, monitoring, and rollback procedures.

Prerequisites

• Staging environment tested
• Production zone credentials in secrets vault
• Monitoring and alerting configured

Instructions

Step 1: Zone and Credential Verification

• [ ] Production zone active in Bright Data CP
• [ ] Zone password stored in secrets vault (not .env)
• [ ] API token scoped to production zone only
• [ ] SSL certificate (brd-ca.crt) deployed
• [ ] Separate zone from development/staging

# Verify production zone is active
curl -s -H "Authorization: Bearer ${BRIGHTDATA_API_TOKEN}" \
  https://api.brightdata.com/zone/get_active_zones \
  | python3 -c "import sys,json; zones=json.load(sys.stdin); print([z['name'] for z in zones])"

# Test production proxy connectivity
curl -x "http://brd-customer-${BRIGHTDATA_CUSTOMER_ID}-zone-${BRIGHTDATA_ZONE}:${BRIGHTDATA_ZONE_PASSWORD}@brd.superproxy.io:33335" \
  -s -w "HTTP %{http_code} in %{time_total}s\n" \
  https://lumtest.com/myip.json

Step 2: Code Quality

• [ ] No hardcoded credentials (grep for passwords, tokens)
• [ ] Retry logic with exponential backoff (see brightdata-rate-limits)
• [ ] Request queuing with concurrency limits (p-queue)
• [ ] Response validation (check for CAPTCHA pages, empty responses)
• [ ] Timeout set to 60-120s for Web Unlocker
• [ ] Error logging includes X-Luminati-Error headers

Step 3: Infrastructure

• [ ] Health check endpoint tests proxy connectivity
• [ ] Monitoring tracks proxy response times, error rates
• [ ] Budget alerts configured in Bright Data CP
• [ ] Circuit breaker for proxy failures

// Health check endpoint
export async function healthCheck() {
  const start = Date.now();
  try {
    const client = getBrightDataClient();
    const res = await client.get('https://lumtest.com/myip.json');
    return {
      status: 'healthy',
      proxy_ip: res.data.ip,
      latency_ms: Date.now() - start,
    };
  } catch (error: any) {
    return {
      status: 'degraded',
      error: error.response?.headers?.['x-luminati-error'] || error.message,
      latency_ms: Date.now() - start,
    };
  }
}

Step 4: Monitoring and Alerts

// src/brightdata/limiter.ts
import PQueue from 'p-queue';

// Match concurrency to your Bright Data plan limits
const scrapeQueue = new PQueue({
  concurrency: 10,       // Max concurrent proxy requests
  interval: 1000,        // Per second
  intervalCap: 20,       // Max 20 requests per second
  timeout: 120000,       // Kill after 2 min
  throwOnTimeout: true,
});

export async function queuedScrape(url: string): Promise<string> {
  return scrapeQueue.add(async () => {
    const client = getBrightDataClient();
    const response = await client.get(url);
    return response.data;
  });
}

// Monitor queue health
scrapeQueue.on('active', () => {
  console.log(`Queue: ${scrapeQueue.size} waiting, ${scrapeQueue.pending} active`);
});

// src/brightdata/backoff.ts
export async function scrapeWithBackoff(
  url: string,
  config = { maxRetries: 5, baseDelay: 2000, maxDelay: 60000 }
): Promise<string> {
  for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
    try {
      const client = getBrightDataClient();
      const response = await client.get(url);
      return response.data;
    } catch (error: any) {
      const status = error.response?.status;
      const luminatiError = error.response?.headers?.['x-luminati-error'];

      // Only retry on transient errors
      const retryable = [502, 503, 429].includes(status)
        || error.code === 'ETIMEDOUT'
        || luminatiEr

my-scraper/
├── src/
│   ├── brightdata/
│   │   ├── proxy.ts            # Proxy config helper (zone, country, session)
│   │   ├── client.ts           # Axios client with proxy + retry
│   │   ├── browser.ts          # Scraping Browser connection manager
│   │   ├── api.ts              # REST API client (trigger, snapshot)
│   │   ├── cache.ts            # Response cache (LRU + optional Redis)
│   │   └── types.ts            # Shared TypeScript interfaces
│   ├── scrapers/
│   │   ├── product-scraper.ts  # Domain-specific scraper
│   │   ├── serp-scraper.ts     # Search result collector
│   │   └── parser.ts           # HTML → structured data (cheerio)
│   ├── pipeline/
│   │   ├── scheduler.ts        # Cron-based scraping scheduler
│   │   ├── processor.ts        # Raw HTML → clean data
│   │   └── storage.ts          # Database/file output
│   ├── webhooks/
│   │   └── brightdata.ts       # Webhook delivery handler
│   └── api/
│       ├── health.ts           # Health check endpoint
│       └── scrape.ts           # On-demand scrape endpoint
├── tests/
│   ├── unit/                   # Mocked tests (no proxy needed)
│   ├── integration/            # Live proxy tests
│   └── fixtures/               # Cached HTML for testing
├── config/
│   ├── zones.json              # Zone configuration per environment
│   └── targets.json            # Target URLs and scraping schedules
└── .env.example

┌──────────────────────────────────────────────────────┐
│                    API / Scheduler                     │
│         (On-demand scrape, cron jobs, webhooks)        │
├──────────────────────────────────────────────────────┤
│                   Scraper Layer                        │
│    (Product scraper, SERP scraper, custom parsers)     │
├────────────┬─────────────────┬───────────────────────┤
│ Web        │  Scraping       │  SERP / Datasets      │
│ Unlocker   │  Browser        │  API                  │
│ (Proxy)    │  (WebSocket)    │  (REST)               │
├────────────┴─────────────────┴───────────────────────┤
│          Bright Data Infrastructure Layer              │
│  (Proxy config, retry, cache, session management)      │
├──────────────────────────────────────────────────────┤
│              Storage / Pipeline                        │
│    (Database, file output, webhook delivery)           │
└──────────────────────────────────────────────────────┘

// src/brightdata/client.ts
import axios, { AxiosInstance, AxiosRequestConfig } from 'axios';
import https from 'https';
import 'dotenv/config';

let instance: AxiosInstance | null = null;

export function getBrightDataClient(options?: {
  country?: string;
  session?: string;
  zone?: string;
}): AxiosInstance {
  const { BRIGHTDATA_CUSTOMER_ID, BRIGHTDATA_ZONE, BRIGHTDATA_ZONE_PASSWORD } = process.env;
  const zone = options?.zone || BRIGHTDATA_ZONE!;

  let username = `brd-customer-${BRIGHTDATA_CUSTOMER_ID}-zone-${zone}`;
  if (options?.country) username += `-country-${options.country}`;
  if (options?.session) username += `-session-${options.session}`;

  if (!instance || options) {
    instance = axios.create({
      proxy: {
        host: 'brd.superproxy.io',
        port: 33335,
        auth: { username, password: BRIGHTDATA_ZONE_PASSWORD! },
      },
      httpsAgent: new https.Agent({ rejectUnauthorized: false }),
      timeout: 60000,
      headers: { 'User-Agent': 'Mozilla/5.0 (compatible; Scraper/1.0)' },
    });
  }
  return instance;
}

// src/brightdata/retry.ts
export async function scrapeWithRetry<T>(
  url: string,
  parser: (html: string) => T,
  config = { maxRetries: 3, baseDelayMs: 2000, maxDelayMs: 30000 }
): Promise<T> {
  const client = getBrightDataClient();

  for (let attempt = 0; attempt <= config.maxRetries; attempt++) {
    try {
      const response = await client.get(url);
      return parser(response.data);
    } catch (error: any) {
      const status = error.response?.status;
      // Bright Data proxy errors that warrant retry
      const retryable = [502, 503, 407, 429].includes(status) || error.code === 'ETIMEDOUT';

      if (attempt === config.maxRetries || !retryable) throw error;

      const delay = Math.min(
        config.baseDelayMs * Math.pow(2, attempt) + Math.random() * 1000,
        config.maxDelayMs
      );
      console.log(`Attempt ${attempt + 1} failed (${status || error.code}), retrying in ${delay.toFixed(0)}ms`);
      await new Promise(r => setTimeout(r, delay));
    }
  }
  throw new Error('Unreachable');
}

// src

# .env (NEVER commit)
BRIGHTDATA_CUSTOMER_ID=c_abc123
BRIGHTDATA_ZONE=web_unlocker1
BRIGHTDATA_ZONE_PASSWORD=z_pass_xyz
BRIGHTDATA_API_TOKEN=abc123def456

# .gitignore
.env
.env.local
.env.*.local

# .env.example (safe to commit — no real values)
BRIGHTDATA_CUSTOMER_ID=
BRIGHTDATA_ZONE=
BRIGHTDATA_ZONE_PASSWORD=
BRIGHTDATA_API_TOKEN=

// config/brightdata.ts
const ZONE_MAP = {
  development: 'web_unlocker_dev',
  staging: 'web_unlocker_staging',
  production: 'web_unlocker_prod',
} as const;

export function getZone(): string {
  const env = process.env.NODE_ENV || 'development';
  return process.env.BRIGHTDATA_ZONE || ZONE_MAP[env] || ZONE_MAP.development;
}

# 1. Create new API token in Bright Data CP > Settings > API tokens
# 2. Update secrets in your deployment platform
# Vercel
vercel env rm BRIGHTDATA_API_TOKEN production
vercel env add BRIGHTDATA_API_TOKEN production

# AWS
aws secretsmanager update-secret --secret-id brightdata/api-token --secret-string "new_token"

# 3. Test new credentials
curl -H "Authorization: Bearer ${NEW_TOKEN}" \
  https://api.brightdata.com/zone/get_active_zones

# 4. Revoke old token in Bright Data CP

# Pre-commit hook to catch leaked credentials
# .git/hooks/pre-commit
#!/bin/bash
if git diff --cached | grep -iE '(BRIGHTDATA_ZONE_PASSWORD|BRIGHTDATA_API_TOKEN)=.{5,}'; then
  echo "ERROR: Bright Data credentials detected in staged changes"
  exit 1
fi

// BEFORE: Raw residential proxy (manual CAPTCHA handling)
const oldProxy = {
  host: 'brd.superproxy.io',
  port: 22225,  // Old residential port
  auth: {
    username: `brd-customer-${CID}-zone-residential_zone`,
    password: OLD_PASSWORD,
  },
};

// AFTER: Web Unlocker (automatic CAPTCHA, fingerprinting)
const newProxy = {
  host: 'brd.superproxy.io',
  port: 33335,  // Web Unlocker port
  auth: {
    username: `brd-customer-${CID}-zone-web_unlocker1`,
    password: NEW_PASSWORD,
  },
};
// Changes: port 22225 → 33335, zone name, password
// Web Unlocker handles CAPTCHAs automatically — remove manual solving code

// BEFORE: Self-hosted Puppeteer with proxy
import puppeteer from 'puppeteer';
const browser = await puppeteer.launch({
  args: [`--proxy-server=http://brd.superproxy.io:22225`],
});

// AFTER: Bright Data Scraping Browser (managed browser)
import puppeteer from 'puppeteer-core';
const AUTH = `brd-customer-${CID}-zone-scraping_browser1:${PASSWORD}`;
const browser = await puppeteer.connect({
  browserWSEndpoint: `wss://${AUTH}@brd.superproxy.io:9222`,
});
// Changes: launch → connect, local browser → remote WebSocket
// Remove: browser install, proxy args, CAPTCHA solving libraries

// BEFORE: Datasets API v2
const v2Response = await fetch(
  `https://api.brightdata.com/dca/trigger?collector=${collectorId}`,
  { method: 'POST', headers: { 'Authorization': `Bearer ${TOKEN}` }, body: JSON.stringify(input) }
);

// AFTER: Datasets API v3 (current)
const v3Response = await fetch

// trigger-with-webhook.ts
const API_TOKEN = process.env.BRIGHTDATA_API_TOKEN!;

async function triggerWithWebhook(datasetId: string, urls: string[]) {
  const params = new URLSearchParams({
    dataset_id: datasetId,
    format: 'json',
    endpoint: 'https://your-app.com/webhooks/brightdata', // Your webhook URL
    uncompressed_webhook: 'true', // Send uncompressed for easier handling
    auth_header: `Bearer ${process.env.BRIGHTDATA_WEBHOOK_SECRET}`, // Auth header sent with delivery
  });

  // Optional: notification URL (lightweight ping when done)
  params.set('notify', 'https://your-app.com/webhooks/brightdata-notify');

  const response = await fetch(
    `https://api.brightdata.com/datasets/v3/trigger?${params}`,
    {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${API_TOKEN}`,
        'Content-Type': 'application/json',
      },
      body: JSON.stringify(urls.map(url => ({ url }))),
    }
  );

  const result = await response.json();
  console.log('Snapshot ID:', result.snapshot_id);
  return result;
}

// api/webhooks/brightdata.ts
import express from 'express';

const app = express();

// Bright Data sends collected data as JSON array
app.post('/webhooks/brightdata',
  express.json({ limit: '50mb' }), // Collections can be large
  async (req, res) => {
    // Validate Authorization header
    const authHeader = req.headers.authorization;
    if (authHeader !== `Bearer ${process.env.BRIGHTDATA_WEBHOOK_SECRET}`) {
      console.error('Invalid webhook authorization');
      return res.status(401).json({ error: 'Unauthorized' });
    }

    const records = req.body; // Array of scraped records
    console.log(`Received ${records.length} records`);

    // Process records
    for (const record of records) {
      console.log(`URL: ${record.url}`);
      console.log(`Title: ${record.title}`);
      console.log(`Data: ${JSON.stringify(record).substring(0, 200)}`);
    }

    // Store results
    await saveToDatabase(records);

    // Return 200 quickly — Bright Data retries on non-2xx
    res.status(200).json({ received: records.length });
  }
);