Performance Metrics

Access general performance metrics for the Scrapest platform through the metrics API endpoint.

Metrics Endpoint

HTTP Request

GET /metrics
Authorization: Bearer YOUR_API_KEY
Content-Type: application/json

Response Format

{
  "window_hours": 24,
  "count": {
    "source": 15420,
    "internal": 15420
  },
  "source_latency_ms": {
    "p50": 150,
    "p95": 450,
    "p99": 1200
  },
  "internal_latency_ms": {
    "p50": 25,
    "p95": 75,
    "p99": 150
  },
  "timestamp": "2024-01-15T10:30:00Z"
}

Response Fields

Time Window

window_hours: Number of hours included in the metrics calculation (always 24)
timestamp: When the metrics were calculated

Request Counts

count.source: Number of source API requests in the time window
count.internal: Number of internal processing operations

Latency Percentiles

source_latency_ms: Source API latency in milliseconds
- p50: Median latency (50th percentile)
- p95: 95th percentile latency
- p99: 99th percentile latency
internal_latency_ms: Internal processing latency in milliseconds
- p50: Median internal processing time
- p95: 95th percentile internal processing time
- p99: 99th percentile internal processing time

Implementation Examples

JavaScript (Node.js)

const axios = require("axios");

async function getPerformanceMetrics(apiKey) {
  try {
    const response = await axios.get("https://scrape.st/metrics", {
      headers: {
        "x-api-key": apiKey,
        "Content-Type": "application/json",
      },
    });

    const metrics = response.data;
    console.log("Performance Metrics:", metrics);

    // Analyze performance
    analyzePerformance(metrics);

    return metrics;
  } catch (error) {
    console.error("Failed to fetch metrics:", error.response?.data || error.message);
    throw error;
  }
}

function analyzePerformance(metrics) {
  const sourceLatency = metrics.source_latency_ms;
  const internalLatency = metrics.internal_latency_ms;

  console.log("Performance Analysis:");
  console.log(`Source Latency - P50: ${sourceLatency.p50}ms, P95: ${sourceLatency.p95}ms, P99: ${sourceLatency.p99}ms`);
  console.log(
    `Internal Latency - P50: ${internalLatency.p50}ms, P95: ${internalLatency.p95}ms, P99: ${internalLatency.p99}ms`,
  );
  console.log(`Total Requests: ${metrics.count.source} in ${metrics.window_hours} hours`);

  // Performance alerts
  if (sourceLatency.p95 > 1000) {
    console.warn("⚠️ High source latency detected (P95 > 1000ms)");
  }

  if (internalLatency.p95 > 200) {
    console.warn("⚠️ High internal latency detected (P95 > 200ms)");
  }
}

// Usage
getPerformanceMetrics("your_api_key_here");

Python

import requests
import json
from datetime import datetime

def get_performance_metrics(api_key):
    url = "https://scrape.st/metrics"
    headers = {
        "Authorization": f"Bearer {api_key}",
        "Content-Type": "application/json"
    }

    try:
        response = requests.get(url, headers=headers)
        response.raise_for_status()
        metrics = response.json()

        print("Performance Metrics:")
        print(json.dumps(metrics, indent=2))

        # Analyze performance
        analyze_performance(metrics)

        return metrics
    except requests.exceptions.RequestException as error:
        print(f"Failed to fetch metrics: {error}")
        raise

def analyze_performance(metrics):
    source_latency = metrics["source_latency_ms"]
    internal_latency = metrics["internal_latency_ms"]

    print("\nPerformance Analysis:")
    print(f"Source Latency - P50: {source_latency['p50']}ms, P95: {source_latency['p95']}ms, P99: {source_latency['p99']}ms")
    print(f"Internal Latency - P50: {internal_latency['p50']}ms, P95: {internal_latency['p95']}ms, P99: {internal_latency['p99']}ms")
    print(f"Total Requests: {metrics['count']['source']} in {metrics['window_hours']} hours")

    # Performance alerts
    if source_latency['p95'] > 1000:
        print("⚠️ High source latency detected (P95 > 1000ms)")

    if internal_latency['p95'] > 200:
        print("⚠️ High internal latency detected (P95 > 200ms)")

# Usage
metrics = get_performance_metrics("your_api_key_here")

cURL

curl -X GET https://scrape.st/metrics \
  -H "x-api-key: YOUR_API_KEY" \
  -H "Content-Type: application/json"

Performance Monitoring

Setting Up Monitoring

class PerformanceMonitor {
  constructor(apiKey, checkInterval = 300000) {
    // 5 minutes
    this.apiKey = apiKey;
    this.checkInterval = checkInterval;
    this.alertThresholds = {
      sourceLatencyP95: 1000,
      internalLatencyP95: 200,
      errorRate: 0.05,
    };
  }

  start() {
    this.monitorInterval = setInterval(() => {
      this.checkPerformance();
    }, this.checkInterval);

    console.log("Performance monitoring started");
  }

  stop() {
    if (this.monitorInterval) {
      clearInterval(this.monitorInterval);
      console.log("Performance monitoring stopped");
    }
  }

  async checkPerformance() {
    try {
      const metrics = await getPerformanceMetrics(this.apiKey);
      this.evaluateAlerts(metrics);
    } catch (error) {
      console.error("Performance check failed:", error);
    }
  }

  evaluateAlerts(metrics) {
    const sourceLatency = metrics.source_latency_ms;
    const internalLatency = metrics.internal_latency_ms;

    // Check source latency
    if (sourceLatency.p95 > this.alertThresholds.sourceLatencyP95) {
      this.sendAlert("High source latency", {
        current: sourceLatency.p95,
        threshold: this.alertThresholds.sourceLatencyP95,
      });
    }

    // Check internal latency
    if (internalLatency.p95 > this.alertThresholds.internalLatencyP95) {
      this.sendAlert("High internal latency", {
        current: internalLatency.p95,
        threshold: this.alertThresholds.internalLatencyP95,
      });
    }
  }

  sendAlert(type, details) {
    console.warn(`🚨 Alert: ${type}`, details);
    // Implement your alert notification logic here
    // Could send email, Slack message, etc.
  }
}

// Usage
const monitor = new PerformanceMonitor("your_api_key_here");
monitor.start();

Data Visualization

Creating Dashboards

// Example using Chart.js for visualization
function createMetricsChart(metrics) {
  const ctx = document.getElementById("metricsChart").getContext("2d");

  new Chart(ctx, {
    type: "bar",
    data: {
      labels: ["P50", "P95", "P99"],
      datasets: [
        {
          label: "Source Latency (ms)",
          data: [metrics.source_latency_ms.p50, metrics.source_latency_ms.p95, metrics.source_latency_ms.p99],
          backgroundColor: "rgba(54, 162, 235, 0.6)",
        },
        {
          label: "Internal Latency (ms)",
          data: [metrics.internal_latency_ms.p50, metrics.internal_latency_ms.p95, metrics.internal_latency_ms.p99],
          backgroundColor: "rgba(255, 99, 132, 0.6)",
        },
      ],
    },
    options: {
      responsive: true,
      scales: {
        y: {
          beginAtZero: true,
          title: {
            display: true,
            text: "Latency (ms)",
          },
        },
      },
    },
  });
}

Historical Analysis

Tracking Trends

class MetricsHistory {
  constructor() {
    this.history = [];
    this.maxHistorySize = 100; // Keep last 100 data points
  }

  addMetrics(metrics) {
    this.history.push({
      timestamp: new Date(),
      metrics: metrics,
    });

    // Trim history if needed
    if (this.history.length > this.maxHistorySize) {
      this.history.shift();
    }
  }

  getTrend(metricPath, hours = 24) {
    const cutoffTime = new Date(Date.now() - hours * 60 * 60 * 1000);
    const recentData = this.history.filter((entry) => entry.timestamp > cutoffTime);

    return recentData.map((entry) => ({
      timestamp: entry.timestamp,
      value: this.getNestedValue(entry.metrics, metricPath),
    }));
  }

  getNestedValue(obj, path) {
    return path.split(".").reduce((current, key) => current?.[key], obj);
  }

  analyzeTrend(metricPath) {
    const trend = this.getTrend(metricPath);
    if (trend.length < 2) return "insufficient_data";

    const firstValue = trend[0].value;
    const lastValue = trend[trend.length - 1].value;
    const change = ((lastValue - firstValue) / firstValue) * 100;

    if (change > 10) return "improving";
    if (change < -10) return "degrading";
    return "stable";
  }
}

// Usage
const history = new MetricsHistory();
history.addMetrics(metrics);

const sourceLatencyTrend = history.analyzeTrend("source_latency_ms.p95");
console.log("Source latency trend:", sourceLatencyTrend);

Error Handling

Common Error Responses

Authentication Error

{
  "error": "Invalid API key",
  "code": 401
}

Rate Limit Error

{
  "error": "Rate limit exceeded",
  "code": 429,
  "retryAfter": 60
}

Service Unavailable

{
  "error": "Metrics service temporarily unavailable",
  "code": 503
}

Error Handling Implementation

async function getMetricsWithRetry(apiKey, maxRetries = 3) {
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      return await getPerformanceMetrics(apiKey);
    } catch (error) {
      if (error.response?.status === 429) {
        const retryAfter = error.response.data?.retryAfter || 60;
        console.log(`Rate limited, retrying after ${retryAfter} seconds...`);
        await new Promise((resolve) => setTimeout(resolve, retryAfter * 1000));
        continue;
      }

      if (error.response?.status === 401) {
        throw new Error("Invalid API key");
      }

      if (attempt === maxRetries) {
        throw error;
      }

      // Exponential backoff for other errors
      const delay = Math.pow(2, attempt) * 1000;
      console.log(`Attempt ${attempt} failed, retrying in ${delay}ms...`);
      await new Promise((resolve) => setTimeout(resolve, delay));
    }
  }
}

Best Practices

For Monitoring

Regular Checks: Monitor metrics every 5-15 minutes
Alert Thresholds: Set appropriate alert thresholds for your use case
Historical Analysis: Track trends over time to identify patterns
Baseline Establishment: Establish performance baselines for comparison

For Performance Optimization

Focus on P95: P95 latency represents most user experiences
Monitor Trends: Watch for gradual performance degradation
Correlate Events: Correlate performance issues with system events
Proactive Monitoring: Set up alerts before issues become critical

Next: Source-Specific Metrics

​Performance Metrics

​Metrics Endpoint

​HTTP Request

​Response Format

​Response Fields

​Time Window

​Request Counts

​Latency Percentiles

​Implementation Examples

​JavaScript (Node.js)

​Python

​cURL

​Performance Monitoring

​Setting Up Monitoring

​Data Visualization

​Creating Dashboards

​Historical Analysis

​Tracking Trends

​Error Handling

​Common Error Responses

​Authentication Error

​Rate Limit Error

​Service Unavailable

​Error Handling Implementation

​Best Practices

​For Monitoring

​For Performance Optimization

Performance Metrics

Metrics Endpoint

HTTP Request

Response Format

Response Fields

Time Window

Request Counts

Latency Percentiles

Implementation Examples

JavaScript (Node.js)

Python

cURL

Performance Monitoring

Setting Up Monitoring

Data Visualization

Creating Dashboards

Historical Analysis

Tracking Trends

Error Handling

Common Error Responses

Authentication Error

Rate Limit Error

Service Unavailable

Error Handling Implementation

Best Practices

For Monitoring

For Performance Optimization