Webhook Failure Triage Template

Systematic troubleshooting framework for diagnosing and resolving broken webhook integrations in agentic workflow automation systems.

Tier 1: Immediate Checks (0-5 minutes)

# Quick Visual Inspection Checklist
WEBHOOK_FAILURE_TIER_1 = {
    "timestamp": "Check when failure first appeared",
    "scope": "Single webhook vs. all webhooks in system",
    "recent_changes": [
        "Code deployments in last 24h",
        "Configuration changes (API keys, endpoints, secrets)",
        "Network/firewall rule updates",
        "Certificate renewals or expirations",
        "Third-party service changes (webhook URL updates)"
    ],
    "error_patterns": {
        "all_requests_failing": "Systemic issue (network, auth, config)",
        "intermittent_failures": "Rate limiting, timeout, or resource exhaustion",
        "specific_payloads_only": "Data validation or serialization issue",
        "specific_time_windows": "Cron conflicts, maintenance windows"
    }
}

# Critical First Steps
1. Check webhook provider status page (Stripe, HubSpot, Zapier, etc.)
2. Verify endpoint URL accessibility: curl -I 
3. Confirm SSL/TLS certificate validity
4. Check basic authentication credentials
5. Review recent server logs for 5xx errors
6. Verify server is reachable (ping, traceroute)
7. Confirm firewall/security groups allow inbound traffic on webhook port
8. Check DNS resolution for webhook domain

Tier 2: HTTP Layer Diagnosis (5-15 minutes)

# HTTP Request/Response Analysis
def diagnose_http_layer(webhook_request, webhook_response):
    diagnostics = {
        'status_code': webhook_response.status_code,
        'headers': webhook_response.headers,
        'body': webhook_response.body,
        'timing': webhook_response.elapsed_time
    }
    
    # Status Code Interpretation
    status_patterns = {
        400: {
            'meaning': 'Bad Request',
            'likely_causes': [
                'Invalid payload format (JSON malformed)',
                'Missing required fields',
                'Data type mismatches',
                'Payload size exceeds limits'
            ],
            'investigation': [
                'Validate JSON syntax',
                'Check required fields against API documentation',
                'Verify data types (string vs number)',
                'Measure payload size'
            ]
        },
        401: {
            'meaning': 'Unauthorized',
            'likely_causes': [
                'Invalid or expired API key',
                'Incorrect authentication scheme',
                'Missing authentication header'
            ],
            'investigation': [
                'Verify API key/credentials',
                'Check Authorization header format',
                'Test credentials directly with provider API'
            ]
        },
        403: {
            'meaning': 'Forbidden',
            'likely_causes': [
                'IP address not whitelisted',
                'Insufficient permissions/scopes',
                'Account restrictions or suspensions'
            ],
            'investigation': [
                'Check IP whitelist configuration',
                'Verify OAuth scopes/permissions',
                'Review account status with provider'
            ]
        },
        404: {
            'meaning': 'Not Found',
            'likely_causes': [
                'Incorrect webhook endpoint URL',
                'Webhook path removed or changed',
                'Account or resource deleted'
            ],
            'investigation': [
                'Verify URL matches documentation exactly',
                'Check for trailing slashes',
                'Test endpoint manually with GET request'
            ]
        },
        408: {'meaning': 'Request Timeout', 'likely_causes': ['Server processing too slow']},
        429: {
            'meaning': 'Too Many Requests',
            'likely_causes': ['Rate limit exceeded'],
            'investigation': [
                'Check rate limit headers',
                'Review request frequency',
                'Implement exponential backoff'
            ]
        },
        500: {'meaning': 'Internal Server Error', 'likely_causes': ['Provider service error']},
        502: {'meaning': 'Bad Gateway', 'likely_causes': ['Provider infrastructure issue']},
        503: {'meaning': 'Service Unavailable', 'likely_causes': ['Provider maintenance or overload']},
        504: {'meaning': 'Gateway Timeout', 'likely_causes': ['Provider processing timeout']}
    }
    
    return status_patterns.get(diagnostics['status_code'], 
                              {'meaning': 'Unknown', 'likely_causes': []})

# cURL Diagnostic Commands
DIAGNOSTIC_COMMANDS = [
    '# Test basic connectivity',
    'curl -I https://webhook.example.com/endpoint',
    
    '# Test with sample payload',
    'curl -X POST https://webhook.example.com/endpoint \\\\',
    '  -H "Content-Type: application/json" \\\\',
    '  -d '{"test": "payload"}',
    
    '# Include authentication',
    'curl -X POST https://webhook.example.com/endpoint \\\\',
    '  -H "Authorization: Bearer YOUR_TOKEN" \\\\',
    '  -H "Content-Type: application/json" \\\\',
    '  -d @payload.json',
    
    '# Verbose output for debugging',
    'curl -v -X POST https://webhook.example.com/endpoint \\\\',
    '  -H "Content-Type: application/json" \\\\',
    '  -d @payload.json',
    
    '# Follow redirects',
    'curl -L -X POST https://webhook.example.com/endpoint \\\\',
    '  -d @payload.json',
    
    '# Check SSL certificate',
    'curl -v https://webhook.example.com 2>&1 | grep -E "(SSL|certificate|expire)"',
    
    '# Measure response time',
    'curl -o /dev/null -s -w "Time: %{time_total}s\\n" \\\\',
    '  https://webhook.example.com/endpoint'
]

Tier 3: Payload & Serialization Analysis (15-30 minutes)

# Payload Validation Checklist
PAYLOAD_ISSUES = {
    "json_syntax": {
        "checks": [
            'Valid JSON structure (no trailing commas)',
            'Properly escaped special characters',
            'Correct Unicode handling',
            'No circular references'
        ],
        "tools": [
            'python -m json.tool payload.json',
            'jq . payload.json',
            'Online JSONLint validation'
        ]
    },
    
    "data_types": {
        "common_issues": {
            "number_as_string": "API expects 123, sending "123"",
            "boolean_as_string": "API expects true, sending "true"",
            "date_format": "API expects ISO 8601, sending MM/DD/YYYY",
            "null_handling": "API treats null as deletion vs. empty"
        },
        "validation": "Compare against API schema documentation"
    },
    
    "size_limits": {
        "typical_limits": {
            "stripe": "8MB per webhook",
            "github": "10MB per payload",
            "shopify": "64KB for order webhooks"
        },
        "check": "Content-Length header vs. documented limits"
    },
    
    "encoding": {
        "issues": [
            'UTF-8 vs. ISO-8859-1 character encoding',
            'Base64 encoded binary data requirements',
            'Multipart form-data vs. application/json'
        ]
    }
}

# Python Payload Validation Script
import json
import jsonschema
from datetime import datetime

def validate_webhook_payload(payload_file, schema_file):
    # Load payload
    with open(payload_file, 'r') as f:
        try:
            payload = json.load(f)
            print(f"JSON Valid: Yes")
        except json.JSONDecodeError as e:
            print(f"JSON Valid: No - {e}")
            return False
    
    # Load schema
    with open(schema_file, 'r') as f:
        schema = json.load(f)
    
    # Validate against schema
    try:
        jsonschema.validate(payload, schema)
        print(f"Schema Valid: Yes")
    except jsonschema.ValidationError as e:
        print(f"Schema Valid: No - {e.message}")
        print(f"Failed at: {list(e.path)}")
        return False
    
    # Check payload size
    size_bytes = len(json.dumps(payload).encode('utf-8'))
    print(f"Payload Size: {size_bytes} bytes ({size_bytes/1024:.2f} KB)")
    
    # Validate timestamps
    date_fields = find_date_fields(payload)
    for field, value in date_fields:
        try:
            datetime.fromisoformat(value.replace('Z', '+00:00'))
            print(f"Date {field}: Valid ISO 8601")
        except:
            print(f"Date {field}: Invalid format - {value}")
    
    return True

# Compare payloads (before/after breaking change)
def diff_payloads(old_payload, new_payload):
    differences = deep_diff(old_payload, new_payload)
    for diff in differences:
        print(f"Difference: {diff.path()}")
        print(f"  Old: {diff.t1}")
        print(f"  New: {diff.t2}")

Tier 4: Network & Infrastructure (30-60 minutes)

# Network Connectivity Diagnostics
NETWORK_CHECKS = {
    "dns_resolution": {
        "command": "nslookup webhook.example.com",
        "expected": "Returns IP address, no timeout",
        "failure_indicator": "NXDOMAIN, timeout, wrong IP"
    },
    
    "port_connectivity": {
        "commands": [
            "telnet webhook.example.com 443",
            "nc -zv webhook.example.com 443",
            "nmap -p 443 webhook.example.com"
        ],
        "expected": "Connection established",
        "failure_indicator": "Connection refused, timeout"
    },
    
    "tls_handshake": {
        "command": "openssl s_client -connect webhook.example.com:443",
        "checks": [
            "Certificate chain complete",
            "Certificate not expired",
            "CN/SAN matches domain",
            "Supported TLS version (1.2+)"
        ]
    },
    
    "routing": {
        "command": "traceroute webhook.example.com",
        "purpose": "Identify network hops and potential blocks"
    },
    
    "latency": {
        "command": "mtr -rw webhook.example.com",
        "purpose": "Continuous latency and packet loss monitoring"
    }
}

# Firewall Rule Verification
FIREWALL_CHECKS = [
    {
        'check': 'Inbound rule for webhook port',
        'linux': 'sudo iptables -L INPUT -n -v | grep :443',
        'aws': 'Security Group inbound rules for port 443',
        'gcp': 'VPC firewall rules allowing tcp:443'
    },
    {
        'check': 'Outbound rule for internet access',
        'linux': 'sudo iptables -L OUTPUT -n -v | grep :443',
        'purpose': 'Server can reach external APIs'
    },
    {
        'check': 'Proxy configuration',
        'env_vars': 'HTTP_PROXY, HTTPS_PROXY, NO_PROXY',
        'impact': 'Proxy may intercept/modify webhook requests'
    },
    {
        'check': 'Web Application Firewall (WAF)',
        'providers': ['CloudFlare', 'AWS WAF', 'Azure WAF'],
        'checks': [
            'Rate limiting rules not blocking webhooks',
            'Bot protection not flagging webhook requests',
            'Geographic restrictions not blocking source IPs'
        ]
    }
]

# Server Resource Monitoring
SERVER_HEALTH_CHECKS = {
    "cpu": {
        "command": "top -bn1 | grep 'Cpu(s)'",
        "threshold": "> 90% indicates resource exhaustion"
    },
    "memory": {
        "command": "free -h",
        "threshold": "< 10% free memory causing swap/OOM"
    },
    "disk": {
        "command": "df -h",
        "threshold": "> 90% full may prevent log writes"
    },
    "connections": {
        "command": "ss -s",
        "threshold": "TIME_WAIT connections > 10000"
    },
    "process": {
        "command": "ps aux | grep -E '(webhook|web|server|node|python)'",
        "purpose": "Verify webhook handler process is running"
    }
}

# SSL Certificate Check
SSL_VALIDATION = {
    "check_command": """
        echo | openssl s_client -servername webhook.example.com \\
          -connect webhook.example.com:443 2>/dev/null \\
          | openssl x509 -noout -dates
    """,
    "expiry_check": """
        openssl s_client -connect webhook.example.com:443 -servername webhook.example.com \\
          2>/dev/null | openssl x509 -noout -enddate
    """,
    "chain_validation": """
        openssl s_client -connect webhook.example.com:443 -showcerts \\
          2>/dev/null | openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt
    """
}

Tier 5: Application-Level Debugging (1-2 hours)

# Application Log Analysis
LOG_ANALYSIS_PATTERN = {
    "timestamp_correlation": {
        "step": "Match webhook delivery time with application logs",
        "query": "grep '2024-01-15T14:30' /var/log/webhook/app.log",
        "purpose": "Identify exact failure point in request chain"
    },
    
    "error_types": {
        "5xx_errors": {
            "500": "Unhandled exception in webhook handler",
            "502": "Upstream service (database, cache) unavailable",
            "503": "Service overloaded or in maintenance",
            "504": "Upstream timeout (database query too slow)"
        },
        "4xx_errors": {
            "401": "Authentication failure in internal service call",
            "403": "Authorization failure (service account permissions)",
            "404": "Internal API endpoint not found"
        }
    },
    
    "stack_trace_analysis": {
        "common_patterns": [
            "NullPointerException: Check for missing data in payload",
            "TimeoutException: Database or API call taking too long",
            "ConnectionRefused: Internal service not running",
            "JSONParseError: Malformed payload from upstream service",
            "MemoryError: Payload too large or memory leak"
        ]
    }
}

# Database Connection Check
DATABASE_HEALTH = {
    "connection_pool": {
        "check": "Active vs. idle connections",
        "postgres": "SELECT count(*) FROM pg_stat_activity;",
        "mysql": "SHOW STATUS LIKE 'Threads_connected';",
        "redis": "INFO clients | grep connected_clients"
    },
    
    "query_performance": {
        "slow_queries": "Identify queries exceeding 1s execution time",
        "locks": "Check for table/row locks blocking webhook processing",
        "deadlocks": "Review database deadlock logs"
    },
    
    "webhook_transactions": {
        "check": "Verify webhook data is being persisted",
        "sql": "SELECT * FROM webhook_events WHERE created_at > NOW() - INTERVAL '5 minutes' ORDER BY created_at DESC;"
    }
}

# Dependency Service Health
DEPENDENCY_CHECKS = [
    {
        "service": "Database",
        "check": "Can webhook handler query database?",
        "test": "SELECT 1;"
    },
    {
        "service": "Message Queue",
        "check": "Can webhook handler publish events?",
        "test": "Publish test message to queue"
    },
    {
        "service": "Cache (Redis/Memcached)",
        "check": "Can webhook handler read/write cache?",
        "test": "SET test_key test_value && GET test_key"
    },
    {
        "service": "External APIs",
        "check": "Are downstream APIs reachable?",
        "test": "Make authenticated test call to each dependency"
    }
]

# Code-Level Debugging
def debug_webhook_handler(payload):
    """
    Instrument webhook handler to identify failure point
    """
    import traceback
    import logging
    
    try:
        logger.info(f"Webhook received: {payload['id']}")
        
        # Step 1: Validate
        logger.info("Step 1: Validating payload...")
        validate_payload(payload)
        
        # Step 2: Transform
        logger.info("Step 2: Transforming data...")
        transformed = transform_payload(payload)
        
        # Step 3: Enrich
        logger.info("Step 3: Enriching with external data...")
        enriched = enrich_data(transformed)
        
        # Step 4: Persist
        logger.info("Step 4: Saving to database...")
        save_to_database(enriched)
        
        # Step 5: Notify
        logger.info("Step 5: Sending notifications...")
        send_notifications(enriched)
        
        logger.info("Webhook processed successfully")
        return {"status": "success"}
        
    except ValidationError as e:
        logger.error(f"Validation failed: {e}")
        logger.error(traceback.format_exc())
        return {"status": "error", "stage": "validation", "error": str(e)}
        
    except EnrichmentError as e:
        logger.error(f"Enrichment failed: {e}")
        logger.error(traceback.format_exc())
        return {"status": "error", "stage": "enrichment", "error": str(e)}
        
    except DatabaseError as e:
        logger.error(f"Database error: {e}")
        logger.error(traceback.format_exc())
        return {"status": "error", "stage": "persistence", "error": str(e)}
        
    except Exception as e:
        logger.error(f"Unexpected error: {e}")
        logger.error(traceback.format_exc())
        return {"status": "error", "stage": "unknown", "error": str(e)}

Tier 6: Advanced Diagnostics (2+ hours)

# Distributed Tracing
DISTRIBUTED_TRACING_SETUP = {
    "tools": ["Jaeger", "Zipkin", "AWS X-Ray", "Datadog APM"],
    "implementation": {
        "1_instrument": "Add tracing library to webhook handler",
        "2_spans": "Create span for each operation (validate, enrich, persist)",
        "3_baggage": "Propagate trace ID through all service calls",
        "4_export": "Send traces to centralized collector",
        "5_analyze": "Use trace visualization to identify bottlenecks"
    },
    "key_metrics": [
        "Total webhook processing time",
        "Time spent in each dependency",
        "Database query duration",
        "External API latency",
        "Queue wait time"
    ]
}

# Load Testing Webhook Endpoint
LOAD_TEST_SCENARIO = {
    "tool": "k6 / Locust / Artillery",
    "test_parameters": {
        "concurrent_users": [10, 50, 100, 200],
        "ramp_up": "10s",
        "duration": "1m",
        "payload": "Realistic webhook payload"
    },
    "success_criteria": {
        "p95_latency": "< 500ms",
        "p99_latency": "< 1000ms",
        "error_rate": "< 0.1%",
        "throughput": "> 100 req/s"
    },
    "breakpoint_analysis": "Increase load until errors appear, identify bottleneck"
}

# Memory Profiling
MEMORY_PROFILING = {
    "python": {
        "tool": "memory_profiler, tracemalloc",
        "check": "Memory usage during webhook processing",
        "issue_indicators": [
            "Memory grows with each request (memory leak)",
            "Large payloads cause OOM errors",
            "Caches not being cleared"
        ]
    },
    "nodejs": {
        "tool": "node --inspect, clinic.js",
        "check": "Heap usage and garbage collection",
        "issue_indicators": [
            "Frequent GC pauses",
            "Heap growing without bound",
            "Detached DOM elements (if applicable)"
        ]
    }
}

# Thread/Process Deadlock Analysis
DEADLOCK_DIAGNOSIS = {
    "symptoms": [
        "Webhook requests hanging indefinitely",
        "CPU at 0% but requests not completing",
        "Database connections stuck in 'Waiting' state"
    ],
    "investigation": {
        "python": "threading.enumerate(), deadlock detection libraries",
        "java": "jstack  - analyze thread dumps",
        "nodejs": "node --inspect - take heap snapshot",
        "database": "Check for lock waits and blocking queries"
    },
    "common_causes": [
        "Circular dependencies in async code",
        "Database transaction deadlocks",
        "Shared resource contention without proper locking",
        "Async/await misuse causing race conditions"
    ]
}