📊 Advanced Monitoring & Observability with TuskLang & PHP

Introduction

Monitoring and observability are the eyes and ears of production systems. TuskLang and PHP let you implement sophisticated monitoring with config-driven metrics collection, distributed tracing, log aggregation, and intelligent alerting that provides complete visibility into your applications.

Key Features

- Application performance monitoring - Infrastructure monitoring - Distributed tracing - Log aggregation and analysis - Intelligent alerting - Real-time dashboards

Example: Monitoring Configuration

[monitoring]
application: @go("monitoring.ConfigureApplication")
infrastructure: @go("monitoring.ConfigureInfrastructure")
tracing: @go("monitoring.ConfigureTracing")
logging: @go("monitoring.ConfigureLogging")
alerting: @go("monitoring.ConfigureAlerting")

PHP: Application Monitoring

<?php
namespace App\Monitoring;
use TuskLang\Config;
use TuskLang\Operators\Env;
use TuskLang\Operators\Metrics;
use TuskLang\Operators\Go;
class ApplicationMonitor
{
    private $config;
    private $metrics;
    private $tracer;
    private $logger;
    
    public function __construct()
    {
        $this->config = new Config();
        $this->metrics = new MetricsCollector();
        $this->tracer = new DistributedTracer();
        $this->logger = new LogAggregator();
    }
    
    public function monitorRequest($request, $response, $duration)
    {
        $startTime = microtime(true);
        
        try {
            // Start trace
            $traceId = $this->tracer->startTrace($request);
            
            // Collect request metrics
            $this->collectRequestMetrics($request, $response, $duration);
            
            // Monitor performance
            $this->monitorPerformance($request, $duration);
            
            // Check for errors
            $this->checkForErrors($response);
            
            // End trace
            $this->tracer->endTrace($traceId, $response);
            
            $monitoringDuration = (microtime(true) - $startTime) * 1000;
            
            // Record monitoring overhead
            Metrics::record("monitoring_overhead", $monitoringDuration);
            
        } catch (\Exception $e) {
            $this->logger->error("Monitoring failed", [
                'error' => $e->getMessage(),
                'request' => $request->getUri()
            ]);
        }
    }
    
    public function monitorDatabase($query, $duration, $result = null)
    {
        // Record query metrics
        Metrics::record("database_query_duration", $duration, [
            'query' => $this->normalizeQuery($query),
            'result_count' => $result ? count($result) : 0
        ]);
        
        // Check for slow queries
        $slowThreshold = $this->config->get('monitoring.database.slow_query_threshold', 100);
        if ($duration > $slowThreshold) {
            $this->logger->warning("Slow database query detected", [
                'query' => $query,
                'duration' => $duration,
                'threshold' => $slowThreshold
            ]);
        }
        
        // Monitor connection pool
        $this->monitorConnectionPool();
    }
    
    public function monitorCache($operation, $key, $duration, $hit = null)
    {
        // Record cache metrics
        Metrics::record("cache_operation_duration", $duration, [
            'operation' => $operation,
            'hit' => $hit
        ]);
        
        // Monitor cache hit rate
        if ($hit !== null) {
            Metrics::record("cache_hit_rate", $hit ? 1 : 0, [
                'operation' => $operation
            ]);
        }
        
        // Check for cache misses
        if ($hit === false) {
            $this->logger->info("Cache miss", [
                'operation' => $operation,
                'key' => $key
            ]);
        }
    }
    
    public function monitorMemory()
    {
        $memoryUsage = memory_get_usage(true);
        $peakMemory = memory_get_peak_usage(true);
        $memoryLimit = $this->getMemoryLimit();
        
        // Record memory metrics
        Metrics::record("memory_usage", $memoryUsage);
        Metrics::record("peak_memory", $peakMemory);
        Metrics::record("memory_limit", $memoryLimit);
        
        // Check for memory issues
        $usagePercentage = ($memoryUsage / $memoryLimit) * 100;
        $warningThreshold = $this->config->get('monitoring.memory.warning_threshold', 80);
        
        if ($usagePercentage > $warningThreshold) {
            $this->logger->warning("High memory usage detected", [
                'usage' => $memoryUsage,
                'limit' => $memoryLimit,
                'percentage' => $usagePercentage
            ]);
        }
    }
    
    private function collectRequestMetrics($request, $response, $duration)
    {
        // Record request metrics
        Metrics::record("request_duration", $duration, [
            'method' => $request->getMethod(),
            'endpoint' => $request->getPathInfo(),
            'status_code' => $response->getStatusCode()
        ]);
        
        // Record request count
        Metrics::record("request_count", 1, [
            'method' => $request->getMethod(),
            'endpoint' => $request->getPathInfo()
        ]);
        
        // Record response size
        $responseSize = strlen($response->getContent());
        Metrics::record("response_size", $responseSize, [
            'endpoint' => $request->getPathInfo()
        ]);
    }
    
    private function monitorPerformance($request, $duration)
    {
        $performanceThresholds = $this->config->get('monitoring.performance.thresholds', []);
        
        foreach ($performanceThresholds as $endpoint => $threshold) {
            if (strpos($request->getPathInfo(), $endpoint) !== false) {
                if ($duration > $threshold) {
                    $this->logger->warning("Performance threshold exceeded", [
                        'endpoint' => $endpoint,
                        'duration' => $duration,
                        'threshold' => $threshold
                    ]);
                }
                break;
            }
        }
    }
    
    private function checkForErrors($response)
    {
        $statusCode = $response->getStatusCode();
        
        if ($statusCode >= 400) {
            Metrics::record("error_count", 1, [
                'status_code' => $statusCode
            ]);
            
            if ($statusCode >= 500) {
                $this->logger->error("Server error occurred", [
                    'status_code' => $statusCode,
                    'content' => $response->getContent()
                ]);
            }
        }
    }
    
    private function normalizeQuery($query)
    {
        // Remove specific values to group similar queries
        $normalized = preg_replace('/\d+/', 'N', $query);
        $normalized = preg_replace('/\'[^\']*\'/', '?', $normalized);
        
        return $normalized;
    }
    
    private function monitorConnectionPool()
    {
        $poolStats = $this->getConnectionPoolStats();
        
        Metrics::record("connection_pool_active", $poolStats['active']);
        Metrics::record("connection_pool_idle", $poolStats['idle']);
        Metrics::record("connection_pool_total", $poolStats['total']);
    }
    
    private function getMemoryLimit()
    {
        $limit = ini_get('memory_limit');
        
        if ($limit === '-1') {
            return PHP_INT_MAX;
        }
        
        $value = (int) $limit;
        $unit = strtolower(substr($limit, -1));
        
        switch ($unit) {
            case 'k':
                return $value * 1024;
            case 'm':
                return $value  1024  1024;
            case 'g':
                return $value  1024  1024 * 1024;
            default:
                return $value;
        }
    }
}
class MetricsCollector
{
    private $config;
    private $storage;
    
    public function __construct()
    {
        $this->config = new Config();
        $this->storage = $this->getStorage();
    }
    
    public function record($metric, $value, $tags = [])
    {
        $metricData = [
            'name' => $metric,
            'value' => $value,
            'tags' => $tags,
            'timestamp' => time()
        ];
        
        $this->storage->store($metricData);
        
        // Check for alerts
        $this->checkAlerts($metricData);
    }
    
    public function getMetric($name, $timeRange = '1h')
    {
        return $this->storage->query($name, $timeRange);
    }
    
    public function getMetrics($names, $timeRange = '1h')
    {
        $metrics = [];
        
        foreach ($names as $name) {
            $metrics[$name] = $this->getMetric($name, $timeRange);
        }
        
        return $metrics;
    }
    
    private function getStorage()
    {
        $type = $this->config->get('monitoring.metrics.storage', 'influxdb');
        
        switch ($type) {
            case 'influxdb':
                return new InfluxDBStorage($this->config);
            case 'prometheus':
                return new PrometheusStorage($this->config);
            case 'redis':
                return new RedisStorage($this->config);
            default:
                throw new \Exception("Unknown metrics storage: {$type}");
        }
    }
    
    private function checkAlerts($metricData)
    {
        $alerts = $this->config->get('monitoring.alerts', []);
        
        foreach ($alerts as $alert) {
            if ($alert['metric'] === $metricData['name']) {
                $this->evaluateAlert($alert, $metricData);
            }
        }
    }
    
    private function evaluateAlert($alert, $metricData)
    {
        $threshold = $alert['threshold'];
        $operator = $alert['operator'] ?? 'gt';
        
        $triggered = false;
        
        switch ($operator) {
            case 'gt':
                $triggered = $metricData['value'] > $threshold;
                break;
            case 'lt':
                $triggered = $metricData['value'] < $threshold;
                break;
            case 'eq':
                $triggered = $metricData['value'] == $threshold;
                break;
        }
        
        if ($triggered) {
            $this->triggerAlert($alert, $metricData);
        }
    }
    
    private function triggerAlert($alert, $metricData)
    {
        $alertManager = new AlertManager();
        $alertManager->send($alert, $metricData);
    }
}

Distributed Tracing

<?php
namespace App\Monitoring\Tracing;
use TuskLang\Config;
class DistributedTracer
{
    private $config;
    private $currentTrace;
    private $spans = [];
    
    public function __construct()
    {
        $this->config = new Config();
    }
    
    public function startTrace($request)
    {
        $traceId = $this->generateTraceId();
        
        $this->currentTrace = [
            'trace_id' => $traceId,
            'span_id' => $this->generateSpanId(),
            'parent_id' => $this->extractParentSpanId($request),
            'start_time' => microtime(true),
            'operation_name' => $request->getMethod() . ' ' . $request->getPathInfo(),
            'tags' => [
                'http.method' => $request->getMethod(),
                'http.url' => $request->getUri(),
                'http.user_agent' => $request->getUserAgent(),
                'http.remote_ip' => $request->getClientIp()
            ]
        ];
        
        return $traceId;
    }
    
    public function startSpan($operationName, $tags = [])
    {
        $spanId = $this->generateSpanId();
        
        $span = [
            'trace_id' => $this->currentTrace['trace_id'],
            'span_id' => $spanId,
            'parent_id' => $this->currentTrace['span_id'],
            'operation_name' => $operationName,
            'start_time' => microtime(true),
            'tags' => $tags
        ];
        
        $this->spans[] = $span;
        
        return $spanId;
    }
    
    public function endSpan($spanId, $tags = [])
    {
        foreach ($this->spans as &$span) {
            if ($span['span_id'] === $spanId) {
                $span['end_time'] = microtime(true);
                $span['duration'] = ($span['end_time'] - $span['start_time']) * 1000;
                $span['tags'] = array_merge($span['tags'], $tags);
                break;
            }
        }
    }
    
    public function endTrace($traceId, $response = null)
    {
        if ($this->currentTrace && $this->currentTrace['trace_id'] === $traceId) {
            $this->currentTrace['end_time'] = microtime(true);
            $this->currentTrace['duration'] = ($this->currentTrace['end_time'] - $this->currentTrace['start_time']) * 1000;
            
            if ($response) {
                $this->currentTrace['tags']['http.status_code'] = $response->getStatusCode();
            }
            
            // Add spans to trace
            $this->currentTrace['spans'] = $this->spans;
            
            // Send trace to storage
            $this->sendTrace($this->currentTrace);
            
            // Clear current trace
            $this->currentTrace = null;
            $this->spans = [];
        }
    }
    
    public function addTag($key, $value)
    {
        if ($this->currentTrace) {
            $this->currentTrace['tags'][$key] = $value;
        }
    }
    
    public function injectTraceHeaders($headers)
    {
        if ($this->currentTrace) {
            $headers['X-Trace-ID'] = $this->currentTrace['trace_id'];
            $headers['X-Span-ID'] = $this->currentTrace['span_id'];
        }
        
        return $headers;
    }
    
    private function generateTraceId()
    {
        return bin2hex(random_bytes(16));
    }
    
    private function generateSpanId()
    {
        return bin2hex(random_bytes(8));
    }
    
    private function extractParentSpanId($request)
    {
        return $request->headers->get('X-Trace-ID');
    }
    
    private function sendTrace($trace)
    {
        $storage = $this->getTraceStorage();
        $storage->store($trace);
    }
    
    private function getTraceStorage()
    {
        $type = $this->config->get('monitoring.tracing.storage', 'jaeger');
        
        switch ($type) {
            case 'jaeger':
                return new JaegerStorage($this->config);
            case 'zipkin':
                return new ZipkinStorage($this->config);
            case 'elastic':
                return new ElasticStorage($this->config);
            default:
                throw new \Exception("Unknown trace storage: {$type}");
        }
    }
}
class JaegerStorage
{
    private $config;
    private $client;
    
    public function __construct($config)
    {
        $this->config = $config;
        $this->client = new HttpClient();
    }
    
    public function store($trace)
    {
        $endpoint = $this->config->get('monitoring.tracing.jaeger.endpoint');
        
        $traceData = $this->formatTraceForJaeger($trace);
        
        $response = $this->client->post($endpoint, $traceData, [
            'Content-Type' => 'application/json'
        ]);
        
        if ($response->getStatusCode() !== 200) {
            throw new \Exception("Failed to send trace to Jaeger");
        }
    }
    
    private function formatTraceForJaeger($trace)
    {
        $spans = [];
        
        // Add root span
        $spans[] = [
            'traceID' => $trace['trace_id'],
            'spanID' => $trace['span_id'],
            'operationName' => $trace['operation_name'],
            'startTime' => $trace['start_time'] * 1000000, // Convert to microseconds
            'duration' => $trace['duration'] * 1000, // Convert to microseconds
            'tags' => $this->formatTags($trace['tags'])
        ];
        
        // Add child spans
        foreach ($trace['spans'] as $span) {
            $spans[] = [
                'traceID' => $span['trace_id'],
                'spanID' => $span['span_id'],
                'parentSpanID' => $span['parent_id'],
                'operationName' => $span['operation_name'],
                'startTime' => $span['start_time'] * 1000000,
                'duration' => $span['duration'] * 1000,
                'tags' => $this->formatTags($span['tags'])
            ];
        }
        
        return $spans;
    }
    
    private function formatTags($tags)
    {
        $formattedTags = [];
        
        foreach ($tags as $key => $value) {
            $formattedTags[] = [
                'key' => $key,
                'type' => is_numeric($value) ? 'int64' : 'string',
                'value' => $value
            ];
        }
        
        return $formattedTags;
    }
}

Log Aggregation

<?php
namespace App\Monitoring\Logging;
use TuskLang\Config;
class LogAggregator
{
    private $config;
    private $storage;
    private $filters = [];
    
    public function __construct()
    {
        $this->config = new Config();
        $this->storage = $this->getStorage();
        $this->loadFilters();
    }
    
    public function log($level, $message, $context = [])
    {
        $logEntry = [
            'timestamp' => date('c'),
            'level' => $level,
            'message' => $message,
            'context' => $context,
            'trace_id' => $this->getCurrentTraceId(),
            'span_id' => $this->getCurrentSpanId(),
            'host' => gethostname(),
            'pid' => getmypid()
        ];
        
        // Apply filters
        $logEntry = $this->applyFilters($logEntry);
        
        // Store log entry
        $this->storage->store($logEntry);
        
        // Check for log-based alerts
        $this->checkLogAlerts($logEntry);
    }
    
    public function emergency($message, $context = [])
    {
        $this->log('emergency', $message, $context);
    }
    
    public function alert($message, $context = [])
    {
        $this->log('alert', $message, $context);
    }
    
    public function critical($message, $context = [])
    {
        $this->log('critical', $message, $context);
    }
    
    public function error($message, $context = [])
    {
        $this->log('error', $message, $context);
    }
    
    public function warning($message, $context = [])
    {
        $this->log('warning', $message, $context);
    }
    
    public function notice($message, $context = [])
    {
        $this->log('notice', $message, $context);
    }
    
    public function info($message, $context = [])
    {
        $this->log('info', $message, $context);
    }
    
    public function debug($message, $context = [])
    {
        $this->log('debug', $message, $context);
    }
    
    public function search($query, $filters = [], $limit = 100)
    {
        return $this->storage->search($query, $filters, $limit);
    }
    
    public function getLogStats($timeRange = '1h')
    {
        return $this->storage->getStats($timeRange);
    }
    
    private function getStorage()
    {
        $type = $this->config->get('monitoring.logging.storage', 'elasticsearch');
        
        switch ($type) {
            case 'elasticsearch':
                return new ElasticsearchStorage($this->config);
            case 'fluentd':
                return new FluentdStorage($this->config);
            case 'syslog':
                return new SyslogStorage($this->config);
            default:
                throw new \Exception("Unknown log storage: {$type}");
        }
    }
    
    private function loadFilters()
    {
        $filters = $this->config->get('monitoring.logging.filters', []);
        
        foreach ($filters as $filter) {
            $this->filters[] = new $filter'class';
        }
    }
    
    private function applyFilters($logEntry)
    {
        foreach ($this->filters as $filter) {
            $logEntry = $filter->apply($logEntry);
        }
        
        return $logEntry;
    }
    
    private function checkLogAlerts($logEntry)
    {
        $alerts = $this->config->get('monitoring.logging.alerts', []);
        
        foreach ($alerts as $alert) {
            if ($this->matchesAlert($logEntry, $alert)) {
                $this->triggerLogAlert($alert, $logEntry);
            }
        }
    }
    
    private function matchesAlert($logEntry, $alert)
    {
        // Check level
        if (isset($alert['level']) && $logEntry['level'] !== $alert['level']) {
            return false;
        }
        
        // Check message pattern
        if (isset($alert['message_pattern'])) {
            if (!preg_match($alert['message_pattern'], $logEntry['message'])) {
                return false;
            }
        }
        
        // Check context filters
        if (isset($alert['context_filters'])) {
            foreach ($alert['context_filters'] as $key => $value) {
                if (!isset($logEntry['context'][$key]) || $logEntry['context'][$key] !== $value) {
                    return false;
                }
            }
        }
        
        return true;
    }
    
    private function triggerLogAlert($alert, $logEntry)
    {
        $alertManager = new AlertManager();
        $alertManager->send($alert, $logEntry);
    }
    
    private function getCurrentTraceId()
    {
        // Get from current trace context
        return null; // Implementation depends on trace context
    }
    
    private function getCurrentSpanId()
    {
        // Get from current trace context
        return null; // Implementation depends on trace context
    }
}
class ElasticsearchStorage
{
    private $config;
    private $client;
    
    public function __construct($config)
    {
        $this->config = $config;
        $this->client = new ElasticsearchClient($config);
    }
    
    public function store($logEntry)
    {
        $index = $this->getIndexName();
        
        $this->client->index([
            'index' => $index,
            'body' => $logEntry
        ]);
    }
    
    public function search($query, $filters = [], $limit = 100)
    {
        $index = $this->getIndexName();
        
        $searchBody = [
            'query' => [
                'bool' => [
                    'must' => [
                        ['query_string' => ['query' => $query]]
                    ]
                ]
            ],
            'size' => $limit,
            'sort' => [
                ['timestamp' => ['order' => 'desc']]
            ]
        ];
        
        // Add filters
        if (!empty($filters)) {
            foreach ($filters as $field => $value) {
                $searchBody['query']['bool']['filter'][] = [
                    'term' => [$field => $value]
                ];
            }
        }
        
        $response = $this->client->search([
            'index' => $index,
            'body' => $searchBody
        ]);
        
        return $response['hits']['hits'];
    }
    
    public function getStats($timeRange = '1h')
    {
        $index = $this->getIndexName();
        
        $response = $this->client->search([
            'index' => $index,
            'body' => [
                'size' => 0,
                'aggs' => [
                    'log_levels' => [
                        'terms' => ['field' => 'level']
                    ],
                    'log_count' => [
                        'date_histogram' => [
                            'field' => 'timestamp',
                            'interval' => '1m'
                        ]
                    ]
                ]
            ]
        ]);
        
        return [
            'levels' => $response['aggregations']['log_levels']['buckets'],
            'timeline' => $response['aggregations']['log_count']['buckets']
        ];
    }
    
    private function getIndexName()
    {
        $prefix = $this->config->get('monitoring.logging.elasticsearch.index_prefix', 'logs');
        $date = date('Y.m.d');
        
        return "{$prefix}-{$date}";
    }
}

Alerting System

<?php
namespace App\Monitoring\Alerting;
use TuskLang\Config;
class AlertManager
{
    private $config;
    private $channels = [];
    
    public function __construct()
    {
        $this->config = new Config();
        $this->loadChannels();
    }
    
    public function send($alert, $data)
    {
        $channels = $alert['channels'] ?? ['default'];
        
        foreach ($channels as $channelName) {
            if (isset($this->channels[$channelName])) {
                $this->channels[$channelName]->send($alert, $data);
            }
        }
    }
    
    public function sendCustom($message, $level = 'info', $channels = ['default'])
    {
        $alert = [
            'name' => 'custom_alert',
            'level' => $level,
            'message' => $message,
            'channels' => $channels
        ];
        
        $this->send($alert, ['message' => $message]);
    }
    
    private function loadChannels()
    {
        $channels = $this->config->get('monitoring.alerting.channels', []);
        
        foreach ($channels as $name => $config) {
            $this->channels[$name] = new $config'class';
        }
    }
}
class EmailAlertChannel
{
    private $config;
    private $mailer;
    
    public function __construct($config)
    {
        $this->config = $config;
        $this->mailer = new Mailer($config);
    }
    
    public function send($alert, $data)
    {
        $recipients = $this->config->get('recipients', []);
        $subject = $this->formatSubject($alert);
        $body = $this->formatBody($alert, $data);
        
        foreach ($recipients as $recipient) {
            $this->mailer->send($recipient, $subject, $body);
        }
    }
    
    private function formatSubject($alert)
    {
        $level = strtoupper($alert['level']);
        $name = $alert['name'];
        
        return "[{$level}] {$name}";
    }
    
    private function formatBody($alert, $data)
    {
        $body = "Alert: {$alert['name']}\n";
        $body .= "Level: {$alert['level']}\n";
        $body .= "Message: {$alert['message']}\n";
        $body .= "Time: " . date('c') . "\n\n";
        
        if (!empty($data)) {
            $body .= "Data:\n";
            $body .= json_encode($data, JSON_PRETTY_PRINT);
        }
        
        return $body;
    }
}
class SlackAlertChannel
{
    private $config;
    private $client;
    
    public function __construct($config)
    {
        $this->config = $config;
        $this->client = new HttpClient();
    }
    
    public function send($alert, $data)
    {
        $webhookUrl = $this->config->get('webhook_url');
        $channel = $this->config->get('channel', '#alerts');
        
        $message = $this->formatMessage($alert, $data);
        
        $payload = [
            'channel' => $channel,
            'text' => $message,
            'attachments' => [
                [
                    'color' => $this->getColorForLevel($alert['level']),
                    'fields' => $this->formatFields($alert, $data)
                ]
            ]
        ];
        
        $this->client->post($webhookUrl, $payload, [
            'Content-Type' => 'application/json'
        ]);
    }
    
    private function formatMessage($alert, $data)
    {
        $level = strtoupper($alert['level']);
        $name = $alert['name'];
        $message = $alert['message'];
        
        return "[{$level}] {$name}\n{$message}";
    }
    
    private function getColorForLevel($level)
    {
        $colors = [
            'emergency' => '#ff0000',
            'alert' => '#ff6600',
            'critical' => '#ff9900',
            'error' => '#ffcc00',
            'warning' => '#ffff00',
            'notice' => '#00ff00',
            'info' => '#00ccff',
            'debug' => '#cccccc'
        ];
        
        return $colors[$level] ?? '#cccccc';
    }
    
    private function formatFields($alert, $data)
    {
        $fields = [
            [
                'title' => 'Alert Name',
                'value' => $alert['name'],
                'short' => true
            ],
            [
                'title' => 'Level',
                'value' => $alert['level'],
                'short' => true
            ],
            [
                'title' => 'Time',
                'value' => date('c'),
                'short' => true
            ]
        ];
        
        if (!empty($data)) {
            $fields[] = [
                'title' => 'Data',
                'value' => json_encode($data),
                'short' => false
            ];
        }
        
        return $fields;
    }
}

Best Practices

- Monitor everything that matters - Use distributed tracing for request flows - Implement structured logging - Set up intelligent alerting - Create comprehensive dashboards - Monitor business metrics

Performance Optimization

- Use sampling for high-volume traces - Implement log buffering - Use efficient storage backends - Optimize alert queries

Security Considerations

- Sanitize log data - Secure monitoring endpoints - Implement access controls - Monitor security events

Troubleshooting

- Check monitoring infrastructure health - Verify alert delivery - Monitor monitoring system - Debug trace collection

Conclusion

TuskLang + PHP = monitoring and observability that provides complete visibility into your applications. See everything, know everything.