ok
Mini Shell
# -*- coding: utf-8 -*-
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
"""
This module contains FaultDetector class, aimed to track throttling
for incoming requests
"""
import logging
import time
from datetime import datetime, timedelta
from threading import RLock
from typing import Tuple, Any
from xray.internal.constants import drop_after, check_period, throttling_threshold
class FaultDetector:
"""
Fault Detector class
"""
def __init__(self):
self.logger = logging.getLogger('fault_detector')
self.throttling_stat = dict()
self.lock = RLock()
self.time_marker_file = '/usr/share/alt-php-xray/flush.latest'
def __call__(self, t_key: Any, cpu_value: int) -> Tuple[bool, int]:
"""
Perform fault detection
"""
with self.lock:
try:
saved_cpu, _ = self.throttling_stat[t_key]
except KeyError:
hitting_limits, throttled_time = False, 0
else:
hitting_limits, throttled_time = self._was_throttled_since(
saved_cpu, cpu_value)
self._flush_entry(t_key)
return hitting_limits, throttled_time
@property
def timestamp(self) -> int:
"""
Current Unix timestamp
"""
return int(time.time())
@property
def _drop_after(self) -> timedelta:
"""
Drop after value as datetime.timedelta
"""
return self.timedelta_in_minutes(drop_after)
@property
def _check_period(self) -> timedelta:
"""
Check period value as datetime.timedelta
"""
return self.timedelta_in_minutes(check_period)
@property
def should_flush(self) -> bool:
"""
If throttling_stat should be flushed right now
regarding check period value
"""
try:
with open(self.time_marker_file) as latest_info:
latest_flush_time = int(latest_info.read().strip())
return self.current_delta(latest_flush_time) > self._check_period
except (OSError, ValueError):
with self.lock:
self._update_latest_flush()
return False
@staticmethod
def timedelta_in_minutes(mins: int) -> timedelta:
"""
Get timedelta object for given number of minutes
"""
return timedelta(minutes=mins)
def current_delta(self, time_stamp: int) -> timedelta:
"""
Calculate timedelta for given time_stamp from current timestamp
"""
def _cast(ts: int) -> datetime:
"""
Make a datetime object from ts (int)
"""
return datetime.fromtimestamp(ts)
return _cast(self.timestamp) - _cast(time_stamp)
def save(self, key: Any, cpu: int) -> None:
"""
Save throttling entry for given key
"""
stamp = self.timestamp
with self.lock:
self.logger.debug('Adding new throttling entry %s: (%i, %i)',
key, cpu, stamp)
self.throttling_stat[key] = (cpu, stamp)
self.logger.debug('Throttling entries total %i',
len(self.throttling_stat))
def _flush_entry(self, key: Any) -> None:
"""
Remove throttling entry by given key
"""
with self.lock:
self.logger.debug('Abandon throttling entry %s', key)
self.throttling_stat.pop(key, 0)
self.logger.debug('Throttling entries total %i',
len(self.throttling_stat))
def _was_throttled_since(self,
initial_value: int,
current_value: int) -> Tuple[bool, int]:
"""
Check current throttling time of given lve_id vs given initial_value
Return: tuple(fact of throttling, throttling time in ms)
"""
# initial values of throttling time are in nanoseconds!
spent_time = (current_value - initial_value) // 1000000 # cast to ms
throttling_fact = spent_time > throttling_threshold
self.logger.debug('Throttling checked with %i: %s, %i',
throttling_threshold,
throttling_fact, spent_time)
return throttling_fact, spent_time if spent_time > 0 else 0
def flush(self) -> None:
"""
Clear throttling stat dict
"""
if self.should_flush:
with self.lock:
self.logger.debug('Flush started, total entries %i',
len(self.throttling_stat))
for key in self._expired_entries():
self._flush_entry(key)
self._update_latest_flush()
self.logger.debug('Flush finished, total entries %i',
len(self.throttling_stat))
def _update_latest_flush(self) -> None:
"""
Write current timestamp into time marker file
"""
try:
with open(self.time_marker_file, 'w') as latest_info:
latest_info.write(str(self.timestamp))
except OSError:
self.logger.error('Failed to save timestamp of latest flush')
def _expired_entries(self) -> list:
"""
Collect expired throttling stat entries keys
regarding the drop after value
"""
expired = list()
for key, value in self.throttling_stat.items():
_, timestamp = value
_delta = self.current_delta(timestamp)
if _delta > self._drop_after:
self.logger.debug('Expired entry detected %i: %s elapsed',
key, _delta)
expired.append(key)
return expired
Zerion Mini Shell 1.0