ok
Direktori : /proc/thread-self/root/opt/cloudlinux/venv/lib/python3.11/site-packages/clwpos/ |
Current File : //proc/thread-self/root/opt/cloudlinux/venv/lib/python3.11/site-packages/clwpos/report_generator.py |
# -*- coding: utf-8 -*- # # Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved # # Licensed under CLOUD LINUX LICENSE AGREEMENT # http://cloudlinux.com/docs/LICENSE.TXT from __future__ import absolute_import import os import sys import time import fcntl import json from psutil import pid_exists from typing import Optional, Dict, TextIO, Union, List, Tuple from secureio import write_file_via_tempfile, disable_quota from clwpos.data_collector_utils import _add_wp_path_info from clcommon.clpwd import ClPwd, drop_privileges from clwpos.feature_suites.configurations import FeatureStatusEnum, extract_suites from clwpos.utils import acquire_lock, get_server_wide_options from clcommon.cpapi import userdomains, cpusers from clwpos.optimization_features import convert_features_dict_to_interface from clwpos.feature_suites import ( get_admin_suites_config, extract_features, AWPSuite, PremiumSuite, CDNSuite, CDNSuitePro ) from clwpos.user.config import UserConfig from clwpos.logsetup import setup_logging from clwpos.cl_wpos_exceptions import WposError from clwpos import gettext as _ from clwpos.constants import ( SCANNING_STATUS, LAST_SCANNED_TS, SCAN_CACHE ) logger = setup_logging(__name__) class ReportGeneratorError(WposError): """ Raised when some bad happened during report generating/getting """ pass class ScanStatus: """ Type for handly scan status manipulations """ def __init__(self, status, pid: Optional[int] = None): self.pid = pid try: if isinstance(status, str): self.current, self.total = map(int, status.split('/')) elif isinstance(status, tuple): self.current, self.total = map(int, status) else: raise ReportGeneratorError( message=_('Unable to parse scan status: %(status)s, type of: %(type)s'), context={'status': str(status), 'type': type(status)}, ) except ValueError: raise ReportGeneratorError( message=_('Unable to parse scan status: %(status)s'), context={'status': str(status)}, ) def __hash__(self): return hash((self.current, self.total)) def __eq__(self, other): if isinstance(other, ScanStatus): return self.current, self.total == other.current, other.total elif isinstance(other, str): return str(self) == other.strip() elif isinstance(other, tuple): return tuple(self) == other return False def __str__(self): return f'{self.current}/{self.total}' def __iter__(self): for i in self.current, self.total: yield i @staticmethod def read() -> Optional['ScanStatus']: """ Read status file, with shared blocking flock """ st = None if os.path.exists(SCANNING_STATUS): with open(SCANNING_STATUS, 'r') as f: fcntl.flock(f.fileno(), fcntl.LOCK_SH) data = f.read() fcntl.flock(f.fileno(), fcntl.LOCK_UN) # status file was created, but not written with data yet if not data: return st try: data = json.loads(data) pid = data.get('pid') st = ScanStatus( (data['current'], data['total']), int(pid) if pid else None, ) except Exception as e: logger.error('Can\'t parse scan status json: %s', e) return st def to_json(self) -> str: """ Status JSON representation to write """ return json.dumps({ 'current': self.current, 'total': self.total, 'pid': self.pid, }) def check_pid(self) -> bool: """ Return True if scan process is still alive """ return bool(self.pid) and pid_exists(self.pid) class ReportGenerator: """ Class for admin report generation/getting """ def __init__(self): self._clpwd = ClPwd() self.status = ScanStatus((0, 0)) self.result = { 'version': '1', 'users': {}, } self.scan_status_fd = None self.logger = logger @staticmethod def is_scan_running() -> bool: """ Checks: 1) SCANNING_STATUS file exists 2) SCANNING_STATUS has the scan process pid and it is still alive """ status = ScanStatus.read() if status: if status.check_pid(): return True return False @staticmethod def get_scan_status() -> Optional[ScanStatus]: """ Read status file and get status """ return ScanStatus.read() def scan(self, user_list: Optional[List[str]] = None) -> Dict[str, int]: """ Public entry point for the scan procedure Do some checks, prepare users list, fork process to actually do scan, and return initial scan progress `user_list` - users to scan; if None - scan all """ if self.is_scan_running(): raise ReportGeneratorError(message=_('Another scan in progress: %(status)s'), context={'status': str(self.get_scan_status())}) usernames = self.__get_usernames() self.logger.debug('Found usernames: %s', usernames) if user_list: usernames = self.__filter_usernames(user_list, usernames) self.logger.debug('Filtered usernames: %s', usernames) self.status.total = len(usernames) # fork the scan to another process and return initial status immediately fp = os.fork() if fp: self.logger.debug('Scan forked to: %d', fp) return {'total_users_scanned': 0, 'total_users_to_scan': self.status.total} else: try: self._scan(usernames) sys.exit(0) except Exception as e: self.logger.exception('Error during user sites scan: %s', e) sys.exit(1) def _scan(self, usernames: List[str]) -> None: """ Executes in forked process Get domain and docroots list for usernames Count user sites, write to SCAN_CACHE Create and remove SCANNING_STATUS file `userdomains` examples: [{'domain': 'res1.com', 'docroot': '/home/res1/public_html'}], [{'domain': 'cltest1.com', 'docroot': '/home/cltest1/public_html'}, {'domain': 'mk.cltest1.com', 'docroot': '/home/cltest1/public_html/mk'}] """ self.scan_status_fd = open(SCANNING_STATUS, 'w') self.status.pid = os.getpid() self.logger.debug('Set status pid to: %d', self.status.pid) for username in usernames: user_data = {} self.status.current += 1 self.__update_scan_status() for domain_name, doc_root in userdomains(username): user_data.setdefault(doc_root, []).append(domain_name) with drop_privileges(username): user_data = _add_wp_path_info(user_data) # set-comprehension and rstrip() used to delete duplicated paths in case of subdomains # ex: {'/dr1': {'wp_paths': ['', 'wordpress', 'sub']}, '/dr1/sub': {'wp_paths': ['']}} wp_paths = { os.path.join(_doc_root, _wp_path).rstrip('/') for _doc_root, values in user_data.items() for _wp_path in values.get('wp_paths', []) } self.result['users'].update({ username: {'wp_sites_count': len(wp_paths)}, }) self.__write_scan_result() self.scan_status_fd.close() os.unlink(SCANNING_STATUS) _ts = self.__update_scan_timestamp() self.logger.debug('New scan timestamp: %d', _ts) def get_status(self) -> dict: """ CLI command to get scan status Cases: 1) no previous scans 2) scan in progress 3) scan finished (success) 4) scan process crashed """ result = { 'scan_status': 'idle', 'total_users_scanned': 0, 'total_users_to_scan': None, 'last_scan_time': None, } if not os.path.exists(SCAN_CACHE): result.update({'total_users_to_scan': len(cpusers())}) return result if self.is_scan_running(): status = self.get_scan_status() result.update({ 'scan_status': 'in_progress', 'total_users_scanned': status.current, 'total_users_to_scan': status.total, }) else: if not os.path.exists(SCANNING_STATUS): result.update({ 'total_users_to_scan': len(cpusers()), 'last_scan_time': self._get_scan_timestamp(), }) else: result.update({ 'scan_status': 'error', 'scan_error': _('Failed to generate report'), 'total_users_to_scan': len(cpusers()), 'last_scan_time': self._get_scan_timestamp(), }) os.unlink(SCANNING_STATUS) return result @staticmethod def _get_scan_timestamp() -> Optional[int]: """ Read timestamp from LAST_SCANNED_TS Returns None in case of any error """ try: with open(LAST_SCANNED_TS, 'rt') as f: t = f.read().strip() return int(t) except Exception as e: logger.error('Can\'t read last scan timestamp: %s', e) return None @staticmethod def __filter_usernames(to_scan, usernames: List[str]) -> List[str]: """ Return usernames from --users cli argument """ return list(filter(lambda x: x in to_scan, usernames)) @staticmethod def __get_usernames() -> List[str]: """ Get usernames from cpapi """ return list(cpusers()) @staticmethod def __update_scan_timestamp() -> int: """ Write current timestamp to LAST_SCANNED_TS Return timestamp for the later use, if needed """ ts = int(time.time()) write_file_via_tempfile(str(ts), LAST_SCANNED_TS, 0o600) return ts def __update_scan_status(self) -> None: """ Write current status to SCANNING_STATUS Data example: "0/10", without any other symbols """ f = self.scan_status_fd # type: TextIO f_no = f.fileno() f.seek(0) fcntl.flock(f_no, fcntl.LOCK_EX | fcntl.LOCK_NB) f.write(self.status.to_json()) f.flush() fcntl.flock(f_no, fcntl.LOCK_UN) def __write_scan_result(self) -> None: """ Write final scan result to SCANNING_STATUS via temp file """ write_file_via_tempfile(json.dumps(self.result), SCAN_CACHE, 0o600) # get-report part def get(self, target_users=None): """ 1. get-report when there is no cache file: - start generating report - return total_users_scanned, total_users_to_scan keys 2. get-report when there is no cache file, but scanning in progress: - return total_users_scanned, total_users_to_scan keys 3. get-report when cache present, no scanning running - return data from cache - no keys total_users_scanned, total_users_to_scan in response 4. get-report when cache present, scanning is running - return data from cache - return total_users_scanned, total_users_to_scan keys """ report = {} scanned_cache = self._get_from_file_with_locking(SCAN_CACHE, is_json=True) scanning_status = self.get_scan_status() # no cache and scanning is not running right now -> # run scanning if not scanned_cache and not scanning_status: scan = self.scan() return scan # no cache, but scanning is running right now -> # returns status from status file if scanning_status: report['total_users_scanned'] = scanning_status.current report['total_users_to_scan'] = scanning_status.total # if scanned cache -> append it to report if scanned_cache: report['last_scan_time'] = self._get_scan_timestamp() # cache already generated, let`s read it users_cache = scanned_cache['users'] ( report['users'], report['total_wordpress_count'], report['total_users_count'], total_users_with_active_awp, total_users_with_active_premium, total_users_with_active_cdn, total_users_with_active_cdn_pro, total_sites_with_active_awp, total_sites_with_active_premium, total_sites_with_active_cdn, total_sites_with_active_cdn_pro ) = self._get_users_report(users_cache, target_users) report['total_users_count_active'] = { AWPSuite.name: total_users_with_active_awp, PremiumSuite.name: total_users_with_active_premium, CDNSuite.name: total_users_with_active_cdn, CDNSuitePro.name: total_users_with_active_cdn_pro } report['total_sites_count_active'] = { AWPSuite.name: total_sites_with_active_awp, PremiumSuite.name: total_sites_with_active_premium, CDNSuite.name: total_sites_with_active_cdn, CDNSuitePro.name: total_sites_with_active_cdn_pro } return report @staticmethod def _is_allowed_suite(user_suites_data, user_features_data, suite): is_suite_allowed = user_suites_data.get(suite.name).status == FeatureStatusEnum.ALLOWED return is_suite_allowed and any([user_features_data.get(feature.NAME.lower()) and user_features_data.get(feature.NAME.lower()).status == FeatureStatusEnum.ALLOWED for feature in suite.features]) @staticmethod def _sites_with_active_features(username): with drop_privileges(username), disable_quota(): wps_with_enabled_awp_features = UserConfig(username). \ wp_paths_with_active_suite_features(AWPSuite.features) wps_with_enabled_awp_premium_features = UserConfig(username). \ wp_paths_with_active_suite_features(PremiumSuite.features) wps_with_enabled_cdn_features = UserConfig(username). \ wp_paths_with_active_suite_features(CDNSuite.primary) return wps_with_enabled_awp_features, wps_with_enabled_awp_premium_features, wps_with_enabled_cdn_features def _get_users_report( self, user_scanned_data: Dict, target_users: Union[None, List] ) -> Tuple[List, int, int, int, int, int, int, int, int, int, int]: """ Returns list of final users data: example: [ {"username": "user1", "features": {"object_cache": true}, "wp_sites_count": "1"}, .... ] """ all_users_info = [] total_wp_sites_count, total_users_count = 0, 0 total_users_with_active_awp = 0 total_users_with_active_premium = 0 total_users_with_active_cdn = 0 total_users_with_active_cdn_pro = 0 total_sites_with_active_awp = 0 total_sites_with_active_premium = 0 total_sites_with_active_cdn = 0 total_sites_with_active_cdn_pro = 0 default_config = get_server_wide_options() for username, scanned_data in user_scanned_data.items(): wps_per_user_with_enabled_awp_features = 0 wps_per_user_with_enabled_awp_premium_features = 0 wps_per_user_with_enabled_cdn = 0 wps_per_user_with_enabled_cdn_pro = 0 if target_users and username not in target_users: continue uid = self._uid_by_name(username) if not uid: self.logger.warning('Cannot get user %s uid', username) continue # let`s not break report collection for all users try: user_features_data = extract_features(get_admin_suites_config(uid), server_wide_options=default_config) user_suites_data = extract_suites(get_admin_suites_config(uid), server_wide_options=default_config) except Exception: self.logger.exception('Unable to get features data from admin config') user_suites_data = {} user_features_data = {} # broken users must not fail whole report try: wps_with_enabled_awp_features, wps_with_enabled_awp_premium_features, wps_with_enabled_cdn_features = \ self._sites_with_active_features(username) except Exception as e: self.logger.warning('Unable to get information for report from user config: %s', str(e)) wps_with_enabled_awp_features, wps_with_enabled_awp_premium_features, wps_with_enabled_cdn_features = \ [], [], [] total_users_count += 1 total_wp_sites_count += int(scanned_data['wp_sites_count']) # if feature is allowed and activated by end-user if self._is_allowed_suite(user_suites_data, user_features_data, AWPSuite) and wps_with_enabled_awp_features: total_users_with_active_awp += 1 total_sites_with_active_awp += len(wps_with_enabled_awp_features) wps_per_user_with_enabled_awp_features = len(wps_with_enabled_awp_features) if self._is_allowed_suite(user_suites_data, user_features_data, PremiumSuite) and wps_with_enabled_awp_premium_features: total_users_with_active_premium += 1 total_sites_with_active_premium += len(wps_with_enabled_awp_premium_features) wps_per_user_with_enabled_awp_premium_features = len(wps_with_enabled_awp_premium_features) if wps_with_enabled_cdn_features: if self._is_allowed_suite(user_suites_data, user_features_data, CDNSuitePro): total_users_with_active_cdn_pro += 1 total_sites_with_active_cdn_pro += len(wps_with_enabled_cdn_features) wps_per_user_with_enabled_cdn_pro = len(wps_with_enabled_cdn_features) elif self._is_allowed_suite(user_suites_data, user_features_data, CDNSuite): total_users_with_active_cdn += 1 total_sites_with_active_cdn += len(wps_with_enabled_cdn_features) wps_per_user_with_enabled_cdn = len(wps_with_enabled_cdn_features) all_users_info.append({ 'username': username, 'suites': user_suites_data, 'wp_sites_count': scanned_data['wp_sites_count'], 'accelerate_wp_active_sites_count': wps_per_user_with_enabled_awp_features, 'accelerate_wp_premium_sites_count': wps_per_user_with_enabled_awp_premium_features, 'accelerate_wp_cdn_sites_count': wps_per_user_with_enabled_cdn, 'accelerate_wp_cdn_pro_sites_count': wps_per_user_with_enabled_cdn_pro, }) # stop iterating through users if targets already found if target_users and len(target_users) == len(all_users_info): break return (all_users_info, total_wp_sites_count, total_users_count, total_users_with_active_awp, total_users_with_active_premium, total_users_with_active_cdn, total_users_with_active_cdn_pro, total_sites_with_active_awp, total_sites_with_active_premium, total_sites_with_active_cdn, total_sites_with_active_cdn_pro) def _get_from_file_with_locking(self, path: str, is_json: bool, with_lock: bool = False) -> Union[Dict, str, None]: """ Reads file with locking if needed """ if not os.path.exists(path): return None try: if with_lock: with acquire_lock(path): return self._read_report_file(path, is_json) else: return self._read_report_file(path, is_json) except Exception as e: raise ReportGeneratorError(message=_('Can`t read %(file)s. Reason: %(error)s'), context={'file': path, 'error': str(e)}) @staticmethod def _read_report_file(path: str, is_json: bool) -> Union[Dict, str, None]: """ Reads file for report """ with open(path, "r") as file: if is_json: return json.load(file) else: return file.read().strip() def _uid_by_name(self, name): try: return self._clpwd.get_uid(name) except ClPwd.NoSuchUserException: return None