ok
Mini Shell
# -*- coding: utf-8 -*-
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
"""
This module contains X Ray Smart Advice local utility main class
"""
import json
import logging
import os
import hashlib
import pwd
import subprocess
from contextlib import contextmanager
from dataclasses import asdict, dataclass
from typing import Any, Tuple, Optional, List, Dict
from enum import Enum
from clwpos.user.config import LicenseApproveStatus
from clwpos.papi import (
is_feature_allowed,
is_subscription_pending,
get_subscription_upgrade_url,
get_license_approve_status,
approve_license_agreement,
get_license_agreement_text,
get_subscriptions_info,
get_user_auth_key,
is_smart_advice_notifications_disabled_server_wide,
is_smart_advice_reminders_disabled_server_wide,
is_smart_advice_wordpress_plugin_disabled_server_wide
)
from clcommon.cpapi import (
docroot,
is_panel_feature_supported,
get_user_emails_list,
panel_awp_link,
getCPName,
userdomains
)
from clcommon.clwpos_lib import is_wp_path
from clcommon.clpwd import drop_privileges
from clcommon.const import Feature
from xray import gettext as _
from .advice_types import get_advice_instance
from .progress import SmartAdviceProgress
from ..apiclient import get_client, api_client
from ..internal.constants import (
advice_pending_storage,
advice_processed_storage,
advice_list_cache,
advice_reason_max_len
)
from ..internal.user_limits import get_lve_limits, get_lve_usage
from ..internal.exceptions import XRayError, SmartAdvicePluginError
from ..internal.user_plugin_utils import user_mode_advice_verification, username_verification, get_xray_exec_user
from ..internal.utils import timestamp, safe_move, get_user_php_version, filelock
from ..analytics.utils import report_usage_action_or_error
from xray.adviser.clwpos_get import ClWposGetter
from xray.smart_advice_plugin_helpers import (
get_mu_directory,
create_mu_plugins_dir_if_not_exist
)
from xray.smart_advice_plugin_manager import plugin_installed, prepare_system_analytics_data
from xray.analytics import report_analytics
from xray.imunify_manager import ImunifyManager, ADV_TYPE
advice_cache_separator = ';'
class AdviceActions(Enum):
APPLY = 'apply'
ROLLBACK = 'rollback'
@dataclass
class SmartAdviceOptions:
panel_type: str
panel_url: str
panel_emails: str
upgrade_url: str
upgrade_url_cdn: str
subscription: dict
notifications: dict
class SmartAdviceUtil:
"""Main Smart Advice local utility class"""
def __init__(self):
self.logger = logging.getLogger('smart_advice')
# check existence of pending and apply storage and create it if missing
for stor in (advice_pending_storage, advice_processed_storage):
self.create_dir(stor)
# initialize Adviser API client
adviser_client_object = get_client('adviser')
self.adviser_client: api_client.SmartAdviceAPIClient = adviser_client_object()
self.imunify_manager = ImunifyManager()
@staticmethod
def create_dir(dpath: str) -> None:
"""Create dir if missing"""
if not os.path.isdir(dpath):
os.mkdir(dpath)
@staticmethod
def response(**kwargs) -> str:
"""
Create JSON response message with result field == success
and given keyword arguments in other fields
:return: json packed string
"""
initial = {'result': 'success', 'timestamp': timestamp()}
if kwargs:
initial.update(kwargs)
return json.dumps(dict(sorted(initial.items())))
@staticmethod
def _apply_datafile(a_id: int) -> str:
"""Per-advice file with results of apply"""
return f'{advice_processed_storage}/{a_id}'
def _apply_results(self, a_id: int) -> Optional[str]:
"""Retrieve data stored in per-advice file with results of apply"""
datafile = self._apply_datafile(a_id)
if os.path.isfile(datafile):
try:
with open(datafile) as _f:
data = json.load(_f)
except OSError:
data = None
except json.JSONDecodeError:
with open(datafile) as _f:
data = _f.read()
finally:
os.unlink(datafile)
return data
@staticmethod
def _pending_flag(a_id: int) -> str:
"""Per-advice pending flag"""
return f'{advice_pending_storage}/{a_id}'
def is_advice_pending(self, advice_id: int) -> bool:
"""Is advice in pending state"""
return os.path.exists(self._pending_flag(advice_id))
def _progress_file(self, a_id: int) -> str:
"""Per-advice progress storage"""
return self._pending_flag(a_id) + '.progress'
def _get_user_awp_link(self, username):
try:
return panel_awp_link(username)
except Exception:
self.logger.exception('Error while getting user login link')
return ''
@user_mode_advice_verification
def get_detailed_advice(self, advice_id: int) -> Tuple[dict, object]:
"""
Get advice details from API along with an
appropriate Advice object by obtained advice type
"""
response = self.adviser_client.advice_details(advice_id)
return response, get_advice_instance(response['advice']['type'])
@staticmethod
def dump_to_file(dst: str, data: Any, as_json: bool = False) -> None:
"""Dump data inside given dst using .bkp file and then move"""
_tmp_dst = dst + '.bkp'
with open(_tmp_dst, 'w') as _f:
if as_json:
try:
json.dump(data, _f)
except TypeError:
_f.write(data)
else:
_f.write(data)
safe_move(_tmp_dst, dst)
def progress(self, advice_id: int, current: SmartAdviceProgress) -> SmartAdviceProgress:
"""
Smart Advice apply own progress,
based on current and previous results of 'clwpos get-progress'.
Returns the maximum progress values among current and previous.
Previous (latest) is stored inside a pending file-flag.
"""
@contextmanager
def resolve_progress() -> SmartAdviceProgress:
"""
Get progress stored in file and return the maximum one among
current and stored results
"""
self.logger.debug('Current progress value: %s', current)
progress_dst = self._progress_file(advice_id)
try:
# read stored result
with open(progress_dst) as prev:
prev_stages = SmartAdviceProgress(**json.load(prev))
except (OSError, json.JSONDecodeError) as e:
self.logger.debug('Error during reading stored value: %s',
str(e))
# or set a dummy one
prev_stages = SmartAdviceProgress()
self.logger.debug('Stored progress value: %s', prev_stages)
yield current if current > prev_stages else prev_stages
if self.is_advice_pending(advice_id) and current > prev_stages:
# update stored result only if pending file exists
self.logger.debug('Updating stored progress: %s', current)
self.dump_to_file(progress_dst, asdict(current),
as_json=True)
return resolve_progress()
def get_current_status(self, _id: int, advice_obj: object,
advice_data: dict) -> Tuple[str, dict]:
"""
Resolve if advice is in pending status.
Progress is retrieved for pending advice only.
Dummy result (0, 0) is returned for other advice states.
"""
if self.is_advice_pending(_id):
status = 'pending'
with self.progress(_id, advice_obj.get_progress(
advice_data['metadata']['username'])) as p:
stages = p
else:
status = advice_data['advice']['status']
stages = SmartAdviceProgress()
return status, asdict(stages)
def advice_list(self, extended=False) -> str:
"""Load validated advice list and update it before returning"""
api_response = self.adviser_client.advice_list()
response_advice_list = self.prepare_advices_response(api_response, extended=extended)
# cl-smart-advice plugin syncing should not break whole command
try:
self.sync_advices_wordpress_plugin(current_advices=api_response)
except Exception:
self.logger.exception('Unable to sync cl-smart-advice plugin during getting list of advices')
return self.response(data=response_advice_list)
@username_verification
def get_site_statuses(self, username) -> str:
api_response = self.adviser_client.site_info(username)
site_statuses_data = {}
for website in api_response:
domain = website['domain']
if domain not in site_statuses_data:
site_statuses_data[domain] = []
site_statuses_data[domain].append({
'website': website['website'],
'urls_scanned': len(website['urls']),
'advices': self.prepare_advices_response(website['advices'])
})
result = [{'domain': domain, 'websites': websites} for domain, websites in site_statuses_data.items()]
return self.response(data=result)
def prepare_advices_response(self, from_api, status_from_microservice=False, extended=False):
advices = from_api.copy()
visible_advices = []
for item in advices:
# TODO: would be great to rewrite this part and avoid
# changing array inplace, instead create new one with proper typing
advice_itself = item['advice']
advice_type = advice_itself['type']
# It's only for smart advice wp plugin
if advice_type.startswith(ADV_TYPE):
visible_advices.append(item)
continue
advice_instance = get_advice_instance(advice_type)
advice_itself['description'] = advice_instance.short_description
# when we need real response from microservice - only before sending json to wp plugin
if status_from_microservice:
status = advice_itself['status']
stages = {'total_stages': 0, 'completed_stages': 0}
else:
status, stages = self.get_current_status(advice_itself['id'],
advice_instance, item)
if extended:
advice_itself['detailed_description'] = advice_instance.detailed_description
advice_itself['status'] = status
advice_itself['is_premium'] = advice_instance.is_premium_feature
advice_itself['module_name'] = advice_instance.module_name
advice_itself['license_status'] = get_license_approve_status(
advice_instance.module_name, item['metadata']['username']).name
if advice_itself['is_premium']:
subscription_status = self._get_subscription_status(
advice_instance.module_name, item['metadata']['username'])
advice_itself['subscription'] = dict(
status=subscription_status,
upgrade_url=get_subscription_upgrade_url(
advice_instance.module_name, item['metadata']['username'])
)
advice_itself.update(stages)
visible_advices.append(item)
return visible_advices
def filter_plugin_advices(self, username, website, domain, advice_list):
return [advice for advice in advice_list
if advice.get('metadata', {}).get('domain') == domain
and advice.get('metadata', {}).get('username') == username
and advice.get('metadata', {}).get('website') == website
and advice.get('advice', {}).get('status') != 'outdated']
def _get_subscription_status(self, module_name, username):
"""
Determines current subscription status based on feature status.
"""
subscription_status = 'no'
if is_feature_allowed(module_name, username):
subscription_status = 'active'
if is_subscription_pending(module_name, username):
subscription_status = 'pending'
return subscription_status
def advice_details(self, advice_id: int) -> str:
"""Load validated advice details and update it before returning"""
api_response, advice_instance = self.get_detailed_advice(advice_id)
advice_itself = api_response['advice']
"""
The microservice responds with field names: title and description.
We must assign:
[local advice] detailed_description = [microservice] description
[local advice] description = [microservice] title
"""
# Until we won't rename detailed_description to description
if advice_itself['description']:
advice_itself['detailed_description'] = advice_itself['description']
else:
advice_itself['detailed_description'] = advice_instance.detailed_description
# Until we won't rename description to title
if advice_itself['title']:
advice_itself['description'] = advice_itself['title']
else:
advice_itself['description'] = advice_instance.short_description
del advice_itself['title']
return self.response(data=api_response)
def manage_advice(self, action, advice_id: int, ignore_errors: bool = False,
async_mode: bool = False, source: str = 'ACCELERATE_WP',
reason: str = None, accept_terms: bool = False,
analytics_data: str = None) -> Optional[str]:
open(self._pending_flag(advice_id), 'w').close()
if async_mode:
# put itself to background in async mode
# and return current status
child = os.fork()
if child:
return self.manage_advice_status(advice_id)
output = self._exec_advice_managing(action, advice_id, ignore_errors, async_mode,
source, reason, accept_terms)
if analytics_data is not None:
report_usage_action_or_error(analytics_data, advice_id, source, output, action)
if async_mode:
# retrun no output in async mode
output = None
return output
def _exec_advice_managing(self, action: str, advice_id: int, ignore_errors: bool = False,
async_mode: bool = False, source: str = 'ACCELERATE_WP',
reason: str = None, accept_terms: bool = False) -> Optional[str]:
"""Execute managing advice with passed action: apply/rollback"""
try:
api_response, advice_instance = self.get_detailed_advice(advice_id)
if action == AdviceActions.APPLY.value:
if accept_terms:
approve_license_agreement(
advice_instance.module_name, api_response['metadata']['username'])
if get_license_approve_status(
advice_instance.module_name,
api_response['metadata']['username']) == LicenseApproveStatus.NOT_APPROVED:
return json.dumps({
'result': 'LICENCE_TERMS_APPROVE_REQUIRED',
'text': _('License approve required to use this feature. '
'Open AccelerateWP plugin in your control panel, apply advice and '
'accept terms and conditions to proceed with installation.')
})
action_result, output = advice_instance.apply(**api_response['metadata'],
ignore_errors=ignore_errors)
if action_result:
self.logger.debug('Applied successfully')
self.adviser_client.update_advice(advice_id=advice_id,
status='applied',
source=source)
try:
self._sync_advice(api_response['metadata']['username'],
api_response['metadata']['domain'],
api_response['metadata']['website'])
except Exception:
self.logger.exception('Error while syncing cl-smart-advice plugin on advice apply')
elif action == AdviceActions.ROLLBACK.value:
action_result, output = advice_instance.rollback(**api_response['metadata'])
if action_result:
self.logger.debug('Rollback successfully')
if reason:
self.adviser_client.update_advice(advice_id=advice_id,
status='review',
source=source,
reason=reason[:advice_reason_max_len])
else:
self.adviser_client.update_advice(advice_id=advice_id,
status='review',
source=source)
try:
self._sync_advice(api_response['metadata']['username'],
api_response['metadata']['domain'],
api_response['metadata']['website'])
except Exception:
self.logger.exception('Error while syncing cl-smart-advice plugin on advice rollback')
else:
raise ValueError(_('Unsupported action with advice, passed action: %s') % str(action))
finally:
if os.path.isfile(self._pending_flag(advice_id)):
os.unlink(self._pending_flag(advice_id))
if async_mode:
# in async mode write result to a special file
self.dump_to_file(self._apply_datafile(advice_id),
output)
# return output for analytics will be cleared after reporting
return output
else:
# in sync mode return output
return output
def advice_counters(self) -> str:
"""Return advice counters for a server"""
try:
api_response = self.adviser_client.advice_list()
except XRayError:
# bad API responses (500 for example)
# or JWT token failed check (CL Shared for example)
result = dict.fromkeys(['total', 'applied'], None)
else:
result = dict(
total=len(api_response),
applied=len([advice for advice in api_response if
advice['advice']['status'] == 'applied'])
)
return self.response(data=result)
def manage_advice_status(self, advice_id: int) -> str:
"""
Return current status and progress of managing particular advice.
"""
api_response, advice_instance = self.get_detailed_advice(advice_id)
status, stages = self.get_current_status(advice_id, advice_instance,
api_response)
subscription_status = self._get_subscription_status(
advice_instance.module_name, api_response['metadata']['username'])
if status != 'pending':
# include result of apply/rollback for non-pending advice only
result = self._apply_results(advice_id)
# drop stored progress
_progress = self._progress_file(advice_id)
if os.path.isfile(_progress):
os.unlink(_progress)
if result:
return self.response(
status=status,
subscription=dict(
object_cache=subscription_status
),
upgrade_url=get_subscription_upgrade_url(
advice_instance.module_name,
api_response['metadata']['username']),
**stages,
data=result)
return self.response(
status=status,
subscription=dict(
object_cache=subscription_status
),
upgrade_url=get_subscription_upgrade_url(
advice_instance.module_name,
api_response['metadata']['username']),
**stages
)
def update_advices_metadata(self):
"""
Updates incompatibilities only, but could be extended to more magic
how to make microservice know that users/domains/websites no longer exist
"""
current_advices = self.adviser_client.advice_list(filtered=False, show_all=True)
getter = ClWposGetter()
user_domain_pairs = {}
for advice_object in current_advices:
current_adv_metadata = advice_object.get('metadata')
if not current_adv_metadata:
self.logger.error('Malformed advice metadata, does not have metadata key: %s',
str(advice_object))
continue
username, domain = current_adv_metadata.get('username'), current_adv_metadata.get('domain')
if not all([domain, username]):
self.logger.error('Malformed advice metadata, does not have required username or domain field: %s',
str(current_adv_metadata))
user_domain_pairs[username] = domain
for user, user_domain in user_domain_pairs.items():
try:
updated_metadata = getter.get_updated_extended_metadata(user, user_domain, current_advices)
if not updated_metadata:
logging.warning('Updated metadata for user: %s will not be sent', str(user))
continue
getter.send(updated_metadata)
except Exception as e:
logging.error('Failed to update advice metadata with Smart Advice: %s',
e,
extra={
'fingerprint': e,
})
continue
return self.response(data=_('Advices metadata update finished'))
def _sync_advice(self, username, domain, website):
current_advices = self.adviser_client.advice_list()
# Get IM360 advice cache
im360_advice_list = self.imunify_manager.im360_advice(cache_only=True)
current_advices.extend(im360_advice_list)
advice_hash_map = self.get_advice_hash_map(current_advices)
try:
filtered_advice_list = self.prepare_advices_response(current_advices,
status_from_microservice=True, extended=True)
filtered_advice_list = self.filter_plugin_advices(username, website, domain, filtered_advice_list)
self._run_smart_advice_script(username, website, domain, filtered_advice_list)
except Exception:
self.logger.exception('Smart Advice plugin sync failed for website: %s', str(website))
return
cached_advices_by_site = self._read_advices_cache()
key = f'{domain}{advice_cache_separator}{username}{advice_cache_separator}{website}'
if key not in cached_advices_by_site:
cached_advices_by_site[key] = {}
cached_advices_by_site[key] = advice_hash_map[key]
self._update_advices_cache(cached_advices_by_site)
def _get_cdn_usage_statistics(self, username):
from xray.adviser.awp_provision_api import AWPProvisionAPI
acc_id = get_user_auth_key(username)
cdn_usage = AWPProvisionAPI().awp_client_api.get_usage(acc_id)
return cdn_usage
@username_verification
def analytics_report(self, username, feature, source, event, advice_id=None,
journey_id=None, user_hash=None, variant_id=None):
analytics_data = prepare_system_analytics_data(username, user_hash, journey_id)
report_analytics(data=analytics_data,
advice_id=advice_id,
source=source.lower(),
event=event,
feature=feature,
variant_id=variant_id)
return self.response(data=_('Analytics sent'))
@username_verification
def get_options(self, username):
"""
gets options:
"panel_type": "cpanel",
"panel_url": "https:/cpanel-foo.com/", # or https://plesk-bar.com/
"panel_emails": "root@hosting.com,manager@hostring.com,foo@bar.com",
"upgrade_url": "https://...",
"upgrade_url_cdn": "https://...",
"subscription": dict,
"notifications": dict,
"""
try:
# [("domain.com"),("/path/to/docroot")]
domain = userdomains(username)[0][0]
email = get_user_emails_list(username, domain)
except Exception:
self.logger.exception('Unable to get user emails')
email = ''
try:
subscription = get_subscriptions_info(username)
except Exception:
self.logger.exception('Cannot obtain subscription info')
subscription = {}
try:
im360_advice_notification = self.imunify_manager.get_im360_settings().get('advice_email_notification')
except Exception:
self.logger.exception('Cannot obtain im360 advice notification setting')
im360_advice_notification = False
try:
protection_status = 'active' if self.imunify_manager.get_im360_protection_status(username) is True else 'no'
except Exception:
self.logger.exception('Cannot obtain im360 protection status')
protection_status = 'no'
subscription['protection'] = {'status': protection_status}
notifications = dict(
email_status="disabled_server" if is_smart_advice_notifications_disabled_server_wide() is True else "enabled",
reminders_status="disabled_server" if is_smart_advice_reminders_disabled_server_wide() is True else "enabled",
imunify_email_status="enabled" if im360_advice_notification is True else "disabled_server",
)
return self.response(**asdict(
SmartAdviceOptions(panel_type=getCPName(),
panel_url=self._get_user_awp_link(username),
panel_emails=email,
upgrade_url=get_subscription_upgrade_url('object_cache', username),
upgrade_url_cdn=get_subscription_upgrade_url('cdn', username),
subscription=subscription,
notifications=notifications)
))
@username_verification
def get_limits(self, username):
"""
Retrieves the LVE limits for a specified user.
{
"lve_cpu": {
"limit": int
},
"lve_ep": {
"limit": int
},
"lve_pmem": {
"limit": int
},
"lve_iops": {
"limit": int
},
"lve_io": {
"limit": int
},
"lve_nproc": {
"limit": int
},
}
or
{
"error": str
}
"""
result = get_lve_limits(username)
return json.dumps(result, indent=2)
@username_verification
def get_usage(self, username):
"""
Retrieves the current usage for a specified user.
{
"lve_cpu": {
"usage": int
},
"lve_ep": {
"usage": int
},
"lve_pmem": {
"usage": int
},
"lve_iops": {
"usage": int
},
"lve_io": {
"usage": int
},
"lve_nproc": {
"usage": int
},
}
or
{
"error": str
}
"""
result = get_lve_usage(username)
return json.dumps(result, indent=2)
def sync_advices_wordpress_plugin(self, current_advices=None, im360_cached=True):
if not current_advices:
current_advices = self.adviser_client.advice_list()
# Get IM360 advice fresh data by cron, or return cache
im360_advice_list = self.imunify_manager.im360_advice(cache_only=im360_cached)
current_advices.extend(im360_advice_list)
new_hash_cache_by_site = self.get_advice_hash_map(current_advices)
existing_hash_cache_by_site = self._read_advices_cache()
if not new_hash_cache_by_site and not existing_hash_cache_by_site:
return self.response(data='No advices found on the server')
updated_cache = self._sync_sa_plugin(
new_hash_cache_by_site,
existing_hash_cache_by_site,
current_advices_json=self.prepare_advices_response(current_advices, extended=True)
)
self._update_advices_cache(updated_cache)
return self.response(data='Plugin installed')
def uninstall_wordpress_plugins(self):
"""
Uninstalls all Smart Advice WordPress plugins on server.
"""
existing_hash_cache_by_site = self._read_advices_cache()
if not existing_hash_cache_by_site:
return self.response(data=_('No plugins found on the server'))
for key in existing_hash_cache_by_site:
(domain, username, website) = key.split(advice_cache_separator)
try:
self._run_smart_advice_script(username, website, domain, advices_json=None)
except Exception:
self.logger.exception('Smart advice plugin removal failed for website: %s', str(website))
continue
self._update_advices_cache(new_cached_advices={})
return self.response(data=_('Plugin uninstalled'))
def _sync_sa_plugin(self, new_hash_advice_cache, existing_hash_advice_cache, current_advices_json=None):
logging.info('Synchronization of smart advice plugin begin')
updated_advices = {}
for key, id_status_hash in new_hash_advice_cache.items():
(domain, username, website) = key.split(advice_cache_separator)
if not self._should_sync_website(domain, username, website, id_status_hash,
existing_hash_advice_cache.get(key)):
logging.debug('Website %s does not need update because cache hash matches', key)
updated_advices[key] = id_status_hash
continue
# keep only current user/website advices
if current_advices_json:
filtered_advice_list = self.filter_plugin_advices(username, website, domain, current_advices_json)
else:
filtered_advice_list = None
try:
self._run_smart_advice_script(username, website, domain, filtered_advice_list)
except Exception:
self.logger.exception('Smart advice plugin sync failed for website: %s', str(website))
continue
updated_advices[key] = id_status_hash
# keep tracking of requests that no longer exist in new cache
orphan_hash_records = set(existing_hash_advice_cache) - set(new_hash_advice_cache)
if orphan_hash_records:
# loop over orphan records and remove plugins on those websites
logging.info('Found outdated records in cache, '
'removing wordpress plugins for those websites')
for key in orphan_hash_records:
(domain, username, website) = key.split(advice_cache_separator)
logging.info('Removing outdated cache record %s', key)
try:
self._run_smart_advice_script(username, website, domain, advices_json=None)
except Exception:
self.logger.exception('Smart advice plugin sync failed for website: %s', str(website))
continue
return updated_advices
@staticmethod
def _update_advices_cache(new_cached_advices: dict):
with open(f'{advice_list_cache}.lock', 'a+') as lock_fd, \
filelock(lock_fd), \
open(advice_list_cache, 'w') as f:
logging.debug('Updating advice hash map cache with values %s', new_cached_advices)
json.dump(new_cached_advices, f, indent=2)
def _read_advices_cache(self) -> dict:
cached_advices_by_site = {}
if os.path.exists(advice_list_cache):
try:
with open(advice_list_cache) as f:
cached_advices_by_site = json.load(f)
except Exception:
self.logger.exception('Unable to read advices cache json')
cached_advices_by_site = {}
return cached_advices_by_site
@staticmethod
def _should_sync_website(domain, username, website, current_hash, cached_hash):
if current_hash != cached_hash:
return True
full_website_path = docroot(domain)[0] + website
mu_plugin_dir = get_mu_directory(username, full_website_path)
if not plugin_installed(mu_plugin_dir):
return True
return False
@staticmethod
def make_hash(id_status_pairs):
h = hashlib.md5()
for advice_id, status in id_status_pairs:
h.update(str(advice_id).encode())
h.update(status.encode())
return h.hexdigest()
def get_advice_hash_map(self, advices) -> Dict[str, str]:
"""
Prepare fingerprints for each of the advices generated,
grouped by the website that advice is assigned to.
Result example:
{
"demo-10-194-0-9.traefik.me;demowp;/": "f43c7bcdccc28fc1ac92b2384f35f9dd"
}
Key is combination of domain, usernane and webiste path and
value is hash against list of advices and their current statuses.
"""
websites = {}
for advice in advices:
metadata = advice['metadata']
advice_data = advice['advice']
key = f'{metadata["domain"]}{advice_cache_separator}{metadata["username"]}' \
f'{advice_cache_separator}{metadata["website"]}'
if key not in websites:
websites[key] = []
websites[key].append((advice_data['id'], advice_data['status']))
return {
key: self.make_hash(tuple(sorted(id_status_pairs, key=lambda item: str(item[0]))))
for key, id_status_pairs in websites.items()
}
@staticmethod
def _save_custom_php(path, content):
"""
For easy mocking
"""
with open(path, 'w') as f:
f.write(content)
def _prepare_php_for_plesk(self, username, domain):
clwpos_dir = f'{pwd.getpwnam(username).pw_dir}/.clwpos'
custom_php = '.php-ini'
custom_php_full = os.path.join(clwpos_dir, custom_php)
extra_paths = [
'/opt/alt/php-xray/',
'/opt/cloudlinux-site-optimization-module/',
'/opt/cloudlinux/'
]
updated_setting = None
open_basedir_setting = subprocess.run(
f'/usr/sbin/plesk bin site --show-php-settings {domain} | /usr/bin/grep open_basedir',
text=True, capture_output=True, shell=True)
open_basedir_setting_sanitized = open_basedir_setting.stdout.replace('open_basedir =', '').replace(
'open_basedir=', '').strip()
self.logger.info('Current open_basedir settings: %s', str(open_basedir_setting.stdout))
# empty string means no restrictions; no need to add the extra paths
if len(open_basedir_setting_sanitized) == 0:
updated_setting = 'open_basedir ='
# non-empty string and not "none"
elif len(open_basedir_setting_sanitized) > 0 and open_basedir_setting_sanitized != 'none':
# if 'none' is in the list of values obtained by splitting the string
if 'none' in open_basedir_setting_sanitized.split(':'):
updated_setting = 'open_basedir = none'
elif open_basedir_setting_sanitized.startswith(':'):
updated_setting = 'open_basedir = {WEBSPACEROOT}{/}{:}{TMP}{/}:' + ":".join(extra_paths)
else:
# collect missing extra paths
missing_paths = []
for extra_path in extra_paths:
# extra path not present
if extra_path not in open_basedir_setting.stdout:
missing_paths.append(extra_path)
# some extra paths are missing, settings update is required
if missing_paths:
updated_setting = f'open_basedir = {open_basedir_setting_sanitized}:{":".join(missing_paths)}'
if updated_setting:
self.logger.info('Updating open_basedir setting: %s for domain: %s', updated_setting, domain)
# Both folder for custom PHP ini file and the file itself should be owned by appropriate user. We don't want
# to leave files owned by root in user's home directory.
with drop_privileges(username):
if not os.path.exists(custom_php_full):
if not os.path.isdir(clwpos_dir):
os.mkdir(clwpos_dir, mode=0o700)
self._save_custom_php(custom_php_full, updated_setting)
result = subprocess.run(['/usr/sbin/plesk', 'bin', 'site', '--update-php-settings', domain, '-settings',
custom_php_full],
text=True, capture_output=True)
self.logger.info('Plesk settings update result: %s', str(result.stdout))
self.logger.error('open_basedir setting updated for domain "%s" from "%s" to "%s"',
str(domain),
str(open_basedir_setting.stdout),
str(updated_setting))
def _uninstall_wordpress_plugin(self, username, website, domain):
"""
Uninstall WordPress SmartAdvice plugin.
"""
try:
json_string = self.response(data=[])
attrs = self._prepare_smart_advice_script_args(username, website, domain, json_string, 'true')
except ValueError:
# fixme: why?
return
command = self._get_smart_advice_script_cmd(username, attrs)
logging.debug('Uninstalling Smart Advice plugin for website %s with advices: %s',
website, json_string)
result = subprocess.run(command, text=True, capture_output=True)
if result.returncode != 0:
self.logger.error('Smart advice plugin uninstalling failed with exit code: %s, stdout: %s, stderr: %s',
str(result.returncode),
str(result.stdout),
str(result.stderr),
extra={
'fingerprint': f'smart-advice-plugin-sh-returncode-{result.returncode}',
})
raise SmartAdvicePluginError(_('Smart Advice plugin uninstalling failed'))
def _install_wordpress_plugin(self, username, website, domain, advices_json):
"""
Install plugin and sync advice
plugin creation is part of main process -> to handle quota exceeded
"""
try:
json_string = self.response(data=advices_json)
attrs = self._prepare_smart_advice_script_args(username, website, domain, json_string, 'false')
except ValueError:
return
command = self._get_smart_advice_script_cmd(username, attrs)
logging.info('Updating Smart Advice plugin for website %s with advices: %s',
website, json_string)
result = subprocess.run(command, text=True, capture_output=True)
if result.returncode != 0:
self.logger.error('Smart advice plugin failed with exit code: %s, stdout: %s, stderr: %s',
str(result.returncode),
str(result.stdout),
str(result.stderr),
extra={
'fingerprint': f'smart-advice-plugin-sh-returncode-{result.returncode}',
})
raise SmartAdvicePluginError(_('Smart Advice plugin installation failed'))
def _should_wordpress_plugin_installed(self):
is_smart_advice_disabled = is_smart_advice_wordpress_plugin_disabled_server_wide()
is_im360_mu_disabled = self.imunify_manager.is_im360_mu_plugin_disabled_server_wide()
self.logger.info("Checking whether to install MU plugin: "
"is_smart_advice_disabled=%s, "
"is_im360_mu_disabled=%s",
str(is_smart_advice_disabled),
str(is_im360_mu_disabled))
return not is_smart_advice_disabled or not is_im360_mu_disabled
def _run_smart_advice_script(self, username, website, domain, advices_json=None):
def _filter_by_advice_type(advices_data):
if not advices_data:
return advices_data
filtered_data = advices_data
if is_smart_advice_wordpress_plugin_disabled_server_wide():
# leave only im360 advices
filtered_data = [item for item in filtered_data if item.get("advice", {}).get("type") == ADV_TYPE]
self.logger.info("AccelerateWP advices are filtered, advices=%s", str(filtered_data))
if self.imunify_manager.is_im360_mu_plugin_disabled_server_wide():
# leave only non-im360 advices
filtered_data = [item for item in filtered_data if item.get("advice", {}).get("type") != ADV_TYPE]
self.logger.info("IM360 advices are filtered, advices=%s", str(filtered_data))
self.logger.info("Filtered advices for Smart Advice plugin, advices=%s", str(filtered_data))
return filtered_data
filtered = _filter_by_advice_type(advices_json)
# Uninstall when none or empty list
if not filtered:
self._uninstall_wordpress_plugin(username, website, domain)
# install only if wordpress plugin feature is enabled by hoster
elif self._should_wordpress_plugin_installed():
self.logger.info("Smart Advice plugin installation is allowed, advices list=%s",
str(advices_json))
self._install_wordpress_plugin(username, website, domain, filtered)
def _prepare_smart_advice_script_args(self, username, website, domain: List[str], advices_json: str,
uninstall='false') -> List[str]:
panel = getCPName()
script_name = 'run.py'
script_path = f'/opt/alt/php-xray/php/smart-advice-plugin/{script_name}'
panel_user_awp_link = self._get_user_awp_link(username)
self.logger.info('Got panel login link: %s', panel_user_awp_link)
try:
email = get_user_emails_list(username, domain)
except Exception:
self.logger.exception('Unable to get user emails')
email = ''
full_website_path = docroot(domain)[0] + website
if not is_wp_path(full_website_path):
self.logger.info('[Smart Advice Plugin]: wordpress: %s does not exist, skipped',
full_website_path)
raise ValueError(_('Non-existing wordpress site'))
php_info = get_user_php_version(username)
for php_data in php_info:
if php_data.get('vhost') == domain:
php_binary_path = php_data.get('php_binary')
break
else:
self.logger.error('Php data for domain %s was not found in %s', domain, str(php_info))
raise SmartAdvicePluginError(_('Unable to get php version for domain %s') % domain)
if not php_binary_path:
self.logger.error('Php version for user: %s was not obtained',
username)
raise SmartAdvicePluginError(_('Php binary was not identified for domain %s') % domain)
if panel == 'Plesk':
try:
self._prepare_php_for_plesk(username, domain)
except Exception:
self.logger.exception('Setting up php for Plesk for SA plugin failed')
mu_plugin_dir = get_mu_directory(username, full_website_path)
if not mu_plugin_dir:
raise SmartAdvicePluginError(_('Unable to detect MU plugin directory'))
# mu_plugins dir creation is part of main process -> to handle quota exceeded
create_mu_plugins_dir_if_not_exist(username, mu_plugin_dir)
# ATTENTION: this order is important in further code, add new atts with caution
res = [
script_path,
php_binary_path,
full_website_path,
mu_plugin_dir,
advices_json,
email,
panel_user_awp_link,
uninstall,
]
return res
@staticmethod
def _get_smart_advice_script_cmd(username: str, attrs: List[str]) -> List[str]:
if is_panel_feature_supported(Feature.CAGEFS):
res = ['/sbin/cagefs_enter_user', username]
res.extend(attrs)
else:
attrs[3] = "'" + attrs[3] + "'" # escape json string
command_str = ' '.join(attrs)
res = ['sudo', '-u', username, '-s', '/bin/bash', '-c', command_str]
return res
def get_agreement_text(self, feature):
"""
Method with handles work with agreements.
"""
user_context = get_xray_exec_user()
if not user_context:
raise SmartAdvicePluginError(_('This command can only be executed as user'))
return json.dumps({
'result': 'success',
'timestamp': timestamp(),
'text': get_license_agreement_text(feature, user_context)
})
def create_subscription(self, advice_id):
"""
Proxy method to AWP which creates subscription
listening for the status of module and automatic advice appy.
"""
user_context = get_xray_exec_user()
if not user_context:
raise SmartAdvicePluginError(_('This command can only be executed as user'))
resp, advice_instance = self.get_detailed_advice(advice_id)
result = subprocess.run([
'cloudlinux-awp-user',
'--user', str(user_context),
'subscription',
'--listen',
'--advice-id', str(advice_id),
'--feature', advice_instance.module_name
], text=True, capture_output=True)
if result.returncode != 0:
self.logger.error('Create subscription smart advice plugin failed with exit code: '
'%s, stdout: %s, stderr: %s',
str(result.returncode),
str(result.stdout),
str(result.stderr),
extra={
'fingerprint': f'smart-advice-plugin-subscription-returncode-{result.returncode}',
})
raise SmartAdvicePluginError(_('cloudlinux-awp-user call failed %s') % result.stderr)
Zerion Mini Shell 1.0