ok

Mini Shell

Direktori : /opt/cloudlinux/venv/lib/python3.11/site-packages/xray/agent/__pycache__/
Upload File :
Current File : //opt/cloudlinux/venv/lib/python3.11/site-packages/xray/agent/__pycache__/daemon.cpython-311.pyc

�

��f�J���dZddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlm
Z
mZmZddlmZmZmZmZddlmZddlZddlmZddlmZdd	lmZdd
lmZddl m!Z!m"Z"ddl#m$Z$dd
l%m&Z&m'Z'm(Z(ddl)m*Z*ddl+m,Z,ddl-m.Z.m/Z/ej0r
ddl1m2Z2m3Z3m4Z4eGd�d����Z5ej6e7��Z8Gd�d��Z9dS)zJ
This module contains contains classes implementing X-Ray Agent behaviour
�N)�Thread�current_thread�Lock)�Any�Optional�Dict�Tuple)�	dataclass�)�BoundedThreadExecutor)�gettext)�
get_client)�local_tasks_storage)�	XRayError�XRayAPIError)�
FaultDetector)�open_local_storage�flush_memory_storage�get_task_ids)�Task)�
extract_creds)�dbm_storage�get_current_cpu_throttling_time)�
SendClient�SmartAdviceAPIClient�	APIClientc�$�eZdZUded<eed<dS)�APIDataContainerr�client�taskN)�__name__�
__module__�__qualname__�__annotations__r���H/opt/cloudlinux/venv/lib64/python3.11/site-packages/xray/agent/daemon.pyrr6s%�����������
�J�J�J�J�Jr&rc	��eZdZdZdZdZdZej��dzej��dzej��dzfd�Z	d	�Z
d
�Zd2d�Zd
�Z
		d3dejdedeeddfd�Zd�Zdedefd�Zdededefd�Zdejdeddfd�Zdejdefd�Zdedd fd!�Zdeded e ffd"�Z!d4d#�Z"deddfd$�Z#d%ed&e defd'�Z$	d2d(d)d*ed+eeddfd,�Z%d-e ddfd.�Z&d/�Z'e(d0edefd1���Z)dS)5�Agentz
    X-Ray Agent class
    ��<i�����c���||_||_||_||_t	jtjtjh��td��}||j���|_	td��}|��|_
t��|_t��|_
t��|_t!��|_d|_d|_dS)N�api_req)�	system_id�adviser)�sys_id�maxqueuesize�max_connections�max_workers�signal�pthread_sigmask�	SIG_BLOCK�SIGUSR2r�send_client�adviser_client�task_client_objectr�api_data_cache_lock�dict�api_data_cacher�fault_detector�signal_handler_thread�flusher_thread)�selfr1r5r6r4�clickhouse_client_object�adviser_client_objects       r'�__init__zAgent.__init__Hs��� ���(���.���&���	��v�/�&�.�1A�B�B�B�=G�y�<Q�<Q� �'?�'?�$�+�'V�'V�'V���CM�i�CX�CX��4I�4I�4K�4K���:D�,�,���#'�6�6�� �;?�6�6���+�o�o���7;��"�04����r&c�x�tjtjh��}tjd|j��|���t��tjd��	tj	|jtj��dS#t$rtjd��YdSwxYw)Nz?Received SIGUSR2 from pid=%s, flushing database storage on diskz;Sending signal back to process that requested storage flushz5Process that requested storage flush no longer exists)r7�sigwaitinfor:�logging�info�si_pid�_flush_mongodb_countersr�os�kill�OSError�warning)rD�siginfos  r'�_wait_for_sigusr2zAgent._wait_for_sigusr2hs����$�f�n�%5�6�6����9�:A�.�	J�	J�	J�	
�$�$�&�&�&�������R�S�S�S�	U��G�G�N�F�N�3�3�3�3�3���	U�	U�	U��O�S�T�T�T�T�T�T�	U���s�1$B�B9�8B9c���tjtjtjh��		|���n$#t
$rt
jd��YnwxYw�;)z�
        Setup SIGUSR2 handler that starts in-memory
        storage flush when received.

        When flushed, send SIGUSR2 back to the process that send signal.
        Tz4Unable to process signal, see traceback for details.)r7r8r9r:rS�	ExceptionrJ�	exception)rDs r'�_setup_signal_handlerzAgent._setup_signal_handlervs}��	��v�/�&�.�1A�B�B�B�	Z�
Z��&�&�(�(�(�(���
Z�
Z�
Z��!�"X�Y�Y�Y�Y�Y�
Z����	Zs�A�A#�"A#Nc��|�|gn
t��}|D]�}t�d|��	|�|��\}}n%#t$rtjd|��Y�XwxYwt|��5}|�|j	���ddd��n#1swxYwY|j
dkr|�||j����|�||j|j
����dS)Nz7Updating task requests counters in mongo for task_id=%sz Unable to get client and task %s��next_request_id�time)r�loggerrK�get_cached_or_loadrrJrQr�update_with_local_datarZ�
tracing_by�
update_counts�
request_count�
tracing_count)rD�task_id�tasks_to_flush�	apiclientr �storages      r'rMzAgent._flush_mongodb_counters�s{��&-�&9�'���|�~�~��%�	7�	7�G��K�K�Q�SZ�[�[�[�
�"&�"9�"9�'�"B�"B��	�4�4���
�
�
��� B�G�L�L�L���
����
$�G�,�,�
U���+�+�G�<S�+�T�T�T�
U�
U�
U�
U�
U�
U�
U�
U�
U�
U�
U����
U�
U�
U�
U���&�(�(��"�"�9�d�.@�A�A�A�A��"�"�9�d�.@�#'�#5�7�7�7�7�+	7�	7s#�A
�
A/�.A/�B)�)B-	�0B-	c��d}d}	tj|j��tj��|z
|jkr||���t��tj��}tj��|z
|jkr'|���tj��}ntd�����)z]
        This method flushes data from memory to
        local storage periodically.
        rTF)�remove)r[�sleep�COUNTERS_FLUSH_INTERVAL�MONGO_FLUSH_INTERVALrMr�CLEANUP_INTERVAL�cleanup_api_data_cache)rD�last_mongo_flush_time�last_api_data_cache_cleanups   r'�_flusherzAgent._flusher�s���
!"��&'�#�	3��J�t�3�4�4�4��y�{�{�2�2�T�5N�N�N��,�,�.�.�.�$�&�&�&�(,�	���%��9�;�;�!<�<�t�?T�T�T��/�/�1�1�1�26�)�+�+�/��$�E�2�2�2�2�	3r&F�sock�background_routine�loops�returnc���t�d��|rft|j���|_|j���t|j���|_|j���t|j	|j
���5}t|jd���5}|�|r�n	|���\}}	|�
|j||��nC#tj$r1t�d��|���YnwxYw|�|dz}|�|��n��ddd��n#1swxYwYddd��dS#1swxYwYdS)	z(
        Start listening socket
        zStarting daemon)�target)r6r4rNTzxRequest %s was rejected because our connections thread pool is full of tasks. Increase max_connections in configuration.r)r\rKrrWrB�startrprCrr6r4r5�accept�submit�handle_incoming_connection�queue�Full�error�close)rDrqrrrs�workers_pool�connections_pool�
connection�addresss        r'rwzAgent.start�s8��	���%�&�&�&��	(�)/�t�7Q�)R�)R�)R�D�&��&�,�,�.�.�.�#)��
�">�">�">�D����%�%�'�'�'�
"� �,�!�.�0�0�0�	�3?�!� �0��
�
�
�	�"� �,�%�,�,�,�&*�k�k�m�m�#�
�G�'�$�+�+��7�"�L�2�2�2�2���z�'�'�'��L�L�"`�a�a�a��$�$�&�&�&�&�&�'����
�$��Q�J�E�!�,�%�,�,�,�	�	�	�	�	�	�	�	�	�	�	����	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	����	�	�	�	�	�	sZ�E%�5E
�C1�0E
�1=D1�.E
�0D1�1E
�E%�
E	�E%�E	�E%�%E)�,E)c�D�|�||��\|d<|d<dS)zj
        Calculate throttling values and update given data
        with resulting throttling stat
        �hitting_limits�throttled_timeN)rA)rD�data�t_key�	cpu_values    r'�add_limit_faultszAgent.add_limit_faults�s5��
:>�9L�9L��9�:�:�6��
���%5� 6� 6� 6r&�php_pid�	cpu_usagec��t�d||��|j�||��|j���dS)z�
        Called when php request starts and sends us welcome
        request meaning that request started on the php side.
        z;Received request init trigger from php=%s with cpu_usage=%sN)r\rKrA�save�flush)rDr�r�s   r'�_handle_request_initzAgent._handle_request_init�sX��
	���Q��Y�	(�	(�	(�	
�� � ��)�4�4�4���!�!�#�#�#�#�#r&�request_datac�0�|�d���|�|||��t�dt	��j|�d��|�d����|�|��dS)Nr�z&[%s] Processing trace for task %s (%s)�tracing_task_id�url)�getr�r\rKr�name�process_request_data)rDr�r�r�s    r'�_handle_request_endzAgent._handle_request_end�s������,�-�-�5��!�!�,���C�C�C����<�"�$�$�)� �$�$�%6�7�7� �$�$�U�+�+�	-�	-�	-�	
�!�!�,�/�/�/�/�/r&r�rc��t|��\}}}t|��}|�d���}	|�|��}nn#tj$r\}	t�dt|	��dt��j
i���Yd}	~	|���dSd}	~	wwxYw	|���n#|���wxYw	|�|�|j
||���dS|�|j|||���dS#tj$rt�d	��YdSwxYw)
a
        Handle incoming connection
        :param connection: socket object usable to
                           send and receive data on the connection
        :param workers_pool: pool where we can place
                             tasks for the futher processing
        �ignore)�errorszJSON decode failed: %s�t_name��extraN)r�r�)r�r�r�z�Request %s was rejected because our workers thread pool is full of tasks. Increase queuemaxsize or max_threads in configuration.)rr�makefile�
read_input�json�JSONDecodeErrorr\r}�strrr�r~ryr�r�r{r|)
rDr�r�_pid�_uid�_gid�current_cpu�fileobj�
input_data�es
          r'rzz Agent.handle_incoming_connection�s���)��4�4���d�D�5�d�;�;��!+�!4�!4�H�!4�!E�!E��		�����1�1�J�J���#�	�	�	��L�L�1�!�!�f�f�X�~�7G�7G�7L�,M�
�
O�
O�
O��F�F�F�
������������	����
�
��������J����������	e��!��#�#�D�$=�t�.9�$�;�;�;�;�;��#�#�D�$<�d�.9�
�$�T�T�T�T�T���z�	e�	e�	e��L�L�d�
e�
e�
e�
e�
e�
e�	e���sG�A�C�B;�=B6�C�6B;�;C�C*�.D/�D/�/)E�E�fileioc��|���}t�d|��t|�����dkrdStj|���d���S)zy
        Read input data and return decoded json
        :param fileio: a file-like object providing read method
        zI received data: %srNF)�strict)�readr\�debug�len�stripr��loads)rDr�r�s   r'r�zAgent.read_input"sc��
�{�{�}�}�����*�D�1�1�1��t�z�z�|�|����!�!��F��z�$�*�*�,�,�u�5�5�5�5r&�fake_task_idrc���	tt��5}	||���}nA#t$r4t	td��||���d�����wxYw	ddd��n#1swxYwYn>#t$r1}t	td��t|��z���d}~wwxYw|�	|j
|���S)z<
        Initialize MongoDB client for current task
        z9Cannot resolve tracing_task_id: no match found in storage)�id�all_idsr�Nz"Cannot resolve tracing_task_id: %s)r1r�)rr�decode�KeyErrorr�_�keys�RuntimeErrorr�r=r3)rDr��task_storage�real_idr�s     r'�instantiate_mongo_clientzAgent.instantiate_mongo_client-so��

	N��0�1�1�
@�\�@�*�<�8�?�?�A�A�G�G���@�@�@�#��U�V�V�%1�*6�*;�*;�*=�*=�?�?�@�@�@�@�@�����
@�
@�
@�
@�
@�
@�
@�
@�
@�
@�
@����
@�
@�
@�
@����	N�	N�	N��A�B�C�C�c�!�f�f�L�M�M�M�����	N�����&�&���7>�'�@�@�	@sN�B
�B�3�B�>A1�1B�5B
�B�B
�B�	B
�
C�,C�Cc�\�t�d|j��|j�|��}|�|j|jfS|�|��}|���}t�d|||��t||���|j|<||fS)z�
        Returns a client and task from cache of API data or
        initialize client and GET task from MongoDB and add to cache
        zCached API data: %sNz,Adding new container in cache: %s --> %s, %s)rr )	r\r�r@r�rr r��get_taskr)rDr��cached_datare�_ts     r'r]zAgent.get_cached_or_loadBs���
	���*�D�,?�@�@�@��)�-�-�l�;�;���"��%�{�'7�7�7��1�1�,�?�?�	�
�
�
�
!�
!�����C�&��I�	7�	7�	7�,<�I�BD�-F�-F�-F���L�)��"�}�r&c��	tt��5}d�|���D��}ddd��n#1swxYwYn+#t$rt�d��YdSwxYwt
|j�����D]c}|j5||jvr9||vr5t�	d|��|j�
|��ddd��n#1swxYwY�ddS)z�
        Cleanup an API data im-memory cache dict in order not store
        inactive (stopped, already completed) tasks there
        c�6�g|]}|�����Sr%)r�)�.0�ks  r'�
<listcomp>z0Agent.cleanup_api_data_cache.<locals>.<listcomp>\s ��H�H�H�q����
�
�H�H�Hr&Nz,Unable to cleanup cache, storage unavailablez!Cleaning up inactive container %s)rrr�r�r\rQ�listr@r>rK�pop)rDr��active_tasks�_tasks    r'rmzAgent.cleanup_api_data_cacheUs���
	��0�1�1�
I�\�H�H�L�4E�4E�4G�4G�H�H�H��
I�
I�
I�
I�
I�
I�
I�
I�
I�
I�
I����
I�
I�
I�
I����	�	�	��N�N�>�
@�
@�
@��F�F�	����
�$�-�2�2�4�4�5�5�	3�	3�E��)�
3�
3��D�/�/�/�E��4M�4M��K�K� C�U�K�K�K��'�+�+�E�2�2�2�
3�
3�
3�
3�
3�
3�
3�
3�
3�
3�
3����
3�
3�
3�
3��	3�	3sF�A
�A�A
�A�A
�A�	A
�
$A5�4A5�'AC6�6C:	�=C:	c��|�|d��\}}t�d|j��t	|d|j���5}|�|j���|jdkr(t�d��	ddd��dS|�	||��}t�d��|�
|��	t�d	��|�|��n#t$rYnwxYw|xjd
z
c_|�|j���ddd��n#1swxYwY|jr|�
|j��|jdkr"|jdkr|�|��dSdSdS)z�
        Increment request ID in /usr/share/alt-php-xray/requests/{tracing_task_id} file
        Substitute request_id and tracing_task_id in request_data.
        Send request_data to ClickHouse
        :param request_data: original request data
        r�zProcessing task: %s)r�rYrz*Tracing count is 0, nothing should be doneNzSending to ClickHousezSending to SmartAdvicerr[)r]r\rKrcr�	is_manualr^rZrb�update_request_datar;r<rrM�fake_idr_�
complete_task)rDr�r�r rf�updated_request_datas      r'r�zAgent.process_request_datahsZ���)�)�,�7H�*I�J�J���4����)�4�<�8�8�8�
��->� ?�&*�n�6�6�6�	Q�9@��'�'��8O�'�P�P�P��!�Q�&�&����H�I�I�I��	Q�	Q�	Q�	Q�	Q�	Q�	Q�	Q�$(�#;�#;�L�<@�$B�$B� �
�K�K�/�0�0�0����1�2�2�2�
����4�5�5�5��#�#�$8�9�9�9�9���
�
�
���
����

�#�#�q�(�#�#��'�'��8O�'�P�P�P�1	Q�	Q�	Q�	Q�	Q�	Q�	Q�	Q�	Q�	Q�	Q����	Q�	Q�	Q�	Q�4�>�	7��(�(���6�6�6��?�f�$�$��);�q�)@�)@����t�$�$�$�$�$�%�$�)@�)@s>�AE'�*AE'�0/D �E'� 
D-�*E'�,D-�-.E'�'E+�.E+r�r c��|jdz|d<|j|d<|dD]C}|jdz|d<|j|d<|ddkr|�|d��|d<�Dt�d|�d��|�d����t�d	|��|S)
z�
        Substitute request_id and tracing_task_id
        :param data: original input
        :param task: a Task instance
        :return: updated input
        r�
request_idr�r��type�mysql_query�queryz5Input updated: tracing_task_id = %s & request_id = %szFull updated input %s)rarc�hide_symbolsr\rKr�r�)rDr�r �items    r'r�zAgent.update_request_data�s���"�/�!�3��\��"&�,��
����L�	A�	A�D�!%�!3�a�!7�D���&*�l�D�"�#��F�|�}�,�,� $� 1� 1�$�w�-� @� @��W�
�����K����"3�4�4�d�h�h�|�6L�6L�	N�	N�	N����,�d�3�3�3��r&rzAPIClient instancerarbc�4�|�||���dS)z:
        Update task counters in mongodb instance
        )rbraN)�update_counts_only)rDrrarbs    r'r`zAgent.update_counts�s0��	�!�!�
�0=�	"�	?�	?�	?�	?�	?r&r�c��t�d|j��tjd��|�|j��dS)z_
        Stop and complete request_qty task
        :param _task: tracing task to stop
        zTask %s should be completedrN)r\rKrcr[ri�_run_complete_task_cmd)rDr�s  r'r�zAgent.complete_task�sM��
	���1���	(�	(�	(�	
�
�1�
�
�
��#�#�E�M�2�2�2�2�2r&c�D�tjddd|jd|g��dS)Nzcloudlinux-xray-manager�stopz--system_idz--tracing_task_id)�
subprocess�check_outputr3)rDrcs  r'r�zAgent._run_complete_task_cmd�s7����%�v��4�;���!
�	�	�	�	�	r&r�c�\�d�}tjd��}tj|||��S)zA
        Sanitize data in single quotes from MySQL query
        c���|�d��}|�d��}d�}tjd||�d����}|�|�|p|��S)zL
            Works with whole string in single or double quotes
            �quote�truncc�b�|�d��rdS|�d��rdSdS)zE
                Works with characters inside quotes
                �digit�0�symbol�xN)�group)�inner_ms r'�
inner_replz8Agent.hide_symbols.<locals>.replacer.<locals>.inner_repl�s@���=�=��)�)���3��]�]�8�,�,���3��r&z.((?P<digit>\d)|(?P<symbol>[^0-9_:;\-/',. \\]))�in_quote)r��re�sub)�m�q�tr��	sanitizeds     r'�replacerz$Agent.hide_symbols.<locals>.replacer�sz������ � �A����� � �A�
�
�
���P�)�1�7�7�:�+>�+>�@�@�I��,��,�A�F��,�,�,r&zH(?P<quote>['"])(?P<in_quote>.*?)((?<![\\|/])(?P=quote)|(?P<trunc>\.{3})))r��compiler�)r�r��patterns   r'r�zAgent.hide_symbols�s7��
	-�	-�	-�2�*�l�m�m���v�g�x��5�5�5r&)N)FN)rtN)*r!r"r#�__doc__rjrkrl�psutil�	cpu_countrGrSrWrMrp�socket�boolr�intrwr�r�r?r�rrz�io�
TextIOBaserr�r�r�r	rr]rmr�r�r`r�r��staticmethodr�r%r&r'r)r)?s��������!������"2��!1�!3�!3�a�!7�-�V�-�/�/�!�3�.�f�.�0�0�2�5�5�5�5�5�@U�U�U�Z�Z�Z�7�7�7�7�43�3�3�.*/�%)�)�)��M�)�"&�)��c�]�)�/3�)�)�)�)�V���
$�C�
$�C�
$�
$�
$�
$�0�3�0�3�0�d�0�0�0�0�!e�V�]�!e�Rg�!e�lp�!e�!e�!e�!e�F	6���	6�3�	6�	6�	6�	6�@�/2�@�7B�@�@�@�@�*�s��u�[�$�=N�7O�����&3�3�3�3�&&%��&%�$�&%�&%�&%�&%�P���D��T�����,6:�?�?�$8�?�%(�?�%-�c�]�?�>B�?�?�?�?�	3�4�	3�D�	3�	3�	3�	3�����6�#�6�#�6�6�6��\�6�6�6r&r)):r�r�r�rJrNr{r�r7r�r�r[�typing�	threadingrrrrrrr	�dataclassesr
r��executorr�xrayr
r��xray.apiclientr�xray.internal.constantsr�xray.internal.exceptionsrrrAr�xray.internal.local_countersrrr�xray.internal.typesr�xray.internal.user_plugin_utilsr�xray.internal.utilsrr�
TYPE_CHECKING�xray.apiclient.api_clientrrrr�	getLoggerr!r\r)r%r&r'�<module>rs�����
�	�	�	���������	�	�	�	�����	�	�	�	�
�
�
�
�
�
�
�
���������
�
�
�
�2�2�2�2�2�2�2�2�2�2�-�-�-�-�-�-�-�-�-�-�-�-�!�!�!�!�!�!�
�
�
�
�+�+�+�+�+�+�������%�%�%�%�%�%�7�7�7�7�7�7�<�<�<�<�<�<�<�<�)�)�)�)�)�)�����������
%�$�$�$�$�$�9�9�9�9�9�9���������

������������������������

��	�8�	$�	$��_6�_6�_6�_6�_6�_6�_6�_6�_6�_6r&

Zerion Mini Shell 1.0