ok
Mini Shell
"""
This program is free software: you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License,
or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Copyright © 2019 Cloud Linux Software Inc.
This software is also available under ImunifyAV commercial license,
see <https://www.imunify360.com/legal/eula>
"""
import asyncio
import json
import logging
import os
import subprocess
import time
from contextlib import suppress
from pathlib import Path
from defence360agent.contracts.config import (
Core as CoreConfig,
Malware as MalwareConfig,
MalwareScanIntensity,
MalwareSignatures,
get_rapid_rescan_frequency,
)
from defence360agent.contracts.license import LicenseCLN
from defence360agent.utils import resource_limits
from imav.contracts.config import MalwareTune
from imav.malwarelib.config import (
AIBOLIT_SCAN_INTENSITY_KEY,
MalwareScanType,
)
from imav.malwarelib.scan import ScanFailedError
from imav.malwarelib.scan.ai_bolit import AIBOLIT, AIBOLIT_PATH
from imav.malwarelib.scan.ai_bolit.detached import AiBolitDetachedDir
from imav.malwarelib.scan.ai_bolit.report import parse_report_json
from imav.malwarelib.scan.crontab import crontab_path, in_crontab
from imav.malwarelib.utils import get_memory
logger = logging.getLogger(__name__)
class AiBolitError(ScanFailedError):
pass
class AiBolit:
def __init__(self, scan_id=None):
self.cmd = None
self.scan_id = scan_id
def _cmd(
self,
filename,
intensity_ram,
progress_path,
*,
scan_type: str,
scan_path=None,
scan_id=None,
db_dir=None,
detect_elf=None,
exclude_patterns=None,
follow_symlinks=None,
file_patterns=None,
use_filters=True,
json_report_path=None,
csv_report_path=None,
):
"""
:param detect_elf: True - detect as malicious
False - detect as suspicious
None - do nothing
"""
self.scan_id = scan_id
cmd = [
"/opt/ai-bolit/wrapper",
AIBOLIT_PATH,
"--smart",
"--deobfuscate",
"--avdb",
MalwareSignatures.AI_BOLIT_HOSTER,
"--no-html",
"--memory",
get_memory(intensity_ram),
"--progress",
progress_path,
*(["--use-filters"] if use_filters else []),
*(
["--use-heuristics"]
if detect_elf is True
else ["--use-heuristics-suspicious"]
if detect_elf is False
else []
),
*(
# Note: AI-BOLIT will check that HyperScan DB version
# is the same as `--avdb` (and will skip HS with a warning
# if they differ), so we don't have to do any
# race-condition-prone checks here in the Agent.
["--hs", MalwareSignatures.AI_BOLIT_HYPERSCAN]
if MalwareConfig.HYPERSCAN
else []
),
]
if scan_path and filename or (not scan_path and not filename):
raise TypeError(
"Ai-Bolit cmd generation error, cannot select from finder "
"and filelist."
"scan_path: {}, filename: {}".format(scan_path, filename)
)
in_crontabs = False
if scan_path is not None:
if not MalwareConfig.CRONTABS_SCAN_ENABLED:
exclude_crontab = [os.path.join(str(crontab_path()), "*")]
if exclude_patterns:
exclude_crontab.append(exclude_patterns)
exclude_patterns = ",".join(exclude_crontab)
else:
in_crontabs = in_crontab(Path(scan_path))
# Finder
cmd.extend(
[
"--path",
scan_path,
*(["--follow-symlink"] if follow_symlinks else []),
*(
["--ignore-filenames", exclude_patterns]
if exclude_patterns
else []
),
*(
["--only-filepaths", file_patterns]
if file_patterns is not None
else []
),
"--ignore-quarantine",
"--use-template-in-path",
"--skip-imunify360-storage",
]
)
if not in_crontabs:
cmd.extend(["--skip-system-owner"])
else:
# Filelist
cmd.extend(["--listing", filename])
if scan_type == MalwareScanType.MODSEC:
return cmd
cmd.append("--with-suspicious")
cmd.extend(["--size", str(MalwareConfig.MAX_SIGNATURE_SIZE_TO_SCAN)])
if MalwareConfig.CLOUD_ASSISTED_SCAN:
if db_dir is not None and not in_crontabs:
cmd.extend(
[
"--rapid-account-scan",
db_dir,
"--rapid-scan-rescan-frequency",
str(get_rapid_rescan_frequency()),
]
)
cmd.extend(
[
"--cloudscan-size",
str(MalwareConfig.MAX_CLOUDSCAN_SIZE_TO_SCAN),
]
)
if scan_type in (
MalwareScanType.BACKGROUND,
MalwareScanType.ON_DEMAND,
MalwareScanType.USER,
):
cmd.append("--encode-b64-fn")
cmd.extend(["--detached", scan_id])
if MalwareTune.USE_JSON_REPORT:
cmd.extend(["--json_report", json_report_path])
else:
cmd.extend(["--csv_report", csv_report_path])
# Do not print progress data to stdout,
# because special terminal characters clutter the output
# and we don't actually need it.
# NOTE: The typo is in ai-bolit (should be "quiet").
cmd.append("--quite")
else:
cmd.extend(["--json_report", ".", "--json-stdout"])
logger.info(cmd)
return cmd
@staticmethod
def get_updated_environment():
if MalwareConfig.CLOUD_ASSISTED_SCAN:
environment = os.environ.copy()
environment["CLOUD_ASSIST"] = str(LicenseCLN.get_server_id())
return environment
return None
@staticmethod
def _generate_progress_file():
return os.path.join(
CoreConfig.TMPDIR,
"progress_file_{}".format(int(time.time() * 10e6)),
)
async def scan(
self,
file,
*,
scan_type: str,
intensity_cpu=None,
intensity_io=None,
intensity_ram=None,
detect_elf=None,
use_filters=True,
scan_id=None,
db_dir=None,
scan_path=None,
exclude_patterns=None,
follow_symlinks=None,
file_patterns=None,
**_,
):
"""
:param file: path to file with list of paths to scan
:param intensity_cpu: [inverse] niceness level of the scan.
The higher the number the more priority the process
gets (more cpu)
:param intensity_io: [inverse] ioniceness level of the scan.
Higher number means more disk time may be provided
in a given period
:param intensity_ram: memory value
:param detect_elf: enable binary malware (elf) detection
:param use_filters: apply ignore filters to list of scanning files
:param scan_type: type of scan
:param scan_id: id of scan
:param db_dir: path to rapid scan database
:param scan_path: str with scan path (templates allowed)
:param exclude_patterns: patterns of filenames to ignore
:param follow_symlinks: bool, if True -> follow symlinks
:param file_patterns: patterns of filenames to scan
:raise CancelledError: when scan was cancelled
:return iterator: parsed report
"""
self.scan_id = scan_id
intensity_cpu = intensity_cpu or MalwareScanIntensity.CPU
intensity_io = intensity_io or MalwareScanIntensity.IO
intensity_ram = intensity_ram or MalwareScanIntensity.RAM
detached = scan_type in (
MalwareScanType.ON_DEMAND,
MalwareScanType.BACKGROUND,
MalwareScanType.USER,
)
if detached:
assert scan_id
with AiBolitDetachedDir(
self.scan_id,
tmp_listing_file=file,
) as work_dir:
cmd = self._cmd(
str(work_dir.listing_file) if file else None,
intensity_ram,
str(work_dir.progress_file),
scan_type=scan_type,
scan_id=scan_id,
db_dir=db_dir,
detect_elf=detect_elf,
exclude_patterns=exclude_patterns,
follow_symlinks=follow_symlinks,
scan_path=scan_path,
file_patterns=file_patterns,
json_report_path=str(work_dir.json_report_path),
csv_report_path=str(work_dir.csv_report_path),
)
scan_info = {"cmd": cmd, "scan_type": scan_type}
with work_dir.scan_info_file.open(mode="w") as f:
json.dump(scan_info, f)
with work_dir.log_file.open(
"w"
) as l_f, work_dir.err_file.open("w") as e_f:
await resource_limits.create_subprocess(
cmd,
intensity_cpu=intensity_cpu,
intensity_io=intensity_io,
key=AIBOLIT_SCAN_INTENSITY_KEY[scan_type],
start_new_session=True,
stdout=l_f,
stderr=e_f,
cwd=str(work_dir),
env=self.get_updated_environment(),
)
return {}
self.cmd = self._cmd(
file.name if file is not None else None,
intensity_ram,
self._generate_progress_file(),
scan_type=scan_type,
scan_id=scan_id,
db_dir=db_dir,
detect_elf=detect_elf,
exclude_patterns=exclude_patterns,
follow_symlinks=follow_symlinks,
scan_path=scan_path,
file_patterns=file_patterns,
use_filters=use_filters,
)
logger.debug("Executing %s", " ".join(self.cmd))
self.proc = await resource_limits.create_subprocess(
self.cmd,
intensity_cpu=intensity_cpu,
intensity_io=intensity_io,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=CoreConfig.TMPDIR,
env=self.get_updated_environment(),
key=AIBOLIT_SCAN_INTENSITY_KEY[scan_type],
)
try:
self.out, self.err = await self.proc.communicate()
except asyncio.CancelledError:
with suppress(ProcessLookupError):
self.proc.terminate()
raise
try:
report = json.loads(self.out.decode())
except json.JSONDecodeError as err:
raise AiBolitError(
message="JSONDecodeError",
command=self.cmd,
return_code=self.proc.returncode,
out=self.out,
err=self.err,
scan_id=self.scan_id,
path=scan_path,
) from err
logger.debug("%s returned %s", AIBOLIT, report)
# TODO: use base64-encoded paths for non-detached scans too
return parse_report_json(report, base64_path=False)
Zerion Mini Shell 1.0