Edit

[EDIT] FILE: decision_maker.py

# -*- coding: utf-8 -*-

# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT

"""
This module contains DecisionMaker class
"""

__package__ = 'ssa.modules'

import heapq
import json
import logging
import os
from os.path import isfile

import numpy as np

from ssa.db import setup_database
from .common import Common
from .storage import (
    iter_domains_data,
    iter_domain_url_data,
)
from ..configuration import load_tunables
from ..configuration.schemes import ssa_tunables_schema
from ..internal.constants import report_path
from ..internal.utils import previous_day_date, sentry_init

# Defensive cap on the path length we are willing to carry into the report.
# Tenant-controlled URLs are otherwise unbounded in length; truncating keeps
# a single oversized path from dominating the bounded per-domain working set.
MAX_REPORT_URL_LEN = 2048

class DecisionMaker(Common):
    """
    SSA Decision maker implementation.
    """

def __init__(self, engine=None):
        super().__init__()
        self.logger = logging.getLogger('decision_maker')
        self.logger.info('DecisionMaker enabled: %s', __package__)

self.engine = engine if engine else setup_database()

def __call__(self):
        self.logger.info('DecisionMaker started')
        self.logger.debug('DecisionMaker loaded config: %s', self.config)
        self.external_tunables = self.load_external_conf()
        self.logger.debug('DecisionMaker loaded tunables: %s', self.external_tunables)
        report = self.data_processing()
        self.add_json_report(report)
        self.logger.info('DecisionMaker report: %s', report)
        return report

@staticmethod
    def _report_file(name) -> str:
        """
        Full path to given filename in DM reports directory
        """
        return os.path.join(report_path, name)

@property
    def current_report_file(self) -> str:
        """
        Full path to current DM report: report.json in DM reports directory
        """
        return self._report_file('report.json')

@property
    def _empty_report(self) -> dict:
        """
        Returns empty report
        """
        return dict(date=previous_day_date(), domains=[])

@property
    def solo_filtered_options(self) -> set:
        return {'correlation'}

@staticmethod
    def load_external_conf():
        """Load external configuration values"""
        return load_tunables('ssa.json', ssa_tunables_schema)

def data_processing(self) -> dict:
        """
        Going through the list of domains, for each domain we go through
        the list of urls. During data processing, we will form
        the resulting dictionary.

Per-URL data is streamed via :func:`iter_domain_url_data`, so the
        per-domain ``{path: [durations...]}`` working set never lives in
        memory at once. This is what keeps the daily routine inside the
        ``MemoryMax=1G`` cgroup limit on hosts with very many distinct
        URLs per domain (CLPRO-3077).
        """
        report = self._empty_report
        # urls_number is schema-validated as a positive int; fall back to a
        # finite default if config is missing/empty so the bounded heap below
        # always has a real ceiling (Common.__getattr__ yields '' when absent).
        leave_top = self.urls_number if isinstance(self.urls_number, int) and self.urls_number > 0 else 5
        for domain_data in iter_domains_data(self.engine):
            # goes through the list of domains. urls_data is a bounded
            # min-heap holding only the top-`leave_top` qualifying URLs seen so
            # far for this domain, keyed by (reqs_num, -order) so the smallest
            # reqs_num (ties broken by latest insertion) is evicted first. This
            # caps peak memory at ~`leave_top` entries per domain even when a
            # tenant emits very many distinct slow URLs (CLPRO-3165), while the
            # observable report stays identical to a full sort + truncate.
            urls_data = list()
            order = 0
            domain_slow_reqs = 0

for url, durations, url_data in iter_domain_url_data(self.engine, domain_data.domain_name):
                if self.is_ignored(url):
                    self.logger.debug('%s ignored', url)
                    continue
                if url in self.non_url_fields:
                    continue
                if not self.is_throttling_suitable(
                    url_data.get('url_throttled_reqs', list([0] * 24)), url_data['url_total_reqs']
                ):
                    # skip by allowed throttling percentage
                    continue
                correlation_value = self.get_correlation(url_data['url_total_reqs'], domain_data.domain_total_reqs)
                if not durations:
                    self.logger.error('Unable to get durations for %s', str(url))
                    continue
                if self.request_number_exceeded(url_data['url_slow_reqs']) and self.correlation_conditions(
                    correlation_value
                ):
                    average_duration_calculation = np.mean(durations)
                    sum_url_slow_reqs = sum(url_data['url_slow_reqs'])
                    domain_slow_reqs += sum_url_slow_reqs
                    entry = dict(
                        name=url[:MAX_REPORT_URL_LEN],
                        reqs_num=sum_url_slow_reqs,
                        average_duration=int(average_duration_calculation),
                        correlation=float(f'{correlation_value:.2f}'),
                    )
                    # Bounded push: keep at most `leave_top` largest entries.
                    item = (sum_url_slow_reqs, -order, entry)
                    if len(urls_data) < leave_top:
                        heapq.heappush(urls_data, item)
                    elif item > urls_data[0]:
                        heapq.heapreplace(urls_data, item)
                    order += 1
            if urls_data:
                # Emit in the same order a stable descending sort + truncate
                # produced before: by reqs_num desc, ties by insertion order.
                sorted_urls = [entry for _, _, entry in sorted(urls_data, reverse=True)]
                report['domains'].append(
                    dict(
                        name=domain_data.domain_name,
                        slow_urls=len(sorted_urls),
                        slow_reqs=domain_slow_reqs,
                        total_reqs=sum(domain_data.domain_total_reqs),
                        urls=sorted_urls,
                    )
                )
        if report['domains']:
            report['domains'] = self.report_sorting(
                list_to_sort=report['domains'], leave_top=self.domains_number, key_for_sorting='slow_reqs'
            )
        return report

def list_handling_considering_time(self, url_slow_reqs: list) -> list:
        """
        Based on the 'url_slow_reqs' list, a new list will be formed,
        where the elements of the original list will be iteratively
        summed by the number of elements equal to 'time'
        """
        time = self.time or 24
        return [sum(url_slow_reqs[i : time + i]) for i in range(0, len(url_slow_reqs), time)]

def compare_elements_with_request_number(self, url_slow_reqs_by_time: list) -> bool:
        """
        This functions will check if any of elements is greater than "request_number"
        """
        for i in url_slow_reqs_by_time:
            if i >= self.request_number:
                return True
        return False

def get_correlation(self, url_total_reqs: list, domain_total_reqs: list):
        """
        Calculates the correlation coefficient using the "url_total_reqs" and
        the "domain_total_reqs" lists
        """
        if not self.correlation:
            return 0
        return np.amin(np.corrcoef(url_total_reqs, domain_total_reqs))

@staticmethod
    def report_sorting(list_to_sort: list, leave_top: int, key_for_sorting: str) -> list:
        """
        Will sort the domain list by "slow_reqs", the goal is to leave only
        "domains_number" of uppers, also per each domain will sort urls by
        "reqs_num", the goal is to leave only "urls_number" of uppers.
        leave_top == 0 allows to keep the full list
        """
        list_to_sort.sort(key=lambda dict_: dict_[key_for_sorting], reverse=True)
        if leave_top:
            return list_to_sort[:leave_top]
        else:
            return list_to_sort

def rename_old_report(self):
        """
        Rename old report
        """
        old_report = self.current_report_file
        if isfile(old_report):
            with open(old_report) as json_data:
                try:
                    d = json.load(json_data)
                except json.JSONDecodeError:
                    date_from_report = 'unknown'
                else:
                    date_from_report = d.get('date', 'dd.mm.yyyy').replace('.', '_')
            new_report_name = f'report__{date_from_report}.json'
            new_report = self._report_file(new_report_name)
            os.rename(old_report, new_report)

def add_json_report(self, report: dict):
        """
        Makes json report
        """
        self.rename_old_report()
        with open(self.current_report_file, 'w', encoding='utf-8') as f:
            json.dump(report, f, ensure_ascii=False, indent=4)

def get_json_report(self) -> dict:
        """
        Return contents of current report or empty report in case of error
        """
        _filtering_hook = None

try:
            with open(self.current_report_file) as report:
                report_dict = json.load(report, object_hook=_filtering_hook)
        except (OSError, json.JSONDecodeError):
            report_dict = self._empty_report
        return report_dict

def correlation_conditions(self, correlation_value: int) -> bool:
        """
        If correlation flag is enabled - we'll compare correlation_coefficient
        from configuration with calculated correlation coefficient.
        If the calculated value exceeds the configuration value - we return
        True otherwise False. At the same time if correlation flag is disabled -
        we'll also return "True" since in this case the correlation coefficient
        is not checked and its value is specified as zero in final report.
        """
        if not self.correlation:
            return True
        return correlation_value > self.correlation_coefficient

def request_number_exceeded(self, url_slow_reqs):
        """
        At least one element from the received list (url_slow_reqs_by_time)
        must be greater than request_number
        """
        url_slow_reqs_by_time = self.list_handling_considering_time(url_slow_reqs)
        return self.compare_elements_with_request_number(url_slow_reqs_by_time)

def is_throttling_suitable(self, url_throttled_reqs: list, url_total_reqs: list) -> bool:
        """
        Check that percent of throttled requests per URL passes given threshold
        """
        throttled_percent = (sum(url_throttled_reqs) / sum(url_total_reqs)) * 100
        self.logger.debug('Calculated throttled percent %s', throttled_percent)
        return throttled_percent <= self.external_tunables.get('allowed_throttling_percentage', 0)

if __name__ == "__main__":
    sentry_init()
    logging.basicConfig(filename='decision_maker_standalone.log', level=logging.INFO)
    dm = DecisionMaker()
    dm()

CANCEL