Edit

[EDIT] FILE: dbsaver.py

# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT

import logging

from sqlalchemy import insert, select
from sqlalchemy.orm import sessionmaker

from lvestats.core.plugin import LveStatsPlugin
from lvestats.lib.commons.func import get_chunks, reboot_lock
from lvestats.orm import user as user_class
from lvestats.orm.history import history
from lvestats.orm.servers import servers

class DBSaver(LveStatsPlugin):
    def __init__(self):
        self.log = logging.getLogger('plugin.DBSaver')
        self.now = 0  # This changes in MainLoop
        self.config = None
        self.period = 60
        self.server_id = 'localhost'

def set_config(self, config):
        self.period = int(config.get('db_timeout', self.period))
        self.server_id = config.get('server_id', self.server_id)

def execute(self, lve_data):
        sql_insert_query = insert(history)

sql_select_servers = select([servers]).where(servers.server_id == self.server_id)
        sql_insert_servers = insert(servers)
        sql_update_servers = servers.__table__.update().where(servers.server_id == self.server_id)

with reboot_lock():
            conn = self.engine.connect()
            tx = conn.begin()
            try:
                row = conn.execute(sql_select_servers)
                if row.returns_rows:
                    res = row.fetchone()
                    if res is not None:
                        if res['lve_version'] != lve_data['LVE_VERSION']:
                            conn.execute(sql_update_servers.values({'lve_version': lve_data['LVE_VERSION']}))
                    else:
                        conn.execute(sql_insert_servers, server_id=self.server_id, lve_version=lve_data['LVE_VERSION'])
                else:
                    conn.execute(sql_insert_servers, server_id=self.server_id, lve_version=lve_data['LVE_VERSION'])

sql_insert_list = []
                for lve_id, v in lve_data.get('lve_usage', {}).items():
                    sql_insert_list.append(
                        {
                            'id': lve_id,
                            'cpu': int(round(v.cpu_usage)),
                            'cpu_limit': v.lcpu,
                            'cpu_fault': v.cpu_fault,
                            'mep': v.mep,
                            'mep_limit': v.lep,
                            'io': int(round(v.io_usage)),
                            'io_limit': v.io,
                            'mem': int(round(v.mem_usage)),
                            'mem_limit': v.lmem,
                            'mem_fault': v.mem_fault,
                            'mep_fault': v.mep_fault,
                            'created': int(round(self.now)),
                            'server_id': self.server_id,
                            'lmemphy': v.lmemphy,
                            'memphy': int(round(v.memphy)),
                            'memphy_fault': v.memphy_fault,
                            'lnproc': v.lnproc,
                            'nproc': int(round(v.nproc)),
                            'nproc_fault': v.nproc_fault,
                            'io_fault': v.io_fault,
                            'iops_fault': v.iops_fault,
                            'liops': v.liops,
                            'iops': int(round(v.iops)),
                        }
                    )

for chunk in get_chunks(sql_insert_list):
                    try:
                        conn.execute(sql_insert_query, chunk)
                    except OverflowError:
                        self._try_executing_small_chunks(conn, sql_insert_query, chunk)
                        raise
                tx.commit()
            except Exception:
                tx.rollback()
                raise
            finally:
                conn.close()

def _try_executing_small_chunks(self, conn, query, insert_params, small_chunk_size=10):
        """Try executing smaller chunks so that it will be feasible to log the one causing error.
        In case when data contains value which is bigger than column type can handle, OveflowError is raised.

:param sqlalchemy.engine.Connection conn: db engine
        :param sqlalchemy.sql.expression.Insert query: query to execute
        :param typing.List[typing.Dict] insert_params: list of params to insert
        :param int chunk_length: length of small chunks
        """
        for small_chunk in get_chunks(insert_params, small_chunk_size):
            try:
                conn.execute(query, small_chunk)
            except OverflowError:
                self.log.error('Overflow detected in %s', small_chunk)
                break

class DbUsernamesSaver(LveStatsPlugin):
    def __init__(self):
        self.period = 60 * 60  # once an hour
        self.engine = None
        self.enabled = True

def set_config(self, config):
        self.enabled = config.get('collect_usernames', 'false').lower() == 'true'

def set_db_engine(self, engine):
        self.engine = engine

def execute(self, lve_data):
        if self.enabled:
            session = sessionmaker(bind=self.engine)()
            for user_info in lve_data['users']:
                user = user_class()
                user.uid, user.user_name, user.server_id = user_info
                try:
                    session.merge(user)
                    session.flush()
                # workaround for sqlalchemy could not deal with complex key on detecting duplicates when merge
                except Exception:
                    session.rollback()

session.commit()
            session.close()

CANCEL