PATH:
usr
/
share
/
lve
/
dbgovernor
/
Editing: governor_package_limitting.py
#!/opt/cloudlinux/venv/bin/python3 # coding:utf-8 # Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2024 All Rights Reserved # # Licensed under CLOUD LINUX LICENSE AGREEMENT # http://cloudlinux.com/docs/LICENSE.TXT # import argparse import functools import itertools import json import logging import os import subprocess import sys import time from collections import defaultdict from typing import Dict, List, Optional, Tuple from clcommon.cpapi import cpinfo, admin_packages, resellers_packages, cpusers, getCPName from utilities import acquire_lock, LockFailedException CURRENT_PROGRAM_NAME = os.path.basename(sys.argv[0]) DBCTL_BIN = '/usr/share/lve/dbgovernor/utils/dbctl_orig' PACKAGE_LIMIT_CONFIG = '/etc/container/governor_package_limit.json' LOCK_FILE = '/var/run/governor_package_limit' DBCTL_SYNC_LOCK_FILE = '/var/run/governor_package_sync' DEBUG = False ENCODING = 'utf-8' def init_logging(): """ Messages related to governor_package_limitting """ logging.basicConfig( stream = sys.stderr, format=f"{CURRENT_PROGRAM_NAME} %(levelname)s: %(message)s", level=logging.ERROR ) def _process_call_error(call_err): if isinstance(call_err.cmd, list): cmd_str = " ".join(call_err.cmd) else: cmd_str = call_err.cmd logging.error(f'command \'{cmd_str}\' exit code = {call_err.returncode}') logging.info('stdout:\n' + str(call_err.stdout)) if call_err.stderr is not None: logging.info('stderr:\n' + str(call_err.stderr)) return call_err.returncode def debug_log(line, end='\n'): """ Debug output log """ global DEBUG if DEBUG: print(line, end=end) def build_parser(): """ Build CLI parser """ parser = argparse.ArgumentParser( prog="Configure mysqlgovernor limits with Control Panel Packages", description="Description: Configure governor-mysql with Control Panel Package limits", add_help=False, usage='governor_package_limitting.py [COMMAND] [OPTIONS]' ) parser._positionals.title = 'Commands' parser._optionals.title = 'Options' parser.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS, help='Governor package limiting') parser.add_argument('--debug', action='store_true', help='Turn on debug mode') subparser = parser.add_subparsers(dest='command') ################ SET ################ _set = subparser.add_parser( 'set', help='Set limits for governor packages limits', description='Description: Set cpu limit for governor package' ) _set.add_argument('--package', help='Package name', type=str, required=True) _set.add_argument('--cpu', help='limit CPU (pct) usage', nargs='+',) _set.add_argument('--read', help='limit READ (MB/s) usage', nargs='+') _set.add_argument('--write', help='limit WRITE (MB/s) usage', nargs='+') _set.add_argument('--debug', action='store_true', help='Turn on debug mode') ################ DELETE ################ delete = subparser.add_parser( 'delete', help='Delete governor package limits', description='Description: Delete governor package limit', usage='governor_package_limitting.py delete [PACKAGE_NAME]' ) delete._optionals.title = 'Options' delete.add_argument('--package', help='Package name', type=str, required=True) delete.add_argument('--debug', action='store_true', help='Turn on debug mode') ################ GET ################ get = subparser.add_parser( 'get', help='Get governor package limits', description='Description: Get governor package limits', usage='governor_package_limitting.py get [OPTIONS]', formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, width=99999) ) get._optionals.title = 'Options' g = get.add_mutually_exclusive_group() g.add_argument('--package', help='Get package limits', type=str, ) g.add_argument('--all', help="Get all package limits", action='store_true') get.add_argument('--format', help='Show limits in formats: (KB/s), (BB/s), (MB/s). Default is mb', default='mb',choices=['bb', 'kb', 'mb']) get._optionals.title = 'Options' get.add_argument('--debug', action='store_true', help='Turn on debug mode') ################ GET INDIVIDUAL ################ get_individual = subparser.add_parser( 'get_individual', help='Get governor package limits vector', description='The vector describes which limit is an individual and which is not', usage='governor_package_limitting.py get_individual --user=[USER_NAME]' ) get_individual._optionals.title = 'Options' get_individual.add_argument('--user', help='Shows a particular user\'s vector', type=str,) get_individual.add_argument('--debug', action='store_true', help='Turn on debug mode') ################ SET INDIVIDUAL ################ _set_individual = subparser.add_parser( 'set_individual', help='Set governor package limits vector', description='The vector describes which limit is an individual and which is not', usage='governor_package_limitting.py set_individual --user=[USER_NAME] --cpu=[False]*4 --read=[True]*4 --write=[True]*4' ) _set_individual.add_argument('--user', help='Package name', type=str, required=True) _set_individual.add_argument('--cpu', help='True/False vector, four values', nargs='+',) _set_individual.add_argument('--read', help='True/False vector, four values', nargs='+') _set_individual.add_argument('--write', help='True/False vector, four values', nargs='+') _set_individual.add_argument('--debug', action='store_true', help='Turn on debug mode') ################ RESET INDIVIDUAL ################ reset_individual = subparser.add_parser( 'reset_individual', help='Erases information about individual limits', description='The user will use a package limits or default limits as result', usage='governor_package_limitting.py reset_individual --user=[USER_NAME] --limits=mysql-io,mysql-cpu' ) reset_individual._optionals.title = 'Options' reset_individual.add_argument('--user', help='User name', type=str, required=True) reset_individual.add_argument('--limits', help='mysql-io or mysql-cpu', type=str) reset_individual.add_argument('--debug', action='store_true', help='Turn on debug mode') ################ SYNC ################ sync = subparser.add_parser( 'sync', help='Synchronize governor package limits', description='Description: Apply package limitting configuration to governor', usage='governor_package_limitting.py sync' ) sync._optionals.title = 'Options' sync.add_argument('--package', help='Package name', type=str, required=False) sync.add_argument('--user', help='User name', type=str, required=False) sync.add_argument('--debug', action='store_true', help='Turn on debug mode') return parser def cp_packages(user: Optional[str]) -> Dict[str, list]: """ Get packages and it's users. Returns: (dict): package_name and contained users list """ package_users = defaultdict(list) # user's packages for package_user, package in cpinfo(user, ("cplogin", "package")): package_users[package].append(package_user) # reseller's packages for reseller, packages in resellers_packages().items(): for package in packages: package_users[package].append(reseller) return package_users def limits_serializer(args: List, set_vector=False): """Convert user input list of string to list of int <set_vector=True> -> when use the function to prepare vector instead of values For example: ['1,2,3,4'] to -> [1,2,3,4] """ __limits = [] try: if args and isinstance(args, list): for i in args[0].split(','): if set_vector and 'true' in i.lower(): __limits.append(True) elif set_vector and 'false' in i.lower(): __limits.append(False) else: __limits.append(i) except Exception as err: logging.error(f'Some parameters are incorrect: {err}') sys.exit(1) if len(__limits) > 4: logging.error('Some parameters are incorrect. Provided options is more than 4') sys.exit(1) if __limits and set_vector and len(__limits) != 4: logging.error('Some parameters are incorrect. Provided options must be equal 4 when you set the vector!') sys.exit(1) if not __limits and set_vector: while len(__limits) < 4: __limits.append(False) if len(__limits) < 4: while len(__limits) < 4: __limits.append(0) return __limits def write_config_to_json_file(cfg: dict): with acquire_lock(LOCK_FILE, exclusive=True): with open(PACKAGE_LIMIT_CONFIG, 'w', encoding=ENCODING) as jsonfile: json.dump(cfg, jsonfile) def read_config_file(): with acquire_lock(LOCK_FILE, exclusive=True): with open(PACKAGE_LIMIT_CONFIG, 'r', encoding=ENCODING) as jsonfile: try: cfg = json.load(jsonfile) except json.JSONDecodeError: return {} return cfg if cfg else {} def get_item_from_config_dict(name: str, cfg: dict): """Get package from cfg dictionary if exists""" cfg = cfg.get(name) if cfg: cfg = {name: cfg} return cfg def show_default_value_instead_of_zero(name: str, lim_list: list, default_limits: tuple): """ Replaces 'zero' values by default values Args: name - cpu or read or write set of limits lim_list - lim_list is a current set of limits we'll modify if 'zero' values are present default_limits - 'default limits' we'll use instead of 'zero' values """ d_cpu, d_read, d_write = default_limits d_limits = {'cpu': d_cpu, 'read': d_read, 'write': d_write} # to know which values have not been changed indices_of_unchanged_elements = list() for _indx, num in enumerate(lim_list): if num < 1: lim_list[_indx] = d_limits[name][_indx] else: indices_of_unchanged_elements.append(_indx) return indices_of_unchanged_elements def print_individual_in_json_format(cfg: dict): print(json.dumps(cfg, ensure_ascii=False)) def print_config_in_json_format(cfg: dict, size_format, only_section=True): default_limits = get_dbctl_limits(format=size_format) if only_section: cfg = cfg['package_limits'] for key, val in cfg.items(): for k, v in val.items(): # returns the default values instead of 'zero' index_list = show_default_value_instead_of_zero( k, v, default_limits) if k != 'cpu': for i in index_list: cfg[key][k][i] = byte_size_convertor( v[i], from_format='bb', to_format=size_format ) print(json.dumps(cfg, ensure_ascii=False)) def check_if_values_are_not_less_than_zero(cfg): for k, v in cfg.items(): for i in v: if i < 0: print(f'Value for {k} must be >= 0') debug_log(f'Incorrect parameters for {k}') debug_log(f'Applied config is {cfg}') sys.exit(1) def fill_gpl_json(entity_name: str, cpu: list = None, io_read: list = None, io_write: list = None, serialize: bool = True, set_vector: bool = False) -> None: """ Setting 'package limits' or 'individual limits' to the governor_package_limit.json Args: entity_name (str): Package or user name cpu (list): cpu limits io_read (list): io read limits io_write (list): io write limits serialize (bool): To serialize or not set_vector (bool): True if setting individual limits Return: None """ # check package existence if not set_vector and entity_name not in get_all_packages(): logging.error(f'Package name {entity_name} not found in {getCPName()} packages') sys.exit(1) # check user existence if set_vector and entity_name not in cpusers(): logging.error(f'User {entity_name} not found in {getCPName()} users') sys.exit(1) cfg = { entity_name: { 'cpu': limits_serializer(cpu, set_vector) if serialize else cpu, 'read': limits_serializer(io_read, set_vector) if serialize else io_read, 'write': limits_serializer(io_write, set_vector) if serialize else io_write, } } if not set_vector: convert_io_rw_to_bb(cfg[entity_name]) check_if_values_are_not_less_than_zero(cfg[entity_name]) section = 'package_limits' else: section = 'individual_limits' config = get_package_limit() if config[section].get(entity_name): if cpu: config[section][entity_name]['cpu'] = cfg[entity_name]['cpu'] if io_read: config[section][entity_name]['read'] = cfg[entity_name]['read'] if io_write: config[section][entity_name]['write'] = cfg[entity_name]['write'] else: config[section][entity_name] = cfg[entity_name] debug_log(f'Setting package limit with config: {config}\n') write_config_to_json_file(config) return def update_default_limits(): """ Update default values in our json from the dbctl list """ config = get_package_limit() try: config['package_limits'].update({'default': {"cpu": [0] * 4, "read": [0] * 4, "write": [0] * 4} } ) write_config_to_json_file(config) except (KeyError, AttributeError): return def delete_package_limit(package: str): """Delete package limits Args: package (str): Name of package limit to delete Returns: None """ config = get_package_limit() try: config['package_limits'].pop(package) write_config_to_json_file(config) debug_log(f'Deleting package {package} from config') except (KeyError, AttributeError) as err: logging.error(f'Package name {package} not found') debug_log(err) sys.exit(1) def reset_individual(username: str, certain_limits: str = None): """ Delete user individual limits vector from the governor_package_limit.json, 'individual_limits' section """ help_msg = f"""Vector {username} not found""" def reset_all(): # remove the individual limits from the mysql-governor.xml run_dbctl_command([username], 'delete') config = get_package_limit() try: config['individual_limits'].pop(username) write_config_to_json_file(config) debug_log(f'Deleting vector {username} from config') except (KeyError, AttributeError) as err: logging.error(help_msg) debug_log(err) sys.exit(1) if not certain_limits or certain_limits.lower() == 'all': reset_all() else: _limits = certain_limits.split(',') if len(_limits) > 1: reset_all() else: reset_a_certain_limit(username, _limits) def change_vector_for_a_user(_user: str, limits: list): """ Marks that a particular limits are not individual anymore. We can reset individual limits for mysql-cpu or mysql-io. Args: _user: is a particular user we makes changes for limits: vector where non individual limits have to be set """ config = get_package_limit() dictionary_to_update = {'mysql-cpu': {'cpu': [False] * 4}, 'mysql-io': {'read': [False] * 4, 'write': [False] * 4} } for lim in limits: update_vector = dictionary_to_update.get(lim) try: config['individual_limits'][_user].update(update_vector) except (KeyError, AttributeError) as err: return write_config_to_json_file(config) def return_the_individual_limits_to_dbctl(_user: str, limits: list, saved_limits: tuple): """ Sets the individual limits as they were before because the individual limits must not be changed Args: _user: the user for which individual limits should be set limits: mysql-io (read/write) or mysql-cpu set of limits saved_limits: individual limits that were set before the general reset of limits for a particular user """ _cpu_limit, _read_limit, _write_limit = saved_limits for lim in limits: # if we change mysql read/write limits - the individual cpu limits # must not be changed! if lim == 'mysql-io': _cpu_limits = ",".join([str(i) + 'b' for i in _cpu_limit]) _limit = f'--cpu={_cpu_limits}' # the same situation as described above elif lim == 'mysql-cpu': _read_limit = ",".join([str(i) + 'b' for i in _read_limit]) _read = f'--read={_read_limit}' _write_limit = ",".join([str(i) + 'b' for i in _write_limit]) _write = f'--write={_write_limit}' _limit = f'{_read} {_write}' else: return dbctlset = f'{DBCTL_BIN} set {_user} {_limit}' try: subprocess.run(dbctlset, shell=True, text=True, check=True, capture_output=True) except subprocess.CalledProcessError as call_err: return _process_call_error(call_err) def reset_a_certain_limit(username: str, limits: list): """ Makes possible to reset the individual limits for a particular limit. For instance we can reset only mysql-cpu or only mysql-io (read/write), not both. """ # Saves current individual limits saved_limits = get_dbctl_limits(username) # Removes all limits (dbctl individual) for a particular user run_dbctl_command([username], 'delete') # Marks non individual limits (set as <False>) # governor_package_limit.json -> individual_limits -> # user name -> cpu or read/write change_vector_for_a_user(username, limits) # Return individual limits for a set of limits which still individual return_the_individual_limits_to_dbctl(username, limits, saved_limits) def get_package_limit(package: str = None, size_format: str = 'mb', print_to_stdout=False, cfg=None): """Get package limits Args: package (str): name of package to get print_to_stdout (bool): Print to stdout is used for cli. size_format: Print values in specified format. mb, kb, bb Returns: Package limit configurations or provided package configuration """ cfg = cfg or read_config_file() if package: cfg = get_item_from_config_dict(package, cfg['package_limits']) only_section = False else: only_section = True if print_to_stdout and cfg: print_config_in_json_format(cfg, size_format, only_section) return cfg def get_individual(username: str = None, print_to_stdout=False): """ Get individual limits Args: username: name of user to get print_to_stdout (bool): Print to stdout is used for cli. Returns: Individual limits vector -> {"individual_limits": {"user1": {"cpu": [false, false, false, false], "read": [true, true, true, true], "write": [true, true, true, true]}}} """ cfg = read_config_file() if username: cfg = get_item_from_config_dict(username, cfg['individual_limits']) if not cfg: cfg = {username: {'cpu': [False] * 4, 'read': [False] * 4, 'write': [False] * 4} } else: cfg = cfg['individual_limits'] if print_to_stdout: print_individual_in_json_format(cfg) return cfg def run_dbctl_command(users: list, action: str, limits: dict = None) -> int: """Run dbctl command in os, return exit code Set or delete limits for all users specified in package Args: users (list): List of users to apply configuration action (str): Set or delete. Set is used both for set and update. limits (dict): cpu, read, write in format [int] """ return_code = 0 if action == 'set' and not limits: logging.error("Limits for dbctl have not been set") sys.exit(1) if action == 'delete' and users: for user in users: command = f'{DBCTL_BIN} delete {user}' debug_log(f'Running command: {command}') try: command_result = subprocess.run(command, shell=True, text=True, check=True, capture_output=True) return_code += command_result.returncode except subprocess.CalledProcessError as call_err: # we shouldn't do exit here because this function is called in loop in dbctl_sync return_code += _process_call_error(call_err) if action == 'set' and limits and users: for user in users: # Returns an empty list when user is absent in dbctl # so we need to skip all dbctl actions prepare_limits_out = prepare_limits( user, package_limit=limits, individual_limit=get_individual(user) ) if prepare_limits_out: cpu, io_read, io_write = prepare_limits_out command = f'{DBCTL_BIN} set {user} --cpu={cpu} --read={io_read} --write={io_write}' debug_log(f'Running command: {command}') try: command_result = subprocess.run(command, shell=True, text=True, check=True, capture_output=True) return_code += command_result.returncode except subprocess.CalledProcessError as call_err: # we shouldn't do exit here because this function is called in loop in dbctl_sync return_code += _process_call_error(call_err) return return_code def dbctl_sync(action: str, package: str = None, user: str = None): """Sync package configuration with dbgovernor Args: action (str): Set or Delete package (str): Package name is used with action delete. package (str): User name is used with action 'set' to synchronise for a specific user only """ if not action: logging.error("sync action not specified") sys.exit(1) exit_code = 0 debug_log("Syncing with dbctl") __cp_packages = cp_packages(user) if action == 'delete' and package: users_to_apply_package = __cp_packages.get(package) if users_to_apply_package: debug_log(f'Deleting package limits for users: {users_to_apply_package}') exit_code += run_dbctl_command(users_to_apply_package, action) sys.exit(exit_code) if action == 'set': _package_limits = get_package_limit(package) # Use a specific package (including all users of these package) # if package for sync is specified # Use all packages otherwise package_limits = _package_limits if package else _package_limits['package_limits'] # To avoid traceback if incorrect package name has been set if not package_limits: logging.warning(f'package limits for \'{package}\' are empty, probably incorrect package name') return for package_name, limits in package_limits.items(): users_to_apply_package = __cp_packages.get(package_name) if users_to_apply_package: if not user: # Sets package limits for all package's users debug_log(f'Setting package limits for users: {users_to_apply_package}') exit_code += run_dbctl_command(users_to_apply_package, action, limits) continue if user in users_to_apply_package: # Sets package limits only for a specific user # if user for sync is specified debug_log(f'Setting package limits for user: {user}') exit_code += run_dbctl_command([user], action, limits) sys.exit(exit_code) def byte_size_convertor(value: int, from_format: str, to_format: str): """Converting integer between formats mb|kb|bb Args: value (int): Integer value to convert from_format (str): To convert from mb|kb|bb to_format (str): To convert to mb|kb|bb """ if value < 0: return value if from_format == 'bb': if to_format == 'kb': value /= 1024 elif to_format == 'mb': value = value / 1024 / 1024 elif from_format == 'kb': if to_format == 'bb': value *= 1024 elif to_format == 'mb': value /= 1024 elif from_format == 'mb': if to_format == 'kb': value *= 1024 elif to_format == 'bb': value = value * 1024 * 1024 return int(value) def convert_io_rw_to_bb(cfg: dict): """Converting MB or KB to bytes Parameters from user cli can be bytes, MB or KB. Symbol 'b' - bytes, symbol 'k' - kilobytes, symbol 'm' or no symbols - megabytes [1, 50m, 52428800b, 100k] Args: cfg (dict): {cpu: [x,x,x,x], read: [x,x,x,x], write: [x,x,x,x,]} Returns: cfg (dict) """ for k, v in cfg.items(): if k != 'cpu': for i in range(4): if isinstance(v[i], str): if 'b' in v[i]: v[i] = int(v[i].replace('b', '')) elif 'k' in v[i]: debug_log(f'Converting kilobytes `{v[i]}` to bytes') v[i] = byte_size_convertor( int(v[i].replace('k', '')), from_format='kb', to_format='bb' ) elif 'm' in v[i]: debug_log(f'Converting megabytes `{v[i]}` to bytes') v[i] = byte_size_convertor( int(v[i].replace('m', '')), from_format='mb', to_format='bb' ) else: debug_log(f'Converting megabytes `{v[i]}` to bytes') v[i] = byte_size_convertor( int(v[i]), from_format='mb', to_format='bb' ) else: for i in range(4): v[i] = int(v[i]) debug_log('\n') return cfg def trying_to_get_user_in_dbctl_list(user: str, format: str) -> Dict: """ Extract user limits from dbctl list output Make additional attempt in case of fail and fallback to default limits Args: user -> web panel user for which we perform actions format -> formats that we use in dbctl (bb, kb, mb) Returns: specific user limits (cpu, read, write) """ data = _get_dbctl_list_json(format).get(user) # make one more attempt in case of new user that wasn't added to dbctl list yet if not data: default_limits = { "cpu": {"current": 0, "short": 0, "mid": 0, "long": 0}, "read": {"current": 0, "short": 0, "mid": 0, "long": 0}, "write": {"current": 0, "short": 0, "mid": 0, "long": 0} } # restart db_governor service to add the user to the dbctl list immediately os.system('/usr/share/lve/dbgovernor/mysqlgovernor.py --dbupdate') time.sleep(1) os.system('service db_governor restart &> /dev/null') time.sleep(1) # if user limits still unavailable - just return a structure with zero values data = _get_dbctl_list_json(format).get(user) or default_limits return data def _get_dbctl_list_json(format: str) -> Dict: dbctl_limits = f'{DBCTL_BIN} list-json --{format}' output = subprocess.run(dbctl_limits, shell=True, text=True, capture_output=True) _data = output.stdout try: data = json.loads(_data) except json.JSONDecodeError: logging.debug(f"<{DBCTL_BIN} list-json --{format}> returns non json data!") logging.debug(f"stdout -> {output.stdout}") logging.debug(f"stderr -> {output.stderr}") sys.exit(1) return data def get_dbctl_limits(user: str = 'default', format: str = 'bb') -> Tuple: """ Gets dbctl limits list and returns a tuple of individual limits (cpu, read, write) for a specific user. Running this function without 'user' argument returns the default limits. Args: user -> to get an individual user limits, returns 'default' if hasn't been set format -> all available formats we use in dbctl: bb - bytes kb - kilobytes mb - megabytes Returns: a tuple of lists ([cpu],[read],[write]), or blank tuple if user has not been found """ individual_limits = trying_to_get_user_in_dbctl_list(user, format) individual_cpu_limit = ( individual_limits['cpu']['current'], individual_limits['cpu']['short'], individual_limits['cpu']['mid'], individual_limits['cpu']['long'] ) individual_read_limit = ( individual_limits['read']['current'], individual_limits['read']['short'], individual_limits['read']['mid'], individual_limits['read']['long'] ) individual_write_limit = ( individual_limits['write']['current'], individual_limits['write']['short'], individual_limits['write']['mid'], individual_limits['write']['long'] ) debug_log(f'{user}\'s individual limits are: ', end='') debug_log(f'cpu: {individual_cpu_limit}, read: {individual_read_limit}, write: {individual_write_limit}') return individual_cpu_limit, individual_read_limit, individual_write_limit def get_the_limits_set(limits_set: list) -> Tuple: """ Here we have twelve values: -> four per cpu -> four per read -> four per write Here we divide them to conviniate representation """ cpu = tuple(limits_set[0:4]) read = tuple(limits_set[4:8]) write = tuple(limits_set[8:]) return cpu, read, write def set_default_limit(package_limit: dict) -> Tuple: """ Defines the limits (cpu, read, write). Uses package limits or default limits depending on values. If package value is greater than zero - use it, otherwise use default value. """ package_cpu_limit = package_limit.get('cpu') package_read_limit = package_limit.get('read') package_write_limit = package_limit.get('write') all_limits = list() for tuple_of_limits_lists in (package_cpu_limit, package_read_limit, package_write_limit): for n in tuple_of_limits_lists: # values of package limit and default limit (0) respectively all_limits.append(n) if n > 0 else all_limits.append(0) return get_the_limits_set(all_limits) def ensures_the_individual_limits_still_set( vector: dict, default_or_package_limit: tuple, individual_limits: tuple) -> Tuple: """ Sets the individual limit instead of <package limit/default limit> if vector value is <True> for the certain limit. """ all_limits = list() individual_cpu_limit, individual_read_limit, individual_write_limit = individual_limits dorp_cpu_limit, dorp_read_limit, dorp_write_limit = default_or_package_limit vectors = vector.values() for vector_dict in vectors: cpu_vectors = vector_dict['cpu'] read_vectors = vector_dict['read'] write_vectors = vector_dict['write'] # <data_set> contains a 'vector list', an 'individual limits' list # and a current list of limits collected from the package limits # or default limits for data_set in ( (cpu_vectors, individual_cpu_limit, dorp_cpu_limit), (read_vectors, individual_read_limit, dorp_read_limit), (write_vectors, individual_write_limit, dorp_write_limit) ): for n in zip(data_set[0], data_set[1], data_set[2]): # If vector is True if n[0]: all_limits.append(n[1]) # If vector is False else: all_limits.append(n[2]) return get_the_limits_set(all_limits) def prepare_limits(user: str, package_limit: Dict, individual_limit: Dict) -> List: """Prepare Limits with algorithm First of all sets the package limits or default limits, than sets the individual limits (dependin on vector values) For example: Pack1 Package limit for cpu is: [100, 0, 65, 40] Default Package limit is: [50, 90, 0, 50] Individual limit is: [150, not_set, not_set, not_set] Result will be: [150, 90, 65, 40] """ cpu_limit, read_limit, write_limit = ensures_the_individual_limits_still_set( individual_limit, set_default_limit(package_limit), get_dbctl_limits(user) ) try: # with 'b' symbol to set bytes using 'dbctl' tool cpu = ','.join(str(x) for x in cpu_limit) io_read = ','.join(str(x) + 'b' for x in read_limit) io_write = ','.join(str(x) + 'b' for x in write_limit) debug_log(f'Limits after calculation for {user} is:') debug_log(f'cpu: [{cpu}], read: [{io_read}], write: [{io_write}]') return cpu, io_read, io_write except TypeError as err: logging.error(f'Some limits are not given: {err}') sys.exit(1) @functools.cache def get_all_packages(): """ Gets all packages: admin packages and resellers packages either. """ all_packages = set(admin_packages()) reseller_packages_iter = itertools.chain.from_iterable(resellers_packages().values()) all_packages.update(reseller_packages_iter) return list(all_packages) def sync_with_panel(): # won't do MYSQLG-789 """ Just getting package names and applying default values [0,0,0,0] """ cfg = read_config_file() for package in get_all_packages(): if not get_package_limit(package, cfg=cfg): fill_gpl_json( entity_name=package ) cfg = read_config_file() def turn_on_debug_if_user_enabled_debug_mode(opts): if opts.debug: global DEBUG DEBUG = True def ensure_json_presence(): """ The governor_package_limit.json file must always be present! Even if someone delete the json config and its content. """ content = {"package_limits": {}, "individual_limits": {}} if not os.path.exists(PACKAGE_LIMIT_CONFIG): write_config_to_json_file(content) return json_ = read_config_file() for element in content.keys(): if not json_.get(element): json_.update({element: {}}) write_config_to_json_file(json_) def main(argv): """ Run main actions """ parser = build_parser() if not argv: parser.print_help() sys.exit(1) opts = parser.parse_args(argv) turn_on_debug_if_user_enabled_debug_mode(opts) ensure_json_presence() init_logging() if opts.command == 'set': fill_gpl_json(opts.package, opts.cpu, opts.read, opts.write) dbctl_sync('set', package=opts.package) elif opts.command == 'get': sync_with_panel() update_default_limits() get_package_limit(opts.package, opts.format, print_to_stdout=True) elif opts.command == 'delete': delete_package_limit(opts.package) dbctl_sync('delete', opts.package) elif opts.command == 'sync': # to avoid excessive calls of sync command just ignore too frequent calls try: with acquire_lock(DBCTL_SYNC_LOCK_FILE, exclusive=True, attempts=1): dbctl_sync('set', opts.package, opts.user) except LockFailedException: debug_log('Excessive sync call ignored') elif opts.command == 'get_individual': get_individual(opts.user, print_to_stdout=True) elif opts.command == 'set_individual': fill_gpl_json(opts.user, opts.cpu, opts.read, opts.write, serialize=True, set_vector=True) # There is no call to 'sync' after this, since in the main case of using this command, # 'dbctl set' is called immediately after it # https://docs.google.com/document/d/1KH3MiHVcqJduvw6Vid8UiOvv9-6-CHDAtwkaUQh8_vU elif opts.command == 'reset_individual': reset_individual(opts.user, opts.limits) dbctl_sync('set', user=opts.user) else: parser.print_help() sys.exit(1) if "__main__" == __name__: main(sys.argv[1:])
SAVE
CANCEL