In [1]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib

from lib.collectors.stat_collector import DiskResourcesStatCollector, DiskWauForecast022020StatCollector
import logging
import yaml
from lib.analyzers.calc import *
from lib.constants import *

loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict]
for logger in loggers:
    logger.setLevel(logging.WARNING)
In [2]:
import os
import sys
sys.path.append(os.path.realpath('..'))
sys.path
Out[2]:
['',
 '/home/kolyann/arcadia/disk/admin/capacity_planner',
 '/home/kolyann/anaconda2/envs/conda_stat/lib/python27.zip',
 '/home/kolyann/anaconda2/envs/conda_stat/lib/python2.7',
 '/home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/plat-linux2',
 '/home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/lib-tk',
 '/home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/lib-old',
 '/home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/lib-dynload',
 '/home/kolyann/.local/lib/python2.7/site-packages',
 '/home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages',
 '/home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages/IPython/extensions',
 '/home/kolyann/.ipython',
 '/home/kolyann/arcadia/disk/admin/capacity_planner']
In [3]:
!{sys.executable} -m pip install xmltodict
!{sys.executable} -m pip install -r ../bin/requirements.txt
DEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 won't be maintained after that date. A future version of pip will drop support for Python 2.7. More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support
Looking in indexes: https://pypi.yandex-team.ru/simple/
Requirement already satisfied: xmltodict in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (0.12.0)
DEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 won't be maintained after that date. A future version of pip will drop support for Python 2.7. More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support
Looking in indexes: https://pypi.yandex-team.ru/simple/
Requirement already satisfied: kiwisolver==1.1.0 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from -r ../bin/requirements.txt (line 1)) (1.1.0)
Requirement already satisfied: scipy==1.2.2 in /home/kolyann/.local/lib/python2.7/site-packages (from -r ../bin/requirements.txt (line 2)) (1.2.2)
Requirement already satisfied: matplotlib==2.2.5 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from -r ../bin/requirements.txt (line 3)) (2.2.5)
Collecting numpy==1.16.5 (from -r ../bin/requirements.txt (line 4))
  Downloading https://pypi.yandex-team.ru/repo/default/download/numpy/393385/numpy-1.16.5-cp27-cp27mu-manylinux1_x86_64.whl (17.0MB)
     |████████████████████████████████| 17.0MB 3.3MB/s 
Requirement already satisfied: pandas==0.24.2 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from -r ../bin/requirements.txt (line 5)) (0.24.2)
Requirement already satisfied: python-dateutil==2.8.1 in /home/kolyann/.local/lib/python2.7/site-packages (from -r ../bin/requirements.txt (line 6)) (2.8.1)
Requirement already satisfied: python-statface-client==0.152.0 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from -r ../bin/requirements.txt (line 7)) (0.152.0)
Requirement already satisfied: PyYAML==5.3 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from -r ../bin/requirements.txt (line 8)) (5.3)
Requirement already satisfied: requests==2.22.0 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from -r ../bin/requirements.txt (line 9)) (2.22.0)
Requirement already satisfied: urllib3==1.25.8 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from -r ../bin/requirements.txt (line 10)) (1.25.8)
Requirement already satisfied: xmltodict==0.12.0 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from -r ../bin/requirements.txt (line 11)) (0.12.0)
Requirement already satisfied: yasmapi==0.5.2 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from -r ../bin/requirements.txt (line 12)) (0.5.2)
Requirement already satisfied: setuptools in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from kiwisolver==1.1.0->-r ../bin/requirements.txt (line 1)) (44.0.0)
Requirement already satisfied: subprocess32 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from matplotlib==2.2.5->-r ../bin/requirements.txt (line 3)) (3.5.4)
Requirement already satisfied: cycler>=0.10 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from matplotlib==2.2.5->-r ../bin/requirements.txt (line 3)) (0.10.0)
Requirement already satisfied: six>=1.10 in /home/kolyann/.local/lib/python2.7/site-packages (from matplotlib==2.2.5->-r ../bin/requirements.txt (line 3)) (1.14.0)
Requirement already satisfied: backports.functools-lru-cache in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from matplotlib==2.2.5->-r ../bin/requirements.txt (line 3)) (1.6.1)
Requirement already satisfied: pytz in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from matplotlib==2.2.5->-r ../bin/requirements.txt (line 3)) (2019.3)
Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from matplotlib==2.2.5->-r ../bin/requirements.txt (line 3)) (2.4.6)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from requests==2.22.0->-r ../bin/requirements.txt (line 9)) (3.0.4)
Requirement already satisfied: idna<2.9,>=2.5 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from requests==2.22.0->-r ../bin/requirements.txt (line 9)) (2.8)
Requirement already satisfied: certifi>=2017.4.17 in /home/kolyann/anaconda2/envs/conda_stat/lib/python2.7/site-packages (from requests==2.22.0->-r ../bin/requirements.txt (line 9)) (2019.11.28)
Installing collected packages: numpy
  Found existing installation: numpy 1.16.6
    Uninstalling numpy-1.16.6:
      Successfully uninstalled numpy-1.16.6
Successfully installed numpy-1.16.5
In [4]:
# Интервал импорта данных
data_sd = '2019-12-01'
data_ed = '2020-04-01'

# Интервал предсказывания 
fc_start = pd.to_datetime(data_ed)
fc_end = pd.to_datetime('2021-01-01')

# Импорт предзаказов
orders = yaml.safe_load(open('../conf/orders.yaml'))
# Импорт железных кондукторных групп, по которым строются предсказания
disk_cnames = yaml.safe_load(open('../conf/cnames.yaml'))

# Фильтрация окружений кулауда, частные графики строятся по окружениям потребляющим больше X% указанного ресурса
qloud_filter_column = 'cpu_total'
qloud_min_perc = 1.5
In [5]:
# Загрузка данных по заданному интервалу
disk_data = DiskResourcesStatCollector().download_data((data_sd, data_ed))
df = pd.DataFrame(disk_data)

# Очистка данных от сущности fqdn, дальшейший анализ производится по кондукторным группам / qloud-prj
tdf = df.copy()
tdf[fd] = pd.to_datetime(tdf[fd])
tdf = tdf.sort_values(fd).set_index(fd).drop(fqdn, axis=1)
tdf = tdf.groupby([tdf.index, cg]).sum().reset_index(cg)
In [6]:
# Загрузка предсказанных данных по недельной аудитории, используется для предсказания
pred_df = DiskWauForecast022020StatCollector().download_data((fc_start, fc_end))

Disk qloud resources

In [7]:
name = 'disk'
prepared = concat_df_with_wau(tdf[tdf[cg] == name], pred_df, fc_start, fc_end)
plot_df_by_cg(prepared, 'qloud disk', fc_start, figsize=(24, 6), orders=orders['qloud'])

qloud_names = df[df[fqdn] == 'qloud'][cg]
qloud_names = qloud_names[qloud_names.apply(lambda x: len(x.split('.')) == 2)].unique()

qloud_names = filter_by_percentage(tdf[tdf[cg].isin(qloud_names)], cg, qloud_filter_column, qloud_min_perc)[cg].values

for name in qloud_names:
    prepared = concat_df_with_wau(tdf[tdf[cg] == name], pred_df, fc_start, fc_end)
    plot_df_by_cg(prepared, name, fc_start)

Disk total hardware resources

In [8]:
hw_cnames = disk_cnames['hw']
prepared = concat_df_with_wau(sum_df_by_names(tdf, cg, hw_cnames), pred_df, fc_start, fc_end)
plot_df_by_cg(prepared, 'HW resources (%s)' % ', '.join(hw_cnames), fc_start, figsize=(24, 6), orders=orders['hw'])

for name in hw_cnames:
    prepared = concat_df_with_wau(tdf[tdf[cg] == name], pred_df, fc_start, fc_end)
    plot_df_by_cg(prepared, name, fc_start)

Disk DB hardware resources

In [9]:
db_names = disk_cnames['db']
prepared = concat_df_with_wau(sum_df_by_names(tdf, cg, db_names), pred_df, fc_start, fc_end)
plot_df_by_cg(prepared, 'DB resources (%s)' % ', '.join(db_names), fc_start, figsize=(24, 6), orders=orders['hw'])

for name in db_names:
    prepared = concat_df_with_wau(tdf[tdf[cg] == name], pred_df, fc_start, fc_end)
    plot_df_by_cg(prepared, name, fc_start)
    
In [9]: