2020-11-05 15:58:11 +00:00
{
"cells": [
{
"cell_type": "code",
2020-11-27 04:35:06 +00:00
"execution_count": 1,
2020-11-05 15:58:11 +00:00
"metadata": {},
2020-11-27 04:35:06 +00:00
"outputs": [],
2020-11-05 15:58:11 +00:00
"source": [
"import os\n",
"import time\n",
"import glob\n",
"from pathlib import Path\n",
"import pandas as pd\n",
"import plotly.graph_objects as go\n",
"import plotly.express as px\n",
"pd.options.plotting.backend = \"plotly\" \n",
"import numpy as np\n",
"import seaborn as sns\n",
"from scipy import stats"
]
},
{
"cell_type": "code",
2020-11-27 04:35:06 +00:00
"execution_count": 2,
2020-11-05 15:58:11 +00:00
"metadata": {},
"outputs": [],
"source": [
"if not os.path.exists('output'):\n",
" os.makedirs('output')\n",
"if not os.path.exists('plots'):\n",
" os.makedirs('plots')\n",
"if not os.path.exists('plots/CPU'):\n",
" os.makedirs('plots/CPU')\n",
"if not os.path.exists('plots/Disc'):\n",
" os.makedirs('plots/Disc')\n",
"if not os.path.exists('plots/Energy'):\n",
" os.makedirs('plots/Energy')\n",
"if not os.path.exists('plots/MEM'):\n",
" os.makedirs('plots/MEM')\n",
"if not os.path.exists('plots/Network'):\n",
" os.makedirs('plots/Network')\n",
"if not os.path.exists('plots/Network/received'):\n",
" os.makedirs('plots/Network/received')\n",
"if not os.path.exists('plots/Network/transmitted'):\n",
" os.makedirs('plots/Network/transmitted')"
]
},
{
"cell_type": "code",
2020-11-27 04:35:06 +00:00
"execution_count": 3,
2020-11-05 15:58:11 +00:00
"metadata": {},
"outputs": [],
"source": [
"def plotFigure(dataToUse, colNamesArray, unit, titleText, filePath):\n",
" if len(colNamesArray) > 3:\n",
" print(\"Too much cols\")\n",
" return\n",
" data = dataToUse[colNamesArray]\n",
" data.columns =['L1','L2','L3']\n",
" iteration_plot = data.plot.line(line_shape='hv') # other options: spline / vhv\n",
" iteration_plot.update_xaxes(\n",
" title_text=\"Time (s)\",\n",
" ticktext=[\"30\", \"60\", \"90\", \"120\"],\n",
" tickvals=[data.index[29], data.index[59], data.index[89], data.index.max()]\n",
" )\n",
" iteration_plot.update_yaxes(title_text=unit)\n",
" iteration_plot.update_layout(\n",
" title_text=titleText,\n",
" legend=dict(\n",
" title=\"Server\"\n",
" )\n",
" )\n",
" iteration_plot.write_image(filePath)"
]
},
{
"cell_type": "code",
2020-11-27 04:35:06 +00:00
"execution_count": 4,
2020-11-05 15:58:11 +00:00
"metadata": {},
"outputs": [],
"source": [
"def plotBoxplotFigure(data, colNamesArray, title, fileName):\n",
" if len(colNamesArray) > 3:\n",
" print(\"Too much cols\")\n",
" return\n",
" fig = go.Figure()\n",
" fig.add_trace(go.Box(\n",
" x=data[colNamesArray[0]],\n",
" name='L1'\n",
" ))\n",
" fig.add_trace(go.Box(\n",
" x=data[colNamesArray[1]],\n",
" name='L2'\n",
" ))\n",
" fig.add_trace(go.Box(\n",
" x=data[colNamesArray[2]],\n",
" name='L3'\n",
" ))\n",
"\n",
" fig.update_layout(\n",
" title_text=title,\n",
" xaxis=dict(title='W', zeroline=False),\n",
" boxmode='group',\n",
" legend=dict(\n",
" title=\"Server\"\n",
" )\n",
" )\n",
" fig.update_traces(orientation='h')\n",
" fig.write_image(fileName)"
]
},
{
"cell_type": "code",
2020-11-27 04:35:06 +00:00
"execution_count": 5,
2020-11-05 15:58:11 +00:00
"metadata": {},
"outputs": [],
"source": [
2020-11-27 04:35:06 +00:00
"def cleanDataIQR(data, factor):\n",
2020-11-05 15:58:11 +00:00
" Q1=data.quantile(0.25)\n",
" Q3=data.quantile(0.75)\n",
" IQR=Q3-Q1\n",
2020-11-27 04:35:06 +00:00
" lowqe_bound=Q1 - factor * IQR\n",
" upper_bound=Q3 + factor * IQR\n",
2020-11-05 15:58:11 +00:00
" return data[~((data < lowqe_bound) |(data > upper_bound))]"
]
},
{
"cell_type": "code",
2020-11-27 04:35:06 +00:00
"execution_count": 6,
2020-11-05 15:58:11 +00:00
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"(123251, 9)\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
" AVG[W] L1 Min[W] L1 Max[W] L1 AVG[W] L2 Min[W] L2 \\\n",
"Time \n",
"14.10.20 00:00:01 0.076 0.009 0.148 -0.117 -0.196 \n",
"14.10.20 00:00:02 0.100 0.021 0.178 -0.068 -0.209 \n",
"14.10.20 00:00:03 0.090 0.002 0.186 0.457 -0.222 \n",
"14.10.20 00:00:04 0.073 -0.018 0.131 -0.085 -0.169 \n",
"14.10.20 00:00:05 0.106 0.029 0.169 -0.100 -0.223 \n",
"\n",
" Max[W] L2 AVG[W] L3 Min[W] L3 Max[W] L3 \n",
"Time \n",
"14.10.20 00:00:01 0.037 0.052 -0.075 0.207 \n",
"14.10.20 00:00:02 0.007 0.030 -0.132 0.176 \n",
"14.10.20 00:00:03 2.782 0.027 -0.130 0.182 \n",
"14.10.20 00:00:04 0.019 0.055 -0.066 0.186 \n",
"14.10.20 00:00:05 0.027 0.005 -0.139 0.176 "
],
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>AVG[W] L1</th>\n <th>Min[W] L1</th>\n <th>Max[W] L1</th>\n <th>AVG[W] L2</th>\n <th>Min[W] L2</th>\n <th>Max[W] L2</th>\n <th>AVG[W] L3</th>\n <th>Min[W] L3</th>\n <th>Max[W] L3</th>\n </tr>\n <tr>\n <th>Time</th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>14.10.20 00:00:01</th>\n <td>0.076</td>\n <td>0.009</td>\n <td>0.148</td>\n <td>-0.117</td>\n <td>-0.196</td>\n <td>0.037</td>\n <td>0.052</td>\n <td>-0.075</td>\n <td>0.207</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:02</th>\n <td>0.100</td>\n <td>0.021</td>\n <td>0.178</td>\n <td>-0.068</td>\n <td>-0.209</td>\n <td>0.007</td>\n <td>0.030</td>\n <td>-0.132</td>\n <td>0.176</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:03</th>\n <td>0.090</td>\n <td>0.002</td>\n <td>0.186</td>\n <td>0.457</td>\n <td>-0.222</td>\n <td>2.782</td>\n <td>0.027</td>\n <td>-0.130</td>\n <td>0.182</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:04</th>\n <td>0.073</td>\n <td>-0.018</td>\n <td>0.131</td>\n <td>-0.085</td>\n <td>-0.169</td>\n <td>0.019</td>\n <td>0.055</td>\n <td>-0.066</td>\n <td>0.186</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:05</th>\n <td>0.106</td>\n <td>0.029</td>\n <td>0.169</td>\n <td>-0.100</td>\n <td>-0.223</td>\n <td>0.027</td>\n <td>0.005</td>\n <td>-0.139</td>\n <td>0.176</td>\n </tr>\n </tbody>\n</table>\n</div>"
},
"metadata": {},
2020-11-27 04:35:06 +00:00
"execution_count": 6
2020-11-05 15:58:11 +00:00
}
],
"source": [
"l1_matrix = pd.read_csv(\"./MeasurementFiles/Matrix/Messung_1_L1.csv\", sep=\";\", decimal=\",\").dropna(axis=1).drop('Nummer', axis=1)\n",
"l1_matrix.columns =['Time', 'AVG[W] L1', 'Min[W] L1', 'Max[W] L1']\n",
"l1_matrix.index = l1_matrix['Time']\n",
"l1_matrix = l1_matrix.drop(\"Time\", axis=1)\n",
"l2_matrix = pd.read_csv(\"./MeasurementFiles/Matrix/Messung_1_L2.csv\", sep=\";\", decimal=\",\").dropna(axis=1).drop('Nummer', axis=1)\n",
"l2_matrix.columns =['Time', 'AVG[W] L2', 'Min[W] L2', 'Max[W] L2']\n",
"l2_matrix.index = l2_matrix['Time']\n",
"l2_matrix = l2_matrix.drop(\"Time\", axis=1)\n",
"l3_matrix = pd.read_csv(\"./MeasurementFiles/Matrix/Messung_1_L3.csv\", sep=\";\", decimal=\",\").dropna(axis=1).drop('Nummer', axis=1)\n",
"l3_matrix.columns =['Time', 'AVG[W] L3', 'Min[W] L3', 'Max[W] L3']\n",
"l3_matrix.index = l3_matrix['Time']\n",
"l3_matrix = l3_matrix.drop(\"Time\", axis=1)\n",
"matrix = l1_matrix.merge(l2_matrix, on='Time').merge(l3_matrix, on='Time')\n",
"print(matrix.shape)\n",
"matrix.head()"
]
},
{
"cell_type": "code",
2020-11-27 04:35:06 +00:00
"execution_count": 7,
2020-11-05 15:58:11 +00:00
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"(123251, 27)\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
" AVG[W] L1 Min[W] L1 Max[W] L1 AVG[W] L2 Min[W] L2 \\\n",
"Time \n",
"14.10.20 00:00:01 0.076 0.009 0.148 -0.117 -0.196 \n",
"14.10.20 00:00:02 0.100 0.021 0.178 -0.068 -0.209 \n",
"14.10.20 00:00:03 0.090 0.002 0.186 0.457 -0.222 \n",
"14.10.20 00:00:04 0.073 -0.018 0.131 -0.085 -0.169 \n",
"14.10.20 00:00:05 0.106 0.029 0.169 -0.100 -0.223 \n",
"\n",
" Max[W] L2 AVG[W] L3 Min[W] L3 Max[W] L3 CPU% L1 ... \\\n",
"Time ... \n",
"14.10.20 00:00:01 0.037 0.052 -0.075 0.207 0.0 ... \n",
"14.10.20 00:00:02 0.007 0.030 -0.132 0.176 0.0 ... \n",
"14.10.20 00:00:03 2.782 0.027 -0.130 0.182 0.0 ... \n",
"14.10.20 00:00:04 0.019 0.055 -0.066 0.186 0.0 ... \n",
"14.10.20 00:00:05 0.027 0.005 -0.139 0.176 0.0 ... \n",
"\n",
" KB Transmitted L2 KB Received L2 KB Read L2 KB Write L2 \\\n",
"Time \n",
"14.10.20 00:00:01 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:02 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:03 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:04 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:05 0.0 0.0 0.0 0.0 \n",
"\n",
" CPU% L3 MEM Used L3 KB Transmitted L3 KB Received L3 \\\n",
"Time \n",
"14.10.20 00:00:01 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:02 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:03 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:04 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:05 0.0 0.0 0.0 0.0 \n",
"\n",
" KB Read L3 KB Write L3 \n",
"Time \n",
"14.10.20 00:00:01 0.0 0.0 \n",
"14.10.20 00:00:02 0.0 0.0 \n",
"14.10.20 00:00:03 0.0 0.0 \n",
"14.10.20 00:00:04 0.0 0.0 \n",
"14.10.20 00:00:05 0.0 0.0 \n",
"\n",
"[5 rows x 27 columns]"
],
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>AVG[W] L1</th>\n <th>Min[W] L1</th>\n <th>Max[W] L1</th>\n <th>AVG[W] L2</th>\n <th>Min[W] L2</th>\n <th>Max[W] L2</th>\n <th>AVG[W] L3</th>\n <th>Min[W] L3</th>\n <th>Max[W] L3</th>\n <th>CPU% L1</th>\n <th>...</th>\n <th>KB Transmitted L2</th>\n <th>KB Received L2</th>\n <th>KB Read L2</th>\n <th>KB Write L2</th>\n <th>CPU% L3</th>\n <th>MEM Used L3</th>\n <th>KB Transmitted L3</th>\n <th>KB Received L3</th>\n <th>KB Read L3</th>\n <th>KB Write L3</th>\n </tr>\n <tr>\n <th>Time</th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>14.10.20 00:00:01</th>\n <td>0.076</td>\n <td>0.009</td>\n <td>0.148</td>\n <td>-0.117</td>\n <td>-0.196</td>\n <td>0.037</td>\n <td>0.052</td>\n <td>-0.075</td>\n <td>0.207</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:02</th>\n <td>0.100</td>\n <td>0.021</td>\n <td>0.178</td>\n <td>-0.068</td>\n <td>-0.209</td>\n <td>0.007</td>\n <td>0.030</td>\n <td>-0.132</td>\n <td>0.176</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:03</th>\n <td>0.090</td>\n <td>0.002</td>\n <td>0.186</td>\n <td>0.457</td>\n <td>-0.222</td>\n <td>2.782</td>\n <td>0.027</td>\n <td>-0.130</td>\n <td>0.182</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:04</th>\n <td>0.073</td>\n <td>-0.018</td>\n <td>0.131</td>\n <td>-0.085</td>\n <td>-0.169</td>\n <td>0.019</td>\n <td>0.055</td>\n <td>-0.066</td>\n <td>0.186</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:05</th>\n <td>0.106</td>\n <td>0.029</td>\n <td>0.169</td>\n <td>-0.100</td>\n <td>-0.223</td>\n <td>0.027</td>\n <td>0.005</td>\n <td>-0.139</td>\n <td>0.176</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n </tbody>\n</table>\n<p>5 rows ×
},
"metadata": {},
2020-11-27 04:35:06 +00:00
"execution_count": 7
2020-11-05 15:58:11 +00:00
}
],
"source": [
"cols = ['Date Time','[CPU]Totl%','[MEM]Used','[NET]TxKBTot','[NET]RxKBTot','[DSK]ReadKBTot','[DSK]WriteKBTot']\n",
"f1_matrix = pd.read_csv(\"./MeasurementFiles/Matrix/Network-server-207-F1.csv\", sep=\";\", usecols=cols).dropna(axis=1)#.drop('Nummer', axis=1)\n",
"f1_matrix.columns =['Time','CPU% L1','MEM Used L1','KB Transmitted L1','KB Received L1','KB Read L1','KB Write L1']\n",
"f1_matrix['Time'] = [time.strftime('%d.%m.%y %H:%M:%S', time.strptime(time_string, '%Y%m%d %H:%M:%S')) for time_string in f1_matrix['Time']]\n",
"f1_matrix.index = f1_matrix['Time']\n",
"f1_matrix = f1_matrix.drop(\"Time\", axis=1)\n",
"f2_matrix = pd.read_csv(\"./MeasurementFiles/Matrix/Network-server-209-F2.csv\", sep=\";\", usecols=cols).dropna(axis=1)#.drop('Nummer', axis=1)\n",
"f2_matrix.columns =['Time','CPU% L2','MEM Used L2','KB Transmitted L2','KB Received L2','KB Read L2','KB Write L2']\n",
"f2_matrix['Time'] = [time.strftime('%d.%m.%y %H:%M:%S', time.strptime(time_string, '%Y%m%d %H:%M:%S')) for time_string in f2_matrix['Time']]\n",
"f2_matrix.index = f2_matrix['Time']\n",
"f2_matrix = f2_matrix.drop(\"Time\", axis=1)\n",
"f3_matrix = pd.read_csv(\"./MeasurementFiles/Matrix/Network-server-208-F3.csv\", sep=\";\", usecols=cols).dropna(axis=1)#.drop('Nummer', axis=1)\n",
"f3_matrix.columns =['Time','CPU% L3','MEM Used L3','KB Transmitted L3','KB Received L3','KB Read L3','KB Write L3']\n",
"f3_matrix['Time'] = [time.strftime('%d.%m.%y %H:%M:%S', time.strptime(time_string, '%Y%m%d %H:%M:%S')) for time_string in f3_matrix['Time']]\n",
"f3_matrix.index = f3_matrix['Time']\n",
"f3_matrix = f3_matrix.drop(\"Time\", axis=1)\n",
"matrix = matrix.merge(f1_matrix, on='Time', how='outer').merge(f2_matrix, on='Time', how='outer').merge(f3_matrix, on='Time', how='outer').fillna(0)\n",
"print(matrix.shape)\n",
"matrix.head()"
]
},
{
"cell_type": "code",
2020-11-27 04:35:06 +00:00
"execution_count": 8,
2020-11-05 15:58:11 +00:00
"metadata": {},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" AVG[W] L1 Min[W] L1 Max[W] L1 AVG[W] L2 Min[W] L2 \\\n",
"Time \n",
"15.10.20 00:00:01 9.157 7.878 13.265 6.515 6.004 \n",
"15.10.20 00:00:02 8.819 7.424 12.104 6.643 5.986 \n",
"15.10.20 00:00:03 8.238 7.494 8.660 6.199 5.304 \n",
"15.10.20 00:00:04 8.269 7.554 8.916 6.522 6.175 \n",
"15.10.20 00:00:05 8.378 7.808 8.852 6.450 6.222 \n",
"\n",
" Max[W] L2 AVG[W] L3 Min[W] L3 Max[W] L3 \n",
"Time \n",
"15.10.20 00:00:01 6.963 7.210 6.939 7.701 \n",
"15.10.20 00:00:02 8.129 9.163 7.199 13.603 \n",
"15.10.20 00:00:03 6.595 7.501 7.057 8.369 \n",
"15.10.20 00:00:04 6.811 7.221 6.728 7.801 \n",
"15.10.20 00:00:05 6.800 7.229 6.982 7.485 "
],
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>AVG[W] L1</th>\n <th>Min[W] L1</th>\n <th>Max[W] L1</th>\n <th>AVG[W] L2</th>\n <th>Min[W] L2</th>\n <th>Max[W] L2</th>\n <th>AVG[W] L3</th>\n <th>Min[W] L3</th>\n <th>Max[W] L3</th>\n </tr>\n <tr>\n <th>Time</th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>15.10.20 00:00:01</th>\n <td>9.157</td>\n <td>7.878</td>\n <td>13.265</td>\n <td>6.515</td>\n <td>6.004</td>\n <td>6.963</td>\n <td>7.210</td>\n <td>6.939</td>\n <td>7.701</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:02</th>\n <td>8.819</td>\n <td>7.424</td>\n <td>12.104</td>\n <td>6.643</td>\n <td>5.986</td>\n <td>8.129</td>\n <td>9.163</td>\n <td>7.199</td>\n <td>13.603</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:03</th>\n <td>8.238</td>\n <td>7.494</td>\n <td>8.660</td>\n <td>6.199</td>\n <td>5.304</td>\n <td>6.595</td>\n <td>7.501</td>\n <td>7.057</td>\n <td>8.369</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:04</th>\n <td>8.269</td>\n <td>7.554</td>\n <td>8.916</td>\n <td>6.522</td>\n <td>6.175</td>\n <td>6.811</td>\n <td>7.221</td>\n <td>6.728</td>\n <td>7.801</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:05</th>\n <td>8.378</td>\n <td>7.808</td>\n <td>8.852</td>\n <td>6.450</td>\n <td>6.222</td>\n <td>6.800</td>\n <td>7.229</td>\n <td>6.982</td>\n <td>7.485</td>\n </tr>\n </tbody>\n</table>\n</div>"
},
"metadata": {},
2020-11-27 04:35:06 +00:00
"execution_count": 8
2020-11-05 15:58:11 +00:00
}
],
"source": [
"l1_activitypub = pd.read_csv(\"./MeasurementFiles/ActivityPub/Messung_2_L1.csv\", sep=\";\", decimal=\",\").dropna(axis=1).drop('Nummer', axis=1)\n",
"l1_activitypub.columns =['Time', 'AVG[W] L1', 'Min[W] L1', 'Max[W] L1']\n",
"l1_activitypub.index = l1_activitypub['Time']\n",
"l1_activitypub = l1_activitypub.drop(\"Time\", axis=1)\n",
"l2_activitypub = pd.read_csv(\"./MeasurementFiles/ActivityPub/Messung_2_L2.csv\", sep=\";\", decimal=\",\").dropna(axis=1).drop('Nummer', axis=1)\n",
"l2_activitypub.columns =['Time', 'AVG[W] L2', 'Min[W] L2', 'Max[W] L2']\n",
"l2_activitypub.index = l2_activitypub['Time']\n",
"l2_activitypub = l2_activitypub.drop(\"Time\", axis=1)\n",
"l3_activitypub = pd.read_csv(\"./MeasurementFiles/ActivityPub/Messung_2_L3.csv\", sep=\";\", decimal=\",\").dropna(axis=1).drop('Nummer', axis=1)\n",
"l3_activitypub.columns =['Time', 'AVG[W] L3', 'Min[W] L3', 'Max[W] L3']\n",
"l3_activitypub.index = l3_activitypub['Time']\n",
"l3_activitypub = l3_activitypub.drop(\"Time\", axis=1)\n",
"activitypub = l1_activitypub.merge(l2_activitypub, on='Time').merge(l3_activitypub, on='Time')\n",
"activitypub.head()"
]
},
{
"cell_type": "code",
2020-11-27 04:35:06 +00:00
"execution_count": 9,
2020-11-05 15:58:11 +00:00
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"(142191, 27)\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
" AVG[W] L1 Min[W] L1 Max[W] L1 AVG[W] L2 Min[W] L2 \\\n",
"Time \n",
"15.10.20 00:00:01 9.157 7.878 13.265 6.515 6.004 \n",
"15.10.20 00:00:02 8.819 7.424 12.104 6.643 5.986 \n",
"15.10.20 00:00:03 8.238 7.494 8.660 6.199 5.304 \n",
"15.10.20 00:00:04 8.269 7.554 8.916 6.522 6.175 \n",
"15.10.20 00:00:05 8.378 7.808 8.852 6.450 6.222 \n",
"\n",
" Max[W] L2 AVG[W] L3 Min[W] L3 Max[W] L3 CPU% L1 ... \\\n",
"Time ... \n",
"15.10.20 00:00:01 6.963 7.210 6.939 7.701 0.0 ... \n",
"15.10.20 00:00:02 8.129 9.163 7.199 13.603 0.0 ... \n",
"15.10.20 00:00:03 6.595 7.501 7.057 8.369 0.0 ... \n",
"15.10.20 00:00:04 6.811 7.221 6.728 7.801 0.0 ... \n",
"15.10.20 00:00:05 6.800 7.229 6.982 7.485 0.0 ... \n",
"\n",
" KB Transmitted L2 KB Received L2 KB Read L2 KB Write L2 \\\n",
"Time \n",
"15.10.20 00:00:01 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:02 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:03 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:04 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:05 0.0 0.0 0.0 0.0 \n",
"\n",
" CPU% L3 MEM Used L3 KB Transmitted L3 KB Received L3 \\\n",
"Time \n",
"15.10.20 00:00:01 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:02 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:03 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:04 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:05 0.0 0.0 0.0 0.0 \n",
"\n",
" KB Read L3 KB Write L3 \n",
"Time \n",
"15.10.20 00:00:01 0.0 0.0 \n",
"15.10.20 00:00:02 0.0 0.0 \n",
"15.10.20 00:00:03 0.0 0.0 \n",
"15.10.20 00:00:04 0.0 0.0 \n",
"15.10.20 00:00:05 0.0 0.0 \n",
"\n",
"[5 rows x 27 columns]"
],
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>AVG[W] L1</th>\n <th>Min[W] L1</th>\n <th>Max[W] L1</th>\n <th>AVG[W] L2</th>\n <th>Min[W] L2</th>\n <th>Max[W] L2</th>\n <th>AVG[W] L3</th>\n <th>Min[W] L3</th>\n <th>Max[W] L3</th>\n <th>CPU% L1</th>\n <th>...</th>\n <th>KB Transmitted L2</th>\n <th>KB Received L2</th>\n <th>KB Read L2</th>\n <th>KB Write L2</th>\n <th>CPU% L3</th>\n <th>MEM Used L3</th>\n <th>KB Transmitted L3</th>\n <th>KB Received L3</th>\n <th>KB Read L3</th>\n <th>KB Write L3</th>\n </tr>\n <tr>\n <th>Time</th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>15.10.20 00:00:01</th>\n <td>9.157</td>\n <td>7.878</td>\n <td>13.265</td>\n <td>6.515</td>\n <td>6.004</td>\n <td>6.963</td>\n <td>7.210</td>\n <td>6.939</td>\n <td>7.701</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:02</th>\n <td>8.819</td>\n <td>7.424</td>\n <td>12.104</td>\n <td>6.643</td>\n <td>5.986</td>\n <td>8.129</td>\n <td>9.163</td>\n <td>7.199</td>\n <td>13.603</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:03</th>\n <td>8.238</td>\n <td>7.494</td>\n <td>8.660</td>\n <td>6.199</td>\n <td>5.304</td>\n <td>6.595</td>\n <td>7.501</td>\n <td>7.057</td>\n <td>8.369</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:04</th>\n <td>8.269</td>\n <td>7.554</td>\n <td>8.916</td>\n <td>6.522</td>\n <td>6.175</td>\n <td>6.811</td>\n <td>7.221</td>\n <td>6.728</td>\n <td>7.801</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:05</th>\n <td>8.378</td>\n <td>7.808</td>\n <td>8.852</td>\n <td>6.450</td>\n <td>6.222</td>\n <td>6.800</td>\n <td>7.229</td>\n <td>6.982</td>\n <td>7.485</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n </tbody>\n</table>\n<p>5 rows ×
},
"metadata": {},
2020-11-27 04:35:06 +00:00
"execution_count": 9
2020-11-05 15:58:11 +00:00
}
],
"source": [
"cols = ['Date Time','[CPU]Totl%','[MEM]Used','[NET]TxKBTot','[NET]RxKBTot','[DSK]ReadKBTot','[DSK]WriteKBTot']\n",
"f1_activitypub = pd.read_csv(\"./MeasurementFiles/ActivityPub/Network-server-207-F1.csv\", sep=\";\", usecols=cols).dropna(axis=1)\n",
"f1_activitypub.columns =['Time','CPU% L1','MEM Used L1','KB Transmitted L1','KB Received L1','KB Read L1','KB Write L1']\n",
"f1_activitypub['Time'] = [time.strftime('%d.%m.%y %H:%M:%S', time.strptime(time_string, '%Y%m%d %H:%M:%S')) for time_string in f1_activitypub['Time']]\n",
"f1_activitypub.index = f1_activitypub['Time']\n",
"f1_activitypub = f1_activitypub.drop(\"Time\", axis=1)\n",
"f2_activitypub = pd.read_csv(\"./MeasurementFiles/ActivityPub/Network-server-209-F2.csv\", sep=\";\", usecols=cols).dropna(axis=1)\n",
"f2_activitypub.columns =['Time','CPU% L2','MEM Used L2','KB Transmitted L2','KB Received L2','KB Read L2','KB Write L2']\n",
"f2_activitypub['Time'] = [time.strftime('%d.%m.%y %H:%M:%S', time.strptime(time_string, '%Y%m%d %H:%M:%S')) for time_string in f2_activitypub['Time']]\n",
"f2_activitypub.index = f2_activitypub['Time']\n",
"f2_activitypub = f2_activitypub.drop(\"Time\", axis=1)\n",
"f3_activitypub = pd.read_csv(\"./MeasurementFiles/ActivityPub/Network-server-208-F3.csv\", sep=\";\", usecols=cols).dropna(axis=1)\n",
"f3_activitypub.columns =['Time','CPU% L3','MEM Used L3','KB Transmitted L3','KB Received L3','KB Read L3','KB Write L3']\n",
"f3_activitypub['Time'] = [time.strftime('%d.%m.%y %H:%M:%S', time.strptime(time_string, '%Y%m%d %H:%M:%S')) for time_string in f3_activitypub['Time']]\n",
"f3_activitypub.index = f3_activitypub['Time']\n",
"f3_activitypub = f3_activitypub.drop(\"Time\", axis=1)\n",
"activitypub = activitypub.merge(f1_activitypub, on='Time', how='outer').merge(f2_activitypub, on='Time', how='outer').merge(f3_activitypub, on='Time', how='outer').fillna(0)\n",
"print(activitypub.shape)\n",
"activitypub.head()"
]
},
{
"cell_type": "code",
2020-11-27 04:35:06 +00:00
"execution_count": 10,
2020-11-05 15:58:11 +00:00
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"(228591, 27)\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
" AVG[W] L1 Min[W] L1 Max[W] L1 AVG[W] L2 Min[W] L2 \\\n",
"Time \n",
"14.10.20 00:00:01 0.076 0.009 0.148 -0.117 -0.196 \n",
"14.10.20 00:00:02 0.100 0.021 0.178 -0.068 -0.209 \n",
"14.10.20 00:00:03 0.090 0.002 0.186 0.457 -0.222 \n",
"14.10.20 00:00:04 0.073 -0.018 0.131 -0.085 -0.169 \n",
"14.10.20 00:00:05 0.106 0.029 0.169 -0.100 -0.223 \n",
"\n",
" Max[W] L2 AVG[W] L3 Min[W] L3 Max[W] L3 CPU% L1 ... \\\n",
"Time ... \n",
"14.10.20 00:00:01 0.037 0.052 -0.075 0.207 0.0 ... \n",
"14.10.20 00:00:02 0.007 0.030 -0.132 0.176 0.0 ... \n",
"14.10.20 00:00:03 2.782 0.027 -0.130 0.182 0.0 ... \n",
"14.10.20 00:00:04 0.019 0.055 -0.066 0.186 0.0 ... \n",
"14.10.20 00:00:05 0.027 0.005 -0.139 0.176 0.0 ... \n",
"\n",
" KB Transmitted L2 KB Received L2 KB Read L2 KB Write L2 \\\n",
"Time \n",
"14.10.20 00:00:01 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:02 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:03 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:04 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:05 0.0 0.0 0.0 0.0 \n",
"\n",
" CPU% L3 MEM Used L3 KB Transmitted L3 KB Received L3 \\\n",
"Time \n",
"14.10.20 00:00:01 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:02 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:03 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:04 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:05 0.0 0.0 0.0 0.0 \n",
"\n",
" KB Read L3 KB Write L3 \n",
"Time \n",
"14.10.20 00:00:01 0.0 0.0 \n",
"14.10.20 00:00:02 0.0 0.0 \n",
"14.10.20 00:00:03 0.0 0.0 \n",
"14.10.20 00:00:04 0.0 0.0 \n",
"14.10.20 00:00:05 0.0 0.0 \n",
"\n",
"[5 rows x 27 columns]"
],
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>AVG[W] L1</th>\n <th>Min[W] L1</th>\n <th>Max[W] L1</th>\n <th>AVG[W] L2</th>\n <th>Min[W] L2</th>\n <th>Max[W] L2</th>\n <th>AVG[W] L3</th>\n <th>Min[W] L3</th>\n <th>Max[W] L3</th>\n <th>CPU% L1</th>\n <th>...</th>\n <th>KB Transmitted L2</th>\n <th>KB Received L2</th>\n <th>KB Read L2</th>\n <th>KB Write L2</th>\n <th>CPU% L3</th>\n <th>MEM Used L3</th>\n <th>KB Transmitted L3</th>\n <th>KB Received L3</th>\n <th>KB Read L3</th>\n <th>KB Write L3</th>\n </tr>\n <tr>\n <th>Time</th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>14.10.20 00:00:01</th>\n <td>0.076</td>\n <td>0.009</td>\n <td>0.148</td>\n <td>-0.117</td>\n <td>-0.196</td>\n <td>0.037</td>\n <td>0.052</td>\n <td>-0.075</td>\n <td>0.207</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:02</th>\n <td>0.100</td>\n <td>0.021</td>\n <td>0.178</td>\n <td>-0.068</td>\n <td>-0.209</td>\n <td>0.007</td>\n <td>0.030</td>\n <td>-0.132</td>\n <td>0.176</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:03</th>\n <td>0.090</td>\n <td>0.002</td>\n <td>0.186</td>\n <td>0.457</td>\n <td>-0.222</td>\n <td>2.782</td>\n <td>0.027</td>\n <td>-0.130</td>\n <td>0.182</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:04</th>\n <td>0.073</td>\n <td>-0.018</td>\n <td>0.131</td>\n <td>-0.085</td>\n <td>-0.169</td>\n <td>0.019</td>\n <td>0.055</td>\n <td>-0.066</td>\n <td>0.186</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:05</th>\n <td>0.106</td>\n <td>0.029</td>\n <td>0.169</td>\n <td>-0.100</td>\n <td>-0.223</td>\n <td>0.027</td>\n <td>0.005</td>\n <td>-0.139</td>\n <td>0.176</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n </tbody>\n</table>\n<p>5 rows ×
},
"metadata": {},
2020-11-27 04:35:06 +00:00
"execution_count": 10
2020-11-05 15:58:11 +00:00
}
],
"source": [
"frames = [matrix, activitypub]\n",
"result = pd.concat(frames)\n",
"result = result[~result.index.duplicated(keep='first')]\n",
"result.to_csv(\"output/MeasurementData.csv\")\n",
"print(result.shape)\n",
"result.head()"
]
},
{
"cell_type": "code",
2020-11-27 04:35:06 +00:00
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"#plotBoxplotFigure(result, [\"KB Received L1\", \"KB Received L2\", \"KB Received L3\"], \"Network before cleaning\", \"Network Boxplot.png\")\n",
"#result[\"KB Received L1\"] = result.loc[stats.zscore(result[\"KB Received L1\"]) <= 3][\"KB Received L1\"]\n",
"#result[\"KB Received L2\"] = result.loc[stats.zscore(result[\"KB Received L2\"]) <= 3][\"KB Received L2\"]\n",
"#result[\"KB Received L3\"] = result.loc[stats.zscore(result[\"KB Received L3\"]) <= 3][\"KB Received L3\"]\n",
"#plotBoxplotFigure(result, [\"KB Received L1\", \"KB Received L2\", \"KB Received L3\"], \"Network before cleaning\", \"Network Boxplot cleaned.png\")"
]
},
{
"cell_type": "code",
"execution_count": 12,
2020-11-05 15:58:11 +00:00
"metadata": {
"tags": []
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"111111111 Windows\n",
"Baseline Windows\n",
"Baseline\n",
2020-11-27 04:35:06 +00:00
"110111111 ActivityPub\n",
"101111111 ActivityPub\n",
"211111111 ActivityPub\n",
"011111111 ActivityPub\n",
"111211111 ActivityPub\n",
"111111111 ActivityPub\n",
"121111111 ActivityPub\n",
"112111111 ActivityPub\n",
"111011111 ActivityPub\n",
2020-11-05 15:58:11 +00:00
"211111111 Matrix\n",
"121111111 Matrix\n",
"111111011 Matrix\n",
"011111111 Matrix\n",
"111011111 Matrix\n",
"112111111 Matrix\n",
"111111101 Matrix\n",
"101111111 Matrix\n",
"111110111 Matrix\n",
"111211111 Matrix\n",
"111111111 Matrix\n",
"110111111 Matrix\n",
"baseline\n",
"Iteration 5.500000e+00\n",
"Start 1.602788e+09\n",
"End 1.602788e+09\n",
"AVG[W] L1 5.950951e+00\n",
"AVG[W] L2 4.132172e+00\n",
"AVG[W] L3 5.818045e+00\n",
"CPU% L1 5.080473e-02\n",
"CPU% L2 2.116508e-02\n",
"CPU% L3 4.742914e-02\n",
"Energy L1 5.950951e+00\n",
"Energy L2 4.132172e+00\n",
"Energy L3 5.818045e+00\n",
"KB Read L1 0.000000e+00\n",
"KB Read L2 0.000000e+00\n",
"KB Read L3 0.000000e+00\n",
"KB Received L1 0.000000e+00\n",
"KB Received L2 0.000000e+00\n",
"KB Received L3 0.000000e+00\n",
"KB Transmitted L1 0.000000e+00\n",
"KB Transmitted L2 0.000000e+00\n",
"KB Transmitted L3 0.000000e+00\n",
"KB Write L1 3.793847e+00\n",
"KB Write L2 5.399402e+00\n",
"KB Write L3 5.540977e+00\n",
"MEM Used L1 4.075354e+06\n",
"MEM Used L2 3.891142e+06\n",
"MEM Used L3 3.822240e+06\n",
"Max[W] L1 6.101333e+00\n",
"Max[W] L2 4.379694e+00\n",
"Max[W] L3 5.993006e+00\n",
"Min[W] L1 5.836322e+00\n",
"Min[W] L2 3.933443e+00\n",
"Min[W] L3 5.669529e+00\n",
"dtype: float64\n",
"min\n",
2020-11-27 04:35:06 +00:00
"AVG[W] L1 5.997093e+00\n",
"Min[W] L1 5.811203e+00\n",
"Max[W] L1 6.292356e+00\n",
"AVG[W] L2 4.037085e+00\n",
"Min[W] L2 3.727415e+00\n",
"Max[W] L2 4.554542e+00\n",
"AVG[W] L3 5.826669e+00\n",
"Min[W] L3 5.622593e+00\n",
"Max[W] L3 6.123331e+00\n",
"CPU% L1 4.322034e-01\n",
"MEM Used L1 4.288508e+06\n",
"KB Transmitted L1 5.677966e-01\n",
2020-11-05 15:58:11 +00:00
"KB Received L1 0.000000e+00\n",
"KB Read L1 0.000000e+00\n",
2020-11-27 04:35:06 +00:00
"KB Write L1 6.166102e+01\n",
"CPU% L2 3.135593e-01\n",
"MEM Used L2 4.091692e+06\n",
"KB Transmitted L2 6.694915e-01\n",
2020-11-05 15:58:11 +00:00
"KB Received L2 0.000000e+00\n",
"KB Read L2 0.000000e+00\n",
2020-11-27 04:35:06 +00:00
"KB Write L2 5.467797e+01\n",
"CPU% L3 3.983051e-01\n",
"MEM Used L3 4.013788e+06\n",
"KB Transmitted L3 1.864407e-01\n",
"KB Received L3 1.016949e-01\n",
2020-11-05 15:58:11 +00:00
"KB Read L3 0.000000e+00\n",
2020-11-27 04:35:06 +00:00
"KB Write L3 3.966949e+01\n",
"Energy L1 5.997093e+00\n",
"Energy L2 4.037085e+00\n",
"Energy L3 6.159462e+00\n",
"Iteration 1.100000e+01\n",
"Energy total 2.924719e-01\n",
"Actions Send 1.200000e+01\n",
2020-11-05 15:58:11 +00:00
"dtype: float64\n",
2020-11-27 04:35:06 +00:00
"2.924719339609503\n",
2020-11-05 15:58:11 +00:00
"max\n",
"AVG[W] L1 6.501847e+00\n",
"Min[W] L1 6.013449e+00\n",
"Max[W] L1 7.691280e+00\n",
"AVG[W] L2 4.705025e+00\n",
"Min[W] L2 3.876602e+00\n",
"Max[W] L2 6.766508e+00\n",
"AVG[W] L3 6.201432e+00\n",
"Min[W] L3 5.670492e+00\n",
"Max[W] L3 7.394390e+00\n",
"CPU% L1 3.728814e+00\n",
"MEM Used L1 4.571048e+06\n",
"KB Transmitted L1 8.279661e+00\n",
"KB Received L1 1.099153e+01\n",
"KB Read L1 0.000000e+00\n",
"KB Write L1 4.452542e+02\n",
"CPU% L2 1.957627e+00\n",
"MEM Used L2 4.403195e+06\n",
"KB Transmitted L2 5.525424e+00\n",
2020-11-27 04:35:06 +00:00
"KB Received L2 3.646552e+00\n",
2020-11-05 15:58:11 +00:00
"KB Read L2 0.000000e+00\n",
"KB Write L2 4.514915e+02\n",
"CPU% L3 3.084746e+00\n",
"MEM Used L3 4.377153e+06\n",
"KB Transmitted L3 5.516949e+00\n",
"KB Received L3 3.686441e+00\n",
"KB Read L3 0.000000e+00\n",
"KB Write L3 4.467458e+02\n",
2020-11-27 04:35:06 +00:00
"Energy L1 4.247116e+01\n",
"Energy L2 1.663821e+01\n",
"Energy L3 1.826516e+01\n",
2020-11-05 15:58:11 +00:00
"Iteration 2.600000e+01\n",
2020-11-27 04:35:06 +00:00
"Energy total 6.147336e+01\n",
2020-11-05 15:58:11 +00:00
"Actions Send 1.090000e+02\n",
"dtype: float64\n",
2020-11-27 04:35:06 +00:00
"67.67709118463543\n"
2020-11-05 15:58:11 +00:00
]
}
],
"source": [
"measurement_files = Path(\"MeasurementFiles\").rglob('*.csv')\n",
"summary = pd.DataFrame()\n",
"baseline = None\n",
"maxW = None\n",
"minW = None\n",
"for measurement_file in measurement_files:\n",
" measurement_avg = None\n",
" if \"Messung\" in measurement_file.name or \"Network\" in measurement_file.name:\n",
" continue\n",
" measurement_name = measurement_file.name.split(\" \")[1]\n",
" if \"Windows\" in measurement_file.name:\n",
" measurement_name = measurement_name + \" Windows\"\n",
" elif \"Matrix\" in measurement_file.name:\n",
" measurement_name = measurement_name + \" Matrix\"\n",
" elif \"ActivityPub\" in measurement_file.name:\n",
" measurement_name = measurement_name + \" ActivityPub\"\n",
" print(measurement_name)\n",
" csv_data = pd.read_csv(measurement_file.absolute()).dropna(axis=1)\n",
" start_measurement_string = time.strftime('%d.%m.%y %H:%M:%S', time.localtime(int(csv_data[\"Start\"][0])))\n",
" end_measurement_string = time.strftime('%d.%m.%y %H:%M:%S', time.localtime(int(csv_data[\"End\"][len(csv_data[\"End\"]) - 1])))\n",
" measurement_data = result.loc[start_measurement_string:end_measurement_string]\n",
"\n",
2020-11-27 04:35:06 +00:00
" plotBoxplotFigure(measurement_data, [\"AVG[W] L1\", \"AVG[W] L2\", \"AVG[W] L3\"], measurement_name + \" Energy before cleaning\", \"plots/Energy/\" + measurement_name.replace(\" \", \"_\") + \"_Boxplot.png\")\n",
" measurement_data[\"AVG[W] L1\"] = cleanDataIQR(measurement_data[\"AVG[W] L1\"], 10)\n",
" measurement_data[\"AVG[W] L2\"] = cleanDataIQR(measurement_data[\"AVG[W] L2\"], 10)\n",
" measurement_data[\"AVG[W] L3\"] = cleanDataIQR(measurement_data[\"AVG[W] L3\"], 10)\n",
" plotBoxplotFigure(measurement_data, [\"AVG[W] L1\", \"AVG[W] L2\", \"AVG[W] L3\"], measurement_name + \" Energy after cleaning\", \"plots/Energy/\" + measurement_name.replace(\" \", \"_\") + \"_Boxplot_cleaned.png\")\n",
2020-11-05 15:58:11 +00:00
" \n",
2020-11-27 04:35:06 +00:00
" plotBoxplotFigure(measurement_data, [\"KB Received L1\", \"KB Received L2\", \"KB Received L3\"], measurement_name + \" Network before cleaning\", \"plots/Network/received/\" + measurement_name.replace(\" \", \"_\") + \"_Boxplot.png\")\n",
2020-11-05 15:58:11 +00:00
" if \"Baseline\" not in measurement_name and \"Windows\" not in measurement_name:\n",
2020-11-27 04:35:06 +00:00
" measurement_data[\"KB Received L1\"] = measurement_data.loc[stats.zscore(measurement_data[\"KB Received L1\"]) <= 1.5][\"KB Received L1\"]\n",
" measurement_data[\"KB Received L2\"] = measurement_data.loc[stats.zscore(measurement_data[\"KB Received L2\"]) <= 1.5][\"KB Received L2\"]\n",
" measurement_data[\"KB Received L3\"] = measurement_data.loc[stats.zscore(measurement_data[\"KB Received L3\"]) <= 1.5][\"KB Received L3\"]\n",
" plotBoxplotFigure(measurement_data, [\"KB Received L1\", \"KB Received L2\", \"KB Received L3\"], measurement_name + \" Network after cleaning\", \"plots/Network/received/\" + measurement_name.replace(\" \", \"_\") + \"_Boxplot_cleaned.png\")\n",
2020-11-05 15:58:11 +00:00
"\n",
" measurement_results = pd.DataFrame()\n",
" iteration_count = 0\n",
" for measurement_row in csv_data.to_numpy():\n",
" title = measurement_name + \" \" + str(measurement_row[0])\n",
" start_string = time.strftime('%d.%m.%y %H:%M:%S', time.localtime(int(measurement_row[1]) + 2))\n",
" end_string = time.strftime('%d.%m.%y %H:%M:%S', time.localtime(int(measurement_row[2]) - 1))\n",
" iteration_data = measurement_data.loc[start_string:end_string]\n",
"\n",
2020-11-27 04:35:06 +00:00
" plotFigure(iteration_data, [\"AVG[W] L1\",\"AVG[W] L2\",\"AVG[W] L3\"], \"Energy (W)\", \"Energy \" + title, \"plots/Energy/\" + title.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(iteration_data, [\"CPU% L1\",\"CPU% L2\",\"CPU% L3\"], \"Workload (%)\", \"CPU \" + title, \"plots/CPU/\" + title.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(iteration_data, [\"MEM Used L1\",\"MEM Used L2\",\"MEM Used L3\"], \"Usage (MByte)\", \"Memory \" + title, \"plots/MEM/\" + title.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(iteration_data, [\"KB Transmitted L1\",\"KB Transmitted L2\",\"KB Transmitted L3\"], \"Transmitted (KB)\", \"Network Send \" + title, \"plots/Network/transmitted/\" + title.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(iteration_data, [\"KB Received L1\",\"KB Received L2\",\"KB Received L3\"], \"Received (KB)\", \"Network Received \" + title, \"plots/Network/received/\" + title.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(iteration_data, [\"KB Read L1\",\"KB Read L2\",\"KB Read L3\"], \"Read (KB)\", \"Disc Read \" + title, \"plots/Disc/\" + title.replace(\" \", \"_\") + \"_read.png\")\n",
" plotFigure(iteration_data, [\"KB Write L1\",\"KB Write L2\",\"KB Write L3\"], \"Write (KB)\", \"Disc Write \" + title, \"plots/Disc/\" + title.replace(\" \", \"_\") + \"_write.png\")\n",
2020-11-05 15:58:11 +00:00
" iteration = iteration_data.mean()\n",
"\n",
2020-11-27 04:35:06 +00:00
" iteration[\"Energy L1\"] = iteration[\"AVG[W] L1\"] + (((iteration.fillna(0)[\"KB Received L1\"] / 1000000) / 0.058034665178572) * 52 * 3600) + ((iteration.fillna(0)[\"KB Received L1\"] / 1000000) * 0.052 * 0.25 * 3600000)\n",
" iteration[\"Energy L2\"] = iteration[\"AVG[W] L2\"] + (((iteration.fillna(0)[\"KB Received L2\"] / 1000000) / 0.058034665178572) * 52 * 3600) + ((iteration.fillna(0)[\"KB Received L2\"] / 1000000) * 0.052 * 0.25 * 3600000)\n",
" iteration[\"Energy L3\"] = iteration[\"AVG[W] L3\"] + (((iteration.fillna(0)[\"KB Received L3\"] / 1000000) / 0.058034665178572) * 52 * 3600) + ((iteration.fillna(0)[\"KB Received L3\"] / 1000000) * 0.052 * 0.25 * 3600000)\n",
2020-11-05 15:58:11 +00:00
"\n",
" iteration['Iteration'] = int(measurement_row[0])\n",
" measurement_results = measurement_results.append(iteration, ignore_index=True)\n",
"\n",
" if baseline is not None:\n",
" baselineTotal = baseline[\"Energy L1\"] + baseline[\"Energy L2\"] + baseline[\"Energy L3\"]\n",
" iteration[\"Energy total\"] = iteration[\"Energy L1\"] + iteration[\"Energy L2\"] + iteration[\"Energy L3\"] - baselineTotal\n",
" if \"Baseline\" not in measurement_name and \"Windows\" not in measurement_name:\n",
" iteration[\"Actions Send\"] = int(measurement_row[3])\n",
" energyEfficiency = iteration[\"Energy total\"] / (iteration[\"Actions Send\"] / 120)\n",
" if maxW is None or energyEfficiency > (maxW[\"Energy total\"] / (maxW[\"Actions Send\"] / 120)):\n",
" maxW = iteration\n",
"\n",
" if minW is None or energyEfficiency < (minW[\"Energy total\"] / (minW[\"Actions Send\"] / 120)):\n",
" minW = iteration\n",
"\n",
" iteration_data = iteration_data.reset_index()\n",
" del iteration_data[\"Time\"]\n",
" if measurement_avg is None:\n",
2020-11-27 04:35:06 +00:00
" measurement_avg = iteration_data.fillna(0)\n",
2020-11-05 15:58:11 +00:00
" else:\n",
" measurement_avg = measurement_avg + iteration_data.fillna(0)\n",
" iteration_count = iteration_count + 1\n",
" measurement_avg = measurement_avg / iteration_count\n",
2020-11-27 04:35:06 +00:00
" plotFigure(measurement_avg, [\"AVG[W] L1\",\"AVG[W] L2\",\"AVG[W] L3\"], \"Energy (W)\", \"Energy \" + measurement_name, \"plots/Energy/\" + measurement_name.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(measurement_avg, [\"CPU% L1\",\"CPU% L2\",\"CPU% L3\"], \"Workload (%)\", \"CPU \" + measurement_name, \"plots/CPU/\" + measurement_name.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(measurement_avg, [\"MEM Used L1\",\"MEM Used L2\",\"MEM Used L3\"], \"Usage (MByte)\", \"Memory \" + measurement_name, \"plots/MEM/\" + measurement_name.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(measurement_avg, [\"KB Transmitted L1\",\"KB Transmitted L2\",\"KB Transmitted L3\"], \"Transmitted (KB)\", \"Network Send \" + measurement_name, \"plots/Network/transmitted/\" + measurement_name.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(measurement_avg, [\"KB Received L1\",\"KB Received L2\",\"KB Received L3\"], \"Received (KB)\", \"Network Received \" + measurement_name, \"plots/Network/received/\" + measurement_name.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(measurement_avg, [\"KB Read L1\",\"KB Read L2\",\"KB Read L3\"], \"Read (KB)\", \"Disc Read \" + measurement_name, \"plots/Disc/\" + measurement_name.replace(\" \", \"_\") + \"_read.png\")\n",
" plotFigure(measurement_avg, [\"KB Write L1\",\"KB Write L2\",\"KB Write L3\"], \"Write (KB)\", \"Disc Write \" + measurement_name, \"plots/Disc/\" + measurement_name.replace(\" \", \"_\") + \"_write.png\")\n",
2020-11-05 15:58:11 +00:00
" csv_data = csv_data.merge(measurement_results)\n",
" csv_data_mean = csv_data.mean()\n",
" csv_data = csv_data.append(csv_data_mean, ignore_index=True)\n",
2020-11-27 04:35:06 +00:00
" csv_data.to_csv(\"output/\" + measurement_file.name)\n",
2020-11-05 15:58:11 +00:00
" if \"Windows\" not in measurement_name:\n",
" if \"Baseline\" in measurement_name:\n",
" baseline = csv_data_mean\n",
" else:\n",
" csv_data_mean[\"Measurement\"] = measurement_name\n",
" cols_name = [\"Measurement\",\"Actions Send\",\"AVG[W] L1\",\"AVG[W] L2\",\"AVG[W] L3\",\"Energy L1\",\"Energy L2\",\"Energy L3\",\"CPU% L1\",\"CPU% L2\",\"CPU% L3\",\"MEM Used L1\",\"MEM Used L2\",\"MEM Used L3\",\"KB Received L1\",\"KB Received L2\",\"KB Received L3\",\"KB Write L1\",\"KB Write L2\",\"KB Write L3\"]\n",
" summary = summary.append(csv_data_mean[cols_name], ignore_index=True)\n",
"if baseline is not None:\n",
" print(\"baseline\")\n",
" print(baseline)\n",
" print(\"min\")\n",
" print(minW)\n",
" minW[\"Energy Efficiency\"] = minW[\"Energy total\"] / (minW[\"Actions Send\"] / 120)\n",
" print(minW[\"Energy Efficiency\"])\n",
" print(\"max\")\n",
" print(maxW)\n",
" maxW[\"Energy Efficiency\"] = maxW[\"Energy total\"] / (maxW[\"Actions Send\"] / 120)\n",
" print(maxW[\"Energy Efficiency\"])\n",
" for key in summary:\n",
" if key != \"Measurement\" and key != \"Actions Send\":\n",
" baseline_values = []\n",
" for value in summary[key]:\n",
" baseline_values.append(baseline[key])\n",
" summary[key + \" without Baseline\"] = summary[key] - baseline_values\n",
" summary[\"Energy total\"] = summary[\"Energy L1 without Baseline\"] + summary[\"Energy L2 without Baseline\"] + summary[\"Energy L3 without Baseline\"]\n",
" summary[\"Energy Efficiency\"] = summary[\"Energy total\"] / (summary[\"Actions Send\"] / 120)\n",
" summary[\"Energy Efficiency normalized\"] = (summary[\"Energy Efficiency\"] - minW[\"Energy Efficiency\"])/(maxW[\"Energy Efficiency\"] - minW[\"Energy Efficiency\"])\n",
" summary[\"CPU% total\"] = summary[\"CPU% L1 without Baseline\"] + summary[\"CPU% L2 without Baseline\"] + summary[\"CPU% L3 without Baseline\"]\n",
" summary[\"MEM Used total\"] = summary[\"MEM Used L1 without Baseline\"] + summary[\"MEM Used L2 without Baseline\"] + summary[\"MEM Used L3 without Baseline\"]\n",
" summary[\"KB Received total\"] = summary[\"KB Received L1 without Baseline\"] + summary[\"KB Received L2 without Baseline\"] + summary[\"KB Received L3 without Baseline\"]\n",
" summary[\"KB Write total\"] = summary[\"KB Write L1 without Baseline\"] + summary[\"KB Write L2 without Baseline\"] + summary[\"KB Write L3 without Baseline\"]\n",
2020-11-27 04:35:06 +00:00
"summary.to_csv(\"output/summary.csv\")"
2020-11-05 15:58:11 +00:00
]
},
{
"cell_type": "code",
2020-11-27 04:35:06 +00:00
"execution_count": 21,
2020-11-05 15:58:11 +00:00
"metadata": {},
"outputs": [],
"source": [
"categories = ['Q1.1','Q1.2','Q1.3', 'Q2.1', 'Q2.2', 'Q3.1', 'Q4.1', 'Q4.2', 'Q4.3']\n",
"\n",
"fig = go.Figure()\n",
"\n",
"fig.add_trace(go.Scatterpolar(\n",
2020-11-27 04:35:06 +00:00
" r=[0.3192,0.3192,0.3192,0.3192,0.3192,0.3192,0.3192,0.3192,0.3192],\n",
2020-11-05 15:58:11 +00:00
" theta=categories,\n",
" fill='toself',\n",
" name='Average'\n",
"))\n",
"fig.add_trace(go.Scatterpolar(\n",
2020-11-27 04:35:06 +00:00
" r=[0.3301,0.2590,0.3331,0.3718,0.3192,0.3192,0.3192,0.3192,0.3192],\n",
2020-11-05 15:58:11 +00:00
" theta=categories,\n",
" fill='toself',\n",
" name='Worst'\n",
"))\n",
"fig.add_trace(go.Scatterpolar(\n",
2020-11-27 04:35:06 +00:00
" r=[0.2815,0.3753,0.0225,0.3264,0.3192,0.3194,0.2819,0.2720,0.3192],\n",
2020-11-05 15:58:11 +00:00
" theta=categories,\n",
" fill='toself',\n",
" name='Best'\n",
"))\n",
"\n",
"fig.update_layout(\n",
" title_text=\"Matrix\",\n",
" polar=dict(\n",
" angularaxis=dict(\n",
" direction=\"clockwise\"\n",
" ),\n",
" radialaxis=dict(\n",
" visible=True,\n",
" range=[0, 1],\n",
" tickmode=\"array\",\n",
" tickvals=[0, 1],\n",
" ticktext=[0, 1]\n",
" )),\n",
" showlegend=True\n",
")\n",
"\n",
2020-11-27 04:35:06 +00:00
"fig.write_image(\"plots/Radar Matrix.png\")"
2020-11-05 15:58:11 +00:00
]
},
{
"cell_type": "code",
2020-11-27 04:35:06 +00:00
"execution_count": 22,
2020-11-05 15:58:11 +00:00
"metadata": {},
"outputs": [],
"source": [
"categories = ['Q1.1','Q1.2','Q1.3', 'Q2.1', 'Q2.2', 'Q3.1', 'Q4.1', 'Q4.2', 'Q4.3']\n",
"\n",
"fig = go.Figure()\n",
"\n",
"fig.add_trace(go.Scatterpolar(\n",
2020-11-27 04:35:06 +00:00
" r=[0.2479,0.2479,0.2479,0.2479,0.2479,0.2479,0.2479,0.2479,0.2479],\n",
2020-11-05 15:58:11 +00:00
" theta=categories,\n",
" fill='toself',\n",
" name='Average'\n",
"))\n",
"fig.add_trace(go.Scatterpolar(\n",
2020-11-27 04:35:06 +00:00
" r=[0.2770,0.2556,0.9052,0.1882,0.2479,0.2479,0.2479,0.2479,0.2479],\n",
2020-11-05 15:58:11 +00:00
" theta=categories,\n",
" fill='toself',\n",
" name='Worst'\n",
"))\n",
"fig.add_trace(go.Scatterpolar(\n",
2020-11-27 04:35:06 +00:00
" r=[0.2187,0.0513,0.0030,0.3072,0.2479,0.2479,0.2479,0.2479,0.2479],\n",
2020-11-05 15:58:11 +00:00
" theta=categories,\n",
" fill='toself',\n",
" name='Best'\n",
"))\n",
"\n",
"fig.update_layout(\n",
" title_text=\"ActivityPub\",\n",
" polar=dict(\n",
" angularaxis=dict(\n",
" direction=\"clockwise\"\n",
" ),\n",
" radialaxis=dict(\n",
" visible=True,\n",
" range=[0, 1],\n",
" tickmode=\"array\",\n",
" tickvals=[0, 1],\n",
" ticktext=[0, 1]\n",
" )),\n",
" showlegend=True\n",
")\n",
"\n",
2020-11-27 04:35:06 +00:00
"fig.write_image(\"plots/Radar ActivityPub.png\")"
2020-11-05 15:58:11 +00:00
]
},
{
"cell_type": "code",
2020-11-27 04:35:06 +00:00
"execution_count": 15,
2020-11-05 15:58:11 +00:00
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"End\n"
]
}
],
"source": [
"print(\"End\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
2020-11-27 04:35:06 +00:00
"version": "3.7.5-final"
2020-11-05 15:58:11 +00:00
}
},
"nbformat": 4,
"nbformat_minor": 4
}
