MeasurementEvaluator/MeasurementEvaluator.ipynb
2020-11-27 05:35:06 +01:00

894 lines
No EOL
60 KiB
Text
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import time\n",
"import glob\n",
"from pathlib import Path\n",
"import pandas as pd\n",
"import plotly.graph_objects as go\n",
"import plotly.express as px\n",
"pd.options.plotting.backend = \"plotly\" \n",
"import numpy as np\n",
"import seaborn as sns\n",
"from scipy import stats"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"if not os.path.exists('output'):\n",
" os.makedirs('output')\n",
"if not os.path.exists('plots'):\n",
" os.makedirs('plots')\n",
"if not os.path.exists('plots/CPU'):\n",
" os.makedirs('plots/CPU')\n",
"if not os.path.exists('plots/Disc'):\n",
" os.makedirs('plots/Disc')\n",
"if not os.path.exists('plots/Energy'):\n",
" os.makedirs('plots/Energy')\n",
"if not os.path.exists('plots/MEM'):\n",
" os.makedirs('plots/MEM')\n",
"if not os.path.exists('plots/Network'):\n",
" os.makedirs('plots/Network')\n",
"if not os.path.exists('plots/Network/received'):\n",
" os.makedirs('plots/Network/received')\n",
"if not os.path.exists('plots/Network/transmitted'):\n",
" os.makedirs('plots/Network/transmitted')"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"def plotFigure(dataToUse, colNamesArray, unit, titleText, filePath):\n",
" if len(colNamesArray) > 3:\n",
" print(\"Too much cols\")\n",
" return\n",
" data = dataToUse[colNamesArray]\n",
" data.columns =['L1','L2','L3']\n",
" iteration_plot = data.plot.line(line_shape='hv') # other options: spline / vhv\n",
" iteration_plot.update_xaxes(\n",
" title_text=\"Time (s)\",\n",
" ticktext=[\"30\", \"60\", \"90\", \"120\"],\n",
" tickvals=[data.index[29], data.index[59], data.index[89], data.index.max()]\n",
" )\n",
" iteration_plot.update_yaxes(title_text=unit)\n",
" iteration_plot.update_layout(\n",
" title_text=titleText,\n",
" legend=dict(\n",
" title=\"Server\"\n",
" )\n",
" )\n",
" iteration_plot.write_image(filePath)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"def plotBoxplotFigure(data, colNamesArray, title, fileName):\n",
" if len(colNamesArray) > 3:\n",
" print(\"Too much cols\")\n",
" return\n",
" fig = go.Figure()\n",
" fig.add_trace(go.Box(\n",
" x=data[colNamesArray[0]],\n",
" name='L1'\n",
" ))\n",
" fig.add_trace(go.Box(\n",
" x=data[colNamesArray[1]],\n",
" name='L2'\n",
" ))\n",
" fig.add_trace(go.Box(\n",
" x=data[colNamesArray[2]],\n",
" name='L3'\n",
" ))\n",
"\n",
" fig.update_layout(\n",
" title_text=title,\n",
" xaxis=dict(title='W', zeroline=False),\n",
" boxmode='group',\n",
" legend=dict(\n",
" title=\"Server\"\n",
" )\n",
" )\n",
" fig.update_traces(orientation='h')\n",
" fig.write_image(fileName)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"def cleanDataIQR(data, factor):\n",
" Q1=data.quantile(0.25)\n",
" Q3=data.quantile(0.75)\n",
" IQR=Q3-Q1\n",
" lowqe_bound=Q1 - factor * IQR\n",
" upper_bound=Q3 + factor * IQR\n",
" return data[~((data < lowqe_bound) |(data > upper_bound))]"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"(123251, 9)\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
" AVG[W] L1 Min[W] L1 Max[W] L1 AVG[W] L2 Min[W] L2 \\\n",
"Time \n",
"14.10.20 00:00:01 0.076 0.009 0.148 -0.117 -0.196 \n",
"14.10.20 00:00:02 0.100 0.021 0.178 -0.068 -0.209 \n",
"14.10.20 00:00:03 0.090 0.002 0.186 0.457 -0.222 \n",
"14.10.20 00:00:04 0.073 -0.018 0.131 -0.085 -0.169 \n",
"14.10.20 00:00:05 0.106 0.029 0.169 -0.100 -0.223 \n",
"\n",
" Max[W] L2 AVG[W] L3 Min[W] L3 Max[W] L3 \n",
"Time \n",
"14.10.20 00:00:01 0.037 0.052 -0.075 0.207 \n",
"14.10.20 00:00:02 0.007 0.030 -0.132 0.176 \n",
"14.10.20 00:00:03 2.782 0.027 -0.130 0.182 \n",
"14.10.20 00:00:04 0.019 0.055 -0.066 0.186 \n",
"14.10.20 00:00:05 0.027 0.005 -0.139 0.176 "
],
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>AVG[W] L1</th>\n <th>Min[W] L1</th>\n <th>Max[W] L1</th>\n <th>AVG[W] L2</th>\n <th>Min[W] L2</th>\n <th>Max[W] L2</th>\n <th>AVG[W] L3</th>\n <th>Min[W] L3</th>\n <th>Max[W] L3</th>\n </tr>\n <tr>\n <th>Time</th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>14.10.20 00:00:01</th>\n <td>0.076</td>\n <td>0.009</td>\n <td>0.148</td>\n <td>-0.117</td>\n <td>-0.196</td>\n <td>0.037</td>\n <td>0.052</td>\n <td>-0.075</td>\n <td>0.207</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:02</th>\n <td>0.100</td>\n <td>0.021</td>\n <td>0.178</td>\n <td>-0.068</td>\n <td>-0.209</td>\n <td>0.007</td>\n <td>0.030</td>\n <td>-0.132</td>\n <td>0.176</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:03</th>\n <td>0.090</td>\n <td>0.002</td>\n <td>0.186</td>\n <td>0.457</td>\n <td>-0.222</td>\n <td>2.782</td>\n <td>0.027</td>\n <td>-0.130</td>\n <td>0.182</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:04</th>\n <td>0.073</td>\n <td>-0.018</td>\n <td>0.131</td>\n <td>-0.085</td>\n <td>-0.169</td>\n <td>0.019</td>\n <td>0.055</td>\n <td>-0.066</td>\n <td>0.186</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:05</th>\n <td>0.106</td>\n <td>0.029</td>\n <td>0.169</td>\n <td>-0.100</td>\n <td>-0.223</td>\n <td>0.027</td>\n <td>0.005</td>\n <td>-0.139</td>\n <td>0.176</td>\n </tr>\n </tbody>\n</table>\n</div>"
},
"metadata": {},
"execution_count": 6
}
],
"source": [
"l1_matrix = pd.read_csv(\"./MeasurementFiles/Matrix/Messung_1_L1.csv\", sep=\";\", decimal=\",\").dropna(axis=1).drop('Nummer', axis=1)\n",
"l1_matrix.columns =['Time', 'AVG[W] L1', 'Min[W] L1', 'Max[W] L1']\n",
"l1_matrix.index = l1_matrix['Time']\n",
"l1_matrix = l1_matrix.drop(\"Time\", axis=1)\n",
"l2_matrix = pd.read_csv(\"./MeasurementFiles/Matrix/Messung_1_L2.csv\", sep=\";\", decimal=\",\").dropna(axis=1).drop('Nummer', axis=1)\n",
"l2_matrix.columns =['Time', 'AVG[W] L2', 'Min[W] L2', 'Max[W] L2']\n",
"l2_matrix.index = l2_matrix['Time']\n",
"l2_matrix = l2_matrix.drop(\"Time\", axis=1)\n",
"l3_matrix = pd.read_csv(\"./MeasurementFiles/Matrix/Messung_1_L3.csv\", sep=\";\", decimal=\",\").dropna(axis=1).drop('Nummer', axis=1)\n",
"l3_matrix.columns =['Time', 'AVG[W] L3', 'Min[W] L3', 'Max[W] L3']\n",
"l3_matrix.index = l3_matrix['Time']\n",
"l3_matrix = l3_matrix.drop(\"Time\", axis=1)\n",
"matrix = l1_matrix.merge(l2_matrix, on='Time').merge(l3_matrix, on='Time')\n",
"print(matrix.shape)\n",
"matrix.head()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"(123251, 27)\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
" AVG[W] L1 Min[W] L1 Max[W] L1 AVG[W] L2 Min[W] L2 \\\n",
"Time \n",
"14.10.20 00:00:01 0.076 0.009 0.148 -0.117 -0.196 \n",
"14.10.20 00:00:02 0.100 0.021 0.178 -0.068 -0.209 \n",
"14.10.20 00:00:03 0.090 0.002 0.186 0.457 -0.222 \n",
"14.10.20 00:00:04 0.073 -0.018 0.131 -0.085 -0.169 \n",
"14.10.20 00:00:05 0.106 0.029 0.169 -0.100 -0.223 \n",
"\n",
" Max[W] L2 AVG[W] L3 Min[W] L3 Max[W] L3 CPU% L1 ... \\\n",
"Time ... \n",
"14.10.20 00:00:01 0.037 0.052 -0.075 0.207 0.0 ... \n",
"14.10.20 00:00:02 0.007 0.030 -0.132 0.176 0.0 ... \n",
"14.10.20 00:00:03 2.782 0.027 -0.130 0.182 0.0 ... \n",
"14.10.20 00:00:04 0.019 0.055 -0.066 0.186 0.0 ... \n",
"14.10.20 00:00:05 0.027 0.005 -0.139 0.176 0.0 ... \n",
"\n",
" KB Transmitted L2 KB Received L2 KB Read L2 KB Write L2 \\\n",
"Time \n",
"14.10.20 00:00:01 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:02 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:03 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:04 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:05 0.0 0.0 0.0 0.0 \n",
"\n",
" CPU% L3 MEM Used L3 KB Transmitted L3 KB Received L3 \\\n",
"Time \n",
"14.10.20 00:00:01 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:02 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:03 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:04 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:05 0.0 0.0 0.0 0.0 \n",
"\n",
" KB Read L3 KB Write L3 \n",
"Time \n",
"14.10.20 00:00:01 0.0 0.0 \n",
"14.10.20 00:00:02 0.0 0.0 \n",
"14.10.20 00:00:03 0.0 0.0 \n",
"14.10.20 00:00:04 0.0 0.0 \n",
"14.10.20 00:00:05 0.0 0.0 \n",
"\n",
"[5 rows x 27 columns]"
],
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>AVG[W] L1</th>\n <th>Min[W] L1</th>\n <th>Max[W] L1</th>\n <th>AVG[W] L2</th>\n <th>Min[W] L2</th>\n <th>Max[W] L2</th>\n <th>AVG[W] L3</th>\n <th>Min[W] L3</th>\n <th>Max[W] L3</th>\n <th>CPU% L1</th>\n <th>...</th>\n <th>KB Transmitted L2</th>\n <th>KB Received L2</th>\n <th>KB Read L2</th>\n <th>KB Write L2</th>\n <th>CPU% L3</th>\n <th>MEM Used L3</th>\n <th>KB Transmitted L3</th>\n <th>KB Received L3</th>\n <th>KB Read L3</th>\n <th>KB Write L3</th>\n </tr>\n <tr>\n <th>Time</th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>14.10.20 00:00:01</th>\n <td>0.076</td>\n <td>0.009</td>\n <td>0.148</td>\n <td>-0.117</td>\n <td>-0.196</td>\n <td>0.037</td>\n <td>0.052</td>\n <td>-0.075</td>\n <td>0.207</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:02</th>\n <td>0.100</td>\n <td>0.021</td>\n <td>0.178</td>\n <td>-0.068</td>\n <td>-0.209</td>\n <td>0.007</td>\n <td>0.030</td>\n <td>-0.132</td>\n <td>0.176</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:03</th>\n <td>0.090</td>\n <td>0.002</td>\n <td>0.186</td>\n <td>0.457</td>\n <td>-0.222</td>\n <td>2.782</td>\n <td>0.027</td>\n <td>-0.130</td>\n <td>0.182</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:04</th>\n <td>0.073</td>\n <td>-0.018</td>\n <td>0.131</td>\n <td>-0.085</td>\n <td>-0.169</td>\n <td>0.019</td>\n <td>0.055</td>\n <td>-0.066</td>\n <td>0.186</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:05</th>\n <td>0.106</td>\n <td>0.029</td>\n <td>0.169</td>\n <td>-0.100</td>\n <td>-0.223</td>\n <td>0.027</td>\n <td>0.005</td>\n <td>-0.139</td>\n <td>0.176</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n </tbody>\n</table>\n<p>5 rows × 27 columns</p>\n</div>"
},
"metadata": {},
"execution_count": 7
}
],
"source": [
"cols = ['Date Time','[CPU]Totl%','[MEM]Used','[NET]TxKBTot','[NET]RxKBTot','[DSK]ReadKBTot','[DSK]WriteKBTot']\n",
"f1_matrix = pd.read_csv(\"./MeasurementFiles/Matrix/Network-server-207-F1.csv\", sep=\";\", usecols=cols).dropna(axis=1)#.drop('Nummer', axis=1)\n",
"f1_matrix.columns =['Time','CPU% L1','MEM Used L1','KB Transmitted L1','KB Received L1','KB Read L1','KB Write L1']\n",
"f1_matrix['Time'] = [time.strftime('%d.%m.%y %H:%M:%S', time.strptime(time_string, '%Y%m%d %H:%M:%S')) for time_string in f1_matrix['Time']]\n",
"f1_matrix.index = f1_matrix['Time']\n",
"f1_matrix = f1_matrix.drop(\"Time\", axis=1)\n",
"f2_matrix = pd.read_csv(\"./MeasurementFiles/Matrix/Network-server-209-F2.csv\", sep=\";\", usecols=cols).dropna(axis=1)#.drop('Nummer', axis=1)\n",
"f2_matrix.columns =['Time','CPU% L2','MEM Used L2','KB Transmitted L2','KB Received L2','KB Read L2','KB Write L2']\n",
"f2_matrix['Time'] = [time.strftime('%d.%m.%y %H:%M:%S', time.strptime(time_string, '%Y%m%d %H:%M:%S')) for time_string in f2_matrix['Time']]\n",
"f2_matrix.index = f2_matrix['Time']\n",
"f2_matrix = f2_matrix.drop(\"Time\", axis=1)\n",
"f3_matrix = pd.read_csv(\"./MeasurementFiles/Matrix/Network-server-208-F3.csv\", sep=\";\", usecols=cols).dropna(axis=1)#.drop('Nummer', axis=1)\n",
"f3_matrix.columns =['Time','CPU% L3','MEM Used L3','KB Transmitted L3','KB Received L3','KB Read L3','KB Write L3']\n",
"f3_matrix['Time'] = [time.strftime('%d.%m.%y %H:%M:%S', time.strptime(time_string, '%Y%m%d %H:%M:%S')) for time_string in f3_matrix['Time']]\n",
"f3_matrix.index = f3_matrix['Time']\n",
"f3_matrix = f3_matrix.drop(\"Time\", axis=1)\n",
"matrix = matrix.merge(f1_matrix, on='Time', how='outer').merge(f2_matrix, on='Time', how='outer').merge(f3_matrix, on='Time', how='outer').fillna(0)\n",
"print(matrix.shape)\n",
"matrix.head()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" AVG[W] L1 Min[W] L1 Max[W] L1 AVG[W] L2 Min[W] L2 \\\n",
"Time \n",
"15.10.20 00:00:01 9.157 7.878 13.265 6.515 6.004 \n",
"15.10.20 00:00:02 8.819 7.424 12.104 6.643 5.986 \n",
"15.10.20 00:00:03 8.238 7.494 8.660 6.199 5.304 \n",
"15.10.20 00:00:04 8.269 7.554 8.916 6.522 6.175 \n",
"15.10.20 00:00:05 8.378 7.808 8.852 6.450 6.222 \n",
"\n",
" Max[W] L2 AVG[W] L3 Min[W] L3 Max[W] L3 \n",
"Time \n",
"15.10.20 00:00:01 6.963 7.210 6.939 7.701 \n",
"15.10.20 00:00:02 8.129 9.163 7.199 13.603 \n",
"15.10.20 00:00:03 6.595 7.501 7.057 8.369 \n",
"15.10.20 00:00:04 6.811 7.221 6.728 7.801 \n",
"15.10.20 00:00:05 6.800 7.229 6.982 7.485 "
],
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>AVG[W] L1</th>\n <th>Min[W] L1</th>\n <th>Max[W] L1</th>\n <th>AVG[W] L2</th>\n <th>Min[W] L2</th>\n <th>Max[W] L2</th>\n <th>AVG[W] L3</th>\n <th>Min[W] L3</th>\n <th>Max[W] L3</th>\n </tr>\n <tr>\n <th>Time</th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>15.10.20 00:00:01</th>\n <td>9.157</td>\n <td>7.878</td>\n <td>13.265</td>\n <td>6.515</td>\n <td>6.004</td>\n <td>6.963</td>\n <td>7.210</td>\n <td>6.939</td>\n <td>7.701</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:02</th>\n <td>8.819</td>\n <td>7.424</td>\n <td>12.104</td>\n <td>6.643</td>\n <td>5.986</td>\n <td>8.129</td>\n <td>9.163</td>\n <td>7.199</td>\n <td>13.603</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:03</th>\n <td>8.238</td>\n <td>7.494</td>\n <td>8.660</td>\n <td>6.199</td>\n <td>5.304</td>\n <td>6.595</td>\n <td>7.501</td>\n <td>7.057</td>\n <td>8.369</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:04</th>\n <td>8.269</td>\n <td>7.554</td>\n <td>8.916</td>\n <td>6.522</td>\n <td>6.175</td>\n <td>6.811</td>\n <td>7.221</td>\n <td>6.728</td>\n <td>7.801</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:05</th>\n <td>8.378</td>\n <td>7.808</td>\n <td>8.852</td>\n <td>6.450</td>\n <td>6.222</td>\n <td>6.800</td>\n <td>7.229</td>\n <td>6.982</td>\n <td>7.485</td>\n </tr>\n </tbody>\n</table>\n</div>"
},
"metadata": {},
"execution_count": 8
}
],
"source": [
"l1_activitypub = pd.read_csv(\"./MeasurementFiles/ActivityPub/Messung_2_L1.csv\", sep=\";\", decimal=\",\").dropna(axis=1).drop('Nummer', axis=1)\n",
"l1_activitypub.columns =['Time', 'AVG[W] L1', 'Min[W] L1', 'Max[W] L1']\n",
"l1_activitypub.index = l1_activitypub['Time']\n",
"l1_activitypub = l1_activitypub.drop(\"Time\", axis=1)\n",
"l2_activitypub = pd.read_csv(\"./MeasurementFiles/ActivityPub/Messung_2_L2.csv\", sep=\";\", decimal=\",\").dropna(axis=1).drop('Nummer', axis=1)\n",
"l2_activitypub.columns =['Time', 'AVG[W] L2', 'Min[W] L2', 'Max[W] L2']\n",
"l2_activitypub.index = l2_activitypub['Time']\n",
"l2_activitypub = l2_activitypub.drop(\"Time\", axis=1)\n",
"l3_activitypub = pd.read_csv(\"./MeasurementFiles/ActivityPub/Messung_2_L3.csv\", sep=\";\", decimal=\",\").dropna(axis=1).drop('Nummer', axis=1)\n",
"l3_activitypub.columns =['Time', 'AVG[W] L3', 'Min[W] L3', 'Max[W] L3']\n",
"l3_activitypub.index = l3_activitypub['Time']\n",
"l3_activitypub = l3_activitypub.drop(\"Time\", axis=1)\n",
"activitypub = l1_activitypub.merge(l2_activitypub, on='Time').merge(l3_activitypub, on='Time')\n",
"activitypub.head()"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"(142191, 27)\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
" AVG[W] L1 Min[W] L1 Max[W] L1 AVG[W] L2 Min[W] L2 \\\n",
"Time \n",
"15.10.20 00:00:01 9.157 7.878 13.265 6.515 6.004 \n",
"15.10.20 00:00:02 8.819 7.424 12.104 6.643 5.986 \n",
"15.10.20 00:00:03 8.238 7.494 8.660 6.199 5.304 \n",
"15.10.20 00:00:04 8.269 7.554 8.916 6.522 6.175 \n",
"15.10.20 00:00:05 8.378 7.808 8.852 6.450 6.222 \n",
"\n",
" Max[W] L2 AVG[W] L3 Min[W] L3 Max[W] L3 CPU% L1 ... \\\n",
"Time ... \n",
"15.10.20 00:00:01 6.963 7.210 6.939 7.701 0.0 ... \n",
"15.10.20 00:00:02 8.129 9.163 7.199 13.603 0.0 ... \n",
"15.10.20 00:00:03 6.595 7.501 7.057 8.369 0.0 ... \n",
"15.10.20 00:00:04 6.811 7.221 6.728 7.801 0.0 ... \n",
"15.10.20 00:00:05 6.800 7.229 6.982 7.485 0.0 ... \n",
"\n",
" KB Transmitted L2 KB Received L2 KB Read L2 KB Write L2 \\\n",
"Time \n",
"15.10.20 00:00:01 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:02 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:03 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:04 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:05 0.0 0.0 0.0 0.0 \n",
"\n",
" CPU% L3 MEM Used L3 KB Transmitted L3 KB Received L3 \\\n",
"Time \n",
"15.10.20 00:00:01 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:02 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:03 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:04 0.0 0.0 0.0 0.0 \n",
"15.10.20 00:00:05 0.0 0.0 0.0 0.0 \n",
"\n",
" KB Read L3 KB Write L3 \n",
"Time \n",
"15.10.20 00:00:01 0.0 0.0 \n",
"15.10.20 00:00:02 0.0 0.0 \n",
"15.10.20 00:00:03 0.0 0.0 \n",
"15.10.20 00:00:04 0.0 0.0 \n",
"15.10.20 00:00:05 0.0 0.0 \n",
"\n",
"[5 rows x 27 columns]"
],
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>AVG[W] L1</th>\n <th>Min[W] L1</th>\n <th>Max[W] L1</th>\n <th>AVG[W] L2</th>\n <th>Min[W] L2</th>\n <th>Max[W] L2</th>\n <th>AVG[W] L3</th>\n <th>Min[W] L3</th>\n <th>Max[W] L3</th>\n <th>CPU% L1</th>\n <th>...</th>\n <th>KB Transmitted L2</th>\n <th>KB Received L2</th>\n <th>KB Read L2</th>\n <th>KB Write L2</th>\n <th>CPU% L3</th>\n <th>MEM Used L3</th>\n <th>KB Transmitted L3</th>\n <th>KB Received L3</th>\n <th>KB Read L3</th>\n <th>KB Write L3</th>\n </tr>\n <tr>\n <th>Time</th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>15.10.20 00:00:01</th>\n <td>9.157</td>\n <td>7.878</td>\n <td>13.265</td>\n <td>6.515</td>\n <td>6.004</td>\n <td>6.963</td>\n <td>7.210</td>\n <td>6.939</td>\n <td>7.701</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:02</th>\n <td>8.819</td>\n <td>7.424</td>\n <td>12.104</td>\n <td>6.643</td>\n <td>5.986</td>\n <td>8.129</td>\n <td>9.163</td>\n <td>7.199</td>\n <td>13.603</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:03</th>\n <td>8.238</td>\n <td>7.494</td>\n <td>8.660</td>\n <td>6.199</td>\n <td>5.304</td>\n <td>6.595</td>\n <td>7.501</td>\n <td>7.057</td>\n <td>8.369</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:04</th>\n <td>8.269</td>\n <td>7.554</td>\n <td>8.916</td>\n <td>6.522</td>\n <td>6.175</td>\n <td>6.811</td>\n <td>7.221</td>\n <td>6.728</td>\n <td>7.801</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>15.10.20 00:00:05</th>\n <td>8.378</td>\n <td>7.808</td>\n <td>8.852</td>\n <td>6.450</td>\n <td>6.222</td>\n <td>6.800</td>\n <td>7.229</td>\n <td>6.982</td>\n <td>7.485</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n </tbody>\n</table>\n<p>5 rows × 27 columns</p>\n</div>"
},
"metadata": {},
"execution_count": 9
}
],
"source": [
"cols = ['Date Time','[CPU]Totl%','[MEM]Used','[NET]TxKBTot','[NET]RxKBTot','[DSK]ReadKBTot','[DSK]WriteKBTot']\n",
"f1_activitypub = pd.read_csv(\"./MeasurementFiles/ActivityPub/Network-server-207-F1.csv\", sep=\";\", usecols=cols).dropna(axis=1)\n",
"f1_activitypub.columns =['Time','CPU% L1','MEM Used L1','KB Transmitted L1','KB Received L1','KB Read L1','KB Write L1']\n",
"f1_activitypub['Time'] = [time.strftime('%d.%m.%y %H:%M:%S', time.strptime(time_string, '%Y%m%d %H:%M:%S')) for time_string in f1_activitypub['Time']]\n",
"f1_activitypub.index = f1_activitypub['Time']\n",
"f1_activitypub = f1_activitypub.drop(\"Time\", axis=1)\n",
"f2_activitypub = pd.read_csv(\"./MeasurementFiles/ActivityPub/Network-server-209-F2.csv\", sep=\";\", usecols=cols).dropna(axis=1)\n",
"f2_activitypub.columns =['Time','CPU% L2','MEM Used L2','KB Transmitted L2','KB Received L2','KB Read L2','KB Write L2']\n",
"f2_activitypub['Time'] = [time.strftime('%d.%m.%y %H:%M:%S', time.strptime(time_string, '%Y%m%d %H:%M:%S')) for time_string in f2_activitypub['Time']]\n",
"f2_activitypub.index = f2_activitypub['Time']\n",
"f2_activitypub = f2_activitypub.drop(\"Time\", axis=1)\n",
"f3_activitypub = pd.read_csv(\"./MeasurementFiles/ActivityPub/Network-server-208-F3.csv\", sep=\";\", usecols=cols).dropna(axis=1)\n",
"f3_activitypub.columns =['Time','CPU% L3','MEM Used L3','KB Transmitted L3','KB Received L3','KB Read L3','KB Write L3']\n",
"f3_activitypub['Time'] = [time.strftime('%d.%m.%y %H:%M:%S', time.strptime(time_string, '%Y%m%d %H:%M:%S')) for time_string in f3_activitypub['Time']]\n",
"f3_activitypub.index = f3_activitypub['Time']\n",
"f3_activitypub = f3_activitypub.drop(\"Time\", axis=1)\n",
"activitypub = activitypub.merge(f1_activitypub, on='Time', how='outer').merge(f2_activitypub, on='Time', how='outer').merge(f3_activitypub, on='Time', how='outer').fillna(0)\n",
"print(activitypub.shape)\n",
"activitypub.head()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"(228591, 27)\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
" AVG[W] L1 Min[W] L1 Max[W] L1 AVG[W] L2 Min[W] L2 \\\n",
"Time \n",
"14.10.20 00:00:01 0.076 0.009 0.148 -0.117 -0.196 \n",
"14.10.20 00:00:02 0.100 0.021 0.178 -0.068 -0.209 \n",
"14.10.20 00:00:03 0.090 0.002 0.186 0.457 -0.222 \n",
"14.10.20 00:00:04 0.073 -0.018 0.131 -0.085 -0.169 \n",
"14.10.20 00:00:05 0.106 0.029 0.169 -0.100 -0.223 \n",
"\n",
" Max[W] L2 AVG[W] L3 Min[W] L3 Max[W] L3 CPU% L1 ... \\\n",
"Time ... \n",
"14.10.20 00:00:01 0.037 0.052 -0.075 0.207 0.0 ... \n",
"14.10.20 00:00:02 0.007 0.030 -0.132 0.176 0.0 ... \n",
"14.10.20 00:00:03 2.782 0.027 -0.130 0.182 0.0 ... \n",
"14.10.20 00:00:04 0.019 0.055 -0.066 0.186 0.0 ... \n",
"14.10.20 00:00:05 0.027 0.005 -0.139 0.176 0.0 ... \n",
"\n",
" KB Transmitted L2 KB Received L2 KB Read L2 KB Write L2 \\\n",
"Time \n",
"14.10.20 00:00:01 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:02 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:03 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:04 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:05 0.0 0.0 0.0 0.0 \n",
"\n",
" CPU% L3 MEM Used L3 KB Transmitted L3 KB Received L3 \\\n",
"Time \n",
"14.10.20 00:00:01 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:02 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:03 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:04 0.0 0.0 0.0 0.0 \n",
"14.10.20 00:00:05 0.0 0.0 0.0 0.0 \n",
"\n",
" KB Read L3 KB Write L3 \n",
"Time \n",
"14.10.20 00:00:01 0.0 0.0 \n",
"14.10.20 00:00:02 0.0 0.0 \n",
"14.10.20 00:00:03 0.0 0.0 \n",
"14.10.20 00:00:04 0.0 0.0 \n",
"14.10.20 00:00:05 0.0 0.0 \n",
"\n",
"[5 rows x 27 columns]"
],
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>AVG[W] L1</th>\n <th>Min[W] L1</th>\n <th>Max[W] L1</th>\n <th>AVG[W] L2</th>\n <th>Min[W] L2</th>\n <th>Max[W] L2</th>\n <th>AVG[W] L3</th>\n <th>Min[W] L3</th>\n <th>Max[W] L3</th>\n <th>CPU% L1</th>\n <th>...</th>\n <th>KB Transmitted L2</th>\n <th>KB Received L2</th>\n <th>KB Read L2</th>\n <th>KB Write L2</th>\n <th>CPU% L3</th>\n <th>MEM Used L3</th>\n <th>KB Transmitted L3</th>\n <th>KB Received L3</th>\n <th>KB Read L3</th>\n <th>KB Write L3</th>\n </tr>\n <tr>\n <th>Time</th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n <th></th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>14.10.20 00:00:01</th>\n <td>0.076</td>\n <td>0.009</td>\n <td>0.148</td>\n <td>-0.117</td>\n <td>-0.196</td>\n <td>0.037</td>\n <td>0.052</td>\n <td>-0.075</td>\n <td>0.207</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:02</th>\n <td>0.100</td>\n <td>0.021</td>\n <td>0.178</td>\n <td>-0.068</td>\n <td>-0.209</td>\n <td>0.007</td>\n <td>0.030</td>\n <td>-0.132</td>\n <td>0.176</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:03</th>\n <td>0.090</td>\n <td>0.002</td>\n <td>0.186</td>\n <td>0.457</td>\n <td>-0.222</td>\n <td>2.782</td>\n <td>0.027</td>\n <td>-0.130</td>\n <td>0.182</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:04</th>\n <td>0.073</td>\n <td>-0.018</td>\n <td>0.131</td>\n <td>-0.085</td>\n <td>-0.169</td>\n <td>0.019</td>\n <td>0.055</td>\n <td>-0.066</td>\n <td>0.186</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n <tr>\n <th>14.10.20 00:00:05</th>\n <td>0.106</td>\n <td>0.029</td>\n <td>0.169</td>\n <td>-0.100</td>\n <td>-0.223</td>\n <td>0.027</td>\n <td>0.005</td>\n <td>-0.139</td>\n <td>0.176</td>\n <td>0.0</td>\n <td>...</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n <td>0.0</td>\n </tr>\n </tbody>\n</table>\n<p>5 rows × 27 columns</p>\n</div>"
},
"metadata": {},
"execution_count": 10
}
],
"source": [
"frames = [matrix, activitypub]\n",
"result = pd.concat(frames)\n",
"result = result[~result.index.duplicated(keep='first')]\n",
"result.to_csv(\"output/MeasurementData.csv\")\n",
"print(result.shape)\n",
"result.head()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"#plotBoxplotFigure(result, [\"KB Received L1\", \"KB Received L2\", \"KB Received L3\"], \"Network before cleaning\", \"Network Boxplot.png\")\n",
"#result[\"KB Received L1\"] = result.loc[stats.zscore(result[\"KB Received L1\"]) <= 3][\"KB Received L1\"]\n",
"#result[\"KB Received L2\"] = result.loc[stats.zscore(result[\"KB Received L2\"]) <= 3][\"KB Received L2\"]\n",
"#result[\"KB Received L3\"] = result.loc[stats.zscore(result[\"KB Received L3\"]) <= 3][\"KB Received L3\"]\n",
"#plotBoxplotFigure(result, [\"KB Received L1\", \"KB Received L2\", \"KB Received L3\"], \"Network before cleaning\", \"Network Boxplot cleaned.png\")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"tags": []
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"111111111 Windows\n",
"Baseline Windows\n",
"Baseline\n",
"110111111 ActivityPub\n",
"101111111 ActivityPub\n",
"211111111 ActivityPub\n",
"011111111 ActivityPub\n",
"111211111 ActivityPub\n",
"111111111 ActivityPub\n",
"121111111 ActivityPub\n",
"112111111 ActivityPub\n",
"111011111 ActivityPub\n",
"211111111 Matrix\n",
"121111111 Matrix\n",
"111111011 Matrix\n",
"011111111 Matrix\n",
"111011111 Matrix\n",
"112111111 Matrix\n",
"111111101 Matrix\n",
"101111111 Matrix\n",
"111110111 Matrix\n",
"111211111 Matrix\n",
"111111111 Matrix\n",
"110111111 Matrix\n",
"baseline\n",
"Iteration 5.500000e+00\n",
"Start 1.602788e+09\n",
"End 1.602788e+09\n",
"AVG[W] L1 5.950951e+00\n",
"AVG[W] L2 4.132172e+00\n",
"AVG[W] L3 5.818045e+00\n",
"CPU% L1 5.080473e-02\n",
"CPU% L2 2.116508e-02\n",
"CPU% L3 4.742914e-02\n",
"Energy L1 5.950951e+00\n",
"Energy L2 4.132172e+00\n",
"Energy L3 5.818045e+00\n",
"KB Read L1 0.000000e+00\n",
"KB Read L2 0.000000e+00\n",
"KB Read L3 0.000000e+00\n",
"KB Received L1 0.000000e+00\n",
"KB Received L2 0.000000e+00\n",
"KB Received L3 0.000000e+00\n",
"KB Transmitted L1 0.000000e+00\n",
"KB Transmitted L2 0.000000e+00\n",
"KB Transmitted L3 0.000000e+00\n",
"KB Write L1 3.793847e+00\n",
"KB Write L2 5.399402e+00\n",
"KB Write L3 5.540977e+00\n",
"MEM Used L1 4.075354e+06\n",
"MEM Used L2 3.891142e+06\n",
"MEM Used L3 3.822240e+06\n",
"Max[W] L1 6.101333e+00\n",
"Max[W] L2 4.379694e+00\n",
"Max[W] L3 5.993006e+00\n",
"Min[W] L1 5.836322e+00\n",
"Min[W] L2 3.933443e+00\n",
"Min[W] L3 5.669529e+00\n",
"dtype: float64\n",
"min\n",
"AVG[W] L1 5.997093e+00\n",
"Min[W] L1 5.811203e+00\n",
"Max[W] L1 6.292356e+00\n",
"AVG[W] L2 4.037085e+00\n",
"Min[W] L2 3.727415e+00\n",
"Max[W] L2 4.554542e+00\n",
"AVG[W] L3 5.826669e+00\n",
"Min[W] L3 5.622593e+00\n",
"Max[W] L3 6.123331e+00\n",
"CPU% L1 4.322034e-01\n",
"MEM Used L1 4.288508e+06\n",
"KB Transmitted L1 5.677966e-01\n",
"KB Received L1 0.000000e+00\n",
"KB Read L1 0.000000e+00\n",
"KB Write L1 6.166102e+01\n",
"CPU% L2 3.135593e-01\n",
"MEM Used L2 4.091692e+06\n",
"KB Transmitted L2 6.694915e-01\n",
"KB Received L2 0.000000e+00\n",
"KB Read L2 0.000000e+00\n",
"KB Write L2 5.467797e+01\n",
"CPU% L3 3.983051e-01\n",
"MEM Used L3 4.013788e+06\n",
"KB Transmitted L3 1.864407e-01\n",
"KB Received L3 1.016949e-01\n",
"KB Read L3 0.000000e+00\n",
"KB Write L3 3.966949e+01\n",
"Energy L1 5.997093e+00\n",
"Energy L2 4.037085e+00\n",
"Energy L3 6.159462e+00\n",
"Iteration 1.100000e+01\n",
"Energy total 2.924719e-01\n",
"Actions Send 1.200000e+01\n",
"dtype: float64\n",
"2.924719339609503\n",
"max\n",
"AVG[W] L1 6.501847e+00\n",
"Min[W] L1 6.013449e+00\n",
"Max[W] L1 7.691280e+00\n",
"AVG[W] L2 4.705025e+00\n",
"Min[W] L2 3.876602e+00\n",
"Max[W] L2 6.766508e+00\n",
"AVG[W] L3 6.201432e+00\n",
"Min[W] L3 5.670492e+00\n",
"Max[W] L3 7.394390e+00\n",
"CPU% L1 3.728814e+00\n",
"MEM Used L1 4.571048e+06\n",
"KB Transmitted L1 8.279661e+00\n",
"KB Received L1 1.099153e+01\n",
"KB Read L1 0.000000e+00\n",
"KB Write L1 4.452542e+02\n",
"CPU% L2 1.957627e+00\n",
"MEM Used L2 4.403195e+06\n",
"KB Transmitted L2 5.525424e+00\n",
"KB Received L2 3.646552e+00\n",
"KB Read L2 0.000000e+00\n",
"KB Write L2 4.514915e+02\n",
"CPU% L3 3.084746e+00\n",
"MEM Used L3 4.377153e+06\n",
"KB Transmitted L3 5.516949e+00\n",
"KB Received L3 3.686441e+00\n",
"KB Read L3 0.000000e+00\n",
"KB Write L3 4.467458e+02\n",
"Energy L1 4.247116e+01\n",
"Energy L2 1.663821e+01\n",
"Energy L3 1.826516e+01\n",
"Iteration 2.600000e+01\n",
"Energy total 6.147336e+01\n",
"Actions Send 1.090000e+02\n",
"dtype: float64\n",
"67.67709118463543\n"
]
}
],
"source": [
"measurement_files = Path(\"MeasurementFiles\").rglob('*.csv')\n",
"summary = pd.DataFrame()\n",
"baseline = None\n",
"maxW = None\n",
"minW = None\n",
"for measurement_file in measurement_files:\n",
" measurement_avg = None\n",
" if \"Messung\" in measurement_file.name or \"Network\" in measurement_file.name:\n",
" continue\n",
" measurement_name = measurement_file.name.split(\" \")[1]\n",
" if \"Windows\" in measurement_file.name:\n",
" measurement_name = measurement_name + \" Windows\"\n",
" elif \"Matrix\" in measurement_file.name:\n",
" measurement_name = measurement_name + \" Matrix\"\n",
" elif \"ActivityPub\" in measurement_file.name:\n",
" measurement_name = measurement_name + \" ActivityPub\"\n",
" print(measurement_name)\n",
" csv_data = pd.read_csv(measurement_file.absolute()).dropna(axis=1)\n",
" start_measurement_string = time.strftime('%d.%m.%y %H:%M:%S', time.localtime(int(csv_data[\"Start\"][0])))\n",
" end_measurement_string = time.strftime('%d.%m.%y %H:%M:%S', time.localtime(int(csv_data[\"End\"][len(csv_data[\"End\"]) - 1])))\n",
" measurement_data = result.loc[start_measurement_string:end_measurement_string]\n",
"\n",
" plotBoxplotFigure(measurement_data, [\"AVG[W] L1\", \"AVG[W] L2\", \"AVG[W] L3\"], measurement_name + \" Energy before cleaning\", \"plots/Energy/\" + measurement_name.replace(\" \", \"_\") + \"_Boxplot.png\")\n",
" measurement_data[\"AVG[W] L1\"] = cleanDataIQR(measurement_data[\"AVG[W] L1\"], 10)\n",
" measurement_data[\"AVG[W] L2\"] = cleanDataIQR(measurement_data[\"AVG[W] L2\"], 10)\n",
" measurement_data[\"AVG[W] L3\"] = cleanDataIQR(measurement_data[\"AVG[W] L3\"], 10)\n",
" plotBoxplotFigure(measurement_data, [\"AVG[W] L1\", \"AVG[W] L2\", \"AVG[W] L3\"], measurement_name + \" Energy after cleaning\", \"plots/Energy/\" + measurement_name.replace(\" \", \"_\") + \"_Boxplot_cleaned.png\")\n",
" \n",
" plotBoxplotFigure(measurement_data, [\"KB Received L1\", \"KB Received L2\", \"KB Received L3\"], measurement_name + \" Network before cleaning\", \"plots/Network/received/\" + measurement_name.replace(\" \", \"_\") + \"_Boxplot.png\")\n",
" if \"Baseline\" not in measurement_name and \"Windows\" not in measurement_name:\n",
" measurement_data[\"KB Received L1\"] = measurement_data.loc[stats.zscore(measurement_data[\"KB Received L1\"]) <= 1.5][\"KB Received L1\"]\n",
" measurement_data[\"KB Received L2\"] = measurement_data.loc[stats.zscore(measurement_data[\"KB Received L2\"]) <= 1.5][\"KB Received L2\"]\n",
" measurement_data[\"KB Received L3\"] = measurement_data.loc[stats.zscore(measurement_data[\"KB Received L3\"]) <= 1.5][\"KB Received L3\"]\n",
" plotBoxplotFigure(measurement_data, [\"KB Received L1\", \"KB Received L2\", \"KB Received L3\"], measurement_name + \" Network after cleaning\", \"plots/Network/received/\" + measurement_name.replace(\" \", \"_\") + \"_Boxplot_cleaned.png\")\n",
"\n",
" measurement_results = pd.DataFrame()\n",
" iteration_count = 0\n",
" for measurement_row in csv_data.to_numpy():\n",
" title = measurement_name + \" \" + str(measurement_row[0])\n",
" start_string = time.strftime('%d.%m.%y %H:%M:%S', time.localtime(int(measurement_row[1]) + 2))\n",
" end_string = time.strftime('%d.%m.%y %H:%M:%S', time.localtime(int(measurement_row[2]) - 1))\n",
" iteration_data = measurement_data.loc[start_string:end_string]\n",
"\n",
" plotFigure(iteration_data, [\"AVG[W] L1\",\"AVG[W] L2\",\"AVG[W] L3\"], \"Energy (W)\", \"Energy \" + title, \"plots/Energy/\" + title.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(iteration_data, [\"CPU% L1\",\"CPU% L2\",\"CPU% L3\"], \"Workload (%)\", \"CPU \" + title, \"plots/CPU/\" + title.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(iteration_data, [\"MEM Used L1\",\"MEM Used L2\",\"MEM Used L3\"], \"Usage (MByte)\", \"Memory \" + title, \"plots/MEM/\" + title.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(iteration_data, [\"KB Transmitted L1\",\"KB Transmitted L2\",\"KB Transmitted L3\"], \"Transmitted (KB)\", \"Network Send \" + title, \"plots/Network/transmitted/\" + title.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(iteration_data, [\"KB Received L1\",\"KB Received L2\",\"KB Received L3\"], \"Received (KB)\", \"Network Received \" + title, \"plots/Network/received/\" + title.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(iteration_data, [\"KB Read L1\",\"KB Read L2\",\"KB Read L3\"], \"Read (KB)\", \"Disc Read \" + title, \"plots/Disc/\" + title.replace(\" \", \"_\") + \"_read.png\")\n",
" plotFigure(iteration_data, [\"KB Write L1\",\"KB Write L2\",\"KB Write L3\"], \"Write (KB)\", \"Disc Write \" + title, \"plots/Disc/\" + title.replace(\" \", \"_\") + \"_write.png\")\n",
" iteration = iteration_data.mean()\n",
"\n",
" iteration[\"Energy L1\"] = iteration[\"AVG[W] L1\"] + (((iteration.fillna(0)[\"KB Received L1\"] / 1000000) / 0.058034665178572) * 52 * 3600) + ((iteration.fillna(0)[\"KB Received L1\"] / 1000000) * 0.052 * 0.25 * 3600000)\n",
" iteration[\"Energy L2\"] = iteration[\"AVG[W] L2\"] + (((iteration.fillna(0)[\"KB Received L2\"] / 1000000) / 0.058034665178572) * 52 * 3600) + ((iteration.fillna(0)[\"KB Received L2\"] / 1000000) * 0.052 * 0.25 * 3600000)\n",
" iteration[\"Energy L3\"] = iteration[\"AVG[W] L3\"] + (((iteration.fillna(0)[\"KB Received L3\"] / 1000000) / 0.058034665178572) * 52 * 3600) + ((iteration.fillna(0)[\"KB Received L3\"] / 1000000) * 0.052 * 0.25 * 3600000)\n",
"\n",
" iteration['Iteration'] = int(measurement_row[0])\n",
" measurement_results = measurement_results.append(iteration, ignore_index=True)\n",
"\n",
" if baseline is not None:\n",
" baselineTotal = baseline[\"Energy L1\"] + baseline[\"Energy L2\"] + baseline[\"Energy L3\"]\n",
" iteration[\"Energy total\"] = iteration[\"Energy L1\"] + iteration[\"Energy L2\"] + iteration[\"Energy L3\"] - baselineTotal\n",
" if \"Baseline\" not in measurement_name and \"Windows\" not in measurement_name:\n",
" iteration[\"Actions Send\"] = int(measurement_row[3])\n",
" energyEfficiency = iteration[\"Energy total\"] / (iteration[\"Actions Send\"] / 120)\n",
" if maxW is None or energyEfficiency > (maxW[\"Energy total\"] / (maxW[\"Actions Send\"] / 120)):\n",
" maxW = iteration\n",
"\n",
" if minW is None or energyEfficiency < (minW[\"Energy total\"] / (minW[\"Actions Send\"] / 120)):\n",
" minW = iteration\n",
"\n",
" iteration_data = iteration_data.reset_index()\n",
" del iteration_data[\"Time\"]\n",
" if measurement_avg is None:\n",
" measurement_avg = iteration_data.fillna(0)\n",
" else:\n",
" measurement_avg = measurement_avg + iteration_data.fillna(0)\n",
" iteration_count = iteration_count + 1\n",
" measurement_avg = measurement_avg / iteration_count\n",
" plotFigure(measurement_avg, [\"AVG[W] L1\",\"AVG[W] L2\",\"AVG[W] L3\"], \"Energy (W)\", \"Energy \" + measurement_name, \"plots/Energy/\" + measurement_name.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(measurement_avg, [\"CPU% L1\",\"CPU% L2\",\"CPU% L3\"], \"Workload (%)\", \"CPU \" + measurement_name, \"plots/CPU/\" + measurement_name.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(measurement_avg, [\"MEM Used L1\",\"MEM Used L2\",\"MEM Used L3\"], \"Usage (MByte)\", \"Memory \" + measurement_name, \"plots/MEM/\" + measurement_name.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(measurement_avg, [\"KB Transmitted L1\",\"KB Transmitted L2\",\"KB Transmitted L3\"], \"Transmitted (KB)\", \"Network Send \" + measurement_name, \"plots/Network/transmitted/\" + measurement_name.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(measurement_avg, [\"KB Received L1\",\"KB Received L2\",\"KB Received L3\"], \"Received (KB)\", \"Network Received \" + measurement_name, \"plots/Network/received/\" + measurement_name.replace(\" \", \"_\") + \".png\")\n",
" plotFigure(measurement_avg, [\"KB Read L1\",\"KB Read L2\",\"KB Read L3\"], \"Read (KB)\", \"Disc Read \" + measurement_name, \"plots/Disc/\" + measurement_name.replace(\" \", \"_\") + \"_read.png\")\n",
" plotFigure(measurement_avg, [\"KB Write L1\",\"KB Write L2\",\"KB Write L3\"], \"Write (KB)\", \"Disc Write \" + measurement_name, \"plots/Disc/\" + measurement_name.replace(\" \", \"_\") + \"_write.png\")\n",
" csv_data = csv_data.merge(measurement_results)\n",
" csv_data_mean = csv_data.mean()\n",
" csv_data = csv_data.append(csv_data_mean, ignore_index=True)\n",
" csv_data.to_csv(\"output/\" + measurement_file.name)\n",
" if \"Windows\" not in measurement_name:\n",
" if \"Baseline\" in measurement_name:\n",
" baseline = csv_data_mean\n",
" else:\n",
" csv_data_mean[\"Measurement\"] = measurement_name\n",
" cols_name = [\"Measurement\",\"Actions Send\",\"AVG[W] L1\",\"AVG[W] L2\",\"AVG[W] L3\",\"Energy L1\",\"Energy L2\",\"Energy L3\",\"CPU% L1\",\"CPU% L2\",\"CPU% L3\",\"MEM Used L1\",\"MEM Used L2\",\"MEM Used L3\",\"KB Received L1\",\"KB Received L2\",\"KB Received L3\",\"KB Write L1\",\"KB Write L2\",\"KB Write L3\"]\n",
" summary = summary.append(csv_data_mean[cols_name], ignore_index=True)\n",
"if baseline is not None:\n",
" print(\"baseline\")\n",
" print(baseline)\n",
" print(\"min\")\n",
" print(minW)\n",
" minW[\"Energy Efficiency\"] = minW[\"Energy total\"] / (minW[\"Actions Send\"] / 120)\n",
" print(minW[\"Energy Efficiency\"])\n",
" print(\"max\")\n",
" print(maxW)\n",
" maxW[\"Energy Efficiency\"] = maxW[\"Energy total\"] / (maxW[\"Actions Send\"] / 120)\n",
" print(maxW[\"Energy Efficiency\"])\n",
" for key in summary:\n",
" if key != \"Measurement\" and key != \"Actions Send\":\n",
" baseline_values = []\n",
" for value in summary[key]:\n",
" baseline_values.append(baseline[key])\n",
" summary[key + \" without Baseline\"] = summary[key] - baseline_values\n",
" summary[\"Energy total\"] = summary[\"Energy L1 without Baseline\"] + summary[\"Energy L2 without Baseline\"] + summary[\"Energy L3 without Baseline\"]\n",
" summary[\"Energy Efficiency\"] = summary[\"Energy total\"] / (summary[\"Actions Send\"] / 120)\n",
" summary[\"Energy Efficiency normalized\"] = (summary[\"Energy Efficiency\"] - minW[\"Energy Efficiency\"])/(maxW[\"Energy Efficiency\"] - minW[\"Energy Efficiency\"])\n",
" summary[\"CPU% total\"] = summary[\"CPU% L1 without Baseline\"] + summary[\"CPU% L2 without Baseline\"] + summary[\"CPU% L3 without Baseline\"]\n",
" summary[\"MEM Used total\"] = summary[\"MEM Used L1 without Baseline\"] + summary[\"MEM Used L2 without Baseline\"] + summary[\"MEM Used L3 without Baseline\"]\n",
" summary[\"KB Received total\"] = summary[\"KB Received L1 without Baseline\"] + summary[\"KB Received L2 without Baseline\"] + summary[\"KB Received L3 without Baseline\"]\n",
" summary[\"KB Write total\"] = summary[\"KB Write L1 without Baseline\"] + summary[\"KB Write L2 without Baseline\"] + summary[\"KB Write L3 without Baseline\"]\n",
"summary.to_csv(\"output/summary.csv\")"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"categories = ['Q1.1','Q1.2','Q1.3', 'Q2.1', 'Q2.2', 'Q3.1', 'Q4.1', 'Q4.2', 'Q4.3']\n",
"\n",
"fig = go.Figure()\n",
"\n",
"fig.add_trace(go.Scatterpolar(\n",
" r=[0.3192,0.3192,0.3192,0.3192,0.3192,0.3192,0.3192,0.3192,0.3192],\n",
" theta=categories,\n",
" fill='toself',\n",
" name='Average'\n",
"))\n",
"fig.add_trace(go.Scatterpolar(\n",
" r=[0.3301,0.2590,0.3331,0.3718,0.3192,0.3192,0.3192,0.3192,0.3192],\n",
" theta=categories,\n",
" fill='toself',\n",
" name='Worst'\n",
"))\n",
"fig.add_trace(go.Scatterpolar(\n",
" r=[0.2815,0.3753,0.0225,0.3264,0.3192,0.3194,0.2819,0.2720,0.3192],\n",
" theta=categories,\n",
" fill='toself',\n",
" name='Best'\n",
"))\n",
"\n",
"fig.update_layout(\n",
" title_text=\"Matrix\",\n",
" polar=dict(\n",
" angularaxis=dict(\n",
" direction=\"clockwise\"\n",
" ),\n",
" radialaxis=dict(\n",
" visible=True,\n",
" range=[0, 1],\n",
" tickmode=\"array\",\n",
" tickvals=[0, 1],\n",
" ticktext=[0, 1]\n",
" )),\n",
" showlegend=True\n",
")\n",
"\n",
"fig.write_image(\"plots/Radar Matrix.png\")"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
"categories = ['Q1.1','Q1.2','Q1.3', 'Q2.1', 'Q2.2', 'Q3.1', 'Q4.1', 'Q4.2', 'Q4.3']\n",
"\n",
"fig = go.Figure()\n",
"\n",
"fig.add_trace(go.Scatterpolar(\n",
" r=[0.2479,0.2479,0.2479,0.2479,0.2479,0.2479,0.2479,0.2479,0.2479],\n",
" theta=categories,\n",
" fill='toself',\n",
" name='Average'\n",
"))\n",
"fig.add_trace(go.Scatterpolar(\n",
" r=[0.2770,0.2556,0.9052,0.1882,0.2479,0.2479,0.2479,0.2479,0.2479],\n",
" theta=categories,\n",
" fill='toself',\n",
" name='Worst'\n",
"))\n",
"fig.add_trace(go.Scatterpolar(\n",
" r=[0.2187,0.0513,0.0030,0.3072,0.2479,0.2479,0.2479,0.2479,0.2479],\n",
" theta=categories,\n",
" fill='toself',\n",
" name='Best'\n",
"))\n",
"\n",
"fig.update_layout(\n",
" title_text=\"ActivityPub\",\n",
" polar=dict(\n",
" angularaxis=dict(\n",
" direction=\"clockwise\"\n",
" ),\n",
" radialaxis=dict(\n",
" visible=True,\n",
" range=[0, 1],\n",
" tickmode=\"array\",\n",
" tickvals=[0, 1],\n",
" ticktext=[0, 1]\n",
" )),\n",
" showlegend=True\n",
")\n",
"\n",
"fig.write_image(\"plots/Radar ActivityPub.png\")"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"End\n"
]
}
],
"source": [
"print(\"End\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.5-final"
}
},
"nbformat": 4,
"nbformat_minor": 4
}