Here, we visualize the performance of three runtimes: PyTorch, our custom-built tinyRuntime (both non-quantized and quantized versions), using the ResNet18 model and 100 images from the Imagenette dataset. We focus on four key metrics: accuracy, execution time, model size and memory usage.
AMD64
Code
import pandas as pdimport plotly.graph_objects as gofrom plotly.subplots import make_subplotsfrom IPython.display import displaydef split_dataframe(df):'''Split dataframe based on Runtime (Pytorch, tinyRuntime (no quant) and tinyRuntime (quant).''' df_pytorch = df[df["Runtime"] =="PyTorch"] df_trv = df[(df["Runtime"] =="tinyRuntime") & (df["Quantization"] ==False)] df_trq = df[(df["Runtime"] =="tinyRuntime") & (df["Quantization"] ==True)]return df_pytorch, df_trv, df_trqdef plot_perf_comp(df, save_image=False):'''Plot latest performance comparisons using Plotly.''' dfs = split_dataframe(df)# Create subplots using Plotly Figure Factory fig = make_subplots(rows=2, cols=2, subplot_titles=("Accuracy", "Time", "Max memory usage", "Model size")) metrics = ["Accuracy", "Time", "Max memory", "Model size"] colors = ['rgba(31, 119, 180, 0.8)', 'rgba(255, 127, 14, 0.8)', 'rgba(44, 160, 44, 0.8)'] names = ["PyTorch", "tinyRuntime (no quant)", "tinyRuntime (quant)"]for i, metric inenumerate(metrics):# Retrieve values for each metric y_values = [df[metric].values[-1] for df in dfs]# Add trace for each runtimefor j, name inenumerate(names): trace = go.Bar(x=[name], y=[y_values[j]], marker_color=colors[j], showlegend=(i ==0), name=name) fig.add_trace(trace, row=i //2+1, col=i %2+1)# Set layout, background color and font size, and disable legend click feature fig.update_layout( legend=dict(orientation="h", yanchor="bottom", y=1.1, xanchor="right", x=1), template="plotly_dark", legend_itemclick=False, legend_itemdoubleclick=False, font=dict(size=14), margin=dict(t=90, b=60, l=80, r=50))# Update axis labels fig.update_yaxes(title_text="Accuracy (%)", row=1, col=1) fig.update_yaxes(title_text="Time (s)", row=1, col=2) fig.update_yaxes(title_text="Max memory usage (MB)", row=2, col=1) fig.update_yaxes(title_text="Model size (MB)", row=2, col=2) fig.update_xaxes(showticklabels=False)# Show the plot with modebar hidden fig.show(config={'displayModeBar': False})if save_image: fig.write_image("images/perf_bar.png")# Create DataFrame data = {"Accuracy (%)": [df["Accuracy"].values[-1] for df in dfs],"Time (s)": [df["Time"].values[-1] for df in dfs],"Max memory usage (MB)": [df["Max memory"].values[-1] for df in dfs],"Model size (MB)": [df["Model size"].values[-1] for df in dfs] } df_results = pd.DataFrame(data, index=["PyTorch", "tinyRuntime (no quant)", "tinyRuntime (quant)"]) display(df_results)df = pd.read_csv('benchmark.csv')df_x86 = df[df["Architecture"] =="x86_64"]plot_perf_comp(df_x86)
Figure 1: Performance comparison on AMD64
Accuracy (%)
Time (s)
Max memory usage (MB)
Model size (MB)
PyTorch
99.0
11.247210
473.507812
44.938353
tinyRuntime (no quant)
99.0
9.644408
98.714844
44.660263
tinyRuntime (quant)
99.0
7.335336
62.062500
11.197475
Code
def plot_radar(df, save_image=False):# Normalize the data df = df.copy() df["Time"] = df["Time"].apply(lambda x: x / df["Time"].max()) df["Accuracy"] = df["Accuracy"].apply(lambda x: x /100) df["Max memory"] = df["Max memory"].apply(lambda x: x / df["Max memory"].max())# Split based on runtime and take three relevant columns dfs = [df_split[['Time', 'Max memory', 'Accuracy']] for df_split in split_dataframe(df)]# Create a radar chart categories = ['Time', 'Memory usage', 'Accuracy'] runtimes = ["PyTorch", "tinyRuntime (no quant)", "tinyRuntime (quant)"] fig = go.Figure()for i, df inenumerate(dfs): fig.add_trace(go.Scatterpolar( r=df.iloc[-1].values, theta=categories, fill='toself', name=runtimes[i] )) fig.update_layout( polar=dict( radialaxis=dict( visible=True,range=[0, 1] ) ), font=dict(size=16), margin=dict(t=50, b=50, l=50, r=50), showlegend=True ) fig.add_annotation( text="*Values are normalized between 0 and 1", xref="paper", yref="paper", x=0.5, y=-0.1, showarrow=False, font=dict(size=15) ) fig.show()if save_image: fig.write_image("images/perf_radar.png")plot_radar(df_x86)
Figure 2: Radar chart of time, memory, and accuracy for AMD64. Quantities are scaled between 0 and 1