diff --git a/lib/benchmark_runner.rb b/lib/benchmark_runner.rb index 98860ca8..a5a85cc8 100644 --- a/lib/benchmark_runner.rb +++ b/lib/benchmark_runner.rb @@ -48,7 +48,7 @@ def write_csv(output_path, ruby_descriptions, table) end # Build output text string with metadata, table, and legend - def build_output_text(ruby_descriptions, table, format, bench_failures) + def build_output_text(ruby_descriptions, table, format, bench_failures, include_rss: false) base_name, *other_names = ruby_descriptions.keys output_str = +"" @@ -65,6 +65,9 @@ def build_output_text(ruby_descriptions, table, format, bench_failures) other_names.each do |name| output_str << "- #{name} 1st itr: ratio of #{base_name}/#{name} time for the first benchmarking iteration.\n" output_str << "- #{base_name}/#{name}: ratio of #{base_name}/#{name} time. Higher is better for #{name}. Above 1 represents a speedup.\n" + if include_rss + output_str << "- RSS #{base_name}/#{name}: ratio of #{base_name}/#{name} RSS. Higher is better for #{name}. Above 1 means lower memory usage.\n" + end end output_str << "- ***: p < 0.001, **: p < 0.01, *: p < 0.05 (Welch's t-test)\n" end diff --git a/lib/benchmark_runner/cli.rb b/lib/benchmark_runner/cli.rb index 2a68e21f..2812f3ee 100644 --- a/lib/benchmark_runner/cli.rb +++ b/lib/benchmark_runner/cli.rb @@ -85,7 +85,7 @@ def run BenchmarkRunner.write_csv(output_path, ruby_descriptions, table) # Save the output in a text file that we can easily refer to - output_str = BenchmarkRunner.build_output_text(ruby_descriptions, table, format, bench_failures) + output_str = BenchmarkRunner.build_output_text(ruby_descriptions, table, format, bench_failures, include_rss: args.rss) out_txt_path = output_path + ".txt" File.open(out_txt_path, "w") { |f| f.write output_str } diff --git a/lib/results_table_builder.rb b/lib/results_table_builder.rb index d066c8c8..34805aac 100644 --- a/lib/results_table_builder.rb +++ b/lib/results_table_builder.rb @@ -54,6 +54,12 @@ def build_header end end + if @include_rss + @other_names.each do |name| + header << "RSS #{@base_name}/#{name}" + end + end + header end @@ -76,6 +82,12 @@ def build_format end end + if @include_rss + @other_names.each do |_name| + format << "%.3f" + end + end + format end @@ -92,6 +104,7 @@ def build_row(bench_name) build_base_columns(row, base_t, base_rss) build_comparison_columns(row, other_ts, other_rsss) build_ratio_columns(row, base_t0, other_t0s, base_t, other_ts) + build_rss_ratio_columns(row, base_rss, other_rsss) row end @@ -124,6 +137,14 @@ def build_ratio_columns(row, base_t0, other_t0s, base_t, other_ts) end end + def build_rss_ratio_columns(row, base_rss, other_rsss) + return unless @include_rss + + other_rsss.each do |other_rss| + row << base_rss / other_rss + end + end + def format_ratio(ratio, pval) sym = significance_symbol(pval) formatted = "%.3f" % ratio diff --git a/test/benchmark_runner_test.rb b/test/benchmark_runner_test.rb index 00fdbd6e..a8eb812f 100644 --- a/test/benchmark_runner_test.rb +++ b/test/benchmark_runner_test.rb @@ -390,6 +390,44 @@ assert_includes result, "- ***: p < 0.001, **: p < 0.01, *: p < 0.05 (Welch's t-test)" end + it 'includes RSS ratio legend when include_rss is true' do + ruby_descriptions = { + 'ruby-base' => 'ruby 3.3.0', + 'ruby-yjit' => 'ruby 3.3.0 +YJIT' + } + table = [ + ['bench', 'ruby-base (ms)', 'stddev (%)', 'RSS (MiB)', 'ruby-yjit (ms)', 'stddev (%)', 'RSS (MiB)', 'ruby-yjit 1st itr', 'ruby-base/ruby-yjit', 'RSS ruby-base/ruby-yjit'], + ['fib', '100.0', '5.0', '10.0', '50.0', '3.0', '12.0', '2.000', '2.000', '0.833'] + ] + format = ['%s', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.3f', '%s', '%.3f'] + bench_failures = {} + + result = BenchmarkRunner.build_output_text( + ruby_descriptions, table, format, bench_failures, include_rss: true + ) + + assert_includes result, '- RSS ruby-base/ruby-yjit: ratio of ruby-base/ruby-yjit RSS. Higher is better for ruby-yjit. Above 1 means lower memory usage.' + end + + it 'omits RSS ratio legend when include_rss is false' do + ruby_descriptions = { + 'ruby-base' => 'ruby 3.3.0', + 'ruby-yjit' => 'ruby 3.3.0 +YJIT' + } + table = [ + ['bench', 'ruby-base (ms)', 'stddev (%)', 'ruby-yjit (ms)', 'stddev (%)'], + ['fib', '100.0', '5.0', '50.0', '3.0'] + ] + format = ['%s', '%.1f', '%.1f', '%.1f', '%.1f'] + bench_failures = {} + + result = BenchmarkRunner.build_output_text( + ruby_descriptions, table, format, bench_failures + ) + + refute_includes result, 'RSS ruby-base/ruby-yjit' + end + it 'includes formatted table in output' do ruby_descriptions = { 'ruby' => 'ruby 3.3.0' } table = [ diff --git a/test/results_table_builder_test.rb b/test/results_table_builder_test.rb index b85ff79a..f3efef6d 100644 --- a/test/results_table_builder_test.rb +++ b/test/results_table_builder_test.rb @@ -87,13 +87,56 @@ table, format = builder.build + # No RSS ratio column with a single executable assert_equal ['bench', 'ruby (ms)', 'stddev (%)', 'RSS (MiB)'], table[0] - assert_equal ['%s', '%.1f', '%.1f', '%.1f'], format - assert_in_delta 10.0, table[1][3], 0.1 end + it 'includes RSS ratio columns when include_rss is true with multiple executables' do + executable_names = ['ruby', 'ruby-yjit'] + bench_data = { + 'ruby' => { + 'fib' => { + 'warmup' => [0.1], + 'bench' => [0.1, 0.11, 0.09], + 'rss' => 1024 * 1024 * 10 + } + }, + 'ruby-yjit' => { + 'fib' => { + 'warmup' => [0.05], + 'bench' => [0.05, 0.06, 0.04], + 'rss' => 1024 * 1024 * 20 + } + } + } + + builder = ResultsTableBuilder.new( + executable_names: executable_names, + bench_data: bench_data, + include_rss: true + ) + + table, format = builder.build + + expected_header = [ + 'bench', + 'ruby (ms)', 'stddev (%)', 'RSS (MiB)', + 'ruby-yjit (ms)', 'stddev (%)', 'RSS (MiB)', + 'ruby-yjit 1st itr', + 'ruby/ruby-yjit', + 'RSS ruby/ruby-yjit' + ] + assert_equal expected_header, table[0] + + expected_format = ['%s', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.3f', '%s', '%.3f'] + assert_equal expected_format, format + + # RSS ratio: 10 MiB / 20 MiB = 0.5 + assert_in_delta 0.5, table[1].last, 0.01 + end + it 'skips benchmarks with missing data' do executable_names = ['ruby', 'ruby-yjit'] bench_data = {