Benchmark Tutorial

The Benchmark module allows you to write easily benchmarks framework to a project for timing critical parts and detecting slow parts of code.

In addition it automatically creates data files of these benchmarks, as well as a gnuplot file which can display the comparison curves of the benchmarks.

Basic Usage

To create a basic benchmark, you have to follow these steps:

• Create a new benchmark
• Write the functions that wraps the functions you want to benchmark.
• Register these wrappers functions.
• Run the benchmark.
• Free the memory.

Here is a basic example of benchmark which creates two functions that will be run. These functions just print a message.

#include <stdlib.h>
#include <stdio.h>
 
#include <Eina.h>
 
static
void work1(int request)
{
  printf ("work1 in progress... Request: %d\n", request);
}
 
static
void work2(int request)
{
  printf ("work2 in progress... Request: %d\n", request);
}
 
int main()
{
  Eina_Benchmark *test;
  Eina_Array     *ea;
 
  if (!eina_init())
    return EXIT_FAILURE;
 
  test = eina_benchmark_new("test", "run");
  if (!test)
    goto shutdown_eina;
 
  eina_benchmark_register(test, "work-1", EINA_BENCHMARK(work1), 200, 300, 10);
  eina_benchmark_register(test, "work-2", EINA_BENCHMARK(work2), 100, 150, 5);
 
  ea = eina_benchmark_run(test);
 
  eina_benchmark_free(test);
  eina_shutdown();
 
  return EXIT_SUCCESS;
 
 shutdown_eina:
  eina_shutdown();
 
  return EXIT_FAILURE;
}

As "test", "run" are passed to eina_benchmark_new() and as the tests "work-1" and "work-2" are registered, the data files bench_test_run.work-1.data and bench_test_run.work-2.data will be created after the eina_benchmark_run() call. They contain four columns. The file bench_test_run.work-1.data contains for example:

# specimen      experiment time starting time   ending time
200     23632   2852446 2876078
210     6924    2883046 2889970
220     6467    2895962 2902429
230     6508    2908271 2914779
240     6278    2920610 2926888
250     6342    2932830 2939172
260     6252    2944954 2951206
270     6463    2956978 2963441
280     6347    2969548 2975895
290     6457    2981702 2988159

The first column (specimen) is the integer passed to the work1() function when the test is run. The second column (experiment time) is the time, in nanosecond, that work1() takes. The third and fourth columns are self-explicit.

You can see that the integer passed work1() starts from 200 and finishes at 290, with a step of 10. These values are computed withe last 3 values passed to eina_benchmark_register(). See the document of that function for the detailed behavior.

The gnuplot file will be named bench_test_run.gnuplot. Just run:

gnuplot bench_test_run.gnuplot

To create the graphic of the comparison curves. The image file is named output_test_run.png.

More Advanced Usage

In this section, several test will be created and run. The idea is exactly the same than in the previous section, but with some basic automatic way to run all the benchmarks. The following code benchmarks some Eina converts functions, and some Eina containers types:

#include <stdlib.h>
#include <stdio.h>
#include <time.h>
 
#include <Eina.h>
 
static void bench_convert(Eina_Benchmark *bench);
static void bench_container(Eina_Benchmark *bench);
 
typedef struct _Benchmark_Case Benchmark_Case;
 
struct _Benchmark_Case
{
   const char *bench_case;
   void (*build)(Eina_Benchmark *bench);
};
 
static const Benchmark_Case benchmarks[] = {
  { "Bench 1", bench_convert },
  { "Bench 2", bench_container },
  { NULL,      NULL }
};
 
static
void convert1(int request)
{
  char tmp[128];
  int i;
 
  for (i = 0; i < request; ++i)
    eina_convert_itoa(rand(), tmp);
}
 
static
void convert2(int request)
{
  char tmp[128];
  int i;
 
  for (i = 0; i < request; ++i)
    eina_convert_xtoa(rand(), tmp);
}
 
static void
bench_convert(Eina_Benchmark *bench)
{
  eina_benchmark_register(bench, "convert-1", EINA_BENCHMARK(convert1), 200, 400, 10);
  eina_benchmark_register(bench, "convert-2", EINA_BENCHMARK(convert2), 200, 400, 10);
}
 
static
void array(int request)
{
  Eina_Array *array;
  Eina_Array_Iterator it;
  int *data;
  int i;
 
  array = eina_array_new(64);
 
  for (i = 0; i < request; ++i)
    {
      data = (int *)malloc(sizeof(int));
      if (!data) continue;
      *data = rand();
      eina_array_push(array, data);
    }
 
  EINA_ARRAY_ITER_NEXT(array, i, data, it)
    free(data);
 
  eina_array_free(array);
}
 
static
void list(int request)
{
  Eina_List *l = NULL;
  int *data;
  int i;
 
  for (i = 0; i < request; ++i)
    {
      data = (int *)malloc(sizeof(int));
      if (!data) continue;
      *data = rand();
      l = eina_list_prepend(l, data);
    }
 
  while (l)
    {
      free(eina_list_data_get(l));
      l = eina_list_remove_list(l, l);
    }
}
 
static void
bench_container(Eina_Benchmark *bench)
{
  eina_benchmark_register(bench, "array", EINA_BENCHMARK(array), 200, 300, 10);
  eina_benchmark_register(bench, "list", EINA_BENCHMARK(list), 200, 300, 10);
}
 
int main()
{
  Eina_Benchmark *test;
  Eina_Array     *ea;
  unsigned int    i;
 
  if (!eina_init())
    return EXIT_FAILURE;
 
  for (i = 0; benchmarks[i].bench_case != NULL; ++i)
    {
      test = eina_benchmark_new(benchmarks[i].bench_case, "Benchmark example");
      if (!test)
        continue;
 
      benchmarks[i].build(test);
 
      ea = eina_benchmark_run(test);
 
      eina_benchmark_free(test);
    }
 
  eina_shutdown();
 
  return EXIT_SUCCESS;
}

gnuplot can be used to see how are performed the convert functions together, as well as how are performed the containers. So it is now easy to see that the hexadecimal convert function is faster than the decimal one, and that arrays are faster than lists.

You can improve all that by executing automatically gnuplot in your program, or integrate the Eina benchmark framework in an autotooled project. See that page for more information.