tools/perf/bench/mem-functions.c

   1 /*
   2  * mem-memcpy.c
   3  *
   4  * Simple memcpy() and memset() benchmarks
   5  *
   6  * Written by Hitoshi Mitake <mitake@dcl.info.waseda.ac.jp>
   7  */
   8
   9 #include "../perf.h"
  10 #include "../util/util.h"
  11 #include "../util/parse-options.h"
  12 #include "../util/header.h"
  13 #include "../util/cloexec.h"
  14 #include "bench.h"
  15 #include "mem-memcpy-arch.h"
  16 #include "mem-memset-arch.h"
  17
  18 #include <stdio.h>
  19 #include <stdlib.h>
  20 #include <string.h>
  21 #include <sys/time.h>
  22 #include <errno.h>
  23
  24 #define K 1024
  25
  26 static const char       *length_str     = "1MB";
  27 static const char       *routine        = "all";
  28 static int              iterations      = 1;
  29 static bool             use_cycle;
  30 static int              cycle_fd;
  31 static bool             only_prefault;
  32 static bool             no_prefault;
  33
  34 static const struct option options[] = {
  35         OPT_STRING('l', "length", &length_str, "1MB",
  36                     "Specify length of memory to copy. "
  37                     "Available units: B, KB, MB, GB and TB (upper and lower)"),
  38         OPT_STRING('r', "routine", &routine, "all",
  39                     "Specify routine to copy, \"all\" runs all available routines"),
  40         OPT_INTEGER('i', "iterations", &iterations,
  41                     "repeat memcpy() invocation this number of times"),
  42         OPT_BOOLEAN('c', "cycle", &use_cycle,
  43                     "Use cycles event instead of gettimeofday() for measuring"),
  44         OPT_BOOLEAN('o', "only-prefault", &only_prefault,
  45                     "Show only the result with page faults before memcpy()"),
  46         OPT_BOOLEAN('n', "no-prefault", &no_prefault,
  47                     "Show only the result without page faults before memcpy()"),
  48         OPT_END()
  49 };
  50
  51 typedef void *(*memcpy_t)(void *, const void *, size_t);
  52 typedef void *(*memset_t)(void *, int, size_t);
  53
  54 struct routine {
  55         const char *name;
  56         const char *desc;
  57         union {
  58                 memcpy_t memcpy;
  59                 memset_t memset;
  60         } fn;
  61 };
  62
  63 struct routine memcpy_routines[] = {
  64         { .name         = "default",
  65           .desc         = "Default memcpy() provided by glibc",
  66           .fn.memcpy    = memcpy },
  67
  68 #ifdef HAVE_ARCH_X86_64_SUPPORT
  69 # define MEMCPY_FN(_fn, _name, _desc) {.name = _name, .desc = _desc, .fn.memcpy = _fn},
  70 # include "mem-memcpy-x86-64-asm-def.h"
  71 # undef MEMCPY_FN
  72 #endif
  73
  74         { NULL, }
  75 };
  76
  77 static const char * const bench_mem_memcpy_usage[] = {
  78         "perf bench mem memcpy <options>",
  79         NULL
  80 };
  81
  82 static struct perf_event_attr cycle_attr = {
  83         .type           = PERF_TYPE_HARDWARE,
  84         .config         = PERF_COUNT_HW_CPU_CYCLES
  85 };
  86
  87 static void init_cycle(void)
  88 {
  89         cycle_fd = sys_perf_event_open(&cycle_attr, getpid(), -1, -1, perf_event_open_cloexec_flag());
  90
  91         if (cycle_fd < 0 && errno == ENOSYS)
  92                 die("No CONFIG_PERF_EVENTS=y kernel support configured?\n");
  93         else
  94                 BUG_ON(cycle_fd < 0);
  95 }
  96
  97 static u64 get_cycle(void)
  98 {
  99         int ret;
 100         u64 clk;
 101
 102         ret = read(cycle_fd, &clk, sizeof(u64));
 103         BUG_ON(ret != sizeof(u64));
 104
 105         return clk;
 106 }
 107
 108 static double timeval2double(struct timeval *ts)
 109 {
 110         return (double)ts->tv_sec + (double)ts->tv_usec / (double)1000000;
 111 }
 112
 113 #define print_bps(x) do {                                       \
 114                 if (x < K)                                      \
 115                         printf(" %14lf B/Sec", x);              \
 116                 else if (x < K * K)                             \
 117                         printf(" %14lfd KB/Sec", x / K);        \
 118                 else if (x < K * K * K)                         \
 119                         printf(" %14lf MB/Sec", x / K / K);     \
 120                 else                                            \
 121                         printf(" %14lf GB/Sec", x / K / K / K); \
 122         } while (0)
 123
 124 struct bench_mem_info {
 125         const struct routine *routines;
 126         u64 (*do_cycle)(const struct routine *r, size_t len, bool prefault);
 127         double (*do_gettimeofday)(const struct routine *r, size_t len, bool prefault);
 128         const char *const *usage;
 129 };
 130
 131 static void __bench_mem_routine(struct bench_mem_info *info, int r_idx, size_t len, double totallen)
 132 {
 133         const struct routine *r = &info->routines[r_idx];
 134         double result_bps[2];
 135         u64 result_cycle[2];
 136         int prefault = no_prefault ? 0 : 1;
 137
 138         result_cycle[0] = result_cycle[1] = 0ULL;
 139         result_bps[0] = result_bps[1] = 0.0;
 140
 141         printf("Routine %s (%s)\n", r->name, r->desc);
 142
 143         if (bench_format == BENCH_FORMAT_DEFAULT)
 144                 printf("# Copying %s Bytes ...\n\n", length_str);
 145
 146         if (!only_prefault && prefault) {
 147                 /* Show both results: */
 148                 if (use_cycle) {
 149                         result_cycle[0] = info->do_cycle(r, len, false);
 150                         result_cycle[1] = info->do_cycle(r, len, true);
 151                 } else {
 152                         result_bps[0]   = info->do_gettimeofday(r, len, false);
 153                         result_bps[1]   = info->do_gettimeofday(r, len, true);
 154                 }
 155         } else {
 156                 if (use_cycle)
 157                         result_cycle[prefault] = info->do_cycle(r, len, only_prefault);
 158                 else
 159                         result_bps[prefault] = info->do_gettimeofday(r, len, only_prefault);
 160         }
 161
 162         switch (bench_format) {
 163         case BENCH_FORMAT_DEFAULT:
 164                 if (!only_prefault && prefault) {
 165                         if (use_cycle) {
 166                                 printf(" %14lf Cycle/Byte\n",
 167                                         (double)result_cycle[0]
 168                                         / totallen);
 169                                 printf(" %14lf Cycle/Byte (with prefault)\n",
 170                                         (double)result_cycle[1]
 171                                         / totallen);
 172                         } else {
 173                                 print_bps(result_bps[0]);
 174                                 printf("\n");
 175                                 print_bps(result_bps[1]);
 176                                 printf(" (with prefault)\n");
 177                         }
 178                 } else {
 179                         if (use_cycle) {
 180                                 printf(" %14lf Cycle/Byte",
 181                                         (double)result_cycle[prefault]
 182                                         / totallen);
 183                         } else
 184                                 print_bps(result_bps[prefault]);
 185
 186                         printf("%s\n", only_prefault ? " (with prefault)" : "");
 187                 }
 188                 break;
 189         case BENCH_FORMAT_SIMPLE:
 190                 if (!only_prefault && prefault) {
 191                         if (use_cycle) {
 192                                 printf("%lf %lf\n",
 193                                         (double)result_cycle[0] / totallen,
 194                                         (double)result_cycle[1] / totallen);
 195                         } else {
 196                                 printf("%lf %lf\n",
 197                                         result_bps[0], result_bps[1]);
 198                         }
 199                 } else {
 200                         if (use_cycle) {
 201                                 printf("%lf\n", (double)result_cycle[prefault]
 202                                         / totallen);
 203                         } else
 204                                 printf("%lf\n", result_bps[prefault]);
 205                 }
 206                 break;
 207         default:
 208                 /* Reaching this means there's some disaster: */
 209                 die("unknown format: %d\n", bench_format);
 210                 break;
 211         }
 212 }
 213
 214 static int bench_mem_common(int argc, const char **argv, struct bench_mem_info *info)
 215 {
 216         int i;
 217         size_t len;
 218         double totallen;
 219
 220         argc = parse_options(argc, argv, options, info->usage, 0);
 221
 222         if (no_prefault && only_prefault) {
 223                 fprintf(stderr, "Invalid options: -o and -n are mutually exclusive\n");
 224                 return 1;
 225         }
 226
 227         if (use_cycle)
 228                 init_cycle();
 229
 230         len = (size_t)perf_atoll((char *)length_str);
 231         totallen = (double)len * iterations;
 232
 233         if ((s64)len <= 0) {
 234                 fprintf(stderr, "Invalid length:%s\n", length_str);
 235                 return 1;
 236         }
 237
 238         /* Same as without specifying either of prefault and no-prefault: */
 239         if (only_prefault && no_prefault)
 240                 only_prefault = no_prefault = false;
 241
 242         if (!strncmp(routine, "all", 3)) {
 243                 for (i = 0; info->routines[i].name; i++)
 244                         __bench_mem_routine(info, i, len, totallen);
 245                 return 0;
 246         }
 247
 248         for (i = 0; info->routines[i].name; i++) {
 249                 if (!strcmp(info->routines[i].name, routine))
 250                         break;
 251         }
 252         if (!info->routines[i].name) {
 253                 printf("Unknown routine:%s\n", routine);
 254                 printf("Available routines...\n");
 255                 for (i = 0; info->routines[i].name; i++) {
 256                         printf("\t%s ... %s\n",
 257                                info->routines[i].name, info->routines[i].desc);
 258                 }
 259                 return 1;
 260         }
 261
 262         __bench_mem_routine(info, i, len, totallen);
 263
 264         return 0;
 265 }
 266
 267 static void memcpy_alloc_mem(void **dst, void **src, size_t length)
 268 {
 269         *dst = zalloc(length);
 270         if (!*dst)
 271                 die("memory allocation failed - maybe length is too large?\n");
 272
 273         *src = zalloc(length);
 274         if (!*src)
 275                 die("memory allocation failed - maybe length is too large?\n");
 276
 277         /* Make sure to always prefault zero pages even if MMAP_THRESH is crossed: */
 278         memset(*src, 0, length);
 279 }
 280
 281 static u64 do_memcpy_cycle(const struct routine *r, size_t len, bool prefault)
 282 {
 283         u64 cycle_start = 0ULL, cycle_end = 0ULL;
 284         void *src = NULL, *dst = NULL;
 285         memcpy_t fn = r->fn.memcpy;
 286         int i;
 287
 288         memcpy_alloc_mem(&dst, &src, len);
 289
 290         if (prefault)
 291                 fn(dst, src, len);
 292
 293         cycle_start = get_cycle();
 294         for (i = 0; i < iterations; ++i)
 295                 fn(dst, src, len);
 296         cycle_end = get_cycle();
 297
 298         free(src);
 299         free(dst);
 300         return cycle_end - cycle_start;
 301 }
 302
 303 static double do_memcpy_gettimeofday(const struct routine *r, size_t len, bool prefault)
 304 {
 305         struct timeval tv_start, tv_end, tv_diff;
 306         memcpy_t fn = r->fn.memcpy;
 307         void *src = NULL, *dst = NULL;
 308         int i;
 309
 310         memcpy_alloc_mem(&dst, &src, len);
 311
 312         if (prefault)
 313                 fn(dst, src, len);
 314
 315         BUG_ON(gettimeofday(&tv_start, NULL));
 316         for (i = 0; i < iterations; ++i)
 317                 fn(dst, src, len);
 318         BUG_ON(gettimeofday(&tv_end, NULL));
 319
 320         timersub(&tv_end, &tv_start, &tv_diff);
 321
 322         free(src);
 323         free(dst);
 324         return (double)(((double)len * iterations) / timeval2double(&tv_diff));
 325 }
 326
 327 int bench_mem_memcpy(int argc, const char **argv, const char *prefix __maybe_unused)
 328 {
 329         struct bench_mem_info info = {
 330                 .routines               = memcpy_routines,
 331                 .do_cycle               = do_memcpy_cycle,
 332                 .do_gettimeofday        = do_memcpy_gettimeofday,
 333                 .usage                  = bench_mem_memcpy_usage,
 334         };
 335
 336         return bench_mem_common(argc, argv, &info);
 337 }
 338
 339 static void memset_alloc_mem(void **dst, size_t length)
 340 {
 341         *dst = zalloc(length);
 342         if (!*dst)
 343                 die("memory allocation failed - maybe length is too large?\n");
 344 }
 345
 346 static u64 do_memset_cycle(const struct routine *r, size_t len, bool prefault)
 347 {
 348         u64 cycle_start = 0ULL, cycle_end = 0ULL;
 349         memset_t fn = r->fn.memset;
 350         void *dst = NULL;
 351         int i;
 352
 353         memset_alloc_mem(&dst, len);
 354
 355         if (prefault)
 356                 fn(dst, -1, len);
 357
 358         cycle_start = get_cycle();
 359         for (i = 0; i < iterations; ++i)
 360                 fn(dst, i, len);
 361         cycle_end = get_cycle();
 362
 363         free(dst);
 364         return cycle_end - cycle_start;
 365 }
 366
 367 static double do_memset_gettimeofday(const struct routine *r, size_t len,
 368                                      bool prefault)
 369 {
 370         struct timeval tv_start, tv_end, tv_diff;
 371         memset_t fn = r->fn.memset;
 372         void *dst = NULL;
 373         int i;
 374
 375         memset_alloc_mem(&dst, len);
 376
 377         if (prefault)
 378                 fn(dst, -1, len);
 379
 380         BUG_ON(gettimeofday(&tv_start, NULL));
 381         for (i = 0; i < iterations; ++i)
 382                 fn(dst, i, len);
 383         BUG_ON(gettimeofday(&tv_end, NULL));
 384
 385         timersub(&tv_end, &tv_start, &tv_diff);
 386
 387         free(dst);
 388         return (double)(((double)len * iterations) / timeval2double(&tv_diff));
 389 }
 390
 391 static const char * const bench_mem_memset_usage[] = {
 392         "perf bench mem memset <options>",
 393         NULL
 394 };
 395
 396 static const struct routine memset_routines[] = {
 397         { .name         = "default",
 398           .desc         = "Default memset() provided by glibc",
 399           .fn.memset    = memset },
 400
 401 #ifdef HAVE_ARCH_X86_64_SUPPORT
 402 # define MEMSET_FN(_fn, _name, _desc) { .name = _name, .desc = _desc, .fn.memset = _fn },
 403 # include "mem-memset-x86-64-asm-def.h"
 404 # undef MEMSET_FN
 405 #endif
 406
 407         { NULL, }
 408 };
 409
 410 int bench_mem_memset(int argc, const char **argv, const char *prefix __maybe_unused)
 411 {
 412         struct bench_mem_info info = {
 413                 .routines               = memset_routines,
 414                 .do_cycle               = do_memset_cycle,
 415                 .do_gettimeofday        = do_memset_gettimeofday,
 416                 .usage                  = bench_mem_memset_usage,
 417         };
 418
 419         return bench_mem_common(argc, argv, &info);
 420 }