src/cpu-op.c

   1 // SPDX-License-Identifier: LGPL-2.1-only
   2 /*
   3  * cpu-op.c
   4  *
   5  * Copyright (C) 2017 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; only
  10  * version 2.1 of the License.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15  * Lesser General Public License for more details.
  16  */
  17
  18 #ifndef _GNU_SOURCE
  19 #define _GNU_SOURCE
  20 #endif
  21 #include <errno.h>
  22 #include <sched.h>
  23 #include <stdio.h>
  24 #include <stdlib.h>
  25 #include <string.h>
  26 #include <unistd.h>
  27 #include <syscall.h>
  28 #include <assert.h>
  29 #include <signal.h>
  30
  31 #include <rseq/cpu-op.h>
  32
  33 #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
  34
  35 #define ACCESS_ONCE(x)          (*(__volatile__  __typeof__(x) *)&(x))
  36 #define WRITE_ONCE(x, v)        __extension__ ({ ACCESS_ONCE(x) = (v); })
  37 #define READ_ONCE(x)            ACCESS_ONCE(x)
  38
  39 int cpu_opv(struct cpu_op *cpu_opv, int cpuopcnt, int cpu, int flags)
  40 {
  41         return syscall(__NR_cpu_opv, cpu_opv, cpuopcnt, cpu, flags);
  42 }
  43
  44 int cpu_op_available(void)
  45 {
  46         int rc;
  47
  48         rc = cpu_opv(NULL, 0, 0, CPU_OP_NR_FLAG);
  49         if (rc >= 0)
  50                 return 1;
  51         return 0;
  52 }
  53
  54 int cpu_op_get_current_cpu(void)
  55 {
  56         int cpu;
  57
  58         cpu = sched_getcpu();
  59         if (cpu < 0) {
  60                 perror("sched_getcpu()");
  61                 abort();
  62         }
  63         return cpu;
  64 }
  65
  66 int cpu_op_cmpxchg(void *v, void *expect, void *old, void *n, size_t len,
  67                    int cpu)
  68 {
  69         struct cpu_op opvec[] = {
  70                 [0] = {
  71                         .op = CPU_MEMCPY_OP,
  72                         .len = len,
  73                         .u.memcpy_op = {
  74                                 .dst = (unsigned long)old,
  75                                 .src = (unsigned long)v,
  76                                 .expect_fault_dst = 0,
  77                                 .expect_fault_src = 0,
  78                         },
  79                 },
  80                 [1] = {
  81                         .op = CPU_COMPARE_EQ_OP,
  82                         .len = len,
  83                         .u.compare_op = {
  84                                 .a = (unsigned long)v,
  85                                 .b = (unsigned long)expect,
  86                                 .expect_fault_a = 0,
  87                                 .expect_fault_b = 0,
  88                         },
  89                 },
  90                 [2] = {
  91                         .op = CPU_MEMCPY_OP,
  92                         .len = len,
  93                         .u.memcpy_op = {
  94                                 .dst = (unsigned long)v,
  95                                 .src = (unsigned long)n,
  96                                 .expect_fault_dst = 0,
  97                                 .expect_fault_src = 0,
  98                         },
  99                 },
 100         };
 101
 102         return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0);
 103 }
 104
 105 int cpu_op_add(void *v, int64_t count, size_t len, int cpu)
 106 {
 107         struct cpu_op opvec[] = {
 108                 [0] = {
 109                         .op = CPU_ADD_OP,
 110                         .len = len,
 111                         .u.arithmetic_op = {
 112                                 .p = (unsigned long)v,
 113                                 .count = count,
 114                                 .expect_fault_p = 0,
 115                         },
 116                 },
 117         };
 118
 119         return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0);
 120 }
 121
 122 int cpu_op_add_release(void *v, int64_t count, size_t len, int cpu)
 123 {
 124         struct cpu_op opvec[] = {
 125                 [0] = {
 126                         .op = CPU_ADD_RELEASE_OP,
 127                         .len = len,
 128                         .u.arithmetic_op = {
 129                                 .p = (unsigned long)v,
 130                                 .count = count,
 131                                 .expect_fault_p = 0,
 132                         },
 133                 },
 134         };
 135
 136         return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0);
 137 }
 138
 139 int cpu_op_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv,
 140                          int cpu)
 141 {
 142         struct cpu_op opvec[] = {
 143                 [0] = {
 144                         .op = CPU_COMPARE_EQ_OP,
 145                         .len = sizeof(intptr_t),
 146                         .u.compare_op = {
 147                                 .a = (unsigned long)v,
 148                                 .b = (unsigned long)&expect,
 149                                 .expect_fault_a = 0,
 150                                 .expect_fault_b = 0,
 151                         },
 152                 },
 153                 [1] = {
 154                         .op = CPU_MEMCPY_OP,
 155                         .len = sizeof(intptr_t),
 156                         .u.memcpy_op = {
 157                                 .dst = (unsigned long)v,
 158                                 .src = (unsigned long)&newv,
 159                                 .expect_fault_dst = 0,
 160                                 .expect_fault_src = 0,
 161                         },
 162                 },
 163         };
 164
 165         return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0);
 166 }
 167
 168 static int cpu_op_cmpeqv_storep_expect_fault(intptr_t *v, intptr_t expect,
 169                                              intptr_t *newp, int cpu)
 170 {
 171         struct cpu_op opvec[] = {
 172                 [0] = {
 173                         .op = CPU_COMPARE_EQ_OP,
 174                         .len = sizeof(intptr_t),
 175                         .u.compare_op = {
 176                                 .a = (unsigned long)v,
 177                                 .b = (unsigned long)&expect,
 178                                 .expect_fault_a = 0,
 179                                 .expect_fault_b = 0,
 180                         },
 181                 },
 182                 [1] = {
 183                         .op = CPU_MEMCPY_OP,
 184                         .len = sizeof(intptr_t),
 185                         .u.memcpy_op = {
 186                                 .dst = (unsigned long)v,
 187                                 .src = (unsigned long)newp,
 188                                 .expect_fault_dst = 0,
 189                                 /* Return EAGAIN on src fault. */
 190                                 .expect_fault_src = 1,
 191                         },
 192                 },
 193         };
 194
 195         return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0);
 196 }
 197
 198 int cpu_op_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 199                                  off_t voffp, intptr_t *load, int cpu)
 200 {
 201         int ret;
 202
 203         do {
 204                 intptr_t oldv = READ_ONCE(*v);
 205                 intptr_t *newp = (intptr_t *)(oldv + voffp);
 206
 207                 if (oldv == expectnot)
 208                         return 1;
 209                 ret = cpu_op_cmpeqv_storep_expect_fault(v, oldv, newp, cpu);
 210                 if (!ret) {
 211                         *load = oldv;
 212                         return 0;
 213                 }
 214         } while (ret > 0);
 215
 216         return -1;
 217 }
 218
 219 int cpu_op_cmpeqv_storev_storev(intptr_t *v, intptr_t expect,
 220                                 intptr_t *v2, intptr_t newv2,
 221                                 intptr_t newv, int cpu)
 222 {
 223         struct cpu_op opvec[] = {
 224                 [0] = {
 225                         .op = CPU_COMPARE_EQ_OP,
 226                         .len = sizeof(intptr_t),
 227                         .u.compare_op = {
 228                                 .a = (unsigned long)v,
 229                                 .b = (unsigned long)&expect,
 230                                 .expect_fault_a = 0,
 231                                 .expect_fault_b = 0,
 232                         },
 233                 },
 234                 [1] = {
 235                         .op = CPU_MEMCPY_OP,
 236                         .len = sizeof(intptr_t),
 237                         .u.memcpy_op = {
 238                                 .dst = (unsigned long)v2,
 239                                 .src = (unsigned long)&newv2,
 240                                 .expect_fault_dst = 0,
 241                                 .expect_fault_src = 0,
 242                         },
 243                 },
 244                 [2] = {
 245                         .op = CPU_MEMCPY_OP,
 246                         .len = sizeof(intptr_t),
 247                         .u.memcpy_op = {
 248                                 .dst = (unsigned long)v,
 249                                 .src = (unsigned long)&newv,
 250                                 .expect_fault_dst = 0,
 251                                 .expect_fault_src = 0,
 252                         },
 253                 },
 254         };
 255
 256         return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0);
 257 }
 258
 259 int cpu_op_cmpeqv_storev_mb_storev(intptr_t *v, intptr_t expect,
 260                                    intptr_t *v2, intptr_t newv2,
 261                                    intptr_t newv, int cpu)
 262 {
 263         struct cpu_op opvec[] = {
 264                 [0] = {
 265                         .op = CPU_COMPARE_EQ_OP,
 266                         .len = sizeof(intptr_t),
 267                         .u.compare_op = {
 268                                 .a = (unsigned long)v,
 269                                 .b = (unsigned long)&expect,
 270                                 .expect_fault_a = 0,
 271                                 .expect_fault_b = 0,
 272                 },
 273                 },
 274                 [1] = {
 275                         .op = CPU_MEMCPY_OP,
 276                         .len = sizeof(intptr_t),
 277                         .u.memcpy_op = {
 278                                 .dst = (unsigned long)v2,
 279                                 .src = (unsigned long)&newv2,
 280                                 .expect_fault_dst = 0,
 281                                 .expect_fault_src = 0,
 282                         },
 283                 },
 284                 [2] = {
 285                         .op = CPU_MEMCPY_RELEASE_OP,
 286                         .len = sizeof(intptr_t),
 287                         .u.memcpy_op = {
 288                                 .dst = (unsigned long)v,
 289                                 .src = (unsigned long)&newv,
 290                                 .expect_fault_dst = 0,
 291                                 .expect_fault_src = 0,
 292                         },
 293                 },
 294         };
 295
 296         return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0);
 297 }
 298
 299 int cpu_op_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 300                                 intptr_t *v2, intptr_t expect2,
 301                                 intptr_t newv, int cpu)
 302 {
 303         struct cpu_op opvec[] = {
 304                 [0] = {
 305                         .op = CPU_COMPARE_EQ_OP,
 306                         .len = sizeof(intptr_t),
 307                         .u.compare_op = {
 308                                 .a = (unsigned long)v,
 309                                 .b = (unsigned long)&expect,
 310                                 .expect_fault_a = 0,
 311                                 .expect_fault_b = 0,
 312                         },
 313                 },
 314                 [1] = {
 315                         .op = CPU_COMPARE_EQ_OP,
 316                         .len = sizeof(intptr_t),
 317                         .u.compare_op = {
 318                                 .a = (unsigned long)v2,
 319                                 .b = (unsigned long)&expect2,
 320                                 .expect_fault_a = 0,
 321                                 .expect_fault_b = 0,
 322                         },
 323                 },
 324                 [2] = {
 325                         .op = CPU_MEMCPY_OP,
 326                         .len = sizeof(intptr_t),
 327                         .u.memcpy_op = {
 328                                 .dst = (unsigned long)v,
 329                                 .src = (unsigned long)&newv,
 330                                 .expect_fault_dst = 0,
 331                                 .expect_fault_src = 0,
 332                         },
 333                 },
 334         };
 335
 336         return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0);
 337 }
 338
 339 int cpu_op_cmpeqv_memcpy_storev(intptr_t *v, intptr_t expect,
 340                                 void *dst, void *src, size_t len,
 341                                 intptr_t newv, int cpu)
 342 {
 343         struct cpu_op opvec[] = {
 344                 [0] = {
 345                         .op = CPU_COMPARE_EQ_OP,
 346                         .len = sizeof(intptr_t),
 347                         .u.compare_op = {
 348                                 .a = (unsigned long)v,
 349                                 .b = (unsigned long)&expect,
 350                                 .expect_fault_a = 0,
 351                                 .expect_fault_b = 0,
 352                         },
 353                 },
 354                 [1] = {
 355                         .op = CPU_MEMCPY_OP,
 356                         .len = len,
 357                         .u.memcpy_op = {
 358                                 .dst = (unsigned long)dst,
 359                                 .src = (unsigned long)src,
 360                                 .expect_fault_dst = 0,
 361                                 .expect_fault_src = 0,
 362                         },
 363                 },
 364                 [2] = {
 365                         .op = CPU_MEMCPY_OP,
 366                         .len = sizeof(intptr_t),
 367                         .u.memcpy_op = {
 368                                 .dst = (unsigned long)v,
 369                                 .src = (unsigned long)&newv,
 370                                 .expect_fault_dst = 0,
 371                                 .expect_fault_src = 0,
 372                         },
 373                 },
 374         };
 375
 376         return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0);
 377 }
 378
 379 int cpu_op_cmpeqv_memcpy_mb_storev(intptr_t *v, intptr_t expect,
 380                                    void *dst, void *src, size_t len,
 381                                    intptr_t newv, int cpu)
 382 {
 383         struct cpu_op opvec[] = {
 384                 [0] = {
 385                         .op = CPU_COMPARE_EQ_OP,
 386                         .len = sizeof(intptr_t),
 387                         .u.compare_op = {
 388                                 .a = (unsigned long)v,
 389                                 .b = (unsigned long)&expect,
 390                                 .expect_fault_a = 0,
 391                                 .expect_fault_b = 0,
 392                         },
 393                 },
 394                 [1] = {
 395                         .op = CPU_MEMCPY_OP,
 396                         .len = len,
 397                         .u.memcpy_op = {
 398                                 .dst = (unsigned long)dst,
 399                                 .src = (unsigned long)src,
 400                                 .expect_fault_dst = 0,
 401                                 .expect_fault_src = 0,
 402                         },
 403                 },
 404                 [2] = {
 405                         .op = CPU_MEMCPY_RELEASE_OP,
 406                         .len = sizeof(intptr_t),
 407                         .u.memcpy_op = {
 408                                 .dst = (unsigned long)v,
 409                                 .src = (unsigned long)&newv,
 410                                 .expect_fault_dst = 0,
 411                                 .expect_fault_src = 0,
 412                         },
 413                 },
 414         };
 415
 416         return cpu_opv(opvec, ARRAY_SIZE(opvec), cpu, 0);
 417 }
 418
 419 int cpu_op_addv(intptr_t *v, int64_t count, int cpu)
 420 {
 421         return cpu_op_add(v, count, sizeof(intptr_t), cpu);
 422 }