[librseq.git] / src / cpu-op.c

// SPDX-License-Identifier: LGPL-2.1-only
/*
 * cpu-op.c
 *
 * Copyright (C) 2017 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; only
 * version 2.1 of the License.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 */

#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <errno.h>
#include <sched.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <syscall.h>
#include <assert.h>
#include <signal.h>
#include <rseq/cpu-op.h>

#include "do-on-cpu-insn.h"

#define ARRAY_SIZE(arr)	(sizeof(arr) / sizeof((arr)[0]))

#define ACCESS_ONCE(x)		(*(__volatile__  __typeof__(x) *)&(x))
#define WRITE_ONCE(x, v)	__extension__ ({ ACCESS_ONCE(x) = (v); })
#define READ_ONCE(x)		ACCESS_ONCE(x)

int do_on_cpu(struct bpf_insn *bytecode, uint32_t len, int64_t *result,
	      int cpu, int flags)
{
	return syscall(__NR_do_on_cpu, bytecode, len, result, cpu, flags);
}

int cpu_op_available(void)
{
	int rc;

	rc = do_on_cpu(NULL, 0, NULL, 0, DO_ON_CPU_LEN_MAX_FLAG);
	if (rc >= 0)
		return 1;
	return 0;
}

int cpu_op_get_current_cpu(void)
{
	int cpu;

	cpu = sched_getcpu();
	if (cpu < 0) {
		perror("sched_getcpu()");
		abort();
	}
	return cpu;
}

static
int __cpu_op_cmpxchg(void *v, void *expect, void *old, void *n, size_t len,
		     int cpu, int acquire, int release)
{
	int ret;
	unsigned int bpf_size, ldx_mode, stx_mode;
	int64_t expectv, nv, res;

	switch (len) {
	case 1:	bpf_size = BPF_B;
		expectv = *(int8_t *) expect;
		nv = *(int8_t *) n;
		break;
	case 2: bpf_size = BPF_H;
		expectv = *(int16_t *) expect;
		nv = *(int16_t *) n;
		break;
	case 4:	bpf_size = BPF_W;
		expectv = *(int32_t *) expect;
		nv = *(int32_t *) n;
		break;
	case 8:	bpf_size = BPF_DW;
		expectv = *(int64_t *) expect;
		nv = *(int64_t *) n;
		break;
	default:
		return -EINVAL;
	}

	ldx_mode = acquire ? BPF_MEM_ACQ_REL : BPF_MEM;
	stx_mode = release ? BPF_MEM_ACQ_REL : BPF_MEM;

	enum {
		BPF_LABEL_BRANCH1 = 6,
		BPF_LABEL_FAIL = 9,
	};

	{
		struct bpf_insn bytecode[] = {
			[0] = BPFI_LD_IMM64(BPF_REG_1, BPF_PTR_TO_V(v)),
			[2] = BPFI_LDX_MODE(bpf_size, ldx_mode, BPF_REG_0, BPF_REG_1, 0),
			[3] = BPFI_LD_IMM64(BPF_REG_2, expectv),
			[5] = BPFI_JNE_X(BPF_REG_2, BPF_REG_0,
					 BPF_LABEL_FAIL - BPF_LABEL_BRANCH1),

			[BPF_LABEL_BRANCH1] = BPFI_LD_IMM64(BPF_REG_3, nv),
			[8] = BPFI_STX_MODE(bpf_size, stx_mode, BPF_REG_1, BPF_REG_3, 0),
			[BPF_LABEL_FAIL] = BPFI_EXIT(),	/* r0 contains old */
		};

		do {
			ret = do_on_cpu(bytecode, ARRAY_SIZE(bytecode),
					&res, cpu, 0);
		} while (ret == -1 && errno == EAGAIN);
	}

	if (!ret) {
		switch (len) {
		case 1:	*(int8_t *) old = (int8_t) res;
			break;
		case 2:	*(int16_t *) old = (int16_t) res;
			break;
		case 4:	*(int32_t *) old = (int32_t) res;
			break;
		case 8:	*(int64_t *) old = res;
			break;
		default:
			return -EINVAL;
		}
	}

	return ret;
}

int cpu_op_cmpxchg(void *v, void *expect, void *old, void *n, size_t len,
		   int cpu)
{
	return __cpu_op_cmpxchg(v, expect, old, n, len, cpu, 1, 1);
}

int cpu_op_cmpxchg_relaxed(void *v, void *expect, void *old, void *n, size_t len,
			   int cpu)
{
	return __cpu_op_cmpxchg(v, expect, old, n, len, cpu, 0, 0);
}

int cpu_op_cmpxchg_acquire(void *v, void *expect, void *old, void *n, size_t len,
			   int cpu)
{
	return __cpu_op_cmpxchg(v, expect, old, n, len, cpu, 1, 0);
}

int cpu_op_cmpxchg_release(void *v, void *expect, void *old, void *n, size_t len,
			   int cpu)
{
	return __cpu_op_cmpxchg(v, expect, old, n, len, cpu, 0, 1);
}

static
int __cpu_op_add(void *v, int64_t count, size_t len, int cpu,
		 int acquire, int release)
{
	int ret;
	unsigned int bpf_size, ldx_mode, stx_mode;

	switch (len) {
	case 1:	bpf_size = BPF_B;
		break;
	case 2: bpf_size = BPF_H;
		break;
	case 4:	bpf_size = BPF_W;
		break;
	case 8:	bpf_size = BPF_DW;
		break;
	default:
		return -EINVAL;
	}

	ldx_mode = acquire ? BPF_MEM_ACQ_REL : BPF_MEM;
	stx_mode = release ? BPF_MEM_ACQ_REL : BPF_MEM;

	{
		struct bpf_insn bytecode[] = {
			BPFI_LD_IMM64(BPF_REG_1, BPF_PTR_TO_V(v)),
			BPFI_LDX_MODE(bpf_size, ldx_mode, BPF_REG_0, BPF_REG_1, 0),
			BPFI_LD_IMM64(BPF_REG_2, count),
			BPFI_ADD64_X(BPF_REG_0, BPF_REG_2),
			BPFI_STX_MODE(bpf_size, stx_mode, BPF_REG_1, BPF_REG_0, 0),
		};

		do {
			ret = do_on_cpu(bytecode, ARRAY_SIZE(bytecode),
					NULL, cpu, 0);
		} while (ret == -1 && errno == EAGAIN);
	}
	return ret;
}

int cpu_op_add(void *v, int64_t count, size_t len, int cpu)
{
	return __cpu_op_add(v, count, len, cpu, 1, 1);
}

int cpu_op_add_relaxed(void *v, int64_t count, size_t len, int cpu)
{
	return __cpu_op_add(v, count, len, cpu, 0, 0);
}

int cpu_op_add_acquire(void *v, int64_t count, size_t len, int cpu)
{
	return __cpu_op_add(v, count, len, cpu, 1, 0);
}

int cpu_op_add_release(void *v, int64_t count, size_t len, int cpu)
{
	return __cpu_op_add(v, count, len, cpu, 0, 1);
}

int cpu_op_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv,
			 int cpu)
{
	intptr_t old;
	int ret;

	ret = cpu_op_cmpxchg_relaxed(v, &expect, &old, &newv, sizeof(*v),
				     cpu);
	if (!ret && old != expect)
		ret = 1;
	return ret;
}

int cpu_op_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
				 off_t voffp, intptr_t *load, int cpu)
{
	int ret;
	int64_t res;
	unsigned int bpf_size1;
	size_t len1 = sizeof(*v);

	switch (len1) {
	case 1:	bpf_size1 = BPF_B;
		break;
	case 2: bpf_size1 = BPF_H;
		break;
	case 4:	bpf_size1 = BPF_W;
		break;
	case 8:	bpf_size1 = BPF_DW;
		break;
	default:
		return -EINVAL;
	}

	enum {
		BPF_LABEL_BRANCH1 = 6,
		BPF_LABEL_FAIL = 16,
	};

	{
		struct bpf_insn bytecode[] = {
			[0] = BPFI_LD_IMM64(BPF_REG_1, BPF_PTR_TO_V(v)),
			[2] = BPFI_LDX(bpf_size1, BPF_REG_2, BPF_REG_1, 0),
			[3] = BPFI_LD_IMM64(BPF_REG_3, expectnot),
			[5] = BPFI_JEQ_X(BPF_REG_2, BPF_REG_3,
					 BPF_LABEL_FAIL - BPF_LABEL_BRANCH1),

			[BPF_LABEL_BRANCH1] = BPFI_LD_IMM64(BPF_REG_3, BPF_PTR_TO_V(load)),
			[8] = BPFI_STX(bpf_size1, BPF_REG_3, BPF_REG_2, 0),

			[9] = BPFI_MOV_X(BPF_REG_3, BPF_REG_2),
			[10] = BPFI_LD_IMM64(BPF_REG_4, voffp),
			[12] = BPFI_ADD64_X(BPF_REG_3, BPF_REG_4),
			[13] = BPFI_LDX(bpf_size1, BPF_REG_3, BPF_REG_3, 0),
			[14] = BPFI_STX(bpf_size1, BPF_REG_1, BPF_REG_3, 0),
			[15] = BPFI_EXIT(),	/* r0 = 0. */
			[BPF_LABEL_FAIL] = BPFI_LD_IMM32(BPF_REG_0, 1),
			[17] = BPFI_EXIT(),	/* r0 = 1. */
		};

		do {
			ret = do_on_cpu(bytecode, ARRAY_SIZE(bytecode),
					&res, cpu, 0);
		} while (ret == -1 && errno == EAGAIN);
	}
	if (!ret)
		ret = (int) !!res;
	return ret;
}

static
int __cpu_op_cmpeqv_storev_storev(intptr_t *v, intptr_t expect,
				  intptr_t *v2, intptr_t newv2,
				  intptr_t newv, int cpu, int release)
{
	int ret;
	int64_t res;
	unsigned int bpf_size1, bpf_size2, stx_mode;
	size_t len1 = sizeof(*v);
	size_t len2 = sizeof(*v2);

	switch (len1) {
	case 1:	bpf_size1 = BPF_B;
		break;
	case 2: bpf_size1 = BPF_H;
		break;
	case 4:	bpf_size1 = BPF_W;
		break;
	case 8:	bpf_size1 = BPF_DW;
		break;
	default:
		return -EINVAL;
	}

	switch (len2) {
	case 1:	bpf_size2 = BPF_B;
		break;
	case 2: bpf_size2 = BPF_H;
		break;
	case 4:	bpf_size2 = BPF_W;
		break;
	case 8:	bpf_size2 = BPF_DW;
		break;
	default:
		return -EINVAL;
	}

	stx_mode = release ? BPF_MEM_ACQ_REL : BPF_MEM;

	enum {
		BPF_LABEL_BRANCH1 = 7,
		BPF_LABEL_FAIL = 16,
	};

	{
		struct bpf_insn bytecode[] = {
			[0] = BPFI_LD_IMM64(BPF_REG_1, BPF_PTR_TO_V(v)),
			[2] = BPFI_LDX(bpf_size1, BPF_REG_2, BPF_REG_1, 0),
			[3] = BPFI_LD_IMM64(BPF_REG_3, BPF_PTR_TO_V(&expect)),
			[5] = BPFI_LDX(bpf_size1, BPF_REG_3, BPF_REG_3, 0),
			[6] = BPFI_JNE_X(BPF_REG_2, BPF_REG_3,
					 BPF_LABEL_FAIL - BPF_LABEL_BRANCH1),

			[BPF_LABEL_BRANCH1] = BPFI_LD_IMM64(BPF_REG_2, BPF_PTR_TO_V(v2)),
			[9] = BPFI_LD_IMM64(BPF_REG_3, newv2),
			[11] = BPFI_LD_IMM64(BPF_REG_4, newv),
			[13] = BPFI_STX(bpf_size2, BPF_REG_2, BPF_REG_3, 0),
			[14] = BPFI_STX_MODE(bpf_size2, stx_mode, BPF_REG_1, BPF_REG_4, 0),
			[15] = BPFI_EXIT(),	/* r0 = 0. */
			[BPF_LABEL_FAIL] = BPFI_LD_IMM32(BPF_REG_0, 1),
			[17] = BPFI_EXIT(),	/* r0 = 1. */
		};

		do {
			ret = do_on_cpu(bytecode, ARRAY_SIZE(bytecode),
					&res, cpu, 0);
		} while (ret == -1 && errno == EAGAIN);
	}
	if (!ret)
		ret = (int) !!res;
	return ret;
}

int cpu_op_cmpeqv_storev_storev(intptr_t *v, intptr_t expect,
				intptr_t *v2, intptr_t newv2,
				intptr_t newv, int cpu)
{
	return __cpu_op_cmpeqv_storev_storev(v, expect, v2, newv2,
					     newv, cpu, 0);
}

int cpu_op_cmpeqv_storev_storev_release(intptr_t *v, intptr_t expect,
				intptr_t *v2, intptr_t newv2,
				intptr_t newv, int cpu)
{
	return __cpu_op_cmpeqv_storev_storev(v, expect, v2, newv2,
					     newv, cpu, 1);
}

static
int __cpu_op_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
				intptr_t *v2, intptr_t expect2,
				intptr_t newv, int cpu, int release)
{
	int ret;
	int64_t res;
	unsigned int bpf_size1, bpf_size2, stx_mode;
	size_t len1 = sizeof(*v);
	size_t len2 = sizeof(*v2);

	switch (len1) {
	case 1:	bpf_size1 = BPF_B;
		break;
	case 2: bpf_size1 = BPF_H;
		break;
	case 4:	bpf_size1 = BPF_W;
		break;
	case 8:	bpf_size1 = BPF_DW;
		break;
	default:
		return -EINVAL;
	}

	switch (len2) {
	case 1:	bpf_size2 = BPF_B;
		break;
	case 2: bpf_size2 = BPF_H;
		break;
	case 4:	bpf_size2 = BPF_W;
		break;
	case 8:	bpf_size2 = BPF_DW;
		break;
	default:
		return -EINVAL;
	}

	stx_mode = release ? BPF_MEM_ACQ_REL : BPF_MEM;

	enum {
		BPF_LABEL_BRANCH1 = 6,
		BPF_LABEL_BRANCH2 = 12,
		BPF_LABEL_FAIL = 16,
	};

	{
		struct bpf_insn bytecode[] = {
			[0] = BPFI_LD_IMM64(BPF_REG_1, BPF_PTR_TO_V(v)),
			[2] = BPFI_LDX(bpf_size1, BPF_REG_2, BPF_REG_1, 0),
			[3] = BPFI_LD_IMM64(BPF_REG_3, expect),
			[5] = BPFI_JNE_X(BPF_REG_2, BPF_REG_3,
					 BPF_LABEL_FAIL - BPF_LABEL_BRANCH1),

			[BPF_LABEL_BRANCH1] = BPFI_LD_IMM64(BPF_REG_2, BPF_PTR_TO_V(v2)),
			[8] = BPFI_LDX(bpf_size1, BPF_REG_2, BPF_REG_2, 0),
			[9] = BPFI_LD_IMM64(BPF_REG_3, expect2),
			[11] = BPFI_JNE_X(BPF_REG_2, BPF_REG_3,
					  BPF_LABEL_FAIL - BPF_LABEL_BRANCH2),

			[BPF_LABEL_BRANCH2] = BPFI_LD_IMM64(BPF_REG_2, newv),
			[14] = BPFI_STX_MODE(bpf_size2, stx_mode, BPF_REG_1, BPF_REG_2, 0),
			[15] = BPFI_EXIT(),	/* r0 = 0. */
			[BPF_LABEL_FAIL] = BPFI_LD_IMM32(BPF_REG_0, 1),
			[17] = BPFI_EXIT(),	/* r0 = 1. */
		};

		do {
			ret = do_on_cpu(bytecode, ARRAY_SIZE(bytecode),
					&res, cpu, 0);
		} while (ret == -1 && errno == EAGAIN);
	}
	if (!ret)
		ret = (int) !!res;
	return ret;

}

int cpu_op_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
				intptr_t *v2, intptr_t expect2,
				intptr_t newv, int cpu)
{
	return __cpu_op_cmpeqv_cmpeqv_storev(v, expect, v2, expect2,
					     newv, cpu, 0);
}

int cpu_op_cmpeqv_cmpeqv_storev_release(intptr_t *v, intptr_t expect,
					intptr_t *v2, intptr_t expect2,
					intptr_t newv, int cpu)
{
	return __cpu_op_cmpeqv_cmpeqv_storev(v, expect, v2, expect2,
					     newv, cpu, 1);
}

static
int __cpu_op_cmpeqv_memcpy_storev(intptr_t *v, intptr_t expect,
				  void *dst, void *src, size_t len,
				  intptr_t newv, int cpu, int release)
{
	int ret;
	int64_t res;
	unsigned int bpf_size2, stx_mode;
	unsigned int len2 = sizeof(*v);

	switch (len2) {
	case 1:	bpf_size2 = BPF_B;
		break;
	case 2: bpf_size2 = BPF_H;
		break;
	case 4:	bpf_size2 = BPF_W;
		break;
	case 8:	bpf_size2 = BPF_DW;
		break;
	default:
		return -EINVAL;
	}

	stx_mode = release ? BPF_MEM_ACQ_REL : BPF_MEM;

	enum {
		BPF_LABEL_BRANCH_TEST = 6,
		BPF_LABEL_LOOP8 = 14,
		BPF_LABEL_BRANCH8_1 = 15,
		BPF_LABEL_LOOP1 = 20,
		BPF_LABEL_BRANCH1_1 = 21,
		BPF_LABEL_BRANCH1_2 = 26,
		BPF_LABEL_FAIL = 30,
	};

	{
		struct bpf_insn bytecode[] = {
			/*
			 * r0 is 0
			 * r1 is temporary register,
			 * r2 is expect
			 * r6 is v
			 */
			[0] = BPFI_LD_IMM64(BPF_REG_6, BPF_PTR_TO_V(v)),
			[2] = BPFI_LDX(bpf_size2, BPF_REG_1, BPF_REG_6, 0),
			[3] = BPFI_LD_IMM64(BPF_REG_2, expect),
			[5] = BPFI_JNE_X(BPF_REG_1, BPF_REG_2,
					 BPF_LABEL_FAIL - BPF_LABEL_BRANCH_TEST),

			/*
			 * r0 is 0
			 * r1 is temporary register,
			 * r2 is dst iterator,
			 * r3 is src iterator,
			 * r4 is src + (len & ~7)	// end of 8-byte copy
			 * r5 is src + len		// end of 1-byte copy
			 * r6 is v
			 */
			[BPF_LABEL_BRANCH_TEST] = BPFI_LD_IMM64(BPF_REG_2,
								BPF_PTR_TO_V(dst)),
			[8] = BPFI_LD_IMM64(BPF_REG_3, BPF_PTR_TO_V(src)),
			[10] = BPFI_LD_IMM64(BPF_REG_4, BPF_PTR_TO_V(src) + (len & ~7)),
			[12] = BPFI_LD_IMM64(BPF_REG_5, BPF_PTR_TO_V(src) + len),

			/* 8-byte copy loop target. */
			[BPF_LABEL_LOOP8] = BPFI_JEQ_X(BPF_REG_3, BPF_REG_4,
						       BPF_LABEL_LOOP1 - BPF_LABEL_BRANCH8_1),

			[BPF_LABEL_BRANCH8_1] = BPFI_LDX(BPF_DW, BPF_REG_1, BPF_REG_3, 0),
			[16] = BPFI_STX(BPF_DW, BPF_REG_2, BPF_REG_1, 0),

			[17] = BPFI_ADD64_K(BPF_REG_2, 8),
			[18] = BPFI_ADD64_K(BPF_REG_3, 8),
			[19] = BPFI_JA_K(BPF_LABEL_LOOP8 - BPF_LABEL_LOOP1),

			/* 1-byte copy loop target. */
			[BPF_LABEL_LOOP1] = BPFI_JEQ_X(BPF_REG_3, BPF_REG_5,
						       BPF_LABEL_BRANCH1_2 - BPF_LABEL_BRANCH1_1),

			[BPF_LABEL_BRANCH1_1] = BPFI_LDX(BPF_B, BPF_REG_1, BPF_REG_3, 0),
			[22] = BPFI_STX(BPF_B, BPF_REG_2, BPF_REG_1, 0),

			[23] = BPFI_ADD64_K(BPF_REG_2, 1),
			[24] = BPFI_ADD64_K(BPF_REG_3, 1),
			[25] = BPFI_JA_K(BPF_LABEL_LOOP1 - BPF_LABEL_BRANCH1_2),

			/* Completed, do store. */

			/*
			 * r0 is 0
			 * r2 is newv
			 * r6 is v
			 */
			[BPF_LABEL_BRANCH1_2] = BPFI_LD_IMM64(BPF_REG_2, newv),
			[28] = BPFI_STX_MODE(bpf_size2, stx_mode, BPF_REG_6,
					     BPF_REG_2, 0),

			[29] = BPFI_EXIT(),	/* r0 = 0. */
			[BPF_LABEL_FAIL] = BPFI_LD_IMM32(BPF_REG_0, 1),
			[31] = BPFI_EXIT(),	/* r0 = 1. */
		};

		do {
			ret = do_on_cpu(bytecode, ARRAY_SIZE(bytecode),
					&res, cpu, 0);
		} while (ret == -1 && errno == EAGAIN);
	}
	if (!ret)
		ret = (int) !!res;
	return ret;
}

int cpu_op_cmpeqv_memcpy_storev(intptr_t *v, intptr_t expect,
				  void *dst, void *src, size_t len,
				  intptr_t newv, int cpu)
{
	return __cpu_op_cmpeqv_memcpy_storev(v, expect, dst, src, len,
					     newv, cpu, 0);
}

int cpu_op_cmpeqv_memcpy_storev_release(intptr_t *v, intptr_t expect,
					void *dst, void *src, size_t len,
					intptr_t newv, int cpu)
{
	return __cpu_op_cmpeqv_memcpy_storev(v, expect, dst, src, len,
					     newv, cpu, 1);
}

int cpu_op_addv(intptr_t *v, int64_t count, int cpu)
{
	return cpu_op_add_relaxed(v, count, sizeof(intptr_t), cpu);
}

int cpu_op_deref_loadoffp(intptr_t *p, off_t voffp, intptr_t *load, int cpu)
{
	int ret;
	int64_t res;
	unsigned int bpf_size1, bpf_size2;
	size_t len1 = sizeof(void *), len2 = sizeof(*load);

	switch (len1) {
	case 1:	bpf_size1 = BPF_B;
		break;
	case 2: bpf_size1 = BPF_H;
		break;
	case 4:	bpf_size1 = BPF_W;
		break;
	case 8:	bpf_size1 = BPF_DW;
		break;
	default:
		return -EINVAL;
	}

	switch (len2) {
	case 1:	bpf_size2 = BPF_B;
		break;
	case 2: bpf_size2 = BPF_H;
		break;
	case 4:	bpf_size2 = BPF_W;
		break;
	case 8:	bpf_size2 = BPF_DW;
		break;
	default:
		return -EINVAL;
	}

	{
		struct bpf_insn bytecode[] = {
			[0] = BPFI_LD_IMM64(BPF_REG_1, BPF_PTR_TO_V(p)),
			[2] = BPFI_LDX(bpf_size1, BPF_REG_1, BPF_REG_1, 0),
			[3] = BPFI_LD_IMM64(BPF_REG_2, voffp),
			[5] = BPFI_ADD64_X(BPF_REG_1, BPF_REG_2),
			[6] = BPFI_LDX(bpf_size2, BPF_REG_0, BPF_REG_1, 0),
		};

		do {
			ret = do_on_cpu(bytecode, ARRAY_SIZE(bytecode),
					&res, cpu, 0);
		} while (ret == -1 && errno == EAGAIN);
	}
	if (!ret)
		*load = (intptr_t) res;
	return ret;
}

int cpu_op_fence(int cpu)
{
	int ret;

	do {
		ret = do_on_cpu(NULL, 0, NULL, cpu, 0);
	} while (ret == -1 && errno == EAGAIN);
	return ret;
}
Commit	Line	Data
744d0b8b	1	// SPDX-License-Identifier: LGPL-2.1-only
b32429a9 MD	2	/*
	3	* cpu-op.c
	4	*
	5	* Copyright (C) 2017 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	6	*
	7	* This library is free software; you can redistribute it and/or
	8	* modify it under the terms of the GNU Lesser General Public
	9	* License as published by the Free Software Foundation; only
	10	* version 2.1 of the License.
	11	*
	12	* This library is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	15	* Lesser General Public License for more details.
	16	*/
	17
2cbca301	18	#ifndef _GNU_SOURCE
b32429a9	19	#define _GNU_SOURCE
2cbca301	20	#endif
b32429a9 MD	21	#include <errno.h>
	22	#include <sched.h>
	23	#include <stdio.h>
	24	#include <stdlib.h>
	25	#include <string.h>
	26	#include <unistd.h>
	27	#include <syscall.h>
	28	#include <assert.h>
	29	#include <signal.h>
b32429a9 MD	30	#include <rseq/cpu-op.h>
b32429a9 MD	31
de28c254 MD	32	#include "do-on-cpu-insn.h"
de28c254 MD	33
b32429a9 MD	34	#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
	35
	36	#define ACCESS_ONCE(x) ((__volatile__ __typeof__(x) )&(x))
	37	#define WRITE_ONCE(x, v) __extension__ ({ ACCESS_ONCE(x) = (v); })
	38	#define READ_ONCE(x) ACCESS_ONCE(x)
	39
de28c254 MD	40	int do_on_cpu(struct bpf_insn bytecode, uint32_t len, int64_t result,
de28c254 MD	41	int cpu, int flags)
b32429a9	42	{
de28c254	43	return syscall(__NR_do_on_cpu, bytecode, len, result, cpu, flags);
b32429a9 MD	44	}
b32429a9 MD	45
52e82b87 MD	46	int cpu_op_available(void)
	47	{
	48	int rc;
	49
de28c254	50	rc = do_on_cpu(NULL, 0, NULL, 0, DO_ON_CPU_LEN_MAX_FLAG);
52e82b87 MD	51	if (rc >= 0)
	52	return 1;
	53	return 0;
	54	}
	55
b32429a9 MD	56	int cpu_op_get_current_cpu(void)
	57	{
	58	int cpu;
	59
	60	cpu = sched_getcpu();
	61	if (cpu < 0) {
	62	perror("sched_getcpu()");
	63	abort();
	64	}
	65	return cpu;
	66	}
	67
de28c254 MD	68	static
	69	int __cpu_op_cmpxchg(void v, void expect, void old, void n, size_t len,
	70	int cpu, int acquire, int release)
	71	{
	72	int ret;
	73	unsigned int bpf_size, ldx_mode, stx_mode;
	74	int64_t expectv, nv, res;
	75
	76	switch (len) {
	77	case 1: bpf_size = BPF_B;
	78	expectv = (int8_t ) expect;
	79	nv = (int8_t ) n;
	80	break;
	81	case 2: bpf_size = BPF_H;
	82	expectv = (int16_t ) expect;
	83	nv = (int16_t ) n;
	84	break;
	85	case 4: bpf_size = BPF_W;
	86	expectv = (int32_t ) expect;
	87	nv = (int32_t ) n;
	88	break;
	89	case 8: bpf_size = BPF_DW;
	90	expectv = (int64_t ) expect;
	91	nv = (int64_t ) n;
	92	break;
	93	default:
	94	return -EINVAL;
	95	}
	96
	97	ldx_mode = acquire ? BPF_MEM_ACQ_REL : BPF_MEM;
	98	stx_mode = release ? BPF_MEM_ACQ_REL : BPF_MEM;
	99
	100	enum {
	101	BPF_LABEL_BRANCH1 = 6,
	102	BPF_LABEL_FAIL = 9,
	103	};
	104
	105	{
	106	struct bpf_insn bytecode[] = {
	107	[0] = BPFI_LD_IMM64(BPF_REG_1, BPF_PTR_TO_V(v)),
	108	[2] = BPFI_LDX_MODE(bpf_size, ldx_mode, BPF_REG_0, BPF_REG_1, 0),
	109	[3] = BPFI_LD_IMM64(BPF_REG_2, expectv),
	110	[5] = BPFI_JNE_X(BPF_REG_2, BPF_REG_0,
	111	BPF_LABEL_FAIL - BPF_LABEL_BRANCH1),
	112
	113	[BPF_LABEL_BRANCH1] = BPFI_LD_IMM64(BPF_REG_3, nv),
	114	[8] = BPFI_STX_MODE(bpf_size, stx_mode, BPF_REG_1, BPF_REG_3, 0),
	115	[BPF_LABEL_FAIL] = BPFI_EXIT(), /* r0 contains old */
	116	};
	117
	118	do {
	119	ret = do_on_cpu(bytecode, ARRAY_SIZE(bytecode),
	120	&res, cpu, 0);
	121	} while (ret == -1 && errno == EAGAIN);
	122	}
	123
	124	if (!ret) {
	125	switch (len) {
	126	case 1: (int8_t ) old = (int8_t) res;
	127	break;
	128	case 2: (int16_t ) old = (int16_t) res;
	129	break;
	130	case 4: (int32_t ) old = (int32_t) res;
	131	break;
132	case 8: (int64_t ) old = res;
133	break;
134	default:
135	return -EINVAL;
136	}
137	}
138
139	return ret;
140	}
141
b32429a9 MD	142	int cpu_op_cmpxchg(void v, void expect, void old, void n, size_t len,
	143	int cpu)
	144	{
de28c254 MD	145	return __cpu_op_cmpxchg(v, expect, old, n, len, cpu, 1, 1);
de28c254 MD	146	}
b32429a9	147
de28c254 MD	148	int cpu_op_cmpxchg_relaxed(void v, void expect, void old, void n, size_t len,
	149	int cpu)
	150	{
	151	return __cpu_op_cmpxchg(v, expect, old, n, len, cpu, 0, 0);
	152	}
	153
	154	int cpu_op_cmpxchg_acquire(void v, void expect, void old, void n, size_t len,
	155	int cpu)
	156	{
	157	return __cpu_op_cmpxchg(v, expect, old, n, len, cpu, 1, 0);
	158	}
	159
	160	int cpu_op_cmpxchg_release(void v, void expect, void old, void n, size_t len,
	161	int cpu)
	162	{
	163	return __cpu_op_cmpxchg(v, expect, old, n, len, cpu, 0, 1);
	164	}
	165
	166	static
	167	int __cpu_op_add(void *v, int64_t count, size_t len, int cpu,
	168	int acquire, int release)
	169	{
	170	int ret;
	171	unsigned int bpf_size, ldx_mode, stx_mode;
	172
	173	switch (len) {
	174	case 1: bpf_size = BPF_B;
	175	break;
	176	case 2: bpf_size = BPF_H;
	177	break;
	178	case 4: bpf_size = BPF_W;
	179	break;
	180	case 8: bpf_size = BPF_DW;
	181	break;
	182	default:
	183	return -EINVAL;
	184	}
	185
	186	ldx_mode = acquire ? BPF_MEM_ACQ_REL : BPF_MEM;
	187	stx_mode = release ? BPF_MEM_ACQ_REL : BPF_MEM;
	188
	189	{
	190	struct bpf_insn bytecode[] = {
	191	BPFI_LD_IMM64(BPF_REG_1, BPF_PTR_TO_V(v)),
	192	BPFI_LDX_MODE(bpf_size, ldx_mode, BPF_REG_0, BPF_REG_1, 0),
	193	BPFI_LD_IMM64(BPF_REG_2, count),
	194	BPFI_ADD64_X(BPF_REG_0, BPF_REG_2),
	195	BPFI_STX_MODE(bpf_size, stx_mode, BPF_REG_1, BPF_REG_0, 0),
	196	};
	197
	198	do {
	199	ret = do_on_cpu(bytecode, ARRAY_SIZE(bytecode),
	200	NULL, cpu, 0);
	201	} while (ret == -1 && errno == EAGAIN);
	202	}
	203	return ret;
b32429a9 MD	204	}
	205
	206	int cpu_op_add(void *v, int64_t count, size_t len, int cpu)
	207	{
de28c254 MD	208	return __cpu_op_add(v, count, len, cpu, 1, 1);
de28c254 MD	209	}
b32429a9	210
de28c254 MD	211	int cpu_op_add_relaxed(void *v, int64_t count, size_t len, int cpu)
	212	{
	213	return __cpu_op_add(v, count, len, cpu, 0, 0);
b32429a9 MD	214	}
b32429a9 MD	215
de28c254	216	int cpu_op_add_acquire(void *v, int64_t count, size_t len, int cpu)
9b7954d0	217	{
de28c254 MD	218	return __cpu_op_add(v, count, len, cpu, 1, 0);
de28c254 MD	219	}
9b7954d0	220
de28c254 MD	221	int cpu_op_add_release(void *v, int64_t count, size_t len, int cpu)
	222	{
	223	return __cpu_op_add(v, count, len, cpu, 0, 1);
9b7954d0 MD	224	}
9b7954d0 MD	225
b32429a9 MD	226	int cpu_op_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv,
	227	int cpu)
	228	{
de28c254 MD	229	intptr_t old;
de28c254 MD	230	int ret;
b32429a9	231
de28c254 MD	232	ret = cpu_op_cmpxchg_relaxed(v, &expect, &old, &newv, sizeof(*v),
	233	cpu);
	234	if (!ret && old != expect)
	235	ret = 1;
	236	return ret;
b32429a9 MD	237	}
b32429a9 MD	238
de28c254 MD	239	int cpu_op_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
de28c254 MD	240	off_t voffp, intptr_t *load, int cpu)
b32429a9	241	{
de28c254 MD	242	int ret;
	243	int64_t res;
	244	unsigned int bpf_size1;
	245	size_t len1 = sizeof(*v);
	246
	247	switch (len1) {
	248	case 1: bpf_size1 = BPF_B;
	249	break;
	250	case 2: bpf_size1 = BPF_H;
	251	break;
	252	case 4: bpf_size1 = BPF_W;
	253	break;
	254	case 8: bpf_size1 = BPF_DW;
	255	break;
	256	default:
	257	return -EINVAL;
	258	}
	259
	260	enum {
	261	BPF_LABEL_BRANCH1 = 6,
	262	BPF_LABEL_FAIL = 16,
b32429a9 MD	263	};
b32429a9 MD	264
de28c254 MD	265	{
	266	struct bpf_insn bytecode[] = {
	267	[0] = BPFI_LD_IMM64(BPF_REG_1, BPF_PTR_TO_V(v)),
	268	[2] = BPFI_LDX(bpf_size1, BPF_REG_2, BPF_REG_1, 0),
	269	[3] = BPFI_LD_IMM64(BPF_REG_3, expectnot),
	270	[5] = BPFI_JEQ_X(BPF_REG_2, BPF_REG_3,
	271	BPF_LABEL_FAIL - BPF_LABEL_BRANCH1),
	272
	273	[BPF_LABEL_BRANCH1] = BPFI_LD_IMM64(BPF_REG_3, BPF_PTR_TO_V(load)),
	274	[8] = BPFI_STX(bpf_size1, BPF_REG_3, BPF_REG_2, 0),
	275
	276	[9] = BPFI_MOV_X(BPF_REG_3, BPF_REG_2),
	277	[10] = BPFI_LD_IMM64(BPF_REG_4, voffp),
	278	[12] = BPFI_ADD64_X(BPF_REG_3, BPF_REG_4),
	279	[13] = BPFI_LDX(bpf_size1, BPF_REG_3, BPF_REG_3, 0),
	280	[14] = BPFI_STX(bpf_size1, BPF_REG_1, BPF_REG_3, 0),
	281	[15] = BPFI_EXIT(), /* r0 = 0. */
	282	[BPF_LABEL_FAIL] = BPFI_LD_IMM32(BPF_REG_0, 1),
	283	[17] = BPFI_EXIT(), /* r0 = 1. */
	284	};
	285
	286	do {
	287	ret = do_on_cpu(bytecode, ARRAY_SIZE(bytecode),
	288	&res, cpu, 0);
	289	} while (ret == -1 && errno == EAGAIN);
	290	}
	291	if (!ret)
	292	ret = (int) !!res;
	293	return ret;
b32429a9 MD	294	}
b32429a9 MD	295
de28c254 MD	296	static
	297	int __cpu_op_cmpeqv_storev_storev(intptr_t *v, intptr_t expect,
	298	intptr_t *v2, intptr_t newv2,
	299	intptr_t newv, int cpu, int release)
b32429a9 MD	300	{
b32429a9 MD	301	int ret;
de28c254 MD	302	int64_t res;
	303	unsigned int bpf_size1, bpf_size2, stx_mode;
	304	size_t len1 = sizeof(*v);
	305	size_t len2 = sizeof(*v2);
	306
	307	switch (len1) {
	308	case 1: bpf_size1 = BPF_B;
	309	break;
	310	case 2: bpf_size1 = BPF_H;
	311	break;
	312	case 4: bpf_size1 = BPF_W;
	313	break;
	314	case 8: bpf_size1 = BPF_DW;
	315	break;
	316	default:
	317	return -EINVAL;
	318	}
b32429a9	319
de28c254 MD	320	switch (len2) {
	321	case 1: bpf_size2 = BPF_B;
	322	break;
	323	case 2: bpf_size2 = BPF_H;
	324	break;
	325	case 4: bpf_size2 = BPF_W;
	326	break;
	327	case 8: bpf_size2 = BPF_DW;
	328	break;
	329	default:
	330	return -EINVAL;
	331	}
	332
	333	stx_mode = release ? BPF_MEM_ACQ_REL : BPF_MEM;
	334
	335	enum {
	336	BPF_LABEL_BRANCH1 = 7,
	337	BPF_LABEL_FAIL = 16,
	338	};
b32429a9	339
de28c254 MD	340	{
	341	struct bpf_insn bytecode[] = {
	342	[0] = BPFI_LD_IMM64(BPF_REG_1, BPF_PTR_TO_V(v)),
	343	[2] = BPFI_LDX(bpf_size1, BPF_REG_2, BPF_REG_1, 0),
	344	[3] = BPFI_LD_IMM64(BPF_REG_3, BPF_PTR_TO_V(&expect)),
	345	[5] = BPFI_LDX(bpf_size1, BPF_REG_3, BPF_REG_3, 0),
	346	[6] = BPFI_JNE_X(BPF_REG_2, BPF_REG_3,
	347	BPF_LABEL_FAIL - BPF_LABEL_BRANCH1),
	348
	349	[BPF_LABEL_BRANCH1] = BPFI_LD_IMM64(BPF_REG_2, BPF_PTR_TO_V(v2)),
	350	[9] = BPFI_LD_IMM64(BPF_REG_3, newv2),
	351	[11] = BPFI_LD_IMM64(BPF_REG_4, newv),
	352	[13] = BPFI_STX(bpf_size2, BPF_REG_2, BPF_REG_3, 0),
	353	[14] = BPFI_STX_MODE(bpf_size2, stx_mode, BPF_REG_1, BPF_REG_4, 0),
	354	[15] = BPFI_EXIT(), /* r0 = 0. */
	355	[BPF_LABEL_FAIL] = BPFI_LD_IMM32(BPF_REG_0, 1),
	356	[17] = BPFI_EXIT(), /* r0 = 1. */
	357	};
	358
	359	do {
	360	ret = do_on_cpu(bytecode, ARRAY_SIZE(bytecode),
	361	&res, cpu, 0);
	362	} while (ret == -1 && errno == EAGAIN);
	363	}
	364	if (!ret)
	365	ret = (int) !!res;
	366	return ret;
b32429a9 MD	367	}
	368
	369	int cpu_op_cmpeqv_storev_storev(intptr_t *v, intptr_t expect,
	370	intptr_t *v2, intptr_t newv2,
	371	intptr_t newv, int cpu)
	372	{
de28c254 MD	373	return __cpu_op_cmpeqv_storev_storev(v, expect, v2, newv2,
de28c254 MD	374	newv, cpu, 0);
b32429a9 MD	375	}
b32429a9 MD	376
d15728d5	377	int cpu_op_cmpeqv_storev_storev_release(intptr_t *v, intptr_t expect,
de28c254 MD	378	intptr_t *v2, intptr_t newv2,
	379	intptr_t newv, int cpu)
	380	{
	381	return __cpu_op_cmpeqv_storev_storev(v, expect, v2, newv2,
	382	newv, cpu, 1);
	383	}
	384
	385	static
	386	int __cpu_op_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
	387	intptr_t *v2, intptr_t expect2,
	388	intptr_t newv, int cpu, int release)
b32429a9	389	{
de28c254 MD	390	int ret;
	391	int64_t res;
	392	unsigned int bpf_size1, bpf_size2, stx_mode;
	393	size_t len1 = sizeof(*v);
	394	size_t len2 = sizeof(*v2);
	395
	396	switch (len1) {
	397	case 1: bpf_size1 = BPF_B;
	398	break;
	399	case 2: bpf_size1 = BPF_H;
	400	break;
	401	case 4: bpf_size1 = BPF_W;
	402	break;
	403	case 8: bpf_size1 = BPF_DW;
	404	break;
	405	default:
	406	return -EINVAL;
	407	}
	408
	409	switch (len2) {
	410	case 1: bpf_size2 = BPF_B;
	411	break;
	412	case 2: bpf_size2 = BPF_H;
	413	break;
	414	case 4: bpf_size2 = BPF_W;
	415	break;
	416	case 8: bpf_size2 = BPF_DW;
	417	break;
	418	default:
	419	return -EINVAL;
	420	}
	421
	422	stx_mode = release ? BPF_MEM_ACQ_REL : BPF_MEM;
	423
	424	enum {
	425	BPF_LABEL_BRANCH1 = 6,
	426	BPF_LABEL_BRANCH2 = 12,
	427	BPF_LABEL_FAIL = 16,
b32429a9 MD	428	};
b32429a9 MD	429
de28c254 MD	430	{
	431	struct bpf_insn bytecode[] = {
	432	[0] = BPFI_LD_IMM64(BPF_REG_1, BPF_PTR_TO_V(v)),
	433	[2] = BPFI_LDX(bpf_size1, BPF_REG_2, BPF_REG_1, 0),
	434	[3] = BPFI_LD_IMM64(BPF_REG_3, expect),
	435	[5] = BPFI_JNE_X(BPF_REG_2, BPF_REG_3,
	436	BPF_LABEL_FAIL - BPF_LABEL_BRANCH1),
	437
	438	[BPF_LABEL_BRANCH1] = BPFI_LD_IMM64(BPF_REG_2, BPF_PTR_TO_V(v2)),
	439	[8] = BPFI_LDX(bpf_size1, BPF_REG_2, BPF_REG_2, 0),
	440	[9] = BPFI_LD_IMM64(BPF_REG_3, expect2),
	441	[11] = BPFI_JNE_X(BPF_REG_2, BPF_REG_3,
	442	BPF_LABEL_FAIL - BPF_LABEL_BRANCH2),
	443
	444	[BPF_LABEL_BRANCH2] = BPFI_LD_IMM64(BPF_REG_2, newv),
	445	[14] = BPFI_STX_MODE(bpf_size2, stx_mode, BPF_REG_1, BPF_REG_2, 0),
	446	[15] = BPFI_EXIT(), /* r0 = 0. */
	447	[BPF_LABEL_FAIL] = BPFI_LD_IMM32(BPF_REG_0, 1),
	448	[17] = BPFI_EXIT(), /* r0 = 1. */
	449	};
	450
	451	do {
	452	ret = do_on_cpu(bytecode, ARRAY_SIZE(bytecode),
	453	&res, cpu, 0);
	454	} while (ret == -1 && errno == EAGAIN);
	455	}
	456	if (!ret)
	457	ret = (int) !!res;
	458	return ret;
	459
b32429a9 MD	460	}
	461
	462	int cpu_op_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
	463	intptr_t *v2, intptr_t expect2,
	464	intptr_t newv, int cpu)
	465	{
de28c254 MD	466	return __cpu_op_cmpeqv_cmpeqv_storev(v, expect, v2, expect2,
	467	newv, cpu, 0);
	468	}
b32429a9	469
de28c254 MD	470	int cpu_op_cmpeqv_cmpeqv_storev_release(intptr_t *v, intptr_t expect,
	471	intptr_t *v2, intptr_t expect2,
	472	intptr_t newv, int cpu)
	473	{
	474	return __cpu_op_cmpeqv_cmpeqv_storev(v, expect, v2, expect2,
	475	newv, cpu, 1);
b32429a9 MD	476	}
b32429a9 MD	477
de28c254 MD	478	static
	479	int __cpu_op_cmpeqv_memcpy_storev(intptr_t *v, intptr_t expect,
	480	void dst, void src, size_t len,
	481	intptr_t newv, int cpu, int release)
b32429a9	482	{
de28c254 MD	483	int ret;
	484	int64_t res;
	485	unsigned int bpf_size2, stx_mode;
	486	unsigned int len2 = sizeof(*v);
	487
	488	switch (len2) {
	489	case 1: bpf_size2 = BPF_B;
	490	break;
	491	case 2: bpf_size2 = BPF_H;
	492	break;
	493	case 4: bpf_size2 = BPF_W;
	494	break;
	495	case 8: bpf_size2 = BPF_DW;
	496	break;
	497	default:
	498	return -EINVAL;
	499	}
	500
	501	stx_mode = release ? BPF_MEM_ACQ_REL : BPF_MEM;
	502
	503	enum {
08ce323a MD	504	BPF_LABEL_BRANCH_TEST = 6,
	505	BPF_LABEL_LOOP8 = 14,
	506	BPF_LABEL_BRANCH8_1 = 15,
	507	BPF_LABEL_LOOP1 = 20,
	508	BPF_LABEL_BRANCH1_1 = 21,
	509	BPF_LABEL_BRANCH1_2 = 26,
	510	BPF_LABEL_FAIL = 30,
b32429a9 MD	511	};
b32429a9 MD	512
de28c254 MD	513	{
de28c254 MD	514	struct bpf_insn bytecode[] = {
08ce323a MD	515	/*
	516	* r0 is 0
	517	* r1 is temporary register,
	518	* r2 is expect
	519	* r6 is v
	520	*/
	521	[0] = BPFI_LD_IMM64(BPF_REG_6, BPF_PTR_TO_V(v)),
	522	[2] = BPFI_LDX(bpf_size2, BPF_REG_1, BPF_REG_6, 0),
de28c254 MD	523	[3] = BPFI_LD_IMM64(BPF_REG_2, expect),
de28c254 MD	524	[5] = BPFI_JNE_X(BPF_REG_1, BPF_REG_2,
08ce323a MD	525	BPF_LABEL_FAIL - BPF_LABEL_BRANCH_TEST),
	526
	527	/*
	528	* r0 is 0
	529	* r1 is temporary register,
	530	* r2 is dst iterator,
	531	* r3 is src iterator,
	532	* r4 is src + (len & ~7) // end of 8-byte copy
	533	* r5 is src + len // end of 1-byte copy
	534	* r6 is v
	535	*/
	536	[BPF_LABEL_BRANCH_TEST] = BPFI_LD_IMM64(BPF_REG_2,
	537	BPF_PTR_TO_V(dst)),
de28c254	538	[8] = BPFI_LD_IMM64(BPF_REG_3, BPF_PTR_TO_V(src)),
08ce323a MD	539	[10] = BPFI_LD_IMM64(BPF_REG_4, BPF_PTR_TO_V(src) + (len & ~7)),
	540	[12] = BPFI_LD_IMM64(BPF_REG_5, BPF_PTR_TO_V(src) + len),
	541
	542	/* 8-byte copy loop target. */
	543	[BPF_LABEL_LOOP8] = BPFI_JEQ_X(BPF_REG_3, BPF_REG_4,
	544	BPF_LABEL_LOOP1 - BPF_LABEL_BRANCH8_1),
de28c254	545
08ce323a MD	546	[BPF_LABEL_BRANCH8_1] = BPFI_LDX(BPF_DW, BPF_REG_1, BPF_REG_3, 0),
08ce323a MD	547	[16] = BPFI_STX(BPF_DW, BPF_REG_2, BPF_REG_1, 0),
de28c254	548
08ce323a MD	549	[17] = BPFI_ADD64_K(BPF_REG_2, 8),
	550	[18] = BPFI_ADD64_K(BPF_REG_3, 8),
	551	[19] = BPFI_JA_K(BPF_LABEL_LOOP8 - BPF_LABEL_LOOP1),
de28c254	552
08ce323a MD	553	/* 1-byte copy loop target. */
	554	[BPF_LABEL_LOOP1] = BPFI_JEQ_X(BPF_REG_3, BPF_REG_5,
	555	BPF_LABEL_BRANCH1_2 - BPF_LABEL_BRANCH1_1),
	556
	557	[BPF_LABEL_BRANCH1_1] = BPFI_LDX(BPF_B, BPF_REG_1, BPF_REG_3, 0),
	558	[22] = BPFI_STX(BPF_B, BPF_REG_2, BPF_REG_1, 0),
	559
	560	[23] = BPFI_ADD64_K(BPF_REG_2, 1),
	561	[24] = BPFI_ADD64_K(BPF_REG_3, 1),
	562	[25] = BPFI_JA_K(BPF_LABEL_LOOP1 - BPF_LABEL_BRANCH1_2),
de28c254 MD	563
de28c254 MD	564	/* Completed, do store. */
08ce323a MD	565
	566	/*
	567	* r0 is 0
	568	* r2 is newv
	569	* r6 is v
	570	*/
	571	[BPF_LABEL_BRANCH1_2] = BPFI_LD_IMM64(BPF_REG_2, newv),
	572	[28] = BPFI_STX_MODE(bpf_size2, stx_mode, BPF_REG_6,
de28c254 MD	573	BPF_REG_2, 0),
de28c254 MD	574
08ce323a	575	[29] = BPFI_EXIT(), /* r0 = 0. */
de28c254	576	[BPF_LABEL_FAIL] = BPFI_LD_IMM32(BPF_REG_0, 1),
08ce323a	577	[31] = BPFI_EXIT(), /* r0 = 1. */
de28c254 MD	578	};
	579
	580	do {
	581	ret = do_on_cpu(bytecode, ARRAY_SIZE(bytecode),
	582	&res, cpu, 0);
	583	} while (ret == -1 && errno == EAGAIN);
	584	}
	585	if (!ret)
	586	ret = (int) !!res;
	587	return ret;
b32429a9 MD	588	}
b32429a9 MD	589
de28c254 MD	590	int cpu_op_cmpeqv_memcpy_storev(intptr_t *v, intptr_t expect,
	591	void dst, void src, size_t len,
	592	intptr_t newv, int cpu)
b32429a9	593	{
de28c254 MD	594	return __cpu_op_cmpeqv_memcpy_storev(v, expect, dst, src, len,
	595	newv, cpu, 0);
	596	}
b32429a9	597
de28c254 MD	598	int cpu_op_cmpeqv_memcpy_storev_release(intptr_t *v, intptr_t expect,
	599	void dst, void src, size_t len,
	600	intptr_t newv, int cpu)
	601	{
	602	return __cpu_op_cmpeqv_memcpy_storev(v, expect, dst, src, len,
	603	newv, cpu, 1);
b32429a9 MD	604	}
	605
	606	int cpu_op_addv(intptr_t *v, int64_t count, int cpu)
	607	{
de28c254 MD	608	return cpu_op_add_relaxed(v, count, sizeof(intptr_t), cpu);
	609	}
	610
	611	int cpu_op_deref_loadoffp(intptr_t p, off_t voffp, intptr_t load, int cpu)
	612	{
	613	int ret;
	614	int64_t res;
	615	unsigned int bpf_size1, bpf_size2;
	616	size_t len1 = sizeof(void ), len2 = sizeof(load);
	617
	618	switch (len1) {
	619	case 1: bpf_size1 = BPF_B;
	620	break;
	621	case 2: bpf_size1 = BPF_H;
	622	break;
	623	case 4: bpf_size1 = BPF_W;
	624	break;
	625	case 8: bpf_size1 = BPF_DW;
	626	break;
	627	default:
	628	return -EINVAL;
	629	}
	630
	631	switch (len2) {
	632	case 1: bpf_size2 = BPF_B;
	633	break;
	634	case 2: bpf_size2 = BPF_H;
	635	break;
	636	case 4: bpf_size2 = BPF_W;
	637	break;
	638	case 8: bpf_size2 = BPF_DW;
	639	break;
	640	default:
	641	return -EINVAL;
	642	}
	643
	644	{
	645	struct bpf_insn bytecode[] = {
	646	[0] = BPFI_LD_IMM64(BPF_REG_1, BPF_PTR_TO_V(p)),
	647	[2] = BPFI_LDX(bpf_size1, BPF_REG_1, BPF_REG_1, 0),
	648	[3] = BPFI_LD_IMM64(BPF_REG_2, voffp),
	649	[5] = BPFI_ADD64_X(BPF_REG_1, BPF_REG_2),
	650	[6] = BPFI_LDX(bpf_size2, BPF_REG_0, BPF_REG_1, 0),
	651	};
	652
	653	do {
	654	ret = do_on_cpu(bytecode, ARRAY_SIZE(bytecode),
	655	&res, cpu, 0);
	656	} while (ret == -1 && errno == EAGAIN);
	657	}
	658	if (!ret)
	659	*load = (intptr_t) res;
	660	return ret;
	661	}
	662
	663	int cpu_op_fence(int cpu)
	664	{
	665	int ret;
	666
	667	do {
	668	ret = do_on_cpu(NULL, 0, NULL, cpu, 0);
	669	} while (ret == -1 && errno == EAGAIN);
	670	return ret;
b32429a9	671	}