/*-
* Copyright (c) 2021 The FreeBSD Foundation
*
* This software were developed by Konstantin Belousov <kib@FreeBSD.org>
* under sponsorship from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/cpuset.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/membarrier.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/syscallsubr.h>
#include <sys/sysproto.h>
#include <vm/vm_param.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#define MEMBARRIER_SUPPORTED_CMDS ( \
MEMBARRIER_CMD_GLOBAL | \
MEMBARRIER_CMD_GLOBAL_EXPEDITED | \
MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED | \
MEMBARRIER_CMD_PRIVATE_EXPEDITED | \
MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED | \
MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE | \
MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE)
static void
membarrier_action_seqcst(void *arg __unused)
{
atomic_thread_fence_seq_cst();
}
static void
membarrier_action_seqcst_sync_core(void *arg __unused)
{
atomic_thread_fence_seq_cst();
cpu_sync_core();
}
static void
do_membarrier_ipi(cpuset_t *csp, void (*func)(void *))
{
atomic_thread_fence_seq_cst();
smp_rendezvous_cpus(*csp, smp_no_rendezvous_barrier, func,
smp_no_rendezvous_barrier, NULL);
atomic_thread_fence_seq_cst();
}
static void
check_cpu_switched(int c, cpuset_t *csp, uint64_t *swt, bool init)
{
struct pcpu *pc;
uint64_t sw;
if (CPU_ISSET(c, csp))
return;
pc = cpuid_to_pcpu[c];
if (pc->pc_curthread == pc->pc_idlethread) {
CPU_SET(c, csp);
return;
}
/*
* Sync with context switch to ensure that override of
* pc_curthread with non-idle thread pointer is visible before
* reading of pc_switchtime.
*/
atomic_thread_fence_acq();
sw = pc->pc_switchtime;
if (init)
swt[c] = sw;
else if (sw != swt[c])
CPU_SET(c, csp);
}
/*
*
* XXXKIB: We execute the requested action (seq_cst and possibly
* sync_core) on current CPU as well. There is no guarantee that
* current thread executes anything with the full fence semantics
* during syscall execution. Similarly, cpu_core_sync() semantics
* might be not provided by the syscall return. E.g. on amd64 we
* typically return without IRET.
*/
int
kern_membarrier(struct thread *td, int cmd, unsigned flags, int cpu_id)
{
struct proc *p, *p1;
struct thread *td1;
cpuset_t cs;
uint64_t *swt;
int c, error;
bool first;
if (flags != 0 || (cmd & ~MEMBARRIER_SUPPORTED_CMDS) != 0)
return (EINVAL);
if (cmd == MEMBARRIER_CMD_QUERY) {
td->td_retval[0] = MEMBARRIER_SUPPORTED_CMDS;
return (0);
}
p = td->td_proc;
error = 0;
switch (cmd) {
case MEMBARRIER_CMD_GLOBAL:
swt = malloc((mp_maxid + 1) * sizeof(*swt), M_TEMP, M_WAITOK);
CPU_ZERO(&cs);
sched_pin();
CPU_SET(PCPU_GET(cpuid), &cs);
for (first = true; error == 0; first = false) {
CPU_FOREACH(c)
check_cpu_switched(c, &cs, swt, first);
if (CPU_CMP(&cs, &all_cpus) == 0)
break;
error = pause_sig("mmbr", 1);
if (error == EWOULDBLOCK)
error = 0;
}
sched_unpin();
free(swt, M_TEMP);
atomic_thread_fence_seq_cst();
break;
case MEMBARRIER_CMD_GLOBAL_EXPEDITED:
if ((td->td_proc->p_flag2 & P2_MEMBAR_GLOBE) == 0) {
error = EPERM;
} else {
CPU_ZERO(&cs);
CPU_FOREACH(c) {
td1 = cpuid_to_pcpu[c]->pc_curthread;
p1 = td1->td_proc;
if (p1 != NULL &&
(p1->p_flag2 & P2_MEMBAR_GLOBE) != 0)
CPU_SET(c, &cs);
}
do_membarrier_ipi(&cs, membarrier_action_seqcst);
}
break;
case MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED:
if ((p->p_flag2 & P2_MEMBAR_GLOBE) == 0) {
PROC_LOCK(p);
p->p_flag2 |= P2_MEMBAR_GLOBE;
PROC_UNLOCK(p);
}
break;
case MEMBARRIER_CMD_PRIVATE_EXPEDITED:
if ((td->td_proc->p_flag2 & P2_MEMBAR_PRIVE) == 0) {
error = EPERM;
} else {
pmap_active_cpus(vmspace_pmap(p->p_vmspace), &cs);
do_membarrier_ipi(&cs, membarrier_action_seqcst);
}
break;
case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED:
if ((p->p_flag2 & P2_MEMBAR_PRIVE) == 0) {
PROC_LOCK(p);
p->p_flag2 |= P2_MEMBAR_PRIVE;
PROC_UNLOCK(p);
}
break;
case MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE:
if ((td->td_proc->p_flag2 & P2_MEMBAR_PRIVE_SYNCORE) == 0) {
error = EPERM;
} else {
/*
* Calculating the IPI multicast mask from
* pmap active mask means that we do not call
* cpu_sync_core() on CPUs that were missed
* from pmap active mask but could be switched
* from or to meantime. This is fine at least
* on amd64 because threads always use slow
* (IRETQ) path to return from syscall after
* context switch.
*/
pmap_active_cpus(vmspace_pmap(p->p_vmspace), &cs);
do_membarrier_ipi(&cs,
membarrier_action_seqcst_sync_core);
}
break;
case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE:
if ((p->p_flag2 & P2_MEMBAR_PRIVE_SYNCORE) == 0) {
PROC_LOCK(p);
p->p_flag2 |= P2_MEMBAR_PRIVE_SYNCORE;
PROC_UNLOCK(p);
}
break;
default:
error = EINVAL;
break;
}
return (error);
}
int
sys_membarrier(struct thread *td, struct membarrier_args *uap)
{
return (kern_membarrier(td, uap->cmd, uap->flags, uap->cpu_id));
}
