xen: Code cleanup and small bug fixes

xen/hypervisor.h:
 - Remove unused helpers: MULTI_update_va_mapping, is_initial_xendomain,
   is_running_on_xen
 - Remove unused define CONFIG_X86_PAE
 - Remove unused variable xen_start_info: note that it's used inpcifront
   which is not built at all
 - Remove forward declaration of HYPERVISOR_crash

xen/xen-os.h:
 - Remove unused define CONFIG_X86_PAE
 - Drop unused helpers: test_and_clear_bit, clear_bit,
   force_evtchn_callback
 - Implement a generic version (based on ofed/include/linux/bitops.h) of
   set_bit and test_bit and prefix them by xen_ to avoid any use by other
   code than Xen. Note that It would be worth to investigate a generic
   implementation in FreeBSD.
 - Replace barrier() by __compiler_membar()
 - Replace cpu_relax() by cpu_spinwait(): it's exactly the same as rep;nop
   = pause

xen/xen_intr.h:
 - Move the prototype of xen_intr_handle_upcall in it: Use by all the
   platform

x86/xen/xen_intr.c:
 - Use BITSET* for the enabledbits: Avoid to use custom helpers
 - test_bit/set_bit has been renamed to xen_test_bit/xen_set_bit
 - Don't export the variable xen_intr_pcpu

dev/xen/blkback/blkback.c:
 - Fix the string format when XBB_DEBUG is enabled: host_addr is typed
   uint64_t

dev/xen/balloon/balloon.c:
 - Remove set but not used variable
 - Use the correct type for frame_list: xen_pfn_t represents the frame
   number on any architecture

dev/xen/control/control.c:
 - Return BUS_PROBE_WILDCARD in xs_probe: Returning 0 in a probe callback
   means the driver can handle this device. If by any chance xenstore is the
   first driver, every new device with the driver is unset will use
   xenstore.

dev/xen/grant-table/grant_table.c:
 - Remove unused cmpxchg
 - Drop unused include opt_pmap.h: Doesn't exist on ARM64 and it doesn't
   contain anything required for the code on x86

dev/xen/netfront/netfront.c:
 - Use the correct type for rx_pfn_array: xen_pfn_t represents the frame
   number on any architecture

dev/xen/netback/netback.c:
 - Use the correct type for gmfn: xen_pfn_t represents the frame number on
   any architecture

dev/xen/xenstore/xenstore.c:
 - Return BUS_PROBE_WILDCARD in xctrl_probe: Returning 0 in a probe callback
   means the driver can handle this device. If by any chance xenstore is the
  first driver, every new device with the driver is unset will use xenstore.

Note that with the changes, x86/include/xen/xen-os.h doesn't contain anymore
arch-specific code. Although, a new series will add some helpers that differ
between x86 and ARM64, so I've kept the headers for now.

Submitted by:		Julien Grall <julien.grall@citrix.com>
Reviewed by:		royger
Differential Revision:	https://reviews.freebsd.org/D3921
Sponsored by:		Citrix Systems R&D

4 files changed, 29 insertions, 58 deletions

diff --git a/sys/xen/blkif.h b/sys/xen/blkif.h
index 8a19d5d01d87..721bb93c3c48 100644
--- a/sys/xen/blkif.h
+++ b/sys/xen/blkif.h
@@ -120,7 +120,7 @@ static void inline blkif_get_x86_32_req(blkif_request_t *dst, blkif_x86_32_reque
 	dst->handle = src->handle;
 	dst->id = src->id;
 	dst->sector_number = src->sector_number;
-	barrier();
+	__compiler_membar();
 	if (n > dst->nr_segments)
 		n = dst->nr_segments;
 	for (i = 0; i < n; i++)
@@ -135,7 +135,7 @@ static void inline blkif_get_x86_64_req(blkif_request_t *dst, blkif_x86_64_reque
 	dst->handle = src->handle;
 	dst->id = src->id;
 	dst->sector_number = src->sector_number;
-	barrier();
+	__compiler_membar();
 	if (n > dst->nr_segments)
 		n = dst->nr_segments;
 	for (i = 0; i < n; i++)
diff --git a/sys/xen/hypervisor.h b/sys/xen/hypervisor.h
index 42cdb3bcf5ee..65d5a3eac978 100644
--- a/sys/xen/hypervisor.h
+++ b/sys/xen/hypervisor.h
@@ -11,22 +11,6 @@
 #ifndef __XEN_HYPERVISOR_H__
 #define __XEN_HYPERVISOR_H__
 
-#ifdef XENHVM
-
-#define is_running_on_xen()	(HYPERVISOR_shared_info != NULL)
-
-#else
-
-#define is_running_on_xen() 1
-
-#endif
-
-#ifdef PAE
-#ifndef CONFIG_X86_PAE
-#define CONFIG_X86_PAE
-#endif
-#endif
-
 #include <sys/cdefs.h>
 #include <sys/systm.h>
 #include <xen/interface/xen.h>
@@ -38,22 +22,6 @@
 #include <xen/interface/memory.h>
 #include <machine/xen/hypercall.h>
 
-#if defined(__amd64__)
-#define MULTI_UVMFLAGS_INDEX 2
-#define MULTI_UVMDOMID_INDEX 3
-#else
-#define MULTI_UVMFLAGS_INDEX 3
-#define MULTI_UVMDOMID_INDEX 4
-#endif
-
-#ifdef CONFIG_XEN_PRIVILEGED_GUEST
-#define is_initial_xendomain() (xen_start_info->flags & SIF_INITDOMAIN)
-#else
-#define is_initial_xendomain() 0
-#endif
-
-extern start_info_t *xen_start_info;
-
 extern uint64_t get_system_time(int ticks);
 
 static inline int 
@@ -62,8 +30,6 @@ HYPERVISOR_console_write(const char *str, int count)
     return HYPERVISOR_console_io(CONSOLEIO_write, count, str); 
 }
 
-static inline void HYPERVISOR_crash(void) __dead2;
-
 static inline int
 HYPERVISOR_yield(void)
 {
@@ -132,23 +98,4 @@ HYPERVISOR_poll(
 	return (rc);
 }
 
-static inline void
-MULTI_update_va_mapping(
-	multicall_entry_t *mcl, unsigned long va,
-        uint64_t new_val, unsigned long flags)
-{
-    mcl->op = __HYPERVISOR_update_va_mapping;
-    mcl->args[0] = va;
-#if defined(__amd64__)
-    mcl->args[1] = new_val;
-#elif defined(PAE)
-    mcl->args[1] = (uint32_t)(new_val & 0xffffffff) ;
-    mcl->args[2] = (uint32_t)(new_val >> 32);
-#else
-    mcl->args[1] = new_val;
-    mcl->args[2] = 0;
-#endif
-    mcl->args[MULTI_UVMFLAGS_INDEX] = flags;
-}
-
 #endif /* __XEN_HYPERVISOR_H__ */
diff --git a/sys/xen/xen-os.h b/sys/xen/xen-os.h
index ab594d437058..eec8244f2d3e 100644
--- a/sys/xen/xen-os.h
+++ b/sys/xen/xen-os.h
@@ -47,9 +47,6 @@
 /* Everything below this point is not included by assembler (.S) files. */
 #ifndef __ASSEMBLY__
 
-/* Force a proper event-channel callback from Xen. */
-void force_evtchn_callback(void);
-
 extern shared_info_t *HYPERVISOR_shared_info;
 extern start_info_t *HYPERVISOR_start_info;
 
@@ -93,6 +90,31 @@ xen_initial_domain(void)
 }
 
 /*
+ * Based on ofed/include/linux/bitops.h
+ *
+ * Those helpers are prefixed by xen_ because xen-os.h is widely included
+ * and we don't want the other drivers using them.
+ *
+ */
+#define NBPL (NBBY * sizeof(long))
+
+static inline bool
+xen_test_bit(int bit, volatile long *addr)
+{
+	unsigned long mask = 1UL << (bit % NBPL);
+
+	return !!(atomic_load_acq_long(&addr[bit / NBPL]) & mask);
+}
+
+static inline void
+xen_set_bit(int bit, volatile long *addr)
+{
+	atomic_set_long(&addr[bit / NBPL], 1UL << (bit % NBPL));
+}
+
+#undef NPBL
+
+/*
  * Functions to allocate/free unused memory in order
  * to map memory from other domains.
  */
diff --git a/sys/xen/xen_intr.h b/sys/xen/xen_intr.h
index b86f94a43b99..5b2f60804eac 100644
--- a/sys/xen/xen_intr.h
+++ b/sys/xen/xen_intr.h
@@ -41,6 +41,8 @@ typedef void * xen_intr_handle_t;
 /** If non-zero, the hypervisor has been configured to use a direct vector */
 extern int xen_vector_callback_enabled;
 
+void xen_intr_handle_upcall(struct trapframe *trap_frame);
+
 /**
  * Associate an already allocated local event channel port an interrupt
  * handler.