vendor/processor-trace/892e12c5a27bda5806d1e63269986bb4171b5a8b

30 files changed, 19296 insertions, 0 deletions

diff --git a/libipt/src/posix/pt_cpuid.c b/libipt/src/posix/pt_cpuid.c
new file mode 100644
index 000000000000..9334e0d4fac2
--- /dev/null
+++ b/libipt/src/posix/pt_cpuid.c
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_cpuid.h"
+
+#include <cpuid.h>
+
+extern void pt_cpuid(uint32_t leaf, uint32_t *eax, uint32_t *ebx,
+		     uint32_t *ecx, uint32_t *edx)
+{
+	__get_cpuid(leaf, eax, ebx, ecx, edx);
+}
diff --git a/libipt/src/posix/pt_section_posix.c b/libipt/src/posix/pt_section_posix.c
new file mode 100644
index 000000000000..651f2546837c
--- /dev/null
+++ b/libipt/src/posix/pt_section_posix.c
@@ -0,0 +1,333 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_section.h"
+#include "pt_section_posix.h"
+#include "pt_section_file.h"
+
+#include "intel-pt.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+
+int pt_section_mk_status(void **pstatus, uint64_t *psize, const char *filename)
+{
+	struct pt_sec_posix_status *status;
+	struct stat buffer;
+	int errcode;
+
+	if (!pstatus || !psize)
+		return -pte_internal;
+
+	errcode = stat(filename, &buffer);
+	if (errcode < 0)
+		return -pte_bad_file;
+
+	if (buffer.st_size < 0)
+		return -pte_bad_image;
+
+	status = malloc(sizeof(*status));
+	if (!status)
+		return -pte_nomem;
+
+	status->stat = buffer;
+
+	*pstatus = status;
+	*psize = (uint64_t) buffer.st_size;
+
+	return 0;
+}
+
+static int check_file_status(struct pt_section *section, int fd)
+{
+	struct pt_sec_posix_status *status;
+	struct stat stat;
+	int errcode;
+
+	if (!section)
+		return -pte_internal;
+
+	errcode = fstat(fd, &stat);
+	if (errcode)
+		return -pte_bad_file;
+
+	status = section->status;
+	if (!status)
+		return -pte_internal;
+
+	if (stat.st_size != status->stat.st_size)
+		return -pte_bad_image;
+
+	if (stat.st_mtime != status->stat.st_mtime)
+		return -pte_bad_image;
+
+	return 0;
+}
+
+int pt_sec_posix_map(struct pt_section *section, int fd)
+{
+	struct pt_sec_posix_mapping *mapping;
+	uint64_t offset, size, adjustment;
+	uint8_t *base;
+	long page_size;
+	int errcode;
+
+	if (!section)
+		return -pte_internal;
+
+	offset = section->offset;
+	size = section->size;
+
+	page_size = sysconf(_SC_PAGESIZE);
+	if (page_size < 0)
+		return -pte_bad_config;
+
+	adjustment = offset % (uint64_t) page_size;
+
+	offset -= adjustment;
+	size += adjustment;
+
+	/* The section is supposed to fit into the file so we shouldn't
+	 * see any overflows, here.
+	 */
+	if (size < section->size)
+		return -pte_internal;
+
+	if (SIZE_MAX < size)
+		return -pte_nomem;
+
+	if (INT_MAX < offset)
+		return -pte_nomem;
+
+	base = mmap(NULL, (size_t) size, PROT_READ, MAP_SHARED, fd,
+		    (off_t) offset);
+	if (base == MAP_FAILED)
+		return -pte_nomem;
+
+	mapping = malloc(sizeof(*mapping));
+	if (!mapping) {
+		errcode = -pte_nomem;
+		goto out_map;
+	}
+
+	mapping->base = base;
+	mapping->size = size;
+	mapping->begin = base + adjustment;
+	mapping->end = base + size;
+
+	section->mapping = mapping;
+	section->unmap = pt_sec_posix_unmap;
+	section->read = pt_sec_posix_read;
+	section->memsize = pt_sec_posix_memsize;
+
+	return 0;
+
+out_map:
+	munmap(base, (size_t) size);
+	return errcode;
+}
+
+static int pt_sec_posix_map_success(struct pt_section *section)
+{
+	uint16_t mcount;
+	int errcode, status;
+
+	if (!section)
+		return -pte_internal;
+
+	mcount = section->mcount + 1;
+	if (!mcount) {
+		(void) pt_section_unlock(section);
+		return -pte_overflow;
+	}
+
+	section->mcount = mcount;
+
+	errcode = pt_section_unlock(section);
+	if (errcode < 0)
+		return errcode;
+
+	status = pt_section_on_map(section);
+	if (status < 0) {
+		/* We had to release the section lock for pt_section_on_map() so
+		 * @section may have meanwhile been mapped by other threads.
+		 *
+		 * We still want to return the error so we release our mapping.
+		 * Our caller does not yet know whether pt_section_map()
+		 * succeeded.
+		 */
+		(void) pt_section_unmap(section);
+		return status;
+	}
+
+	return 0;
+}
+
+int pt_section_map(struct pt_section *section)
+{
+	const char *filename;
+	FILE *file;
+	int fd, errcode;
+
+	if (!section)
+		return -pte_internal;
+
+	errcode = pt_section_lock(section);
+	if (errcode < 0)
+		return errcode;
+
+	if (section->mcount)
+		return pt_sec_posix_map_success(section);
+
+	if (section->mapping)
+		goto out_unlock;
+
+	filename = section->filename;
+	if (!filename)
+		goto out_unlock;
+
+	errcode = -pte_bad_file;
+	fd = open(filename, O_RDONLY);
+	if (fd == -1)
+		goto out_unlock;
+
+	errcode = check_file_status(section, fd);
+	if (errcode < 0)
+		goto out_fd;
+
+	/* We close the file on success.  This does not unmap the section. */
+	errcode = pt_sec_posix_map(section, fd);
+	if (!errcode) {
+		close(fd);
+
+		return pt_sec_posix_map_success(section);
+	}
+
+	/* Fall back to file based sections - report the original error
+	 * if we fail to convert the file descriptor.
+	 */
+	file = fdopen(fd, "rb");
+	if (!file) {
+		errcode = -pte_bad_file;
+		goto out_fd;
+	}
+
+	/* We need to keep the file open on success.  It will be closed when
+	 * the section is unmapped.
+	 */
+	errcode = pt_sec_file_map(section, file);
+	if (!errcode)
+		return pt_sec_posix_map_success(section);
+
+	fclose(file);
+	goto out_unlock;
+
+out_fd:
+	close(fd);
+
+out_unlock:
+	(void) pt_section_unlock(section);
+	return errcode;
+}
+
+int pt_sec_posix_unmap(struct pt_section *section)
+{
+	struct pt_sec_posix_mapping *mapping;
+
+	if (!section)
+		return -pte_internal;
+
+	mapping = section->mapping;
+	if (!mapping || !section->unmap || !section->read || !section->memsize)
+		return -pte_internal;
+
+	section->mapping = NULL;
+	section->unmap = NULL;
+	section->read = NULL;
+	section->memsize = NULL;
+
+	munmap(mapping->base, (size_t) mapping->size);
+	free(mapping);
+
+	return 0;
+}
+
+int pt_sec_posix_read(const struct pt_section *section, uint8_t *buffer,
+		      uint16_t size, uint64_t offset)
+{
+	struct pt_sec_posix_mapping *mapping;
+	const uint8_t *begin;
+
+	if (!buffer || !section)
+		return -pte_internal;
+
+	mapping = section->mapping;
+	if (!mapping)
+		return -pte_internal;
+
+	/* We already checked in pt_section_read() that the requested memory
+	 * lies within the section's boundaries.
+	 *
+	 * And we checked that the entire section was mapped.  There's no need
+	 * to check for overflows, again.
+	 */
+	begin = mapping->begin + offset;
+
+	memcpy(buffer, begin, size);
+	return (int) size;
+}
+
+int pt_sec_posix_memsize(const struct pt_section *section, uint64_t *size)
+{
+	struct pt_sec_posix_mapping *mapping;
+	const uint8_t *begin, *end;
+
+	if (!section || !size)
+		return -pte_internal;
+
+	mapping = section->mapping;
+	if (!mapping)
+		return -pte_internal;
+
+	begin = mapping->base;
+	end = mapping->end;
+
+	if (!begin || !end || end < begin)
+		return -pte_internal;
+
+	*size = (uint64_t) (end - begin);
+
+	return 0;
+}
diff --git a/libipt/src/pt_asid.c b/libipt/src/pt_asid.c
new file mode 100644
index 000000000000..5cb4dd61b69e
--- /dev/null
+++ b/libipt/src/pt_asid.c
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2014-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_asid.h"
+
+#include "intel-pt.h"
+
+#include <string.h>
+
+
+int pt_asid_from_user(struct pt_asid *asid, const struct pt_asid *user)
+{
+	if (!asid)
+		return -pte_internal;
+
+	pt_asid_init(asid);
+
+	if (user) {
+		size_t size;
+
+		size = user->size;
+
+		/* Ignore fields in the user's asid we don't know. */
+		if (sizeof(*asid) < size)
+			size = sizeof(*asid);
+
+		/* Copy (portions of) the user's asid. */
+		memcpy(asid, user, size);
+
+		/* We copied user's size - fix it. */
+		asid->size = sizeof(*asid);
+	}
+
+	return 0;
+}
+
+int pt_asid_to_user(struct pt_asid *user, const struct pt_asid *asid,
+		    size_t size)
+{
+	if (!user || !asid)
+		return -pte_internal;
+
+	/* We need at least space for the size field. */
+	if (size < sizeof(asid->size))
+		return -pte_invalid;
+
+	/* Only provide the fields we actually have. */
+	if (sizeof(*asid) < size)
+		size = sizeof(*asid);
+
+	/* Copy (portions of) our asid to the user's. */
+	memcpy(user, asid, size);
+
+	/* We copied our size - fix it. */
+	user->size = size;
+
+	return 0;
+}
+
+int pt_asid_match(const struct pt_asid *lhs, const struct pt_asid *rhs)
+{
+	uint64_t lcr3, rcr3, lvmcs, rvmcs;
+
+	if (!lhs || !rhs)
+		return -pte_internal;
+
+	lcr3 = lhs->cr3;
+	rcr3 = rhs->cr3;
+
+	if (lcr3 != rcr3 && lcr3 != pt_asid_no_cr3 && rcr3 != pt_asid_no_cr3)
+		return 0;
+
+	lvmcs = lhs->vmcs;
+	rvmcs = rhs->vmcs;
+
+	if (lvmcs != rvmcs && lvmcs != pt_asid_no_vmcs &&
+	    rvmcs != pt_asid_no_vmcs)
+		return 0;
+
+	return 1;
+}
diff --git a/libipt/src/pt_block_cache.c b/libipt/src/pt_block_cache.c
new file mode 100644
index 000000000000..9c12a8ffb69d
--- /dev/null
+++ b/libipt/src/pt_block_cache.c
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2016-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_block_cache.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+
+struct pt_block_cache *pt_bcache_alloc(uint64_t nentries)
+{
+	struct pt_block_cache *bcache;
+	uint64_t size;
+
+	if (!nentries || (UINT32_MAX < nentries))
+		return NULL;
+
+	size = sizeof(*bcache) + (nentries * sizeof(struct pt_bcache_entry));
+	if (SIZE_MAX < size)
+		return NULL;
+
+	bcache = malloc((size_t) size);
+	if (!bcache)
+		return NULL;
+
+	memset(bcache, 0, (size_t) size);
+	bcache->nentries = (uint32_t) nentries;
+
+	return bcache;
+}
+
+void pt_bcache_free(struct pt_block_cache *bcache)
+{
+	free(bcache);
+}
+
+int pt_bcache_add(struct pt_block_cache *bcache, uint64_t index,
+		  struct pt_bcache_entry bce)
+{
+	if (!bcache)
+		return -pte_internal;
+
+	if (bcache->nentries <= index)
+		return -pte_internal;
+
+	/* We rely on guaranteed atomic operations as specified in section 8.1.1
+	 * in Volume 3A of the Intel(R) Software Developer's Manual at
+	 * http://www.intel.com/sdm.
+	 */
+	bcache->entry[(uint32_t) index] = bce;
+
+	return 0;
+}
+
+int pt_bcache_lookup(struct pt_bcache_entry *bce,
+		     const struct pt_block_cache *bcache, uint64_t index)
+{
+	if (!bce || !bcache)
+		return -pte_internal;
+
+	if (bcache->nentries <= index)
+		return -pte_internal;
+
+	/* We rely on guaranteed atomic operations as specified in section 8.1.1
+	 * in Volume 3A of the Intel(R) Software Developer's Manual at
+	 * http://www.intel.com/sdm.
+	 */
+	*bce = bcache->entry[(uint32_t) index];
+
+	return 0;
+}
diff --git a/libipt/src/pt_block_decoder.c b/libipt/src/pt_block_decoder.c
new file mode 100644
index 000000000000..e6890f5bed27
--- /dev/null
+++ b/libipt/src/pt_block_decoder.c
@@ -0,0 +1,3473 @@
+/*
+ * Copyright (c) 2016-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_block_decoder.h"
+#include "pt_block_cache.h"
+#include "pt_section.h"
+#include "pt_image.h"
+#include "pt_insn.h"
+#include "pt_config.h"
+#include "pt_asid.h"
+#include "pt_compiler.h"
+
+#include "intel-pt.h"
+
+#include <string.h>
+#include <stdlib.h>
+
+
+static int pt_blk_proceed_trailing_event(struct pt_block_decoder *,
+					 struct pt_block *);
+
+
+static int pt_blk_status(const struct pt_block_decoder *decoder, int flags)
+{
+	int status;
+
+	if (!decoder)
+		return -pte_internal;
+
+	status = decoder->status;
+
+	/* Indicate whether tracing is disabled or enabled.
+	 *
+	 * This duplicates the indication in struct pt_insn and covers the case
+	 * where we indicate the status after synchronizing.
+	 */
+	if (!decoder->enabled)
+		flags |= pts_ip_suppressed;
+
+	/* Forward end-of-trace indications.
+	 *
+	 * Postpone it as long as we're still processing events, though.
+	 */
+	if ((status & pts_eos) && !decoder->process_event)
+		flags |= pts_eos;
+
+	return flags;
+}
+
+static void pt_blk_reset(struct pt_block_decoder *decoder)
+{
+	if (!decoder)
+		return;
+
+	decoder->mode = ptem_unknown;
+	decoder->ip = 0ull;
+	decoder->status = 0;
+	decoder->enabled = 0;
+	decoder->process_event = 0;
+	decoder->speculative = 0;
+	decoder->process_insn = 0;
+	decoder->bound_paging = 0;
+	decoder->bound_vmcs = 0;
+	decoder->bound_ptwrite = 0;
+
+	memset(&decoder->event, 0, sizeof(decoder->event));
+	pt_retstack_init(&decoder->retstack);
+	pt_asid_init(&decoder->asid);
+}
+
+/* Initialize the query decoder flags based on our flags. */
+
+static int pt_blk_init_qry_flags(struct pt_conf_flags *qflags,
+				 const struct pt_conf_flags *flags)
+{
+	if (!qflags || !flags)
+		return -pte_internal;
+
+	memset(qflags, 0, sizeof(*qflags));
+	qflags->variant.query.keep_tcal_on_ovf =
+		flags->variant.block.keep_tcal_on_ovf;
+
+	return 0;
+}
+
+int pt_blk_decoder_init(struct pt_block_decoder *decoder,
+			const struct pt_config *uconfig)
+{
+	struct pt_config config;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	errcode = pt_config_from_user(&config, uconfig);
+	if (errcode < 0)
+		return errcode;
+
+	/* The user supplied decoder flags. */
+	decoder->flags = config.flags;
+
+	/* Set the flags we need for the query decoder we use. */
+	errcode = pt_blk_init_qry_flags(&config.flags, &decoder->flags);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = pt_qry_decoder_init(&decoder->query, &config);
+	if (errcode < 0)
+		return errcode;
+
+	pt_image_init(&decoder->default_image, NULL);
+	decoder->image = &decoder->default_image;
+
+	errcode = pt_msec_cache_init(&decoder->scache);
+	if (errcode < 0)
+		return errcode;
+
+	pt_blk_reset(decoder);
+
+	return 0;
+}
+
+void pt_blk_decoder_fini(struct pt_block_decoder *decoder)
+{
+	if (!decoder)
+		return;
+
+	pt_msec_cache_fini(&decoder->scache);
+	pt_image_fini(&decoder->default_image);
+	pt_qry_decoder_fini(&decoder->query);
+}
+
+struct pt_block_decoder *
+pt_blk_alloc_decoder(const struct pt_config *config)
+{
+	struct pt_block_decoder *decoder;
+	int errcode;
+
+	decoder = malloc(sizeof(*decoder));
+	if (!decoder)
+		return NULL;
+
+	errcode = pt_blk_decoder_init(decoder, config);
+	if (errcode < 0) {
+		free(decoder);
+		return NULL;
+	}
+
+	return decoder;
+}
+
+void pt_blk_free_decoder(struct pt_block_decoder *decoder)
+{
+	if (!decoder)
+		return;
+
+	pt_blk_decoder_fini(decoder);
+	free(decoder);
+}
+
+/* Maybe synthesize a tick event.
+ *
+ * If we're not already processing events, check the current time against the
+ * last event's time.  If it changed, synthesize a tick event with the new time.
+ *
+ * Returns zero if no tick event has been created.
+ * Returns a positive integer if a tick event has been created.
+ * Returns a negative error code otherwise.
+ */
+static int pt_blk_tick(struct pt_block_decoder *decoder, uint64_t ip)
+{
+	struct pt_event *ev;
+	uint64_t tsc;
+	uint32_t lost_mtc, lost_cyc;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	/* We're not generating tick events if tracing is disabled. */
+	if (!decoder->enabled)
+		return -pte_internal;
+
+	/* Events already provide a timestamp so there is no need to synthesize
+	 * an artificial tick event.  There's no room, either, since this would
+	 * overwrite the in-progress event.
+	 *
+	 * In rare cases where we need to proceed to an event location using
+	 * trace this may cause us to miss a timing update if the event is not
+	 * forwarded to the user.
+	 *
+	 * The only case I can come up with at the moment is a MODE.EXEC binding
+	 * to the TIP IP of a far branch.
+	 */
+	if (decoder->process_event)
+		return 0;
+
+	errcode = pt_qry_time(&decoder->query, &tsc, &lost_mtc, &lost_cyc);
+	if (errcode < 0) {
+		/* If we don't have wall-clock time, we use relative time. */
+		if (errcode != -pte_no_time)
+			return errcode;
+	}
+
+	ev = &decoder->event;
+
+	/* We're done if time has not changed since the last event. */
+	if (tsc == ev->tsc)
+		return 0;
+
+	/* Time has changed so we create a new tick event. */
+	memset(ev, 0, sizeof(*ev));
+	ev->type = ptev_tick;
+	ev->variant.tick.ip = ip;
+
+	/* Indicate if we have wall-clock time or only relative time. */
+	if (errcode != -pte_no_time)
+		ev->has_tsc = 1;
+	ev->tsc = tsc;
+	ev->lost_mtc = lost_mtc;
+	ev->lost_cyc = lost_cyc;
+
+	/* We now have an event to process. */
+	decoder->process_event = 1;
+
+	return 1;
+}
+
+/* Query an indirect branch.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_indirect_branch(struct pt_block_decoder *decoder,
+				  uint64_t *ip)
+{
+	uint64_t evip;
+	int status, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	evip = decoder->ip;
+
+	status = pt_qry_indirect_branch(&decoder->query, ip);
+	if (status < 0)
+		return status;
+
+	if (decoder->flags.variant.block.enable_tick_events) {
+		errcode = pt_blk_tick(decoder, evip);
+		if (errcode < 0)
+			return errcode;
+	}
+
+	return status;
+}
+
+/* Query a conditional branch.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_cond_branch(struct pt_block_decoder *decoder, int *taken)
+{
+	int status, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	status = pt_qry_cond_branch(&decoder->query, taken);
+	if (status < 0)
+		return status;
+
+	if (decoder->flags.variant.block.enable_tick_events) {
+		errcode = pt_blk_tick(decoder, decoder->ip);
+		if (errcode < 0)
+			return errcode;
+	}
+
+	return status;
+}
+
+static int pt_blk_start(struct pt_block_decoder *decoder, int status)
+{
+	if (!decoder)
+		return -pte_internal;
+
+	if (status < 0)
+		return status;
+
+	decoder->status = status;
+	if (!(status & pts_ip_suppressed))
+		decoder->enabled = 1;
+
+	/* We will always have an event.
+	 *
+	 * If we synchronized onto an empty PSB+, tracing is disabled and we'll
+	 * process events until the enabled event.
+	 *
+	 * If tracing is enabled, PSB+ must at least provide the execution mode,
+	 * which we're going to forward to the user.
+	 */
+	return pt_blk_proceed_trailing_event(decoder, NULL);
+}
+
+static int pt_blk_sync_reset(struct pt_block_decoder *decoder)
+{
+	if (!decoder)
+		return -pte_internal;
+
+	pt_blk_reset(decoder);
+
+	return 0;
+}
+
+int pt_blk_sync_forward(struct pt_block_decoder *decoder)
+{
+	int errcode, status;
+
+	if (!decoder)
+		return -pte_invalid;
+
+	errcode = pt_blk_sync_reset(decoder);
+	if (errcode < 0)
+		return errcode;
+
+	status = pt_qry_sync_forward(&decoder->query, &decoder->ip);
+
+	return pt_blk_start(decoder, status);
+}
+
+int pt_blk_sync_backward(struct pt_block_decoder *decoder)
+{
+	int errcode, status;
+
+	if (!decoder)
+		return -pte_invalid;
+
+	errcode = pt_blk_sync_reset(decoder);
+	if (errcode < 0)
+		return errcode;
+
+	status = pt_qry_sync_backward(&decoder->query, &decoder->ip);
+
+	return pt_blk_start(decoder, status);
+}
+
+int pt_blk_sync_set(struct pt_block_decoder *decoder, uint64_t offset)
+{
+	int errcode, status;
+
+	if (!decoder)
+		return -pte_invalid;
+
+	errcode = pt_blk_sync_reset(decoder);
+	if (errcode < 0)
+		return errcode;
+
+	status = pt_qry_sync_set(&decoder->query, &decoder->ip, offset);
+
+	return pt_blk_start(decoder, status);
+}
+
+int pt_blk_get_offset(const struct pt_block_decoder *decoder, uint64_t *offset)
+{
+	if (!decoder)
+		return -pte_invalid;
+
+	return pt_qry_get_offset(&decoder->query, offset);
+}
+
+int pt_blk_get_sync_offset(const struct pt_block_decoder *decoder,
+			   uint64_t *offset)
+{
+	if (!decoder)
+		return -pte_invalid;
+
+	return pt_qry_get_sync_offset(&decoder->query, offset);
+}
+
+struct pt_image *pt_blk_get_image(struct pt_block_decoder *decoder)
+{
+	if (!decoder)
+		return NULL;
+
+	return decoder->image;
+}
+
+int pt_blk_set_image(struct pt_block_decoder *decoder, struct pt_image *image)
+{
+	if (!decoder)
+		return -pte_invalid;
+
+	if (!image)
+		image = &decoder->default_image;
+
+	decoder->image = image;
+	return 0;
+}
+
+const struct pt_config *
+pt_blk_get_config(const struct pt_block_decoder *decoder)
+{
+	if (!decoder)
+		return NULL;
+
+	return pt_qry_get_config(&decoder->query);
+}
+
+int pt_blk_time(struct pt_block_decoder *decoder, uint64_t *time,
+		uint32_t *lost_mtc, uint32_t *lost_cyc)
+{
+	if (!decoder || !time)
+		return -pte_invalid;
+
+	return pt_qry_time(&decoder->query, time, lost_mtc, lost_cyc);
+}
+
+int pt_blk_core_bus_ratio(struct pt_block_decoder *decoder, uint32_t *cbr)
+{
+	if (!decoder || !cbr)
+		return -pte_invalid;
+
+	return pt_qry_core_bus_ratio(&decoder->query, cbr);
+}
+
+int pt_blk_asid(const struct pt_block_decoder *decoder, struct pt_asid *asid,
+		size_t size)
+{
+	if (!decoder || !asid)
+		return -pte_invalid;
+
+	return pt_asid_to_user(asid, &decoder->asid, size);
+}
+
+/* Fetch the next pending event.
+ *
+ * Checks for pending events.  If an event is pending, fetches it (if not
+ * already in process).
+ *
+ * Returns zero if no event is pending.
+ * Returns a positive integer if an event is pending or in process.
+ * Returns a negative error code otherwise.
+ */
+static inline int pt_blk_fetch_event(struct pt_block_decoder *decoder)
+{
+	int status;
+
+	if (!decoder)
+		return -pte_internal;
+
+	if (decoder->process_event)
+		return 1;
+
+	if (!(decoder->status & pts_event_pending))
+		return 0;
+
+	status = pt_qry_event(&decoder->query, &decoder->event,
+			      sizeof(decoder->event));
+	if (status < 0)
+		return status;
+
+	decoder->process_event = 1;
+	decoder->status = status;
+
+	return 1;
+}
+
+static inline int pt_blk_block_is_empty(const struct pt_block *block)
+{
+	if (!block)
+		return 1;
+
+	return !block->ninsn;
+}
+
+static inline int block_to_user(struct pt_block *ublock, size_t size,
+				const struct pt_block *block)
+{
+	if (!ublock || !block)
+		return -pte_internal;
+
+	if (ublock == block)
+		return 0;
+
+	/* Zero out any unknown bytes. */
+	if (sizeof(*block) < size) {
+		memset(ublock + sizeof(*block), 0, size - sizeof(*block));
+
+		size = sizeof(*block);
+	}
+
+	memcpy(ublock, block, size);
+
+	return 0;
+}
+
+static int pt_insn_false(const struct pt_insn *insn,
+			 const struct pt_insn_ext *iext)
+{
+	(void) insn;
+	(void) iext;
+
+	return 0;
+}
+
+/* Determine the next IP using trace.
+ *
+ * Tries to determine the IP of the next instruction using trace and provides it
+ * in @pip.
+ *
+ * Not requiring trace to determine the IP is treated as an internal error.
+ *
+ * Does not update the return compression stack for indirect calls.  This is
+ * expected to have been done, already, when trying to determine the next IP
+ * without using trace.
+ *
+ * Does not update @decoder->status.  The caller is expected to do that.
+ *
+ * Returns a non-negative pt_status_flag bit-vector on success, a negative error
+ * code otherwise.
+ * Returns -pte_internal if @pip, @decoder, @insn, or @iext are NULL.
+ * Returns -pte_internal if no trace is required.
+ */
+static int pt_blk_next_ip(uint64_t *pip, struct pt_block_decoder *decoder,
+			  const struct pt_insn *insn,
+			  const struct pt_insn_ext *iext)
+{
+	int status, errcode;
+
+	if (!pip || !decoder || !insn || !iext)
+		return -pte_internal;
+
+	/* We handle non-taken conditional branches, and compressed returns
+	 * directly in the switch.
+	 *
+	 * All kinds of branches are handled below the switch.
+	 */
+	switch (insn->iclass) {
+	case ptic_cond_jump: {
+		uint64_t ip;
+		int taken;
+
+		status = pt_blk_cond_branch(decoder, &taken);
+		if (status < 0)
+			return status;
+
+		ip = insn->ip + insn->size;
+		if (taken)
+			ip += (uint64_t) (int64_t)
+				iext->variant.branch.displacement;
+
+		*pip = ip;
+		return status;
+	}
+
+	case ptic_return: {
+		int taken;
+
+		/* Check for a compressed return. */
+		status = pt_blk_cond_branch(decoder, &taken);
+		if (status < 0) {
+			if (status != -pte_bad_query)
+				return status;
+
+			break;
+		}
+
+		/* A compressed return is indicated by a taken conditional
+		 * branch.
+		 */
+		if (!taken)
+			return -pte_bad_retcomp;
+
+		errcode = pt_retstack_pop(&decoder->retstack, pip);
+		if (errcode < 0)
+			return errcode;
+
+		return status;
+	}
+
+	case ptic_jump:
+	case ptic_call:
+		/* A direct jump or call wouldn't require trace. */
+		if (iext->variant.branch.is_direct)
+			return -pte_internal;
+
+		break;
+
+	case ptic_far_call:
+	case ptic_far_return:
+	case ptic_far_jump:
+		break;
+
+	case ptic_ptwrite:
+	case ptic_other:
+		return -pte_internal;
+
+	case ptic_error:
+		return -pte_bad_insn;
+	}
+
+	/* Process an indirect branch.
+	 *
+	 * This covers indirect jumps and calls, non-compressed returns, and all
+	 * flavors of far transfers.
+	 */
+	return pt_blk_indirect_branch(decoder, pip);
+}
+
+/* Proceed to the next IP using trace.
+ *
+ * We failed to proceed without trace.  This ends the current block.  Now use
+ * trace to do one final step to determine the start IP of the next block.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_proceed_with_trace(struct pt_block_decoder *decoder,
+				     const struct pt_insn *insn,
+				     const struct pt_insn_ext *iext)
+{
+	int status;
+
+	if (!decoder)
+		return -pte_internal;
+
+	status = pt_blk_next_ip(&decoder->ip, decoder, insn, iext);
+	if (status < 0)
+		return status;
+
+	/* Preserve the query decoder's response which indicates upcoming
+	 * events.
+	 */
+	decoder->status = status;
+
+	/* We do need an IP in order to proceed. */
+	if (status & pts_ip_suppressed)
+		return -pte_noip;
+
+	return 0;
+}
+
+/* Decode one instruction in a known section.
+ *
+ * Decode the instruction at @insn->ip in @msec assuming execution mode
+ * @insn->mode.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_decode_in_section(struct pt_insn *insn,
+				    struct pt_insn_ext *iext,
+				    const struct pt_mapped_section *msec)
+{
+	int status;
+
+	if (!insn || !iext)
+		return -pte_internal;
+
+	/* We know that @ip is contained in @section.
+	 *
+	 * Note that we need to translate @ip into a section offset.
+	 */
+	status = pt_msec_read(msec, insn->raw, sizeof(insn->raw), insn->ip);
+	if (status < 0)
+		return status;
+
+	/* We initialize @insn->size to the maximal possible size.  It will be
+	 * set to the actual size during instruction decode.
+	 */
+	insn->size = (uint8_t) status;
+
+	return pt_ild_decode(insn, iext);
+}
+
+/* Update the return-address stack if @insn is a near call.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static inline int pt_blk_log_call(struct pt_block_decoder *decoder,
+				  const struct pt_insn *insn,
+				  const struct pt_insn_ext *iext)
+{
+	if (!decoder || !insn || !iext)
+		return -pte_internal;
+
+	if (insn->iclass != ptic_call)
+		return 0;
+
+	/* Ignore direct calls to the next instruction that are used for
+	 * position independent code.
+	 */
+	if (iext->variant.branch.is_direct &&
+	    !iext->variant.branch.displacement)
+		return 0;
+
+	return pt_retstack_push(&decoder->retstack, insn->ip + insn->size);
+}
+
+/* Proceed by one instruction.
+ *
+ * Tries to decode the instruction at @decoder->ip and, on success, adds it to
+ * @block and provides it in @pinsn and @piext.
+ *
+ * The instruction will not be added if:
+ *
+ *   - the memory could not be read:  return error
+ *   - it could not be decoded:       return error
+ *   - @block is already full:        return zero
+ *   - @block would switch sections:  return zero
+ *
+ * Returns a positive integer if the instruction was added.
+ * Returns zero if the instruction didn't fit into @block.
+ * Returns a negative error code otherwise.
+ */
+static int pt_blk_proceed_one_insn(struct pt_block_decoder *decoder,
+				   struct pt_block *block,
+				   struct pt_insn *pinsn,
+				   struct pt_insn_ext *piext)
+{
+	struct pt_insn_ext iext;
+	struct pt_insn insn;
+	uint16_t ninsn;
+	int status;
+
+	if (!decoder || !block || !pinsn || !piext)
+		return -pte_internal;
+
+	/* There's nothing to do if there is no room in @block. */
+	ninsn = block->ninsn + 1;
+	if (!ninsn)
+		return 0;
+
+	/* The truncated instruction must be last. */
+	if (block->truncated)
+		return 0;
+
+	memset(&insn, 0, sizeof(insn));
+	memset(&iext, 0, sizeof(iext));
+
+	insn.mode = decoder->mode;
+	insn.ip = decoder->ip;
+
+	status = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);
+	if (status < 0)
+		return status;
+
+	/* We do not switch sections inside a block. */
+	if (insn.isid != block->isid) {
+		if (!pt_blk_block_is_empty(block))
+			return 0;
+
+		block->isid = insn.isid;
+	}
+
+	/* If we couldn't read @insn's memory in one chunk from @insn.isid, we
+	 * provide the memory in @block.
+	 */
+	if (insn.truncated) {
+		memcpy(block->raw, insn.raw, insn.size);
+		block->size = insn.size;
+		block->truncated = 1;
+	}
+
+	/* Log calls' return addresses for return compression. */
+	status = pt_blk_log_call(decoder, &insn, &iext);
+	if (status < 0)
+		return status;
+
+	/* We have a new instruction. */
+	block->iclass = insn.iclass;
+	block->end_ip = insn.ip;
+	block->ninsn = ninsn;
+
+	*pinsn = insn;
+	*piext = iext;
+
+	return 1;
+}
+
+
+/* Proceed to a particular type of instruction without using trace.
+ *
+ * Proceed until we reach an instruction for which @predicate returns a positive
+ * integer or until:
+ *
+ *   - @predicate returns an error:  return error
+ *   - @block is full:               return zero
+ *   - @block would switch sections: return zero
+ *   - we would need trace:          return -pte_bad_query
+ *
+ * Provide the last instruction that was reached in @insn and @iext.
+ *
+ * Update @decoder->ip to point to the last IP that was reached.  If we fail due
+ * to lack of trace or if we reach a desired instruction, this is @insn->ip;
+ * otherwise this is the next instruction's IP.
+ *
+ * Returns a positive integer if a suitable instruction was reached.
+ * Returns zero if no such instruction was reached.
+ * Returns a negative error code otherwise.
+ */
+static int pt_blk_proceed_to_insn(struct pt_block_decoder *decoder,
+				  struct pt_block *block,
+				  struct pt_insn *insn,
+				  struct pt_insn_ext *iext,
+				  int (*predicate)(const struct pt_insn *,
+						   const struct pt_insn_ext *))
+{
+	int status;
+
+	if (!decoder || !insn || !predicate)
+		return -pte_internal;
+
+	for (;;) {
+		status = pt_blk_proceed_one_insn(decoder, block, insn, iext);
+		if (status <= 0)
+			return status;
+
+		/* We're done if this instruction matches the spec (positive
+		 * status) or we run into an error (negative status).
+		 */
+		status = predicate(insn, iext);
+		if (status != 0)
+			return status;
+
+		/* Let's see if we can proceed to the next IP without trace. */
+		status = pt_insn_next_ip(&decoder->ip, insn, iext);
+		if (status < 0)
+			return status;
+
+		/* End the block if the user asked us to.
+		 *
+		 * We only need to take care about direct near branches.
+		 * Indirect and far branches require trace and will naturally
+		 * end a block.
+		 */
+		if ((decoder->flags.variant.block.end_on_call &&
+		     (insn->iclass == ptic_call)) ||
+		    (decoder->flags.variant.block.end_on_jump &&
+		     (insn->iclass == ptic_jump)))
+			return 0;
+	}
+}
+
+/* Proceed to a particular IP without using trace.
+ *
+ * Proceed until we reach @ip or until:
+ *
+ *   - @block is full:               return zero
+ *   - @block would switch sections: return zero
+ *   - we would need trace:          return -pte_bad_query
+ *
+ * Provide the last instruction that was reached in @insn and @iext.  If we
+ * reached @ip, this is the instruction preceding it.
+ *
+ * Update @decoder->ip to point to the last IP that was reached.  If we fail due
+ * to lack of trace, this is @insn->ip; otherwise this is the next instruction's
+ * IP.
+ *
+ * Returns a positive integer if @ip was reached.
+ * Returns zero if no such instruction was reached.
+ * Returns a negative error code otherwise.
+ */
+static int pt_blk_proceed_to_ip(struct pt_block_decoder *decoder,
+				struct pt_block *block, struct pt_insn *insn,
+				struct pt_insn_ext *iext, uint64_t ip)
+{
+	int status;
+
+	if (!decoder || !insn)
+		return -pte_internal;
+
+	for (;;) {
+		/* We're done when we reach @ip.  We may not even have to decode
+		 * a single instruction in some cases.
+		 */
+		if (decoder->ip == ip)
+			return 1;
+
+		status = pt_blk_proceed_one_insn(decoder, block, insn, iext);
+		if (status <= 0)
+			return status;
+
+		/* Let's see if we can proceed to the next IP without trace. */
+		status = pt_insn_next_ip(&decoder->ip, insn, iext);
+		if (status < 0)
+			return status;
+
+		/* End the block if the user asked us to.
+		 *
+		 * We only need to take care about direct near branches.
+		 * Indirect and far branches require trace and will naturally
+		 * end a block.
+		 *
+		 * The call at the end of the block may have reached @ip; make
+		 * sure to indicate that.
+		 */
+		if ((decoder->flags.variant.block.end_on_call &&
+		     (insn->iclass == ptic_call)) ||
+		    (decoder->flags.variant.block.end_on_jump &&
+		     (insn->iclass == ptic_jump))) {
+			return (decoder->ip == ip ? 1 : 0);
+		}
+	}
+}
+
+/* Proceed to a particular IP with trace, if necessary.
+ *
+ * Proceed until we reach @ip or until:
+ *
+ *   - @block is full:               return zero
+ *   - @block would switch sections: return zero
+ *   - we need trace:                return zero
+ *
+ * Update @decoder->ip to point to the last IP that was reached.
+ *
+ * A return of zero ends @block.
+ *
+ * Returns a positive integer if @ip was reached.
+ * Returns zero if no such instruction was reached.
+ * Returns a negative error code otherwise.
+ */
+static int pt_blk_proceed_to_ip_with_trace(struct pt_block_decoder *decoder,
+					   struct pt_block *block,
+					   uint64_t ip)
+{
+	struct pt_insn_ext iext;
+	struct pt_insn insn;
+	int status;
+
+	/* Try to reach @ip without trace.
+	 *
+	 * We're also OK if @block overflowed or we switched sections and we
+	 * have to try again in the next iteration.
+	 */
+	status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext, ip);
+	if (status != -pte_bad_query)
+		return status;
+
+	/* Needing trace is not an error.  We use trace to determine the next
+	 * start IP and end the block.
+	 */
+	return pt_blk_proceed_with_trace(decoder, &insn, &iext);
+}
+
+static int pt_insn_skl014(const struct pt_insn *insn,
+			  const struct pt_insn_ext *iext)
+{
+	if (!insn || !iext)
+		return 0;
+
+	switch (insn->iclass) {
+	default:
+		return 0;
+
+	case ptic_call:
+	case ptic_jump:
+		return iext->variant.branch.is_direct;
+
+	case ptic_other:
+		return pt_insn_changes_cr3(insn, iext);
+	}
+}
+
+/* Proceed to the location of a synchronous disabled event with suppressed IP
+ * considering SKL014.
+ *
+ * We have a (synchronous) disabled event pending.  Proceed to the event
+ * location and indicate whether we were able to reach it.
+ *
+ * With SKL014 a TIP.PGD with suppressed IP may also be generated by a direct
+ * unconditional branch that clears FilterEn by jumping out of a filter region
+ * or into a TraceStop region.  Use the filter configuration to determine the
+ * exact branch the event binds to.
+ *
+ * The last instruction that was reached is stored in @insn/@iext.
+ *
+ * Returns a positive integer if the event location was reached.
+ * Returns zero if the event location was not reached.
+ * Returns a negative error code otherwise.
+ */
+static int pt_blk_proceed_skl014(struct pt_block_decoder *decoder,
+				 struct pt_block *block, struct pt_insn *insn,
+				 struct pt_insn_ext *iext)
+{
+	const struct pt_conf_addr_filter *addr_filter;
+	int status;
+
+	if (!decoder || !block || !insn || !iext)
+		return -pte_internal;
+
+	addr_filter = &decoder->query.config.addr_filter;
+	for (;;) {
+		uint64_t ip;
+
+		status = pt_blk_proceed_to_insn(decoder, block, insn, iext,
+						pt_insn_skl014);
+		if (status <= 0)
+			break;
+
+		/* The erratum doesn't apply if we can bind the event to a
+		 * CR3-changing instruction.
+		 */
+		if (pt_insn_changes_cr3(insn, iext))
+			break;
+
+		/* Check the filter against the branch target. */
+		status = pt_insn_next_ip(&ip, insn, iext);
+		if (status < 0)
+			break;
+
+		status = pt_filter_addr_check(addr_filter, ip);
+		if (status <= 0) {
+			/* We need to flip the indication.
+			 *
+			 * We reached the event location when @ip lies inside a
+			 * tracing-disabled region.
+			 */
+			if (!status)
+				status = 1;
+
+			break;
+		}
+
+		/* This is not the correct instruction.  Proceed past it and try
+		 * again.
+		 */
+		decoder->ip = ip;
+
+		/* End the block if the user asked us to.
+		 *
+		 * We only need to take care about direct near branches.
+		 * Indirect and far branches require trace and will naturally
+		 * end a block.
+		 */
+		if ((decoder->flags.variant.block.end_on_call &&
+		    (insn->iclass == ptic_call)) ||
+		    (decoder->flags.variant.block.end_on_jump &&
+		    (insn->iclass == ptic_jump)))
+			break;
+	}
+
+	return status;
+}
+
+/* Proceed to the event location for a disabled event.
+ *
+ * We have a (synchronous) disabled event pending.  Proceed to the event
+ * location and indicate whether we were able to reach it.
+ *
+ * The last instruction that was reached is stored in @insn/@iext.
+ *
+ * Returns a positive integer if the event location was reached.
+ * Returns zero if the event location was not reached.
+ * Returns a negative error code otherwise.
+ */
+static int pt_blk_proceed_to_disabled(struct pt_block_decoder *decoder,
+				      struct pt_block *block,
+				      struct pt_insn *insn,
+				      struct pt_insn_ext *iext,
+				      const struct pt_event *ev)
+{
+	if (!decoder || !block || !ev)
+		return -pte_internal;
+
+	if (ev->ip_suppressed) {
+		/* Due to SKL014 the TIP.PGD payload may be suppressed also for
+		 * direct branches.
+		 *
+		 * If we don't have a filter configuration we assume that no
+		 * address filters were used and the erratum does not apply.
+		 *
+		 * We might otherwise disable tracing too early.
+		 */
+		if (decoder->query.config.addr_filter.config.addr_cfg &&
+		    decoder->query.config.errata.skl014)
+			return pt_blk_proceed_skl014(decoder, block, insn,
+						     iext);
+
+		/* A synchronous disabled event also binds to far branches and
+		 * CPL-changing instructions.  Both would require trace,
+		 * however, and are thus implicitly handled by erroring out.
+		 *
+		 * The would-require-trace error is handled by our caller.
+		 */
+		return pt_blk_proceed_to_insn(decoder, block, insn, iext,
+					      pt_insn_changes_cr3);
+	} else
+		return pt_blk_proceed_to_ip(decoder, block, insn, iext,
+					    ev->variant.disabled.ip);
+}
+
+/* Set the expected resume address for a synchronous disable.
+ *
+ * On a synchronous disable, @decoder->ip still points to the instruction to
+ * which the event bound.  That's not where we expect tracing to resume.
+ *
+ * For calls, a fair assumption is that tracing resumes after returning from the
+ * called function.  For other types of instructions, we simply don't know.
+ *
+ * Returns zero on success, a negative pt_error_code otherwise.
+ */
+static int pt_blk_set_disable_resume_ip(struct pt_block_decoder *decoder,
+					const struct pt_insn *insn)
+{
+	if (!decoder || !insn)
+		return -pte_internal;
+
+	switch (insn->iclass) {
+	case ptic_call:
+	case ptic_far_call:
+		decoder->ip = insn->ip + insn->size;
+		break;
+
+	default:
+		decoder->ip = 0ull;
+		break;
+	}
+
+	return 0;
+}
+
+/* Proceed to the event location for an async paging event.
+ *
+ * We have an async paging event pending.  Proceed to the event location and
+ * indicate whether we were able to reach it.  Needing trace in order to proceed
+ * is not an error in this case but ends the block.
+ *
+ * Returns a positive integer if the event location was reached.
+ * Returns zero if the event location was not reached.
+ * Returns a negative error code otherwise.
+ */
+static int pt_blk_proceed_to_async_paging(struct pt_block_decoder *decoder,
+					  struct pt_block *block,
+					  const struct pt_event *ev)
+{
+	int status;
+
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	/* Apply the event immediately if we don't have an IP. */
+	if (ev->ip_suppressed)
+		return 1;
+
+	status = pt_blk_proceed_to_ip_with_trace(decoder, block,
+						 ev->variant.async_paging.ip);
+	if (status < 0)
+		return status;
+
+	/* We may have reached the IP. */
+	return (decoder->ip == ev->variant.async_paging.ip ? 1 : 0);
+}
+
+/* Proceed to the event location for an async vmcs event.
+ *
+ * We have an async vmcs event pending.  Proceed to the event location and
+ * indicate whether we were able to reach it.  Needing trace in order to proceed
+ * is not an error in this case but ends the block.
+ *
+ * Returns a positive integer if the event location was reached.
+ * Returns zero if the event location was not reached.
+ * Returns a negative error code otherwise.
+ */
+static int pt_blk_proceed_to_async_vmcs(struct pt_block_decoder *decoder,
+					struct pt_block *block,
+					const struct pt_event *ev)
+{
+	int status;
+
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	/* Apply the event immediately if we don't have an IP. */
+	if (ev->ip_suppressed)
+		return 1;
+
+	status = pt_blk_proceed_to_ip_with_trace(decoder, block,
+						 ev->variant.async_vmcs.ip);
+	if (status < 0)
+		return status;
+
+	/* We may have reached the IP. */
+	return (decoder->ip == ev->variant.async_vmcs.ip ? 1 : 0);
+}
+
+/* Proceed to the event location for an exec mode event.
+ *
+ * We have an exec mode event pending.  Proceed to the event location and
+ * indicate whether we were able to reach it.  Needing trace in order to proceed
+ * is not an error in this case but ends the block.
+ *
+ * Returns a positive integer if the event location was reached.
+ * Returns zero if the event location was not reached.
+ * Returns a negative error code otherwise.
+ */
+static int pt_blk_proceed_to_exec_mode(struct pt_block_decoder *decoder,
+				       struct pt_block *block,
+				       const struct pt_event *ev)
+{
+	int status;
+
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	/* Apply the event immediately if we don't have an IP. */
+	if (ev->ip_suppressed)
+		return 1;
+
+	status = pt_blk_proceed_to_ip_with_trace(decoder, block,
+						 ev->variant.exec_mode.ip);
+	if (status < 0)
+		return status;
+
+	/* We may have reached the IP. */
+	return (decoder->ip == ev->variant.exec_mode.ip ? 1 : 0);
+}
+
+/* Proceed to the event location for a ptwrite event.
+ *
+ * We have a ptwrite event pending.  Proceed to the event location and indicate
+ * whether we were able to reach it.
+ *
+ * In case of the event binding to a ptwrite instruction, we pass beyond that
+ * instruction and update the event to provide the instruction's IP.
+ *
+ * In the case of the event binding to an IP provided in the event, we move
+ * beyond the instruction at that IP.
+ *
+ * Returns a positive integer if the event location was reached.
+ * Returns zero if the event location was not reached.
+ * Returns a negative error code otherwise.
+ */
+static int pt_blk_proceed_to_ptwrite(struct pt_block_decoder *decoder,
+				     struct pt_block *block,
+				     struct pt_insn *insn,
+				     struct pt_insn_ext *iext,
+				     struct pt_event *ev)
+{
+	int status;
+
+	if (!insn || !ev)
+		return -pte_internal;
+
+	/* If we don't have an IP, the event binds to the next PTWRITE
+	 * instruction.
+	 *
+	 * If we have an IP it still binds to the next PTWRITE instruction but
+	 * now the IP tells us where that instruction is.  This makes most sense
+	 * when tracing is disabled and we don't have any other means of finding
+	 * the PTWRITE instruction.  We nevertheless distinguish the two cases,
+	 * here.
+	 *
+	 * In both cases, we move beyond the PTWRITE instruction, so it will be
+	 * the last instruction in the current block and @decoder->ip will point
+	 * to the instruction following it.
+	 */
+	if (ev->ip_suppressed) {
+		status = pt_blk_proceed_to_insn(decoder, block, insn, iext,
+						pt_insn_is_ptwrite);
+		if (status <= 0)
+			return status;
+
+		/* We now know the IP of the PTWRITE instruction corresponding
+		 * to this event.  Fill it in to make it more convenient for the
+		 * user to process the event.
+		 */
+		ev->variant.ptwrite.ip = insn->ip;
+		ev->ip_suppressed = 0;
+	} else {
+		status = pt_blk_proceed_to_ip(decoder, block, insn, iext,
+					      ev->variant.ptwrite.ip);
+		if (status <= 0)
+			return status;
+
+		/* We reached the PTWRITE instruction and @decoder->ip points to
+		 * it; @insn/@iext still contain the preceding instruction.
+		 *
+		 * Proceed beyond the PTWRITE to account for it.  Note that we
+		 * may still overflow the block, which would cause us to
+		 * postpone both instruction and event to the next block.
+		 */
+		status = pt_blk_proceed_one_insn(decoder, block, insn, iext);
+		if (status <= 0)
+			return status;
+	}
+
+	return 1;
+}
+
+/* Try to work around erratum SKD022.
+ *
+ * If we get an asynchronous disable on VMLAUNCH or VMRESUME, the FUP that
+ * caused the disable to be asynchronous might have been bogous.
+ *
+ * Returns a positive integer if the erratum has been handled.
+ * Returns zero if the erratum does not apply.
+ * Returns a negative error code otherwise.
+ */
+static int pt_blk_handle_erratum_skd022(struct pt_block_decoder *decoder,
+					struct pt_event *ev)
+{
+	struct pt_insn_ext iext;
+	struct pt_insn insn;
+	int errcode;
+
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	insn.mode = decoder->mode;
+	insn.ip = ev->variant.async_disabled.at;
+
+	errcode = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);
+	if (errcode < 0)
+		return 0;
+
+	switch (iext.iclass) {
+	default:
+		/* The erratum does not apply. */
+		return 0;
+
+	case PTI_INST_VMLAUNCH:
+	case PTI_INST_VMRESUME:
+		/* The erratum may apply.  We can't be sure without a lot more
+		 * analysis.  Let's assume it does.
+		 *
+		 * We turn the async disable into a sync disable.  Our caller
+		 * will restart event processing.
+		 */
+		ev->type = ptev_disabled;
+		ev->variant.disabled.ip = ev->variant.async_disabled.ip;
+
+		return 1;
+	}
+}
+
+/* Postpone proceeding past @insn/@iext and indicate a pending event.
+ *
+ * There may be further events pending on @insn/@iext.  Postpone proceeding past
+ * @insn/@iext until we processed all events that bind to it.
+ *
+ * Returns a non-negative pt_status_flag bit-vector indicating a pending event
+ * on success, a negative pt_error_code otherwise.
+ */
+static int pt_blk_postpone_insn(struct pt_block_decoder *decoder,
+				const struct pt_insn *insn,
+				const struct pt_insn_ext *iext)
+{
+	if (!decoder || !insn || !iext)
+		return -pte_internal;
+
+	/* Only one can be active. */
+	if (decoder->process_insn)
+		return -pte_internal;
+
+	decoder->process_insn = 1;
+	decoder->insn = *insn;
+	decoder->iext = *iext;
+
+	return pt_blk_status(decoder, pts_event_pending);
+}
+
+/* Remove any postponed instruction from @decoder.
+ *
+ * Returns zero on success, a negative pt_error_code otherwise.
+ */
+static int pt_blk_clear_postponed_insn(struct pt_block_decoder *decoder)
+{
+	if (!decoder)
+		return -pte_internal;
+
+	decoder->process_insn = 0;
+	decoder->bound_paging = 0;
+	decoder->bound_vmcs = 0;
+	decoder->bound_ptwrite = 0;
+
+	return 0;
+}
+
+/* Proceed past a postponed instruction.
+ *
+ * If an instruction has been postponed in @decoder, proceed past it.
+ *
+ * Returns zero on success, a negative pt_error_code otherwise.
+ */
+static int pt_blk_proceed_postponed_insn(struct pt_block_decoder *decoder)
+{
+	int status;
+
+	if (!decoder)
+		return -pte_internal;
+
+	/* There's nothing to do if we have no postponed instruction. */
+	if (!decoder->process_insn)
+		return 0;
+
+	/* There's nothing to do if tracing got disabled. */
+	if (!decoder->enabled)
+		return pt_blk_clear_postponed_insn(decoder);
+
+	status = pt_insn_next_ip(&decoder->ip, &decoder->insn, &decoder->iext);
+	if (status < 0) {
+		if (status != -pte_bad_query)
+			return status;
+
+		status = pt_blk_proceed_with_trace(decoder, &decoder->insn,
+						   &decoder->iext);
+		if (status < 0)
+			return status;
+	}
+
+	return pt_blk_clear_postponed_insn(decoder);
+}
+
+/* Proceed to the next event.
+ *
+ * We have an event pending.  Proceed to the event location and indicate the
+ * event to the user.
+ *
+ * On our way to the event location we may also be forced to postpone the event
+ * to the next block, e.g. if we overflow the number of instructions in the
+ * block or if we need trace in order to reach the event location.
+ *
+ * If we're not able to reach the event location, we return zero.  This is what
+ * pt_blk_status() would return since:
+ *
+ *   - we suppress pts_eos as long as we're processing events
+ *   - we do not set pts_ip_suppressed since tracing must be enabled
+ *
+ * Returns a non-negative pt_status_flag bit-vector on success, a negative error
+ * code otherwise.
+ */
+static int pt_blk_proceed_event(struct pt_block_decoder *decoder,
+				struct pt_block *block)
+{
+	struct pt_insn_ext iext;
+	struct pt_insn insn;
+	struct pt_event *ev;
+	int status;
+
+	if (!decoder || !decoder->process_event || !block)
+		return -pte_internal;
+
+	ev = &decoder->event;
+	switch (ev->type) {
+	case ptev_enabled:
+		break;
+
+	case ptev_disabled:
+		status = pt_blk_proceed_to_disabled(decoder, block, &insn,
+						    &iext, ev);
+		if (status <= 0) {
+			/* A synchronous disable event also binds to the next
+			 * indirect or conditional branch, i.e. to any branch
+			 * that would have required trace.
+			 */
+			if (status != -pte_bad_query)
+				return status;
+
+			status = pt_blk_set_disable_resume_ip(decoder, &insn);
+			if (status < 0)
+				return status;
+		}
+
+		break;
+
+	case ptev_async_disabled:
+		status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
+					      ev->variant.async_disabled.at);
+		if (status <= 0)
+			return status;
+
+		if (decoder->query.config.errata.skd022) {
+			status = pt_blk_handle_erratum_skd022(decoder, ev);
+			if (status != 0) {
+				if (status < 0)
+					return status;
+
+				/* If the erratum hits, we modify the event.
+				 * Try again.
+				 */
+				return pt_blk_proceed_event(decoder, block);
+			}
+		}
+
+		break;
+
+	case ptev_async_branch:
+		status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
+					      ev->variant.async_branch.from);
+		if (status <= 0)
+			return status;
+
+		break;
+
+	case ptev_paging:
+		if (!decoder->enabled)
+			break;
+
+		status = pt_blk_proceed_to_insn(decoder, block, &insn, &iext,
+						pt_insn_binds_to_pip);
+		if (status <= 0)
+			return status;
+
+		/* We bound a paging event.  Make sure we do not bind further
+		 * paging events to this instruction.
+		 */
+		decoder->bound_paging = 1;
+
+		return pt_blk_postpone_insn(decoder, &insn, &iext);
+
+	case ptev_async_paging:
+		status = pt_blk_proceed_to_async_paging(decoder, block, ev);
+		if (status <= 0)
+			return status;
+
+		break;
+
+	case ptev_vmcs:
+		if (!decoder->enabled)
+			break;
+
+		status = pt_blk_proceed_to_insn(decoder, block, &insn, &iext,
+						pt_insn_binds_to_vmcs);
+		if (status <= 0)
+			return status;
+
+		/* We bound a vmcs event.  Make sure we do not bind further vmcs
+		 * events to this instruction.
+		 */
+		decoder->bound_vmcs = 1;
+
+		return pt_blk_postpone_insn(decoder, &insn, &iext);
+
+	case ptev_async_vmcs:
+		status = pt_blk_proceed_to_async_vmcs(decoder, block, ev);
+		if (status <= 0)
+			return status;
+
+		break;
+
+	case ptev_overflow:
+		break;
+
+	case ptev_exec_mode:
+		status = pt_blk_proceed_to_exec_mode(decoder, block, ev);
+		if (status <= 0)
+			return status;
+
+		break;
+
+	case ptev_tsx:
+		if (ev->ip_suppressed)
+			break;
+
+		status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
+					      ev->variant.tsx.ip);
+		if (status <= 0)
+			return status;
+
+		break;
+
+	case ptev_stop:
+		break;
+
+	case ptev_exstop:
+		if (!decoder->enabled || ev->ip_suppressed)
+			break;
+
+		status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
+					      ev->variant.exstop.ip);
+		if (status <= 0)
+			return status;
+
+		break;
+
+	case ptev_mwait:
+		if (!decoder->enabled || ev->ip_suppressed)
+			break;
+
+		status = pt_blk_proceed_to_ip(decoder, block, &insn, &iext,
+					      ev->variant.mwait.ip);
+		if (status <= 0)
+			return status;
+
+		break;
+
+	case ptev_pwre:
+	case ptev_pwrx:
+		break;
+
+	case ptev_ptwrite:
+		if (!decoder->enabled)
+			break;
+
+		status = pt_blk_proceed_to_ptwrite(decoder, block, &insn,
+						   &iext, ev);
+		if (status <= 0)
+			return status;
+
+		/* We bound a ptwrite event.  Make sure we do not bind further
+		 * ptwrite events to this instruction.
+		 */
+		decoder->bound_ptwrite = 1;
+
+		return pt_blk_postpone_insn(decoder, &insn, &iext);
+
+	case ptev_tick:
+	case ptev_cbr:
+	case ptev_mnt:
+		break;
+	}
+
+	return pt_blk_status(decoder, pts_event_pending);
+}
+
+/* Proceed to the next decision point without using the block cache.
+ *
+ * Tracing is enabled and we don't have an event pending.  Proceed as far as
+ * we get without trace.  Stop when we either:
+ *
+ *   - need trace in order to continue
+ *   - overflow the max number of instructions in a block
+ *
+ * We actually proceed one instruction further to get the start IP for the next
+ * block.  This only updates @decoder's internal state, though.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_proceed_no_event_uncached(struct pt_block_decoder *decoder,
+					    struct pt_block *block)
+{
+	struct pt_insn_ext iext;
+	struct pt_insn insn;
+	int status;
+
+	if (!decoder || !block)
+		return -pte_internal;
+
+	/* This is overly conservative, really.  We shouldn't get a bad-query
+	 * status unless we decoded at least one instruction successfully.
+	 */
+	memset(&insn, 0, sizeof(insn));
+	memset(&iext, 0, sizeof(iext));
+
+	/* Proceed as far as we get without trace. */
+	status = pt_blk_proceed_to_insn(decoder, block, &insn, &iext,
+					pt_insn_false);
+	if (status < 0) {
+		if (status != -pte_bad_query)
+			return status;
+
+		return pt_blk_proceed_with_trace(decoder, &insn, &iext);
+	}
+
+	return 0;
+}
+
+/* Check if @ip is contained in @section loaded at @laddr.
+ *
+ * Returns non-zero if it is.
+ * Returns zero if it isn't or of @section is NULL.
+ */
+static inline int pt_blk_is_in_section(const struct pt_mapped_section *msec,
+				       uint64_t ip)
+{
+	uint64_t begin, end;
+
+	begin = pt_msec_begin(msec);
+	end = pt_msec_end(msec);
+
+	return (begin <= ip && ip < end);
+}
+
+/* Insert a trampoline block cache entry.
+ *
+ * Add a trampoline block cache entry at @ip to continue at @nip, where @nip
+ * must be the next instruction after @ip.
+ *
+ * Both @ip and @nip must be section-relative
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static inline int pt_blk_add_trampoline(struct pt_block_cache *bcache,
+					uint64_t ip, uint64_t nip,
+					enum pt_exec_mode mode)
+{
+	struct pt_bcache_entry bce;
+	int64_t disp;
+
+	/* The displacement from @ip to @nip for the trampoline. */
+	disp = (int64_t) (nip - ip);
+
+	memset(&bce, 0, sizeof(bce));
+	bce.displacement = (int32_t) disp;
+	bce.ninsn = 1;
+	bce.mode = mode;
+	bce.qualifier = ptbq_again;
+
+	/* If we can't reach @nip without overflowing the displacement field, we
+	 * have to stop and re-decode the instruction at @ip.
+	 */
+	if ((int64_t) bce.displacement != disp) {
+
+		memset(&bce, 0, sizeof(bce));
+		bce.ninsn = 1;
+		bce.mode = mode;
+		bce.qualifier = ptbq_decode;
+	}
+
+	return pt_bcache_add(bcache, ip, bce);
+}
+
+/* Insert a decode block cache entry.
+ *
+ * Add a decode block cache entry at @ioff.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static inline int pt_blk_add_decode(struct pt_block_cache *bcache,
+				    uint64_t ioff, enum pt_exec_mode mode)
+{
+	struct pt_bcache_entry bce;
+
+	memset(&bce, 0, sizeof(bce));
+	bce.ninsn = 1;
+	bce.mode = mode;
+	bce.qualifier = ptbq_decode;
+
+	return pt_bcache_add(bcache, ioff, bce);
+}
+
+enum {
+	/* The maximum number of steps when filling the block cache. */
+	bcache_fill_steps	= 0x400
+};
+
+/* Proceed to the next instruction and fill the block cache for @decoder->ip.
+ *
+ * Tracing is enabled and we don't have an event pending.  The current IP is not
+ * yet cached.
+ *
+ * Proceed one instruction without using the block cache, then try to proceed
+ * further using the block cache.
+ *
+ * On our way back, add a block cache entry for the IP before proceeding.  Note
+ * that the recursion is bounded by @steps and ultimately by the maximum number
+ * of instructions in a block.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int
+pt_blk_proceed_no_event_fill_cache(struct pt_block_decoder *decoder,
+				   struct pt_block *block,
+				   struct pt_block_cache *bcache,
+				   const struct pt_mapped_section *msec,
+				   size_t steps)
+{
+	struct pt_bcache_entry bce;
+	struct pt_insn_ext iext;
+	struct pt_insn insn;
+	uint64_t nip, dip, ioff, noff;
+	int64_t disp;
+	int status;
+
+	if (!decoder || !steps)
+		return -pte_internal;
+
+	/* Proceed one instruction by decoding and examining it.
+	 *
+	 * Note that we also return on a status of zero that indicates that the
+	 * instruction didn't fit into @block.
+	 */
+	status = pt_blk_proceed_one_insn(decoder, block, &insn, &iext);
+	if (status <= 0)
+		return status;
+
+	ioff = pt_msec_unmap(msec, insn.ip);
+
+	/* Let's see if we can proceed to the next IP without trace.
+	 *
+	 * If we can't, this is certainly a decision point.
+	 */
+	status = pt_insn_next_ip(&decoder->ip, &insn, &iext);
+	if (status < 0) {
+		if (status != -pte_bad_query)
+			return status;
+
+		memset(&bce, 0, sizeof(bce));
+		bce.ninsn = 1;
+		bce.mode = insn.mode;
+		bce.isize = insn.size;
+
+		/* Clear the instruction size in case of overflows. */
+		if ((uint8_t) bce.isize != insn.size)
+			bce.isize = 0;
+
+		switch (insn.iclass) {
+		case ptic_ptwrite:
+		case ptic_error:
+		case ptic_other:
+			return -pte_internal;
+
+		case ptic_jump:
+			/* A direct jump doesn't require trace. */
+			if (iext.variant.branch.is_direct)
+				return -pte_internal;
+
+			bce.qualifier = ptbq_indirect;
+			break;
+
+		case ptic_call:
+			/* A direct call doesn't require trace. */
+			if (iext.variant.branch.is_direct)
+				return -pte_internal;
+
+			bce.qualifier = ptbq_ind_call;
+			break;
+
+		case ptic_return:
+			bce.qualifier = ptbq_return;
+			break;
+
+		case ptic_cond_jump:
+			bce.qualifier = ptbq_cond;
+			break;
+
+		case ptic_far_call:
+		case ptic_far_return:
+		case ptic_far_jump:
+			bce.qualifier = ptbq_indirect;
+			break;
+		}
+
+		/* If the block was truncated, we have to decode its last
+		 * instruction each time.
+		 *
+		 * We could have skipped the above switch and size assignment in
+		 * this case but this is already a slow and hopefully infrequent
+		 * path.
+		 */
+		if (block->truncated)
+			bce.qualifier = ptbq_decode;
+
+		status = pt_bcache_add(bcache, ioff, bce);
+		if (status < 0)
+			return status;
+
+		return pt_blk_proceed_with_trace(decoder, &insn, &iext);
+	}
+
+	/* The next instruction's IP. */
+	nip = decoder->ip;
+	noff = pt_msec_unmap(msec, nip);
+
+	/* Even if we were able to proceed without trace, we might have to stop
+	 * here for various reasons:
+	 *
+	 *   - at near direct calls to update the return-address stack
+	 *
+	 *     We are forced to re-decode @insn to get the branch displacement.
+	 *
+	 *     Even though it is constant, we don't cache it to avoid increasing
+	 *     the size of a cache entry.  Note that the displacement field is
+	 *     zero for this entry and we might be tempted to use it - but other
+	 *     entries that point to this decision point will have non-zero
+	 *     displacement.
+	 *
+	 *     We could proceed after a near direct call but we migh as well
+	 *     postpone it to the next iteration.  Make sure to end the block if
+	 *     @decoder->flags.variant.block.end_on_call is set, though.
+	 *
+	 *   - at near direct backwards jumps to detect section splits
+	 *
+	 *     In case the current section is split underneath us, we must take
+	 *     care to detect that split.
+	 *
+	 *     There is one corner case where the split is in the middle of a
+	 *     linear sequence of instructions that branches back into the
+	 *     originating section.
+	 *
+	 *     Calls, indirect branches, and far branches are already covered
+	 *     since they either require trace or already require us to stop
+	 *     (i.e. near direct calls) for other reasons.  That leaves near
+	 *     direct backward jumps.
+	 *
+	 *     Instead of the decode stop at the jump instruction we're using we
+	 *     could have made sure that other block cache entries that extend
+	 *     this one insert a trampoline to the jump's entry.  This would
+	 *     have been a bit more complicated.
+	 *
+	 *   - if we switched sections
+	 *
+	 *     This ends a block just like a branch that requires trace.
+	 *
+	 *     We need to re-decode @insn in order to determine the start IP of
+	 *     the next block.
+	 *
+	 *   - if the block is truncated
+	 *
+	 *     We need to read the last instruction's memory from multiple
+	 *     sections and provide it to the user.
+	 *
+	 *     We could still use the block cache but then we'd have to handle
+	 *     this case for each qualifier.  Truncation is hopefully rare and
+	 *     having to read the memory for the instruction from multiple
+	 *     sections is already slow.  Let's rather keep things simple and
+	 *     route it through the decode flow, where we already have
+	 *     everything in place.
+	 */
+	switch (insn.iclass) {
+	case ptic_call:
+		return pt_blk_add_decode(bcache, ioff, insn.mode);
+
+	case ptic_jump:
+		/* An indirect branch requires trace and should have been
+		 * handled above.
+		 */
+		if (!iext.variant.branch.is_direct)
+			return -pte_internal;
+
+		if (iext.variant.branch.displacement < 0 ||
+		    decoder->flags.variant.block.end_on_jump)
+			return pt_blk_add_decode(bcache, ioff, insn.mode);
+
+		fallthrough;
+	default:
+		if (!pt_blk_is_in_section(msec, nip) || block->truncated)
+			return pt_blk_add_decode(bcache, ioff, insn.mode);
+
+		break;
+	}
+
+	/* We proceeded one instruction.  Let's see if we have a cache entry for
+	 * the next instruction.
+	 */
+	status = pt_bcache_lookup(&bce, bcache, noff);
+	if (status < 0)
+		return status;
+
+	/* If we don't have a valid cache entry, yet, fill the cache some more.
+	 *
+	 * On our way back, we add a cache entry for this instruction based on
+	 * the cache entry of the succeeding instruction.
+	 */
+	if (!pt_bce_is_valid(bce)) {
+		/* If we exceeded the maximum number of allowed steps, we insert
+		 * a trampoline to the next instruction.
+		 *
+		 * The next time we encounter the same code, we will use the
+		 * trampoline to jump directly to where we left off this time
+		 * and continue from there.
+		 */
+		steps -= 1;
+		if (!steps)
+			return pt_blk_add_trampoline(bcache, ioff, noff,
+						     insn.mode);
+
+		status = pt_blk_proceed_no_event_fill_cache(decoder, block,
+							    bcache, msec,
+							    steps);
+		if (status < 0)
+			return status;
+
+		/* Let's see if we have more luck this time. */
+		status = pt_bcache_lookup(&bce, bcache, noff);
+		if (status < 0)
+			return status;
+
+		/* If we still don't have a valid cache entry, we're done.  Most
+		 * likely, @block overflowed and we couldn't proceed past the
+		 * next instruction.
+		 */
+		if (!pt_bce_is_valid(bce))
+			return 0;
+	}
+
+	/* We must not have switched execution modes.
+	 *
+	 * This would require an event and we're on the no-event flow.
+	 */
+	if (pt_bce_exec_mode(bce) != insn.mode)
+		return -pte_internal;
+
+	/* The decision point IP and the displacement from @insn.ip. */
+	dip = nip + (uint64_t) (int64_t) bce.displacement;
+	disp = (int64_t) (dip - insn.ip);
+
+	/* We may have switched sections if the section was split.  See
+	 * pt_blk_proceed_no_event_cached() for a more elaborate comment.
+	 *
+	 * We're not adding a block cache entry since this won't apply to the
+	 * original section which may be shared with other decoders.
+	 *
+	 * We will instead take the slow path until the end of the section.
+	 */
+	if (!pt_blk_is_in_section(msec, dip))
+		return 0;
+
+	/* Let's try to reach @nip's decision point from @insn.ip.
+	 *
+	 * There are two fields that may overflow: @bce.ninsn and
+	 * @bce.displacement.
+	 */
+	bce.ninsn += 1;
+	bce.displacement = (int32_t) disp;
+
+	/* If none of them overflowed, we're done.
+	 *
+	 * If one or both overflowed, let's try to insert a trampoline, i.e. we
+	 * try to reach @dip via a ptbq_again entry to @nip.
+	 */
+	if (!bce.ninsn || ((int64_t) bce.displacement != disp))
+		return pt_blk_add_trampoline(bcache, ioff, noff, insn.mode);
+
+	/* We're done.  Add the cache entry.
+	 *
+	 * There's a chance that other decoders updated the cache entry in the
+	 * meantime.  They should have come to the same conclusion as we,
+	 * though, and the cache entries should be identical.
+	 *
+	 * Cache updates are atomic so even if the two versions were not
+	 * identical, we wouldn't care because they are both correct.
+	 */
+	return pt_bcache_add(bcache, ioff, bce);
+}
+
+/* Proceed at a potentially truncated instruction.
+ *
+ * We were not able to decode the instruction at @decoder->ip in @decoder's
+ * cached section.  This is typically caused by not having enough bytes.
+ *
+ * Try to decode the instruction again using the entire image.  If this succeeds
+ * we expect to end up with an instruction that was truncated in the section it
+ * started.  We provide the full instruction in this case and end the block.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_proceed_truncated(struct pt_block_decoder *decoder,
+				    struct pt_block *block)
+{
+	struct pt_insn_ext iext;
+	struct pt_insn insn;
+	int errcode;
+
+	if (!decoder || !block)
+		return -pte_internal;
+
+	memset(&iext, 0, sizeof(iext));
+	memset(&insn, 0, sizeof(insn));
+
+	insn.mode = decoder->mode;
+	insn.ip = decoder->ip;
+
+	errcode = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);
+	if (errcode < 0)
+		return errcode;
+
+	/* We shouldn't use this function if the instruction isn't truncated. */
+	if (!insn.truncated)
+		return -pte_internal;
+
+	/* Provide the instruction in the block.  This ends the block. */
+	memcpy(block->raw, insn.raw, insn.size);
+	block->iclass = insn.iclass;
+	block->size = insn.size;
+	block->truncated = 1;
+
+	/* Log calls' return addresses for return compression. */
+	errcode = pt_blk_log_call(decoder, &insn, &iext);
+	if (errcode < 0)
+		return errcode;
+
+	/* Let's see if we can proceed to the next IP without trace.
+	 *
+	 * The truncated instruction ends the block but we still need to get the
+	 * next block's start IP.
+	 */
+	errcode = pt_insn_next_ip(&decoder->ip, &insn, &iext);
+	if (errcode < 0) {
+		if (errcode != -pte_bad_query)
+			return errcode;
+
+		return pt_blk_proceed_with_trace(decoder, &insn, &iext);
+	}
+
+	return 0;
+}
+
+/* Proceed to the next decision point using the block cache.
+ *
+ * Tracing is enabled and we don't have an event pending.  We already set
+ * @block's isid.  All reads are done within @msec as we're not switching
+ * sections between blocks.
+ *
+ * Proceed as far as we get without trace.  Stop when we either:
+ *
+ *   - need trace in order to continue
+ *   - overflow the max number of instructions in a block
+ *
+ * We actually proceed one instruction further to get the start IP for the next
+ * block.  This only updates @decoder's internal state, though.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_proceed_no_event_cached(struct pt_block_decoder *decoder,
+					  struct pt_block *block,
+					  struct pt_block_cache *bcache,
+					  const struct pt_mapped_section *msec)
+{
+	struct pt_bcache_entry bce;
+	uint16_t binsn, ninsn;
+	uint64_t offset, nip;
+	int status;
+
+	if (!decoder || !block)
+		return -pte_internal;
+
+	offset = pt_msec_unmap(msec, decoder->ip);
+	status = pt_bcache_lookup(&bce, bcache, offset);
+	if (status < 0)
+		return status;
+
+	/* If we don't find a valid cache entry, fill the cache. */
+	if (!pt_bce_is_valid(bce))
+		return pt_blk_proceed_no_event_fill_cache(decoder, block,
+							  bcache, msec,
+							  bcache_fill_steps);
+
+	/* If we switched sections, the origianl section must have been split
+	 * underneath us.  A split preserves the block cache of the original
+	 * section.
+	 *
+	 * Crossing sections requires ending the block so we can indicate the
+	 * proper isid for the entire block.
+	 *
+	 * Plus there's the chance that the new section that caused the original
+	 * section to split changed instructions.
+	 *
+	 * This check will also cover changes to a linear sequence of code we
+	 * would otherwise have jumped over as long as the start and end are in
+	 * different sub-sections.
+	 *
+	 * Since we stop on every (backwards) branch (through an artificial stop
+	 * in the case of a near direct backward branch) we will detect all
+	 * section splits.
+	 *
+	 * Switch to the slow path until we reach the end of this section.
+	 */
+	nip = decoder->ip + (uint64_t) (int64_t) bce.displacement;
+	if (!pt_blk_is_in_section(msec, nip))
+		return pt_blk_proceed_no_event_uncached(decoder, block);
+
+	/* We have a valid cache entry.  Let's first check if the way to the
+	 * decision point still fits into @block.
+	 *
+	 * If it doesn't, we end the block without filling it as much as we
+	 * could since this would require us to switch to the slow path.
+	 *
+	 * On the next iteration, we will start with an empty block, which is
+	 * guaranteed to have enough room for at least one block cache entry.
+	 */
+	binsn = block->ninsn;
+	ninsn = binsn + (uint16_t) bce.ninsn;
+	if (ninsn < binsn)
+		return 0;
+
+	/* Jump ahead to the decision point and proceed from there.
+	 *
+	 * We're not switching execution modes so even if @block already has an
+	 * execution mode, it will be the one we're going to set.
+	 */
+	decoder->ip = nip;
+
+	/* We don't know the instruction class so we should be setting it to
+	 * ptic_error.  Since we will be able to fill it back in later in most
+	 * cases, we move the clearing to the switch cases that don't.
+	 */
+	block->end_ip = nip;
+	block->ninsn = ninsn;
+	block->mode = pt_bce_exec_mode(bce);
+
+
+	switch (pt_bce_qualifier(bce)) {
+	case ptbq_again:
+		/* We're not able to reach the actual decision point due to
+		 * overflows so we inserted a trampoline.
+		 *
+		 * We don't know the instruction and it is not guaranteed that
+		 * we will proceed further (e.g. if @block overflowed).  Let's
+		 * clear any previously stored instruction class which has
+		 * become invalid when we updated @block->ninsn.
+		 */
+		block->iclass = ptic_error;
+
+		return pt_blk_proceed_no_event_cached(decoder, block, bcache,
+						      msec);
+
+	case ptbq_cond:
+		/* We're at a conditional branch. */
+		block->iclass = ptic_cond_jump;
+
+		/* Let's first check whether we know the size of the
+		 * instruction.  If we do, we might get away without decoding
+		 * the instruction.
+		 *
+		 * If we don't know the size we might as well do the full decode
+		 * and proceed-with-trace flow we do for ptbq_decode.
+		 */
+		if (bce.isize) {
+			uint64_t ip;
+			int taken;
+
+			/* If the branch is not taken, we don't need to decode
+			 * the instruction at @decoder->ip.
+			 *
+			 * If it is taken, we have to implement everything here.
+			 * We can't use the normal decode and proceed-with-trace
+			 * flow since we already consumed the TNT bit.
+			 */
+			status = pt_blk_cond_branch(decoder, &taken);
+			if (status < 0)
+				return status;
+
+			/* Preserve the query decoder's response which indicates
+			 * upcoming events.
+			 */
+			decoder->status = status;
+
+			ip = decoder->ip;
+			if (taken) {
+				struct pt_insn_ext iext;
+				struct pt_insn insn;
+
+				memset(&iext, 0, sizeof(iext));
+				memset(&insn, 0, sizeof(insn));
+
+				insn.mode = pt_bce_exec_mode(bce);
+				insn.ip = ip;
+
+				status = pt_blk_decode_in_section(&insn, &iext,
+								  msec);
+				if (status < 0)
+					return status;
+
+				ip += (uint64_t) (int64_t)
+					iext.variant.branch.displacement;
+			}
+
+			decoder->ip = ip + bce.isize;
+			break;
+		}
+
+		fallthrough;
+	case ptbq_decode: {
+		struct pt_insn_ext iext;
+		struct pt_insn insn;
+
+		/* We need to decode the instruction at @decoder->ip and decide
+		 * what to do based on that.
+		 *
+		 * We already accounted for the instruction so we can't just
+		 * call pt_blk_proceed_one_insn().
+		 */
+
+		memset(&iext, 0, sizeof(iext));
+		memset(&insn, 0, sizeof(insn));
+
+		insn.mode = pt_bce_exec_mode(bce);
+		insn.ip = decoder->ip;
+
+		status = pt_blk_decode_in_section(&insn, &iext, msec);
+		if (status < 0) {
+			if (status != -pte_bad_insn)
+				return status;
+
+			return pt_blk_proceed_truncated(decoder, block);
+		}
+
+		/* We just decoded @insn so we know the instruction class. */
+		block->iclass = insn.iclass;
+
+		/* Log calls' return addresses for return compression. */
+		status = pt_blk_log_call(decoder, &insn, &iext);
+		if (status < 0)
+			return status;
+
+		/* Let's see if we can proceed to the next IP without trace.
+		 *
+		 * Note that we also stop due to displacement overflows or to
+		 * maintain the return-address stack for near direct calls.
+		 */
+		status = pt_insn_next_ip(&decoder->ip, &insn, &iext);
+		if (status < 0) {
+			if (status != -pte_bad_query)
+				return status;
+
+			/* We can't, so let's proceed with trace, which
+			 * completes the block.
+			 */
+			return pt_blk_proceed_with_trace(decoder, &insn, &iext);
+		}
+
+		/* End the block if the user asked us to.
+		 *
+		 * We only need to take care about direct near branches.
+		 * Indirect and far branches require trace and will naturally
+		 * end a block.
+		 */
+		if ((decoder->flags.variant.block.end_on_call &&
+		     (insn.iclass == ptic_call)) ||
+		    (decoder->flags.variant.block.end_on_jump &&
+		     (insn.iclass == ptic_jump)))
+			break;
+
+		/* If we can proceed without trace and we stay in @msec we may
+		 * proceed further.
+		 *
+		 * We're done if we switch sections, though.
+		 */
+		if (!pt_blk_is_in_section(msec, decoder->ip))
+			break;
+
+		return pt_blk_proceed_no_event_cached(decoder, block, bcache,
+						      msec);
+	}
+
+	case ptbq_ind_call: {
+		uint64_t ip;
+
+		/* We're at a near indirect call. */
+		block->iclass = ptic_call;
+
+		/* We need to update the return-address stack and query the
+		 * destination IP.
+		 */
+		ip = decoder->ip;
+
+		/* If we already know the size of the instruction, we don't need
+		 * to re-decode it.
+		 */
+		if (bce.isize)
+			ip += bce.isize;
+		else {
+			struct pt_insn_ext iext;
+			struct pt_insn insn;
+
+			memset(&iext, 0, sizeof(iext));
+			memset(&insn, 0, sizeof(insn));
+
+			insn.mode = pt_bce_exec_mode(bce);
+			insn.ip = ip;
+
+			status = pt_blk_decode_in_section(&insn, &iext, msec);
+			if (status < 0)
+				return status;
+
+			ip += insn.size;
+		}
+
+		status = pt_retstack_push(&decoder->retstack, ip);
+		if (status < 0)
+			return status;
+
+		status = pt_blk_indirect_branch(decoder, &decoder->ip);
+		if (status < 0)
+			return status;
+
+		/* Preserve the query decoder's response which indicates
+		 * upcoming events.
+		 */
+		decoder->status = status;
+		break;
+	}
+
+	case ptbq_return: {
+		int taken;
+
+		/* We're at a near return. */
+		block->iclass = ptic_return;
+
+		/* Check for a compressed return. */
+		status = pt_blk_cond_branch(decoder, &taken);
+		if (status < 0) {
+			if (status != -pte_bad_query)
+				return status;
+
+			/* The return is not compressed.  We need another query
+			 * to determine the destination IP.
+			 */
+			status = pt_blk_indirect_branch(decoder, &decoder->ip);
+			if (status < 0)
+				return status;
+
+			/* Preserve the query decoder's response which indicates
+			 * upcoming events.
+			 */
+			decoder->status = status;
+			break;
+		}
+
+		/* Preserve the query decoder's response which indicates
+		 * upcoming events.
+		 */
+		decoder->status = status;
+
+		/* A compressed return is indicated by a taken conditional
+		 * branch.
+		 */
+		if (!taken)
+			return -pte_bad_retcomp;
+
+		return pt_retstack_pop(&decoder->retstack, &decoder->ip);
+	}
+
+	case ptbq_indirect:
+		/* We're at an indirect jump or far transfer.
+		 *
+		 * We don't know the exact instruction class and there's no
+		 * reason to decode the instruction for any other purpose.
+		 *
+		 * Indicate that we don't know the instruction class and leave
+		 * it to our caller to decode the instruction if needed.
+		 */
+		block->iclass = ptic_error;
+
+		/* This is neither a near call nor return so we don't need to
+		 * touch the return-address stack.
+		 *
+		 * Just query the destination IP.
+		 */
+		status = pt_blk_indirect_branch(decoder, &decoder->ip);
+		if (status < 0)
+			return status;
+
+		/* Preserve the query decoder's response which indicates
+		 * upcoming events.
+		 */
+		decoder->status = status;
+		break;
+	}
+
+	return 0;
+}
+
+static int pt_blk_msec_fill(struct pt_block_decoder *decoder,
+			    const struct pt_mapped_section **pmsec)
+{
+	const struct pt_mapped_section *msec;
+	struct pt_section *section;
+	int isid, errcode;
+
+	if (!decoder || !pmsec)
+		return -pte_internal;
+
+	isid = pt_msec_cache_fill(&decoder->scache, &msec,  decoder->image,
+				  &decoder->asid, decoder->ip);
+	if (isid < 0)
+		return isid;
+
+	section = pt_msec_section(msec);
+	if (!section)
+		return -pte_internal;
+
+	*pmsec = msec;
+
+	errcode = pt_section_request_bcache(section);
+	if (errcode < 0)
+		return errcode;
+
+	return isid;
+}
+
+static inline int pt_blk_msec_lookup(struct pt_block_decoder *decoder,
+				     const struct pt_mapped_section **pmsec)
+{
+	int isid;
+
+	if (!decoder)
+		return -pte_internal;
+
+	isid = pt_msec_cache_read(&decoder->scache, pmsec, decoder->image,
+				  decoder->ip);
+	if (isid < 0) {
+		if (isid != -pte_nomap)
+			return isid;
+
+		return pt_blk_msec_fill(decoder, pmsec);
+	}
+
+	return isid;
+}
+
+/* Proceed to the next decision point - try using the cache.
+ *
+ * Tracing is enabled and we don't have an event pending.  Proceed as far as
+ * we get without trace.  Stop when we either:
+ *
+ *   - need trace in order to continue
+ *   - overflow the max number of instructions in a block
+ *
+ * We actually proceed one instruction further to get the start IP for the next
+ * block.  This only updates @decoder's internal state, though.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_proceed_no_event(struct pt_block_decoder *decoder,
+				   struct pt_block *block)
+{
+	const struct pt_mapped_section *msec;
+	struct pt_block_cache *bcache;
+	struct pt_section *section;
+	int isid;
+
+	if (!decoder || !block)
+		return -pte_internal;
+
+	isid = pt_blk_msec_lookup(decoder, &msec);
+	if (isid < 0) {
+		if (isid != -pte_nomap)
+			return isid;
+
+		/* Even if there is no such section in the image, we may still
+		 * read the memory via the callback function.
+		 */
+		return pt_blk_proceed_no_event_uncached(decoder, block);
+	}
+
+	/* We do not switch sections inside a block. */
+	if (isid != block->isid) {
+		if (!pt_blk_block_is_empty(block))
+			return 0;
+
+		block->isid = isid;
+	}
+
+	section = pt_msec_section(msec);
+	if (!section)
+		return -pte_internal;
+
+	bcache = pt_section_bcache(section);
+	if (!bcache)
+		return pt_blk_proceed_no_event_uncached(decoder, block);
+
+	return pt_blk_proceed_no_event_cached(decoder, block, bcache, msec);
+}
+
+/* Proceed to the next event or decision point.
+ *
+ * Returns a non-negative pt_status_flag bit-vector on success, a negative error
+ * code otherwise.
+ */
+static int pt_blk_proceed(struct pt_block_decoder *decoder,
+			  struct pt_block *block)
+{
+	int status;
+
+	status = pt_blk_fetch_event(decoder);
+	if (status != 0) {
+		if (status < 0)
+			return status;
+
+		return pt_blk_proceed_event(decoder, block);
+	}
+
+	/* If tracing is disabled we should either be out of trace or we should
+	 * have taken the event flow above.
+	 */
+	if (!decoder->enabled) {
+		if (decoder->status & pts_eos)
+			return -pte_eos;
+
+		return -pte_no_enable;
+	}
+
+	status = pt_blk_proceed_no_event(decoder, block);
+	if (status < 0)
+		return status;
+
+	return pt_blk_proceed_trailing_event(decoder, block);
+}
+
+enum {
+	/* The maximum number of steps to take when determining whether the
+	 * event location can be reached.
+	 */
+	bdm64_max_steps	= 0x100
+};
+
+/* Try to work around erratum BDM64.
+ *
+ * If we got a transaction abort immediately following a branch that produced
+ * trace, the trace for that branch might have been corrupted.
+ *
+ * Returns a positive integer if the erratum was handled.
+ * Returns zero if the erratum does not seem to apply.
+ * Returns a negative error code otherwise.
+ */
+static int pt_blk_handle_erratum_bdm64(struct pt_block_decoder *decoder,
+				       const struct pt_block *block,
+				       const struct pt_event *ev)
+{
+	struct pt_insn_ext iext;
+	struct pt_insn insn;
+	int status;
+
+	if (!decoder || !block || !ev)
+		return -pte_internal;
+
+	/* This only affects aborts. */
+	if (!ev->variant.tsx.aborted)
+		return 0;
+
+	/* This only affects branches that require trace.
+	 *
+	 * If the erratum hits, that branch ended the current block and brought
+	 * us to the trailing event flow.
+	 */
+	if (pt_blk_block_is_empty(block))
+		return 0;
+
+	insn.mode = block->mode;
+	insn.ip = block->end_ip;
+
+	status = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);
+	if (status < 0)
+		return 0;
+
+	if (!pt_insn_is_branch(&insn, &iext))
+		return 0;
+
+	/* Let's check if we can reach the event location from here.
+	 *
+	 * If we can, let's assume the erratum did not hit.  We might still be
+	 * wrong but we're not able to tell.
+	 */
+	status = pt_insn_range_is_contiguous(decoder->ip, ev->variant.tsx.ip,
+					     decoder->mode, decoder->image,
+					     &decoder->asid, bdm64_max_steps);
+	if (status > 0)
+		return status;
+
+	/* We can't reach the event location.  This could either mean that we
+	 * stopped too early (and status is zero) or that the erratum hit.
+	 *
+	 * We assume the latter and pretend that the previous branch brought us
+	 * to the event location, instead.
+	 */
+	decoder->ip = ev->variant.tsx.ip;
+
+	return 1;
+}
+
+/* Check whether a trailing TSX event should be postponed.
+ *
+ * This involves handling erratum BDM64.
+ *
+ * Returns a positive integer if the event is to be postponed.
+ * Returns zero if the event should be processed.
+ * Returns a negative error code otherwise.
+ */
+static inline int pt_blk_postpone_trailing_tsx(struct pt_block_decoder *decoder,
+					       struct pt_block *block,
+					       const struct pt_event *ev)
+{
+	int status;
+
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	if (ev->ip_suppressed)
+		return 0;
+
+	if (block && decoder->query.config.errata.bdm64) {
+		status = pt_blk_handle_erratum_bdm64(decoder, block, ev);
+		if (status < 0)
+			return 1;
+	}
+
+	if (decoder->ip != ev->variant.tsx.ip)
+		return 1;
+
+	return 0;
+}
+
+/* Proceed with events that bind to the current decoder IP.
+ *
+ * This function is used in the following scenarios:
+ *
+ *   - we just synchronized onto the trace stream
+ *   - we ended a block and proceeded to the next IP
+ *   - we processed an event that was indicated by this function
+ *
+ * Check if there is an event at the current IP that needs to be indicated to
+ * the user.
+ *
+ * Returns a non-negative pt_status_flag bit-vector on success, a negative error
+ * code otherwise.
+ */
+static int pt_blk_proceed_trailing_event(struct pt_block_decoder *decoder,
+					 struct pt_block *block)
+{
+	struct pt_event *ev;
+	int status;
+
+	if (!decoder)
+		return -pte_internal;
+
+	status = pt_blk_fetch_event(decoder);
+	if (status <= 0) {
+		if (status < 0)
+			return status;
+
+		status = pt_blk_proceed_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		return pt_blk_status(decoder, 0);
+	}
+
+	ev = &decoder->event;
+	switch (ev->type) {
+	case ptev_disabled:
+		/* Synchronous disable events are normally indicated on the
+		 * event flow.
+		 */
+		if (!decoder->process_insn)
+			break;
+
+		/* A sync disable may bind to a CR3 changing instruction. */
+		if (ev->ip_suppressed &&
+		    pt_insn_changes_cr3(&decoder->insn, &decoder->iext))
+			return pt_blk_status(decoder, pts_event_pending);
+
+		/* Or it binds to the next branch that would require trace.
+		 *
+		 * Try to complete processing the current instruction by
+		 * proceeding past it.  If that fails because it would require
+		 * trace, we can apply the disabled event.
+		 */
+		status = pt_insn_next_ip(&decoder->ip, &decoder->insn,
+					 &decoder->iext);
+		if (status < 0) {
+			if (status != -pte_bad_query)
+				return status;
+
+			status = pt_blk_set_disable_resume_ip(decoder,
+							      &decoder->insn);
+			if (status < 0)
+				return status;
+
+			return pt_blk_status(decoder, pts_event_pending);
+		}
+
+		/* We proceeded past the current instruction. */
+		status = pt_blk_clear_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		/* This might have brought us to the disable IP. */
+		if (!ev->ip_suppressed &&
+		    decoder->ip == ev->variant.disabled.ip)
+			return pt_blk_status(decoder, pts_event_pending);
+
+		break;
+
+	case ptev_enabled:
+		/* This event does not bind to an instruction. */
+		status = pt_blk_proceed_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_async_disabled:
+		/* This event does not bind to an instruction. */
+		status = pt_blk_proceed_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		if (decoder->ip != ev->variant.async_disabled.at)
+			break;
+
+		if (decoder->query.config.errata.skd022) {
+			status = pt_blk_handle_erratum_skd022(decoder, ev);
+			if (status != 0) {
+				if (status < 0)
+					return status;
+
+				/* If the erratum applies, the event is modified
+				 * to a synchronous disable event that will be
+				 * processed on the next pt_blk_proceed_event()
+				 * call.  We're done.
+				 */
+				break;
+			}
+		}
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_async_branch:
+		/* This event does not bind to an instruction. */
+		status = pt_blk_proceed_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		if (decoder->ip != ev->variant.async_branch.from)
+			break;
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_paging:
+		/* We apply the event immediately if we're not tracing. */
+		if (!decoder->enabled)
+			return pt_blk_status(decoder, pts_event_pending);
+
+		/* Synchronous paging events are normally indicated on the event
+		 * flow, unless they bind to the same instruction as a previous
+		 * event.
+		 *
+		 * We bind at most one paging event to an instruction, though.
+		 */
+		if (!decoder->process_insn || decoder->bound_paging)
+			break;
+
+		/* We're done if we're not binding to the currently postponed
+		 * instruction.  We will process the event on the normal event
+		 * flow in the next iteration.
+		 */
+		if (!pt_insn_binds_to_pip(&decoder->insn, &decoder->iext))
+			break;
+
+		/* We bound a paging event.  Make sure we do not bind further
+		 * paging events to this instruction.
+		 */
+		decoder->bound_paging = 1;
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_async_paging:
+		/* This event does not bind to an instruction. */
+		status = pt_blk_proceed_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		if (!ev->ip_suppressed &&
+		    decoder->ip != ev->variant.async_paging.ip)
+			break;
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_vmcs:
+		/* We apply the event immediately if we're not tracing. */
+		if (!decoder->enabled)
+			return pt_blk_status(decoder, pts_event_pending);
+
+		/* Synchronous vmcs events are normally indicated on the event
+		 * flow, unless they bind to the same instruction as a previous
+		 * event.
+		 *
+		 * We bind at most one vmcs event to an instruction, though.
+		 */
+		if (!decoder->process_insn || decoder->bound_vmcs)
+			break;
+
+		/* We're done if we're not binding to the currently postponed
+		 * instruction.  We will process the event on the normal event
+		 * flow in the next iteration.
+		 */
+		if (!pt_insn_binds_to_vmcs(&decoder->insn, &decoder->iext))
+			break;
+
+		/* We bound a vmcs event.  Make sure we do not bind further vmcs
+		 * events to this instruction.
+		 */
+		decoder->bound_vmcs = 1;
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_async_vmcs:
+		/* This event does not bind to an instruction. */
+		status = pt_blk_proceed_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		if (!ev->ip_suppressed &&
+		    decoder->ip != ev->variant.async_vmcs.ip)
+			break;
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_overflow:
+		/* This event does not bind to an instruction. */
+		status = pt_blk_proceed_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_exec_mode:
+		/* This event does not bind to an instruction. */
+		status = pt_blk_proceed_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		if (!ev->ip_suppressed &&
+		    decoder->ip != ev->variant.exec_mode.ip)
+			break;
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_tsx:
+		/* This event does not bind to an instruction. */
+		status = pt_blk_proceed_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		status = pt_blk_postpone_trailing_tsx(decoder, block, ev);
+		if (status != 0) {
+			if (status < 0)
+				return status;
+
+			break;
+		}
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_stop:
+		/* This event does not bind to an instruction. */
+		status = pt_blk_proceed_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_exstop:
+		/* This event does not bind to an instruction. */
+		status = pt_blk_proceed_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		if (!ev->ip_suppressed && decoder->enabled &&
+		    decoder->ip != ev->variant.exstop.ip)
+			break;
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_mwait:
+		/* This event does not bind to an instruction. */
+		status = pt_blk_proceed_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		if (!ev->ip_suppressed && decoder->enabled &&
+		    decoder->ip != ev->variant.mwait.ip)
+			break;
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_pwre:
+	case ptev_pwrx:
+		/* This event does not bind to an instruction. */
+		status = pt_blk_proceed_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_ptwrite:
+		/* We apply the event immediately if we're not tracing. */
+		if (!decoder->enabled)
+			return pt_blk_status(decoder, pts_event_pending);
+
+		/* Ptwrite events are normally indicated on the event flow,
+		 * unless they bind to the same instruction as a previous event.
+		 *
+		 * We bind at most one ptwrite event to an instruction, though.
+		 */
+		if (!decoder->process_insn || decoder->bound_ptwrite)
+			break;
+
+		/* We're done if we're not binding to the currently postponed
+		 * instruction.  We will process the event on the normal event
+		 * flow in the next iteration.
+		 */
+		if (!ev->ip_suppressed ||
+		    !pt_insn_is_ptwrite(&decoder->insn, &decoder->iext))
+			break;
+
+		/* We bound a ptwrite event.  Make sure we do not bind further
+		 * ptwrite events to this instruction.
+		 */
+		decoder->bound_ptwrite = 1;
+
+		return pt_blk_status(decoder, pts_event_pending);
+
+	case ptev_tick:
+	case ptev_cbr:
+	case ptev_mnt:
+		/* This event does not bind to an instruction. */
+		status = pt_blk_proceed_postponed_insn(decoder);
+		if (status < 0)
+			return status;
+
+		return pt_blk_status(decoder, pts_event_pending);
+	}
+
+	/* No further events.  Proceed past any postponed instruction. */
+	status = pt_blk_proceed_postponed_insn(decoder);
+	if (status < 0)
+		return status;
+
+	return pt_blk_status(decoder, 0);
+}
+
+int pt_blk_next(struct pt_block_decoder *decoder, struct pt_block *ublock,
+		size_t size)
+{
+	struct pt_block block, *pblock;
+	int errcode, status;
+
+	if (!decoder || !ublock)
+		return -pte_invalid;
+
+	pblock = size == sizeof(block) ? ublock : &block;
+
+	/* Zero-initialize the block in case of error returns. */
+	memset(pblock, 0, sizeof(*pblock));
+
+	/* Fill in a few things from the current decode state.
+	 *
+	 * This reflects the state of the last pt_blk_next() or pt_blk_start()
+	 * call.  Note that, unless we stop with tracing disabled, we proceed
+	 * already to the start IP of the next block.
+	 *
+	 * Some of the state may later be overwritten as we process events.
+	 */
+	pblock->ip = decoder->ip;
+	pblock->mode = decoder->mode;
+	if (decoder->speculative)
+		pblock->speculative = 1;
+
+	/* Proceed one block. */
+	status = pt_blk_proceed(decoder, pblock);
+
+	errcode = block_to_user(ublock, size, pblock);
+	if (errcode < 0)
+		return errcode;
+
+	return status;
+}
+
+/* Process an enabled event.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_process_enabled(struct pt_block_decoder *decoder,
+				  const struct pt_event *ev)
+{
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	/* This event can't be a status update. */
+	if (ev->status_update)
+		return -pte_bad_context;
+
+	/* We must have an IP in order to start decoding. */
+	if (ev->ip_suppressed)
+		return -pte_noip;
+
+	/* We must currently be disabled. */
+	if (decoder->enabled)
+		return -pte_bad_context;
+
+	decoder->ip = ev->variant.enabled.ip;
+	decoder->enabled = 1;
+	decoder->process_event = 0;
+
+	return 0;
+}
+
+/* Process a disabled event.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_process_disabled(struct pt_block_decoder *decoder,
+				   const struct pt_event *ev)
+{
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	/* This event can't be a status update. */
+	if (ev->status_update)
+		return -pte_bad_context;
+
+	/* We must currently be enabled. */
+	if (!decoder->enabled)
+		return -pte_bad_context;
+
+	/* We preserve @decoder->ip.  This is where we expect tracing to resume
+	 * and we'll indicate that on the subsequent enabled event if tracing
+	 * actually does resume from there.
+	 */
+	decoder->enabled = 0;
+	decoder->process_event = 0;
+
+	return 0;
+}
+
+/* Process an asynchronous branch event.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_process_async_branch(struct pt_block_decoder *decoder,
+				       const struct pt_event *ev)
+{
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	/* This event can't be a status update. */
+	if (ev->status_update)
+		return -pte_bad_context;
+
+	/* We must currently be enabled. */
+	if (!decoder->enabled)
+		return -pte_bad_context;
+
+	/* Jump to the branch destination.  We will continue from there in the
+	 * next iteration.
+	 */
+	decoder->ip = ev->variant.async_branch.to;
+	decoder->process_event = 0;
+
+	return 0;
+}
+
+/* Process a paging event.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_process_paging(struct pt_block_decoder *decoder,
+				 const struct pt_event *ev)
+{
+	uint64_t cr3;
+	int errcode;
+
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	cr3 = ev->variant.paging.cr3;
+	if (decoder->asid.cr3 != cr3) {
+		errcode = pt_msec_cache_invalidate(&decoder->scache);
+		if (errcode < 0)
+			return errcode;
+
+		decoder->asid.cr3 = cr3;
+	}
+
+	decoder->process_event = 0;
+
+	return 0;
+}
+
+/* Process a vmcs event.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_process_vmcs(struct pt_block_decoder *decoder,
+			       const struct pt_event *ev)
+{
+	uint64_t vmcs;
+	int errcode;
+
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	vmcs = ev->variant.vmcs.base;
+	if (decoder->asid.vmcs != vmcs) {
+		errcode = pt_msec_cache_invalidate(&decoder->scache);
+		if (errcode < 0)
+			return errcode;
+
+		decoder->asid.vmcs = vmcs;
+	}
+
+	decoder->process_event = 0;
+
+	return 0;
+}
+
+/* Process an overflow event.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_process_overflow(struct pt_block_decoder *decoder,
+				   const struct pt_event *ev)
+{
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	/* This event can't be a status update. */
+	if (ev->status_update)
+		return -pte_bad_context;
+
+	/* If the IP is suppressed, the overflow resolved while tracing was
+	 * disabled.  Otherwise it resolved while tracing was enabled.
+	 */
+	if (ev->ip_suppressed) {
+		/* Tracing is disabled.  It doesn't make sense to preserve the
+		 * previous IP.  This will just be misleading.  Even if tracing
+		 * had been disabled before, as well, we might have missed the
+		 * re-enable in the overflow.
+		 */
+		decoder->enabled = 0;
+		decoder->ip = 0ull;
+	} else {
+		/* Tracing is enabled and we're at the IP at which the overflow
+		 * resolved.
+		 */
+		decoder->enabled = 1;
+		decoder->ip = ev->variant.overflow.ip;
+	}
+
+	/* We don't know the TSX state.  Let's assume we execute normally.
+	 *
+	 * We also don't know the execution mode.  Let's keep what we have
+	 * in case we don't get an update before we have to decode the next
+	 * instruction.
+	 */
+	decoder->speculative = 0;
+	decoder->process_event = 0;
+
+	return 0;
+}
+
+/* Process an exec mode event.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_process_exec_mode(struct pt_block_decoder *decoder,
+				    const struct pt_event *ev)
+{
+	enum pt_exec_mode mode;
+
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	/* Use status update events to diagnose inconsistencies. */
+	mode = ev->variant.exec_mode.mode;
+	if (ev->status_update && decoder->enabled &&
+	    decoder->mode != ptem_unknown && decoder->mode != mode)
+		return -pte_bad_status_update;
+
+	decoder->mode = mode;
+	decoder->process_event = 0;
+
+	return 0;
+}
+
+/* Process a tsx event.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_process_tsx(struct pt_block_decoder *decoder,
+			      const struct pt_event *ev)
+{
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	decoder->speculative = ev->variant.tsx.speculative;
+	decoder->process_event = 0;
+
+	return 0;
+}
+
+/* Process a stop event.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_blk_process_stop(struct pt_block_decoder *decoder,
+			       const struct pt_event *ev)
+{
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	/* This event can't be a status update. */
+	if (ev->status_update)
+		return -pte_bad_context;
+
+	/* Tracing is always disabled before it is stopped. */
+	if (decoder->enabled)
+		return -pte_bad_context;
+
+	decoder->process_event = 0;
+
+	return 0;
+}
+
+int pt_blk_event(struct pt_block_decoder *decoder, struct pt_event *uevent,
+		 size_t size)
+{
+	struct pt_event *ev;
+	int status;
+
+	if (!decoder || !uevent)
+		return -pte_invalid;
+
+	/* We must currently process an event. */
+	if (!decoder->process_event)
+		return -pte_bad_query;
+
+	ev = &decoder->event;
+	switch (ev->type) {
+	case ptev_enabled:
+		/* Indicate that tracing resumes from the IP at which tracing
+		 * had been disabled before (with some special treatment for
+		 * calls).
+		 */
+		if (ev->variant.enabled.ip == decoder->ip)
+			ev->variant.enabled.resumed = 1;
+
+		status = pt_blk_process_enabled(decoder, ev);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_async_disabled:
+		if (decoder->ip != ev->variant.async_disabled.at)
+			return -pte_bad_query;
+
+		fallthrough;
+	case ptev_disabled:
+
+		status = pt_blk_process_disabled(decoder, ev);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_async_branch:
+		if (decoder->ip != ev->variant.async_branch.from)
+			return -pte_bad_query;
+
+		status = pt_blk_process_async_branch(decoder, ev);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_async_paging:
+		if (!ev->ip_suppressed &&
+		    decoder->ip != ev->variant.async_paging.ip)
+			return -pte_bad_query;
+
+		fallthrough;
+	case ptev_paging:
+		status = pt_blk_process_paging(decoder, ev);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_async_vmcs:
+		if (!ev->ip_suppressed &&
+		    decoder->ip != ev->variant.async_vmcs.ip)
+			return -pte_bad_query;
+
+		fallthrough;
+	case ptev_vmcs:
+		status = pt_blk_process_vmcs(decoder, ev);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_overflow:
+		status = pt_blk_process_overflow(decoder, ev);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_exec_mode:
+		if (!ev->ip_suppressed &&
+		    decoder->ip != ev->variant.exec_mode.ip)
+			return -pte_bad_query;
+
+		status = pt_blk_process_exec_mode(decoder, ev);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_tsx:
+		if (!ev->ip_suppressed && decoder->ip != ev->variant.tsx.ip)
+			return -pte_bad_query;
+
+		status = pt_blk_process_tsx(decoder, ev);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_stop:
+		status = pt_blk_process_stop(decoder, ev);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_exstop:
+		if (!ev->ip_suppressed && decoder->enabled &&
+		    decoder->ip != ev->variant.exstop.ip)
+			return -pte_bad_query;
+
+		decoder->process_event = 0;
+		break;
+
+	case ptev_mwait:
+		if (!ev->ip_suppressed && decoder->enabled &&
+		    decoder->ip != ev->variant.mwait.ip)
+			return -pte_bad_query;
+
+		decoder->process_event = 0;
+		break;
+
+	case ptev_pwre:
+	case ptev_pwrx:
+	case ptev_ptwrite:
+	case ptev_tick:
+	case ptev_cbr:
+	case ptev_mnt:
+		decoder->process_event = 0;
+		break;
+	}
+
+	/* Copy the event to the user.  Make sure we're not writing beyond the
+	 * memory provided by the user.
+	 *
+	 * We might truncate details of an event but only for those events the
+	 * user can't know about, anyway.
+	 */
+	if (sizeof(*ev) < size)
+		size = sizeof(*ev);
+
+	memcpy(uevent, ev, size);
+
+	/* Indicate further events. */
+	return pt_blk_proceed_trailing_event(decoder, NULL);
+}
diff --git a/libipt/src/pt_config.c b/libipt/src/pt_config.c
new file mode 100644
index 000000000000..7c14c5eb181f
--- /dev/null
+++ b/libipt/src/pt_config.c
@@ -0,0 +1,259 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_config.h"
+#include "pt_opcodes.h"
+
+#include "intel-pt.h"
+
+#include <string.h>
+#include <stddef.h>
+
+
+int pt_cpu_errata(struct pt_errata *errata, const struct pt_cpu *cpu)
+{
+	if (!errata || !cpu)
+		return -pte_invalid;
+
+	memset(errata, 0, sizeof(*errata));
+
+	/* We don't know about others. */
+	if (cpu->vendor != pcv_intel)
+		return -pte_bad_cpu;
+
+	switch (cpu->family) {
+	case 0x6:
+		switch (cpu->model) {
+		case 0x3d:
+		case 0x47:
+		case 0x4f:
+		case 0x56:
+			errata->bdm70 = 1;
+			errata->bdm64 = 1;
+			return 0;
+
+		case 0x4e:
+		case 0x5e:
+		case 0x8e:
+		case 0x9e:
+			errata->bdm70 = 1;
+			errata->skd007 = 1;
+			errata->skd022 = 1;
+			errata->skd010 = 1;
+			errata->skl014 = 1;
+			errata->skl168 = 1;
+			return 0;
+
+		case 0x55:
+		case 0x66:
+		case 0x7d:
+		case 0x7e:
+			errata->bdm70 = 1;
+			errata->skl014 = 1;
+			errata->skd022 = 1;
+			return 0;
+
+		case 0x5c:
+		case 0x5f:
+			errata->apl12 = 1;
+			errata->apl11 = 1;
+			return 0;
+
+		case 0x7a:
+		case 0x86:
+			errata->apl11 = 1;
+			return 0;
+		}
+		break;
+	}
+
+	return -pte_bad_cpu;
+}
+
+int pt_config_from_user(struct pt_config *config,
+			const struct pt_config *uconfig)
+{
+	uint8_t *begin, *end;
+	size_t size;
+
+	if (!config)
+		return -pte_internal;
+
+	if (!uconfig)
+		return -pte_invalid;
+
+	size = uconfig->size;
+	if (size < offsetof(struct pt_config, decode))
+		return -pte_bad_config;
+
+	begin = uconfig->begin;
+	end = uconfig->end;
+
+	if (!begin || !end || end < begin)
+		return -pte_bad_config;
+
+	/* Ignore fields in the user's configuration we don't know; zero out
+	 * fields the user didn't know about.
+	 */
+	if (sizeof(*config) <= size)
+		size = sizeof(*config);
+	else
+		memset(((uint8_t *) config) + size, 0, sizeof(*config) - size);
+
+	/* Copy (portions of) the user's configuration. */
+	memcpy(config, uconfig, size);
+
+	/* We copied user's size - fix it. */
+	config->size = size;
+
+	return 0;
+}
+
+/* The maximum number of filter addresses that fit into the configuration. */
+static inline size_t pt_filter_addr_ncfg(void)
+{
+	return (sizeof(struct pt_conf_addr_filter) -
+		offsetof(struct pt_conf_addr_filter, addr0_a)) /
+		(2 * sizeof(uint64_t));
+}
+
+uint32_t pt_filter_addr_cfg(const struct pt_conf_addr_filter *filter, uint8_t n)
+{
+	if (!filter)
+		return 0u;
+
+	if (pt_filter_addr_ncfg() <= n)
+		return 0u;
+
+	return (filter->config.addr_cfg >> (4 * n)) & 0xf;
+}
+
+uint64_t pt_filter_addr_a(const struct pt_conf_addr_filter *filter, uint8_t n)
+{
+	const uint64_t *addr;
+
+	if (!filter)
+		return 0ull;
+
+	if (pt_filter_addr_ncfg() <= n)
+		return 0ull;
+
+	addr = &filter->addr0_a;
+	return addr[2 * n];
+}
+
+uint64_t pt_filter_addr_b(const struct pt_conf_addr_filter *filter, uint8_t n)
+{
+	const uint64_t *addr;
+
+	if (!filter)
+		return 0ull;
+
+	if (pt_filter_addr_ncfg() <= n)
+		return 0ull;
+
+	addr = &filter->addr0_a;
+	return addr[(2 * n) + 1];
+}
+
+static int pt_filter_check_cfg_filter(const struct pt_conf_addr_filter *filter,
+				      uint64_t addr)
+{
+	uint8_t n;
+
+	if (!filter)
+		return -pte_internal;
+
+	for (n = 0; n < pt_filter_addr_ncfg(); ++n) {
+		uint64_t addr_a, addr_b;
+		uint32_t addr_cfg;
+
+		addr_cfg = pt_filter_addr_cfg(filter, n);
+		if (addr_cfg != pt_addr_cfg_filter)
+			continue;
+
+		addr_a = pt_filter_addr_a(filter, n);
+		addr_b = pt_filter_addr_b(filter, n);
+
+		/* Note that both A and B are inclusive. */
+		if ((addr_a <= addr) && (addr <= addr_b))
+			return 1;
+	}
+
+	/* No filter hit.  If we have at least one FilterEn filter, this means
+	 * that tracing is disabled; otherwise, tracing is enabled.
+	 */
+	for (n = 0; n < pt_filter_addr_ncfg(); ++n) {
+		uint32_t addr_cfg;
+
+		addr_cfg = pt_filter_addr_cfg(filter, n);
+		if (addr_cfg == pt_addr_cfg_filter)
+			return 0;
+	}
+
+	return 1;
+}
+
+static int pt_filter_check_cfg_stop(const struct pt_conf_addr_filter *filter,
+				    uint64_t addr)
+{
+	uint8_t n;
+
+	if (!filter)
+		return -pte_internal;
+
+	for (n = 0; n < pt_filter_addr_ncfg(); ++n) {
+		uint64_t addr_a, addr_b;
+		uint32_t addr_cfg;
+
+		addr_cfg = pt_filter_addr_cfg(filter, n);
+		if (addr_cfg != pt_addr_cfg_stop)
+			continue;
+
+		addr_a = pt_filter_addr_a(filter, n);
+		addr_b = pt_filter_addr_b(filter, n);
+
+		/* Note that both A and B are inclusive. */
+		if ((addr_a <= addr) && (addr <= addr_b))
+			return 0;
+	}
+
+	return 1;
+}
+
+int pt_filter_addr_check(const struct pt_conf_addr_filter *filter,
+			 uint64_t addr)
+{
+	int status;
+
+	status = pt_filter_check_cfg_stop(filter, addr);
+	if (status <= 0)
+		return status;
+
+	return pt_filter_check_cfg_filter(filter, addr);
+}
diff --git a/libipt/src/pt_cpu.c b/libipt/src/pt_cpu.c
new file mode 100644
index 000000000000..adcb1ca5cb83
--- /dev/null
+++ b/libipt/src/pt_cpu.c
@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_cpu.h"
+#include "pt_cpuid.h"
+
+#include "intel-pt.h"
+
+#include <limits.h>
+#include <stdlib.h>
+
+
+static const char * const cpu_vendors[] = {
+	"",
+	"GenuineIntel"
+};
+
+enum {
+	pt_cpuid_vendor_size = 12
+};
+
+union cpu_vendor {
+	/* The raw data returned from cpuid. */
+	struct {
+		uint32_t ebx;
+		uint32_t edx;
+		uint32_t ecx;
+	} cpuid;
+
+	/* The resulting vendor string. */
+	char vendor_string[pt_cpuid_vendor_size];
+};
+
+static enum pt_cpu_vendor cpu_vendor(void)
+{
+	union cpu_vendor vendor;
+	uint32_t eax;
+	size_t i;
+
+	memset(&vendor, 0, sizeof(vendor));
+	eax = 0;
+
+	pt_cpuid(0u, &eax, &vendor.cpuid.ebx, &vendor.cpuid.ecx,
+		 &vendor.cpuid.edx);
+
+	for (i = 0; i < sizeof(cpu_vendors)/sizeof(*cpu_vendors); i++)
+		if (strncmp(vendor.vendor_string,
+			    cpu_vendors[i], pt_cpuid_vendor_size) == 0)
+			return (enum pt_cpu_vendor) i;
+
+	return pcv_unknown;
+}
+
+static uint32_t cpu_info(void)
+{
+	uint32_t eax, ebx, ecx, edx;
+
+	eax = 0;
+	ebx = 0;
+	ecx = 0;
+	edx = 0;
+	pt_cpuid(1u, &eax, &ebx, &ecx, &edx);
+
+	return eax;
+}
+
+int pt_cpu_parse(struct pt_cpu *cpu, const char *s)
+{
+	const char sep = '/';
+	char *endptr;
+	long family, model, stepping;
+
+	if (!cpu || !s)
+		return -pte_invalid;
+
+	family = strtol(s, &endptr, 0);
+	if (s == endptr || *endptr == '\0' || *endptr != sep)
+		return -pte_invalid;
+
+	if (family < 0 || family > USHRT_MAX)
+		return -pte_invalid;
+
+	/* skip separator */
+	s = endptr + 1;
+
+	model = strtol(s, &endptr, 0);
+	if (s == endptr || (*endptr != '\0' && *endptr != sep))
+		return -pte_invalid;
+
+	if (model < 0 || model > UCHAR_MAX)
+		return -pte_invalid;
+
+	if (*endptr == '\0')
+		/* stepping was omitted, it defaults to 0 */
+		stepping = 0;
+	else {
+		/* skip separator */
+		s = endptr + 1;
+
+		stepping = strtol(s, &endptr, 0);
+		if (*endptr != '\0')
+			return -pte_invalid;
+
+		if (stepping < 0 || stepping > UCHAR_MAX)
+			return -pte_invalid;
+	}
+
+	cpu->vendor = pcv_intel;
+	cpu->family = (uint16_t) family;
+	cpu->model = (uint8_t) model;
+	cpu->stepping = (uint8_t) stepping;
+
+	return 0;
+}
+
+int pt_cpu_read(struct pt_cpu *cpu)
+{
+	uint32_t info;
+	uint16_t family;
+
+	if (!cpu)
+		return -pte_invalid;
+
+	cpu->vendor = cpu_vendor();
+
+	info = cpu_info();
+
+	cpu->family = family = (info>>8) & 0xf;
+	if (family == 0xf)
+		cpu->family += (info>>20) & 0xf;
+
+	cpu->model = (info>>4) & 0xf;
+	if (family == 0x6 || family == 0xf)
+		cpu->model += (info>>12) & 0xf0;
+
+	cpu->stepping = (info>>0) & 0xf;
+
+	return 0;
+}
diff --git a/libipt/src/pt_decoder_function.c b/libipt/src/pt_decoder_function.c
new file mode 100644
index 000000000000..96734cac6920
--- /dev/null
+++ b/libipt/src/pt_decoder_function.c
@@ -0,0 +1,379 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_decoder_function.h"
+#include "pt_packet_decoder.h"
+#include "pt_query_decoder.h"
+#include "pt_opcodes.h"
+
+#include "intel-pt.h"
+
+
+const struct pt_decoder_function pt_decode_unknown = {
+	/* .packet = */ pt_pkt_decode_unknown,
+	/* .decode = */ pt_qry_decode_unknown,
+	/* .header = */ pt_qry_decode_unknown,
+	/* .flags =  */ pdff_unknown
+};
+
+const struct pt_decoder_function pt_decode_pad = {
+	/* .packet = */ pt_pkt_decode_pad,
+	/* .decode = */ pt_qry_decode_pad,
+	/* .header = */ pt_qry_decode_pad,
+	/* .flags =  */ pdff_pad
+};
+
+const struct pt_decoder_function pt_decode_psb = {
+	/* .packet = */ pt_pkt_decode_psb,
+	/* .decode = */ pt_qry_decode_psb,
+	/* .header = */ NULL,
+	/* .flags =  */ 0
+};
+
+const struct pt_decoder_function pt_decode_tip = {
+	/* .packet = */ pt_pkt_decode_tip,
+	/* .decode = */ pt_qry_decode_tip,
+	/* .header = */ NULL,
+	/* .flags =  */ pdff_tip
+};
+
+const struct pt_decoder_function pt_decode_tnt_8 = {
+	/* .packet = */ pt_pkt_decode_tnt_8,
+	/* .decode = */ pt_qry_decode_tnt_8,
+	/* .header = */ NULL,
+	/* .flags =  */ pdff_tnt
+};
+
+const struct pt_decoder_function pt_decode_tnt_64 = {
+	/* .packet = */ pt_pkt_decode_tnt_64,
+	/* .decode = */ pt_qry_decode_tnt_64,
+	/* .header = */ NULL,
+	/* .flags =  */ pdff_tnt
+};
+
+const struct pt_decoder_function pt_decode_tip_pge = {
+	/* .packet = */ pt_pkt_decode_tip_pge,
+	/* .decode = */ pt_qry_decode_tip_pge,
+	/* .header = */ NULL,
+	/* .flags =  */ pdff_event
+};
+
+const struct pt_decoder_function pt_decode_tip_pgd = {
+	/* .packet = */ pt_pkt_decode_tip_pgd,
+	/* .decode = */ pt_qry_decode_tip_pgd,
+	/* .header = */ NULL,
+	/* .flags =  */ pdff_event
+};
+
+const struct pt_decoder_function pt_decode_fup = {
+	/* .packet = */ pt_pkt_decode_fup,
+	/* .decode = */ pt_qry_decode_fup,
+	/* .header = */ pt_qry_header_fup,
+	/* .flags =  */ pdff_fup
+};
+
+const struct pt_decoder_function pt_decode_pip = {
+	/* .packet = */ pt_pkt_decode_pip,
+	/* .decode = */ pt_qry_decode_pip,
+	/* .header = */ pt_qry_header_pip,
+	/* .flags =  */ pdff_event
+};
+
+const struct pt_decoder_function pt_decode_ovf = {
+	/* .packet = */ pt_pkt_decode_ovf,
+	/* .decode = */ pt_qry_decode_ovf,
+	/* .header = */ NULL,
+	/* .flags =  */ pdff_psbend | pdff_event
+};
+
+const struct pt_decoder_function pt_decode_mode = {
+	/* .packet = */ pt_pkt_decode_mode,
+	/* .decode = */ pt_qry_decode_mode,
+	/* .header = */ pt_qry_header_mode,
+	/* .flags =  */ pdff_event
+};
+
+const struct pt_decoder_function pt_decode_psbend = {
+	/* .packet = */ pt_pkt_decode_psbend,
+	/* .decode = */ pt_qry_decode_psbend,
+	/* .header = */ NULL,
+	/* .flags =  */ pdff_psbend
+};
+
+const struct pt_decoder_function pt_decode_tsc = {
+	/* .packet = */ pt_pkt_decode_tsc,
+	/* .decode = */ pt_qry_decode_tsc,
+	/* .header = */ pt_qry_header_tsc,
+	/* .flags =  */ pdff_timing
+};
+
+const struct pt_decoder_function pt_decode_cbr = {
+	/* .packet = */ pt_pkt_decode_cbr,
+	/* .decode = */ pt_qry_decode_cbr,
+	/* .header = */ pt_qry_header_cbr,
+	/* .flags =  */ pdff_timing | pdff_event
+};
+
+const struct pt_decoder_function pt_decode_tma = {
+	/* .packet = */ pt_pkt_decode_tma,
+	/* .decode = */ pt_qry_decode_tma,
+	/* .header = */ pt_qry_decode_tma,
+	/* .flags =  */ pdff_timing
+};
+
+const struct pt_decoder_function pt_decode_mtc = {
+	/* .packet = */ pt_pkt_decode_mtc,
+	/* .decode = */ pt_qry_decode_mtc,
+	/* .header = */ pt_qry_decode_mtc,
+	/* .flags =  */ pdff_timing
+};
+
+const struct pt_decoder_function pt_decode_cyc = {
+	/* .packet = */ pt_pkt_decode_cyc,
+	/* .decode = */ pt_qry_decode_cyc,
+	/* .header = */ pt_qry_decode_cyc,
+	/* .flags =  */ pdff_timing
+};
+
+const struct pt_decoder_function pt_decode_stop = {
+	/* .packet = */ pt_pkt_decode_stop,
+	/* .decode = */ pt_qry_decode_stop,
+	/* .header = */ NULL,
+	/* .flags =  */ pdff_event
+};
+
+const struct pt_decoder_function pt_decode_vmcs = {
+	/* .packet = */ pt_pkt_decode_vmcs,
+	/* .decode = */ pt_qry_decode_vmcs,
+	/* .header = */ pt_qry_header_vmcs,
+	/* .flags =  */ pdff_event
+};
+
+const struct pt_decoder_function pt_decode_mnt = {
+	/* .packet = */ pt_pkt_decode_mnt,
+	/* .decode = */ pt_qry_decode_mnt,
+	/* .header = */ pt_qry_header_mnt,
+	/* .flags =  */ pdff_event
+};
+
+const struct pt_decoder_function pt_decode_exstop = {
+	/* .packet = */ pt_pkt_decode_exstop,
+	/* .decode = */ pt_qry_decode_exstop,
+	/* .header = */ NULL,
+	/* .flags =  */ pdff_event
+};
+
+const struct pt_decoder_function pt_decode_mwait = {
+	/* .packet = */ pt_pkt_decode_mwait,
+	/* .decode = */ pt_qry_decode_mwait,
+	/* .header = */ NULL,
+	/* .flags =  */ pdff_event
+};
+
+const struct pt_decoder_function pt_decode_pwre = {
+	/* .packet = */ pt_pkt_decode_pwre,
+	/* .decode = */ pt_qry_decode_pwre,
+	/* .header = */ NULL,
+	/* .flags =  */ pdff_event
+};
+
+const struct pt_decoder_function pt_decode_pwrx = {
+	/* .packet = */ pt_pkt_decode_pwrx,
+	/* .decode = */ pt_qry_decode_pwrx,
+	/* .header = */ NULL,
+	/* .flags =  */ pdff_event
+};
+
+const struct pt_decoder_function pt_decode_ptw = {
+	/* .packet = */ pt_pkt_decode_ptw,
+	/* .decode = */ pt_qry_decode_ptw,
+	/* .header = */ NULL,
+	/* .flags =  */ pdff_event
+};
+
+
+int pt_df_fetch(const struct pt_decoder_function **dfun, const uint8_t *pos,
+		const struct pt_config *config)
+{
+	const uint8_t *begin, *end;
+	uint8_t opc, ext, ext2;
+
+	if (!dfun || !config)
+		return -pte_internal;
+
+	/* Clear the decode function in case of errors. */
+	*dfun = NULL;
+
+	begin = config->begin;
+	end = config->end;
+
+	if (!pos || (pos < begin) || (end < pos))
+		return -pte_nosync;
+
+	if (pos == end)
+		return -pte_eos;
+
+	opc = *pos++;
+	switch (opc) {
+	default:
+		/* Check opcodes that require masking. */
+		if ((opc & pt_opm_tnt_8) == pt_opc_tnt_8) {
+			*dfun = &pt_decode_tnt_8;
+			return 0;
+		}
+
+		if ((opc & pt_opm_cyc) == pt_opc_cyc) {
+			*dfun = &pt_decode_cyc;
+			return 0;
+		}
+
+		if ((opc & pt_opm_tip) == pt_opc_tip) {
+			*dfun = &pt_decode_tip;
+			return 0;
+		}
+
+		if ((opc & pt_opm_fup) == pt_opc_fup) {
+			*dfun = &pt_decode_fup;
+			return 0;
+		}
+
+		if ((opc & pt_opm_tip) == pt_opc_tip_pge) {
+			*dfun = &pt_decode_tip_pge;
+			return 0;
+		}
+
+		if ((opc & pt_opm_tip) == pt_opc_tip_pgd) {
+			*dfun = &pt_decode_tip_pgd;
+			return 0;
+		}
+
+		*dfun = &pt_decode_unknown;
+		return 0;
+
+	case pt_opc_pad:
+		*dfun = &pt_decode_pad;
+		return 0;
+
+	case pt_opc_mode:
+		*dfun = &pt_decode_mode;
+		return 0;
+
+	case pt_opc_tsc:
+		*dfun = &pt_decode_tsc;
+		return 0;
+
+	case pt_opc_mtc:
+		*dfun = &pt_decode_mtc;
+		return 0;
+
+	case pt_opc_ext:
+		if (pos == end)
+			return -pte_eos;
+
+		ext = *pos++;
+		switch (ext) {
+		default:
+			/* Check opcodes that require masking. */
+			if ((ext & pt_opm_ptw) == pt_ext_ptw) {
+				*dfun = &pt_decode_ptw;
+				return 0;
+			}
+
+			*dfun = &pt_decode_unknown;
+			return 0;
+
+		case pt_ext_psb:
+			*dfun = &pt_decode_psb;
+			return 0;
+
+		case pt_ext_ovf:
+			*dfun = &pt_decode_ovf;
+			return 0;
+
+		case pt_ext_tnt_64:
+			*dfun = &pt_decode_tnt_64;
+			return 0;
+
+		case pt_ext_psbend:
+			*dfun = &pt_decode_psbend;
+			return 0;
+
+		case pt_ext_cbr:
+			*dfun = &pt_decode_cbr;
+			return 0;
+
+		case pt_ext_pip:
+			*dfun = &pt_decode_pip;
+			return 0;
+
+		case pt_ext_tma:
+			*dfun = &pt_decode_tma;
+			return 0;
+
+		case pt_ext_stop:
+			*dfun = &pt_decode_stop;
+			return 0;
+
+		case pt_ext_vmcs:
+			*dfun = &pt_decode_vmcs;
+			return 0;
+
+		case pt_ext_exstop:
+		case pt_ext_exstop_ip:
+			*dfun = &pt_decode_exstop;
+			return 0;
+
+		case pt_ext_mwait:
+			*dfun = &pt_decode_mwait;
+			return 0;
+
+		case pt_ext_pwre:
+			*dfun = &pt_decode_pwre;
+			return 0;
+
+		case pt_ext_pwrx:
+			*dfun = &pt_decode_pwrx;
+			return 0;
+
+		case pt_ext_ext2:
+			if (pos == end)
+				return -pte_eos;
+
+			ext2 = *pos++;
+			switch (ext2) {
+			default:
+				*dfun = &pt_decode_unknown;
+				return 0;
+
+			case pt_ext2_mnt:
+				*dfun = &pt_decode_mnt;
+				return 0;
+			}
+		}
+	}
+}
diff --git a/libipt/src/pt_encoder.c b/libipt/src/pt_encoder.c
new file mode 100644
index 000000000000..a34ef12466cf
--- /dev/null
+++ b/libipt/src/pt_encoder.c
@@ -0,0 +1,919 @@
+/*
+ * Copyright (c) 2014-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_encoder.h"
+#include "pt_config.h"
+#include "pt_opcodes.h"
+
+#include <string.h>
+#include <stdlib.h>
+
+
+int pt_encoder_init(struct pt_encoder *encoder, const struct pt_config *config)
+{
+	int errcode;
+
+	if (!encoder)
+		return -pte_invalid;
+
+	memset(encoder, 0, sizeof(*encoder));
+
+	errcode = pt_config_from_user(&encoder->config, config);
+	if (errcode < 0)
+		return errcode;
+
+	encoder->pos = encoder->config.begin;
+
+	return 0;
+}
+
+void pt_encoder_fini(struct pt_encoder *encoder)
+{
+	(void) encoder;
+
+	/* Nothing to do. */
+}
+
+struct pt_encoder *pt_alloc_encoder(const struct pt_config *config)
+{
+	struct pt_encoder *encoder;
+	int errcode;
+
+	encoder = malloc(sizeof(*encoder));
+	if (!encoder)
+		return NULL;
+
+	errcode = pt_encoder_init(encoder, config);
+	if (errcode < 0) {
+		free(encoder);
+		return NULL;
+	}
+
+	return encoder;
+}
+
+void pt_free_encoder(struct pt_encoder *encoder)
+{
+	pt_encoder_fini(encoder);
+	free(encoder);
+}
+
+int pt_enc_sync_set(struct pt_encoder *encoder, uint64_t offset)
+{
+	uint8_t *begin, *end, *pos;
+
+	if (!encoder)
+		return -pte_invalid;
+
+	begin = encoder->config.begin;
+	end = encoder->config.end;
+	pos = begin + offset;
+
+	if (end < pos || pos < begin)
+		return -pte_eos;
+
+	encoder->pos = pos;
+	return 0;
+}
+
+int pt_enc_get_offset(const struct pt_encoder *encoder, uint64_t *offset)
+{
+	const uint8_t *raw, *begin;
+
+	if (!encoder || !offset)
+		return -pte_invalid;
+
+	/* The encoder is synchronized at all times. */
+	raw = encoder->pos;
+	if (!raw)
+		return -pte_internal;
+
+	begin = encoder->config.begin;
+	if (!begin)
+		return -pte_internal;
+
+	*offset = (uint64_t) (int64_t) (raw - begin);
+	return 0;
+}
+
+const struct pt_config *pt_enc_get_config(const struct pt_encoder *encoder)
+{
+	if (!encoder)
+		return NULL;
+
+	return &encoder->config;
+}
+
+/* Check the remaining space.
+ *
+ * Returns zero if there are at least \@size bytes of free space available in
+ * \@encoder's Intel PT buffer.
+ *
+ * Returns -pte_eos if not enough space is available.
+ * Returns -pte_internal if \@encoder is NULL.
+ * Returns -pte_internal if \@encoder is not synchronized.
+ */
+static int pt_reserve(const struct pt_encoder *encoder, unsigned int size)
+{
+	const uint8_t *begin, *end, *pos;
+
+	if (!encoder)
+		return -pte_internal;
+
+	/* The encoder is synchronized at all times. */
+	pos = encoder->pos;
+	if (!pos)
+		return -pte_internal;
+
+	begin = encoder->config.begin;
+	end = encoder->config.end;
+
+	pos += size;
+	if (pos < begin || end < pos)
+		return -pte_eos;
+
+	return 0;
+}
+
+/* Return the size of an IP payload based on its IP compression.
+ *
+ * Returns -pte_bad_packet if \@ipc is not a valid IP compression.
+ */
+static int pt_ipc_size(enum pt_ip_compression ipc)
+{
+	switch (ipc) {
+	case pt_ipc_suppressed:
+		return 0;
+
+	case pt_ipc_update_16:
+		return pt_pl_ip_upd16_size;
+
+	case pt_ipc_update_32:
+		return pt_pl_ip_upd32_size;
+
+	case pt_ipc_update_48:
+		return pt_pl_ip_upd48_size;
+
+	case pt_ipc_sext_48:
+		return pt_pl_ip_sext48_size;
+
+	case pt_ipc_full:
+		return pt_pl_ip_full_size;
+	}
+
+	return -pte_invalid;
+}
+
+/* Encode an integer value.
+ *
+ * Writes the \@size least signifficant bytes of \@value starting from \@pos.
+ *
+ * The caller needs to ensure that there is enough space available.
+ *
+ * Returns the updated position.
+ */
+static uint8_t *pt_encode_int(uint8_t *pos, uint64_t val, int size)
+{
+	for (; size; --size, val >>= 8)
+		*pos++ = (uint8_t) val;
+
+	return pos;
+}
+
+/* Encode an IP packet.
+ *
+ * Write an IP packet with opcode \@opc and payload from \@packet if there is
+ * enough space in \@encoder's Intel PT buffer.
+ *
+ * Returns the number of bytes written on success.
+ *
+ * Returns -pte_eos if there is not enough space.
+ * Returns -pte_internal if \@encoder or \@packet is NULL.
+ * Returns -pte_invalid if \@packet.ipc is not valid.
+ */
+static int pt_encode_ip(struct pt_encoder *encoder, enum pt_opcode op,
+			const struct pt_packet_ip *packet)
+{
+	uint8_t *pos;
+	uint8_t opc, ipc;
+	int size, errcode;
+
+	if (!encoder || !packet)
+		return pte_internal;
+
+	size = pt_ipc_size(packet->ipc);
+	if (size < 0)
+		return size;
+
+	errcode = pt_reserve(encoder,
+			     /* opc size = */ 1u + (unsigned int) size);
+	if (errcode < 0)
+		return errcode;
+
+	/* We already checked the ipc in pt_ipc_size(). */
+	ipc = (uint8_t) (packet->ipc << pt_opm_ipc_shr);
+	opc = (uint8_t) op;
+
+	pos = encoder->pos;
+	*pos++ = opc | ipc;
+
+	encoder->pos = pt_encode_int(pos, packet->ip, size);
+	return /* opc size = */ 1 + size;
+}
+
+int pt_enc_next(struct pt_encoder *encoder, const struct pt_packet *packet)
+{
+	uint8_t *pos, *begin;
+	int errcode;
+
+	if (!encoder || !packet)
+		return -pte_invalid;
+
+	pos = begin = encoder->pos;
+	switch (packet->type) {
+	case ppt_pad:
+		errcode = pt_reserve(encoder, ptps_pad);
+		if (errcode < 0)
+			return errcode;
+
+		*pos++ = pt_opc_pad;
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+
+	case ppt_psb: {
+		uint64_t psb;
+
+		errcode = pt_reserve(encoder, ptps_psb);
+		if (errcode < 0)
+			return errcode;
+
+		psb = ((uint64_t) pt_psb_hilo << 48 |
+		       (uint64_t) pt_psb_hilo << 32 |
+		       (uint64_t) pt_psb_hilo << 16 |
+		       (uint64_t) pt_psb_hilo);
+
+		pos = pt_encode_int(pos, psb, 8);
+		pos = pt_encode_int(pos, psb, 8);
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+	}
+
+	case ppt_psbend:
+		errcode = pt_reserve(encoder, ptps_psbend);
+		if (errcode < 0)
+			return errcode;
+
+		*pos++ = pt_opc_ext;
+		*pos++ = pt_ext_psbend;
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+
+	case ppt_ovf:
+		errcode = pt_reserve(encoder, ptps_ovf);
+		if (errcode < 0)
+			return errcode;
+
+		*pos++ = pt_opc_ext;
+		*pos++ = pt_ext_ovf;
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+
+	case ppt_fup:
+		return pt_encode_ip(encoder, pt_opc_fup, &packet->payload.ip);
+
+	case ppt_tip:
+		return pt_encode_ip(encoder, pt_opc_tip, &packet->payload.ip);
+
+	case ppt_tip_pge:
+		return pt_encode_ip(encoder, pt_opc_tip_pge,
+				    &packet->payload.ip);
+
+	case ppt_tip_pgd:
+		return pt_encode_ip(encoder, pt_opc_tip_pgd,
+				    &packet->payload.ip);
+
+	case ppt_tnt_8: {
+		uint8_t opc, stop;
+
+		if (packet->payload.tnt.bit_size >= 7)
+			return -pte_bad_packet;
+
+		errcode = pt_reserve(encoder, ptps_tnt_8);
+		if (errcode < 0)
+			return errcode;
+
+		stop = packet->payload.tnt.bit_size + pt_opm_tnt_8_shr;
+		opc = (uint8_t)
+			(packet->payload.tnt.payload << pt_opm_tnt_8_shr);
+
+		*pos++ = (uint8_t) (opc | (1u << stop));
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+	}
+
+	case ppt_tnt_64: {
+		uint64_t tnt, stop;
+
+		errcode = pt_reserve(encoder, ptps_tnt_64);
+		if (errcode < 0)
+			return errcode;
+
+		if (packet->payload.tnt.bit_size >= pt_pl_tnt_64_bits)
+			return -pte_invalid;
+
+		stop = 1ull << packet->payload.tnt.bit_size;
+		tnt = packet->payload.tnt.payload;
+
+		if (tnt & ~(stop - 1))
+			return -pte_invalid;
+
+		*pos++ = pt_opc_ext;
+		*pos++ = pt_ext_tnt_64;
+		pos = pt_encode_int(pos, tnt | stop, pt_pl_tnt_64_size);
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+	}
+
+	case ppt_mode: {
+		uint8_t mode;
+
+		errcode = pt_reserve(encoder, ptps_mode);
+		if (errcode < 0)
+			return errcode;
+
+		switch (packet->payload.mode.leaf) {
+		default:
+			return -pte_bad_packet;
+
+		case pt_mol_exec:
+			mode = pt_mol_exec;
+
+			if (packet->payload.mode.bits.exec.csl)
+				mode |= pt_mob_exec_csl;
+
+			if (packet->payload.mode.bits.exec.csd)
+				mode |= pt_mob_exec_csd;
+			break;
+
+		case pt_mol_tsx:
+			mode = pt_mol_tsx;
+
+			if (packet->payload.mode.bits.tsx.intx)
+				mode |= pt_mob_tsx_intx;
+
+			if (packet->payload.mode.bits.tsx.abrt)
+				mode |= pt_mob_tsx_abrt;
+			break;
+		}
+
+		*pos++ = pt_opc_mode;
+		*pos++ = mode;
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+	}
+
+	case ppt_pip: {
+		uint64_t cr3;
+
+		errcode = pt_reserve(encoder, ptps_pip);
+		if (errcode < 0)
+			return errcode;
+
+		cr3 = packet->payload.pip.cr3;
+		cr3 >>= pt_pl_pip_shl;
+		cr3 <<= pt_pl_pip_shr;
+
+		if (packet->payload.pip.nr)
+			cr3 |= (uint64_t) pt_pl_pip_nr;
+
+		*pos++ = pt_opc_ext;
+		*pos++ = pt_ext_pip;
+		pos = pt_encode_int(pos, cr3, pt_pl_pip_size);
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+	}
+
+	case ppt_tsc:
+		errcode = pt_reserve(encoder, ptps_tsc);
+		if (errcode < 0)
+			return errcode;
+
+		*pos++ = pt_opc_tsc;
+		pos = pt_encode_int(pos, packet->payload.tsc.tsc,
+				    pt_pl_tsc_size);
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+
+	case ppt_cbr:
+		errcode = pt_reserve(encoder, ptps_cbr);
+		if (errcode < 0)
+			return errcode;
+
+		*pos++ = pt_opc_ext;
+		*pos++ = pt_ext_cbr;
+		*pos++ = packet->payload.cbr.ratio;
+		*pos++ = 0;
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+
+	case ppt_tma: {
+		uint16_t ctc, fc;
+
+		errcode = pt_reserve(encoder, ptps_tma);
+		if (errcode < 0)
+			return errcode;
+
+		ctc = packet->payload.tma.ctc;
+		fc = packet->payload.tma.fc;
+
+		if (fc & ~pt_pl_tma_fc_mask)
+			return -pte_bad_packet;
+
+		*pos++ = pt_opc_ext;
+		*pos++ = pt_ext_tma;
+		pos = pt_encode_int(pos, ctc, pt_pl_tma_ctc_size);
+		*pos++ = 0;
+		pos = pt_encode_int(pos, fc, pt_pl_tma_fc_size);
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+	}
+
+	case ppt_mtc:
+		errcode = pt_reserve(encoder, ptps_mtc);
+		if (errcode < 0)
+			return errcode;
+
+		*pos++ = pt_opc_mtc;
+		*pos++ = packet->payload.mtc.ctc;
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+
+	case ppt_cyc: {
+		uint8_t byte[pt_pl_cyc_max_size], index, end;
+		uint64_t ctc;
+
+		ctc = (uint8_t) packet->payload.cyc.value;
+		ctc <<= pt_opm_cyc_shr;
+
+		byte[0] = pt_opc_cyc;
+		byte[0] |= (uint8_t) ctc;
+
+		ctc = packet->payload.cyc.value;
+		ctc >>= (8 - pt_opm_cyc_shr);
+		if (ctc)
+			byte[0] |= pt_opm_cyc_ext;
+
+		for (end = 1; ctc; ++end) {
+			/* Check if the CYC payload is too big. */
+			if (pt_pl_cyc_max_size <= end)
+				return -pte_bad_packet;
+
+			ctc <<= pt_opm_cycx_shr;
+
+			byte[end] = (uint8_t) ctc;
+
+			ctc >>= 8;
+			if (ctc)
+				byte[end] |= pt_opm_cycx_ext;
+		}
+
+		errcode = pt_reserve(encoder, end);
+		if (errcode < 0)
+			return errcode;
+
+		for (index = 0; index < end; ++index)
+			*pos++ = byte[index];
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+	}
+
+	case ppt_stop:
+		errcode = pt_reserve(encoder, ptps_stop);
+		if (errcode < 0)
+			return errcode;
+
+		*pos++ = pt_opc_ext;
+		*pos++ = pt_ext_stop;
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+
+	case ppt_vmcs:
+		errcode = pt_reserve(encoder, ptps_vmcs);
+		if (errcode < 0)
+			return errcode;
+
+		*pos++ = pt_opc_ext;
+		*pos++ = pt_ext_vmcs;
+		pos = pt_encode_int(pos,
+				    packet->payload.vmcs.base >> pt_pl_vmcs_shl,
+				    pt_pl_vmcs_size);
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+
+	case ppt_mnt:
+		errcode = pt_reserve(encoder, ptps_mnt);
+		if (errcode < 0)
+			return errcode;
+
+		*pos++ = pt_opc_ext;
+		*pos++ = pt_ext_ext2;
+		*pos++ = pt_ext2_mnt;
+		pos = pt_encode_int(pos, packet->payload.mnt.payload,
+				    pt_pl_mnt_size);
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+
+	case ppt_exstop: {
+		uint8_t ext;
+
+		errcode = pt_reserve(encoder, ptps_exstop);
+		if (errcode < 0)
+			return errcode;
+
+		ext = packet->payload.exstop.ip ?
+			pt_ext_exstop_ip : pt_ext_exstop;
+
+		*pos++ = pt_opc_ext;
+		*pos++ = ext;
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+	}
+
+	case ppt_mwait:
+		errcode = pt_reserve(encoder, ptps_mwait);
+		if (errcode < 0)
+			return errcode;
+
+		*pos++ = pt_opc_ext;
+		*pos++ = pt_ext_mwait;
+		pos = pt_encode_int(pos, packet->payload.mwait.hints,
+				    pt_pl_mwait_hints_size);
+		pos = pt_encode_int(pos, packet->payload.mwait.ext,
+				    pt_pl_mwait_ext_size);
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+
+	case ppt_pwre: {
+		uint64_t payload;
+
+		errcode = pt_reserve(encoder, ptps_pwre);
+		if (errcode < 0)
+			return errcode;
+
+		payload = 0ull;
+		payload |= ((uint64_t) packet->payload.pwre.state <<
+			    pt_pl_pwre_state_shr) &
+			(uint64_t) pt_pl_pwre_state_mask;
+		payload |= ((uint64_t) packet->payload.pwre.sub_state <<
+			    pt_pl_pwre_sub_state_shr) &
+			(uint64_t) pt_pl_pwre_sub_state_mask;
+
+		if (packet->payload.pwre.hw)
+			payload |= (uint64_t) pt_pl_pwre_hw_mask;
+
+		*pos++ = pt_opc_ext;
+		*pos++ = pt_ext_pwre;
+		pos = pt_encode_int(pos, payload, pt_pl_pwre_size);
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+	}
+
+	case ppt_pwrx: {
+		uint64_t payload;
+
+		errcode = pt_reserve(encoder, ptps_pwrx);
+		if (errcode < 0)
+			return errcode;
+
+		payload = 0ull;
+		payload |= ((uint64_t) packet->payload.pwrx.last <<
+			    pt_pl_pwrx_last_shr) &
+			(uint64_t) pt_pl_pwrx_last_mask;
+		payload |= ((uint64_t) packet->payload.pwrx.deepest <<
+			    pt_pl_pwrx_deepest_shr) &
+			(uint64_t) pt_pl_pwrx_deepest_mask;
+
+		if (packet->payload.pwrx.interrupt)
+			payload |= (uint64_t) pt_pl_pwrx_wr_int;
+		if (packet->payload.pwrx.store)
+			payload |= (uint64_t) pt_pl_pwrx_wr_store;
+		if (packet->payload.pwrx.autonomous)
+			payload |= (uint64_t) pt_pl_pwrx_wr_hw;
+
+		*pos++ = pt_opc_ext;
+		*pos++ = pt_ext_pwrx;
+		pos = pt_encode_int(pos, payload, pt_pl_pwrx_size);
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+	}
+
+	case ppt_ptw: {
+		uint8_t plc, ext;
+		int size;
+
+		plc = packet->payload.ptw.plc;
+
+		size = pt_ptw_size(plc);
+		if (size < 0)
+			return size;
+
+		errcode = pt_reserve(encoder,
+				     (unsigned int) (pt_opcs_ptw + size));
+		if (errcode < 0)
+			return errcode;
+
+		ext = pt_ext_ptw;
+		ext |= plc << pt_opm_ptw_pb_shr;
+
+		if (packet->payload.ptw.ip)
+			ext |= (uint8_t) pt_opm_ptw_ip;
+
+		*pos++ = pt_opc_ext;
+		*pos++ = ext;
+		pos = pt_encode_int(pos, packet->payload.ptw.payload, size);
+
+		encoder->pos = pos;
+		return (int) (pos - begin);
+	}
+
+	case ppt_unknown:
+	case ppt_invalid:
+		return -pte_bad_opc;
+	}
+
+	return -pte_bad_opc;
+}
+
+int pt_encode_pad(struct pt_encoder *encoder)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_pad;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_psb(struct pt_encoder *encoder)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_psb;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_psbend(struct pt_encoder *encoder)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_psbend;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_tip(struct pt_encoder *encoder, uint64_t ip,
+		  enum pt_ip_compression ipc)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_tip;
+	packet.payload.ip.ip = ip;
+	packet.payload.ip.ipc = ipc;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_tnt_8(struct pt_encoder *encoder, uint8_t tnt, int size)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_tnt_8;
+	packet.payload.tnt.bit_size = (uint8_t) size;
+	packet.payload.tnt.payload = tnt;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_tnt_64(struct pt_encoder *encoder, uint64_t tnt, int size)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_tnt_64;
+	packet.payload.tnt.bit_size = (uint8_t) size;
+	packet.payload.tnt.payload = tnt;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_tip_pge(struct pt_encoder *encoder, uint64_t ip,
+		      enum pt_ip_compression ipc)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_tip_pge;
+	packet.payload.ip.ip = ip;
+	packet.payload.ip.ipc = ipc;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_tip_pgd(struct pt_encoder *encoder, uint64_t ip,
+		      enum pt_ip_compression ipc)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_tip_pgd;
+	packet.payload.ip.ip = ip;
+	packet.payload.ip.ipc = ipc;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_fup(struct pt_encoder *encoder, uint64_t ip,
+		  enum pt_ip_compression ipc)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_fup;
+	packet.payload.ip.ip = ip;
+	packet.payload.ip.ipc = ipc;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_pip(struct pt_encoder *encoder, uint64_t cr3, uint8_t flags)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_pip;
+	packet.payload.pip.cr3 = cr3;
+	packet.payload.pip.nr = (flags & pt_pl_pip_nr) != 0;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_ovf(struct pt_encoder *encoder)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_ovf;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_mode_exec(struct pt_encoder *encoder, enum pt_exec_mode mode)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_mode;
+	packet.payload.mode.leaf = pt_mol_exec;
+	packet.payload.mode.bits.exec = pt_set_exec_mode(mode);
+
+	return pt_enc_next(encoder, &packet);
+}
+
+
+int pt_encode_mode_tsx(struct pt_encoder *encoder, uint8_t bits)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_mode;
+	packet.payload.mode.leaf = pt_mol_tsx;
+
+	if (bits & pt_mob_tsx_intx)
+		packet.payload.mode.bits.tsx.intx = 1;
+	else
+		packet.payload.mode.bits.tsx.intx = 0;
+
+	if (bits & pt_mob_tsx_abrt)
+		packet.payload.mode.bits.tsx.abrt = 1;
+	else
+		packet.payload.mode.bits.tsx.abrt = 0;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_tsc(struct pt_encoder *encoder, uint64_t tsc)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_tsc;
+	packet.payload.tsc.tsc = tsc;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_cbr(struct pt_encoder *encoder, uint8_t cbr)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_cbr;
+	packet.payload.cbr.ratio = cbr;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_tma(struct pt_encoder *encoder, uint16_t ctc, uint16_t fc)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_tma;
+	packet.payload.tma.ctc = ctc;
+	packet.payload.tma.fc = fc;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_mtc(struct pt_encoder *encoder, uint8_t ctc)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_mtc;
+	packet.payload.mtc.ctc = ctc;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_cyc(struct pt_encoder *encoder, uint32_t ctc)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_cyc;
+	packet.payload.cyc.value = ctc;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_stop(struct pt_encoder *encoder)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_stop;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_vmcs(struct pt_encoder *encoder, uint64_t payload)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_vmcs;
+	packet.payload.vmcs.base = payload;
+
+	return pt_enc_next(encoder, &packet);
+}
+
+int pt_encode_mnt(struct pt_encoder *encoder, uint64_t payload)
+{
+	struct pt_packet packet;
+
+	packet.type = ppt_mnt;
+	packet.payload.mnt.payload = payload;
+
+	return pt_enc_next(encoder, &packet);
+}
diff --git a/libipt/src/pt_error.c b/libipt/src/pt_error.c
new file mode 100644
index 000000000000..5208d1f09ba8
--- /dev/null
+++ b/libipt/src/pt_error.c
@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "intel-pt.h"
+
+
+const char *pt_errstr(enum pt_error_code errcode)
+{
+	switch (errcode) {
+	case pte_ok:
+		return "OK";
+
+	case pte_internal:
+		return "internal error";
+
+	case pte_invalid:
+		return "invalid argument";
+
+	case pte_nosync:
+		return "decoder out of sync";
+
+	case pte_bad_opc:
+		return "unknown opcode";
+
+	case pte_bad_packet:
+		return "unknown packet";
+
+	case pte_bad_context:
+		return "unexpected packet context";
+
+	case pte_eos:
+		return "reached end of trace stream";
+
+	case pte_bad_query:
+		return "trace stream does not match query";
+
+	case pte_nomem:
+		return "out of memory";
+
+	case pte_bad_config:
+		return "bad configuration";
+
+	case pte_noip:
+		return "no ip";
+
+	case pte_ip_suppressed:
+		return "ip has been suppressed";
+
+	case pte_nomap:
+		return "no memory mapped at this address";
+
+	case pte_bad_insn:
+		return "unknown instruction";
+
+	case pte_no_time:
+		return "no timing information";
+
+	case pte_no_cbr:
+		return "no core:bus ratio";
+
+	case pte_bad_image:
+		return "bad image";
+
+	case pte_bad_lock:
+		return "locking error";
+
+	case pte_not_supported:
+		return "not supported";
+
+	case pte_retstack_empty:
+		return "compressed return without call";
+
+	case pte_bad_retcomp:
+		return "bad compressed return";
+
+	case pte_bad_status_update:
+		return "bad status update";
+
+	case pte_no_enable:
+		return "expected tracing enabled event";
+
+	case pte_event_ignored:
+		return "event ignored";
+
+	case pte_overflow:
+		return "overflow";
+
+	case pte_bad_file:
+		return "bad file";
+
+	case pte_bad_cpu:
+		return "unknown cpu";
+	}
+
+	/* Should not reach here. */
+	return "internal error.";
+}
diff --git a/libipt/src/pt_event_queue.c b/libipt/src/pt_event_queue.c
new file mode 100644
index 000000000000..e0d449016a97
--- /dev/null
+++ b/libipt/src/pt_event_queue.c
@@ -0,0 +1,203 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_event_queue.h"
+
+#include <string.h>
+
+
+static inline uint8_t pt_evq_inc(uint8_t idx)
+{
+	idx += 1;
+	idx %= evq_max;
+
+	return idx;
+}
+
+static struct pt_event *pt_event_init(struct pt_event *event)
+{
+	if (event)
+		memset(event, 0, sizeof(*event));
+
+	return event;
+}
+
+void pt_evq_init(struct pt_event_queue *evq)
+{
+	if (!evq)
+		return;
+
+	memset(evq, 0, sizeof(*evq));
+}
+
+struct pt_event *pt_evq_standalone(struct pt_event_queue *evq)
+{
+	if (!evq)
+		return NULL;
+
+	return pt_event_init(&evq->standalone);
+}
+
+struct pt_event *pt_evq_enqueue(struct pt_event_queue *evq,
+				enum pt_event_binding evb)
+{
+	uint8_t begin, end, gap, idx;
+
+	if (!evq)
+		return NULL;
+
+	if (evb_max <= evb)
+		return NULL;
+
+	begin = evq->begin[evb];
+	idx = evq->end[evb];
+
+	if (evq_max <= begin)
+		return NULL;
+
+	if (evq_max <= idx)
+		return NULL;
+
+	end = pt_evq_inc(idx);
+	gap = pt_evq_inc(end);
+
+	/* Leave a gap so we don't overwrite the last dequeued event. */
+	if (begin == gap)
+		return NULL;
+
+	evq->end[evb] = end;
+
+	return pt_event_init(&evq->queue[evb][idx]);
+}
+
+struct pt_event *pt_evq_dequeue(struct pt_event_queue *evq,
+				enum pt_event_binding evb)
+{
+	uint8_t begin, end;
+
+	if (!evq)
+		return NULL;
+
+	if (evb_max <= evb)
+		return NULL;
+
+	begin = evq->begin[evb];
+	end = evq->end[evb];
+
+	if (evq_max <= begin)
+		return NULL;
+
+	if (evq_max <= end)
+		return NULL;
+
+	if (begin == end)
+		return NULL;
+
+	evq->begin[evb] = pt_evq_inc(begin);
+
+	return &evq->queue[evb][begin];
+}
+
+int pt_evq_clear(struct pt_event_queue *evq, enum pt_event_binding evb)
+{
+	if (!evq)
+		return -pte_internal;
+
+	if (evb_max <= evb)
+		return -pte_internal;
+
+	evq->begin[evb] = 0;
+	evq->end[evb] = 0;
+
+	return 0;
+}
+
+int pt_evq_empty(const struct pt_event_queue *evq, enum pt_event_binding evb)
+{
+	uint8_t begin, end;
+
+	if (!evq)
+		return -pte_internal;
+
+	if (evb_max <= evb)
+		return -pte_internal;
+
+	begin = evq->begin[evb];
+	end = evq->end[evb];
+
+	if (evq_max <= begin)
+		return -pte_internal;
+
+	if (evq_max <= end)
+		return -pte_internal;
+
+	return begin == end;
+}
+
+int pt_evq_pending(const struct pt_event_queue *evq, enum pt_event_binding evb)
+{
+	int errcode;
+
+	errcode = pt_evq_empty(evq, evb);
+	if (errcode < 0)
+		return errcode;
+
+	return !errcode;
+}
+
+struct pt_event *pt_evq_find(struct pt_event_queue *evq,
+			     enum pt_event_binding evb,
+			     enum pt_event_type evt)
+{
+	uint8_t begin, end;
+
+	if (!evq)
+		return NULL;
+
+	if (evb_max <= evb)
+		return NULL;
+
+	begin = evq->begin[evb];
+	end = evq->end[evb];
+
+	if (evq_max <= begin)
+		return NULL;
+
+	if (evq_max <= end)
+		return NULL;
+
+	for (; begin != end; begin = pt_evq_inc(begin)) {
+		struct pt_event *ev;
+
+		ev = &evq->queue[evb][begin];
+		if (ev->type == evt)
+			return ev;
+	}
+
+	return NULL;
+}
diff --git a/libipt/src/pt_ild.c b/libipt/src/pt_ild.c
new file mode 100644
index 000000000000..748c1ae38153
--- /dev/null
+++ b/libipt/src/pt_ild.c
@@ -0,0 +1,1429 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_ild.h"
+#include "pti-imm-defs.h"
+#include "pti-imm.h"
+#include "pti-modrm-defs.h"
+#include "pti-modrm.h"
+#include "pti-disp-defs.h"
+#include "pti-disp.h"
+#include "pti-disp_default.h"
+#include "pti-sib.h"
+
+#include <string.h>
+
+
+static const uint8_t eamode_table[2][4] = {
+	/* Default: */ {
+		/* ptem_unknown = */	ptem_unknown,
+		/* ptem_16bit = */	ptem_16bit,
+		/* ptem_32bit = */	ptem_32bit,
+		/* ptem_64bit = */	ptem_64bit
+	},
+
+	/* With Address-size prefix (0x67): */ {
+		/* ptem_unknown = */	ptem_unknown,
+		/* ptem_16bit = */	ptem_32bit,
+		/* ptem_32bit = */	ptem_16bit,
+		/* ptem_64bit = */	ptem_32bit
+	}
+};
+
+/* SOME ACCESSORS */
+
+static inline uint8_t get_byte(const struct pt_ild *ild, uint8_t i)
+{
+	return ild->itext[i];
+}
+
+static inline uint8_t const *get_byte_ptr(const struct pt_ild *ild, uint8_t i)
+{
+	return ild->itext + i;
+}
+
+static inline int mode_64b(const struct pt_ild *ild)
+{
+	return ild->mode == ptem_64bit;
+}
+
+static inline int mode_32b(const struct pt_ild *ild)
+{
+	return ild->mode == ptem_32bit;
+}
+
+static inline int bits_match(uint8_t x, uint8_t mask, uint8_t target)
+{
+	return (x & mask) == target;
+}
+
+static inline enum pt_exec_mode
+pti_get_nominal_eosz_non64(const struct pt_ild *ild)
+{
+	if (mode_32b(ild)) {
+		if (ild->u.s.osz)
+			return ptem_16bit;
+		return ptem_32bit;
+	}
+	if (ild->u.s.osz)
+		return ptem_32bit;
+	return ptem_16bit;
+}
+
+static inline enum pt_exec_mode
+pti_get_nominal_eosz(const struct pt_ild *ild)
+{
+	if (mode_64b(ild)) {
+		if (ild->u.s.rex_w)
+			return ptem_64bit;
+		if (ild->u.s.osz)
+			return ptem_16bit;
+		return ptem_32bit;
+	}
+	return pti_get_nominal_eosz_non64(ild);
+}
+
+static inline enum pt_exec_mode
+pti_get_nominal_eosz_df64(const struct pt_ild *ild)
+{
+	if (mode_64b(ild)) {
+		if (ild->u.s.rex_w)
+			return ptem_64bit;
+		if (ild->u.s.osz)
+			return ptem_16bit;
+		/* only this next line of code is different relative
+		   to pti_get_nominal_eosz(), above */
+		return ptem_64bit;
+	}
+	return pti_get_nominal_eosz_non64(ild);
+}
+
+static inline enum pt_exec_mode
+pti_get_nominal_easz_non64(const struct pt_ild *ild)
+{
+	if (mode_32b(ild)) {
+		if (ild->u.s.asz)
+			return ptem_16bit;
+		return ptem_32bit;
+	}
+	if (ild->u.s.asz)
+		return ptem_32bit;
+	return ptem_16bit;
+}
+
+static inline enum pt_exec_mode
+pti_get_nominal_easz(const struct pt_ild *ild)
+{
+	if (mode_64b(ild)) {
+		if (ild->u.s.asz)
+			return ptem_32bit;
+		return ptem_64bit;
+	}
+	return pti_get_nominal_easz_non64(ild);
+}
+
+static inline int resolve_z(uint8_t *pbytes, enum pt_exec_mode eosz)
+{
+	static const uint8_t bytes[] = { 2, 4, 4 };
+	unsigned int idx;
+
+	if (!pbytes)
+		return -pte_internal;
+
+	idx = (unsigned int) eosz - 1;
+	if (sizeof(bytes) <= idx)
+		return -pte_bad_insn;
+
+	*pbytes = bytes[idx];
+	return 0;
+}
+
+static inline int resolve_v(uint8_t *pbytes, enum pt_exec_mode eosz)
+{
+	static const uint8_t bytes[] = { 2, 4, 8 };
+	unsigned int idx;
+
+	if (!pbytes)
+		return -pte_internal;
+
+	idx = (unsigned int) eosz - 1;
+	if (sizeof(bytes) <= idx)
+		return -pte_bad_insn;
+
+	*pbytes = bytes[idx];
+	return 0;
+}
+
+/*  DECODERS */
+
+static int set_imm_bytes(struct pt_ild *ild)
+{
+	/*: set ild->imm1_bytes and  ild->imm2_bytes for maps 0/1 */
+	static uint8_t const *const map_map[] = {
+		/* map 0 */ imm_bytes_map_0x0,
+		/* map 1 */ imm_bytes_map_0x0F
+	};
+	uint8_t map, imm_code;
+
+	if (!ild)
+		return -pte_internal;
+
+	map = ild->map;
+
+	if ((sizeof(map_map) / sizeof(*map_map)) <= map)
+		return 0;
+
+	imm_code = map_map[map][ild->nominal_opcode];
+	switch (imm_code) {
+	case PTI_IMM_NONE:
+	case PTI_0_IMM_WIDTH_CONST_l2:
+	default:
+		return 0;
+
+	case PTI_UIMM8_IMM_WIDTH_CONST_l2:
+		ild->imm1_bytes = 1;
+		return 0;
+
+	case PTI_SIMM8_IMM_WIDTH_CONST_l2:
+		ild->imm1_bytes = 1;
+		return 0;
+
+	case PTI_SIMMz_IMM_WIDTH_OSZ_NONTERM_EOSZ_l2:
+		/* SIMMz(eosz) */
+		return resolve_z(&ild->imm1_bytes, pti_get_nominal_eosz(ild));
+
+	case PTI_UIMMv_IMM_WIDTH_OSZ_NONTERM_EOSZ_l2:
+		/* UIMMv(eosz) */
+		return resolve_v(&ild->imm1_bytes, pti_get_nominal_eosz(ild));
+
+	case PTI_UIMM16_IMM_WIDTH_CONST_l2:
+		ild->imm1_bytes = 2;
+		return 0;
+
+	case PTI_SIMMz_IMM_WIDTH_OSZ_NONTERM_DF64_EOSZ_l2:
+		/* push defaults to eosz64 in 64b mode, then uses SIMMz */
+		return resolve_z(&ild->imm1_bytes,
+				 pti_get_nominal_eosz_df64(ild));
+
+	case PTI_RESOLVE_BYREG_IMM_WIDTH_map0x0_op0xf7_l1:
+		if (ild->map == PTI_MAP_0 && pti_get_modrm_reg(ild) < 2) {
+			return resolve_z(&ild->imm1_bytes,
+					 pti_get_nominal_eosz(ild));
+		}
+		return 0;
+
+	case PTI_RESOLVE_BYREG_IMM_WIDTH_map0x0_op0xc7_l1:
+		if (ild->map == PTI_MAP_0 && pti_get_modrm_reg(ild) == 0) {
+			return resolve_z(&ild->imm1_bytes,
+					 pti_get_nominal_eosz(ild));
+		}
+		return 0;
+
+	case PTI_RESOLVE_BYREG_IMM_WIDTH_map0x0_op0xf6_l1:
+		if (ild->map == PTI_MAP_0 && pti_get_modrm_reg(ild) < 2)
+			ild->imm1_bytes = 1;
+
+		return 0;
+
+	case PTI_IMM_hasimm_map0x0_op0xc8_l1:
+		if (ild->map == PTI_MAP_0) {
+			/*enter -> imm1=2, imm2=1 */
+			ild->imm1_bytes = 2;
+			ild->imm2_bytes = 1;
+		}
+		return 0;
+
+	case PTI_IMM_hasimm_map0x0F_op0x78_l1:
+		/* AMD SSE4a (insertq/extrq use  osz/f2) vs vmread
+		 * (no prefixes)
+		 */
+		if (ild->map == PTI_MAP_1) {
+			if (ild->u.s.osz || ild->u.s.last_f2f3 == 2) {
+				ild->imm1_bytes = 1;
+				ild->imm2_bytes = 1;
+			}
+		}
+		return 0;
+	}
+}
+
+static int imm_dec(struct pt_ild *ild, uint8_t length)
+{
+	int errcode;
+
+	if (!ild)
+		return -pte_internal;
+
+	if (ild->map == PTI_MAP_AMD3DNOW) {
+		if (ild->max_bytes <= length)
+			return -pte_bad_insn;
+
+		ild->nominal_opcode = get_byte(ild, length);
+		return length + 1;
+	}
+
+	errcode = set_imm_bytes(ild);
+	if (errcode < 0)
+		return errcode;
+
+	length += ild->imm1_bytes;
+	length += ild->imm2_bytes;
+	if (ild->max_bytes < length)
+		return -pte_bad_insn;
+
+	return length;
+}
+
+static int compute_disp_dec(struct pt_ild *ild)
+{
+	/* set ild->disp_bytes for maps 0 and 1. */
+	static uint8_t const *const map_map[] = {
+		/* map 0 */ disp_bytes_map_0x0,
+		/* map 1 */ disp_bytes_map_0x0F
+	};
+	uint8_t map, disp_kind;
+
+	if (!ild)
+		return -pte_internal;
+
+	if (0 < ild->disp_bytes)
+		return 0;
+
+	map = ild->map;
+
+	if ((sizeof(map_map) / sizeof(*map_map)) <= map)
+		return 0;
+
+	disp_kind = map_map[map][ild->nominal_opcode];
+	switch (disp_kind) {
+	case PTI_DISP_NONE:
+		ild->disp_bytes = 0;
+		return 0;
+
+	case PTI_PRESERVE_DEFAULT:
+		/* nothing to do */
+		return 0;
+
+	case PTI_BRDISP8:
+		ild->disp_bytes = 1;
+		return 0;
+
+	case PTI_DISP_BUCKET_0_l1:
+		/* BRDISPz(eosz) for 16/32 modes, and BRDISP32 for 64b mode */
+		if (mode_64b(ild)) {
+			ild->disp_bytes = 4;
+			return 0;
+		}
+
+		return resolve_z(&ild->disp_bytes,
+				 pti_get_nominal_eosz(ild));
+
+	case PTI_MEMDISPv_DISP_WIDTH_ASZ_NONTERM_EASZ_l2:
+		/* MEMDISPv(easz) */
+		return resolve_v(&ild->disp_bytes, pti_get_nominal_easz(ild));
+
+	case PTI_BRDISPz_BRDISP_WIDTH_OSZ_NONTERM_EOSZ_l2:
+		/* BRDISPz(eosz) for 16/32/64 modes */
+		return resolve_z(&ild->disp_bytes, pti_get_nominal_eosz(ild));
+
+	case PTI_RESOLVE_BYREG_DISP_map0x0_op0xc7_l1:
+		/* reg=0 -> preserve, reg=7 -> BRDISPz(eosz) */
+		if (ild->map == PTI_MAP_0 && pti_get_modrm_reg(ild) == 7) {
+			return resolve_z(&ild->disp_bytes,
+					 pti_get_nominal_eosz(ild));
+		}
+		return 0;
+
+	default:
+		return -pte_bad_insn;
+	}
+}
+
+static int disp_dec(struct pt_ild *ild, uint8_t length)
+{
+	uint8_t disp_bytes;
+	int errcode;
+
+	if (!ild)
+		return -pte_internal;
+
+	errcode = compute_disp_dec(ild);
+	if (errcode < 0)
+		return errcode;
+
+	disp_bytes = ild->disp_bytes;
+	if (disp_bytes == 0)
+		return imm_dec(ild, length);
+
+	if (length + disp_bytes > ild->max_bytes)
+		return -pte_bad_insn;
+
+	/*Record only position; must be able to re-read itext bytes for actual
+	   value. (SMC/CMC issue). */
+	ild->disp_pos = length;
+
+	return imm_dec(ild, length + disp_bytes);
+}
+
+static int sib_dec(struct pt_ild *ild, uint8_t length)
+{
+	uint8_t sib;
+
+	if (!ild)
+		return -pte_internal;
+
+	if (ild->max_bytes <= length)
+		return -pte_bad_insn;
+
+	sib = get_byte(ild, length);
+	if ((sib & 0x07) == 0x05 && pti_get_modrm_mod(ild) == 0)
+		ild->disp_bytes = 4;
+
+	return disp_dec(ild, length + 1);
+}
+
+static int modrm_dec(struct pt_ild *ild, uint8_t length)
+{
+	static uint8_t const *const has_modrm_2d[2] = {
+		has_modrm_map_0x0,
+		has_modrm_map_0x0F
+	};
+	int has_modrm = PTI_MODRM_FALSE;
+	pti_map_enum_t map;
+
+	if (!ild)
+		return -pte_internal;
+
+	map = pti_get_map(ild);
+	if (map >= PTI_MAP_2)
+		has_modrm = PTI_MODRM_TRUE;
+	else
+		has_modrm = has_modrm_2d[map][ild->nominal_opcode];
+
+	if (has_modrm == PTI_MODRM_FALSE || has_modrm == PTI_MODRM_UNDEF)
+		return disp_dec(ild, length);
+
+	/* really >= here because we have not eaten the byte yet */
+	if (length >= ild->max_bytes)
+		return -pte_bad_insn;
+
+	ild->modrm_byte = get_byte(ild, length);
+
+	if (has_modrm != PTI_MODRM_IGNORE_MOD) {
+		/* set disp_bytes and sib using simple tables */
+
+		uint8_t eamode = eamode_table[ild->u.s.asz][ild->mode];
+		uint8_t mod = (uint8_t) pti_get_modrm_mod(ild);
+		uint8_t rm = (uint8_t) pti_get_modrm_rm(ild);
+		uint8_t sib;
+
+		ild->disp_bytes = disp_default[eamode][mod][rm];
+
+		sib = has_sib[eamode][mod][rm];
+		if (sib)
+			return sib_dec(ild, length + 1);
+	}
+
+	return disp_dec(ild, length + 1);
+}
+
+static inline int get_next_as_opcode(struct pt_ild *ild, uint8_t length)
+{
+	if (!ild)
+		return -pte_internal;
+
+	if (ild->max_bytes <= length)
+		return -pte_bad_insn;
+
+	ild->nominal_opcode = get_byte(ild, length);
+
+	return modrm_dec(ild, length + 1);
+}
+
+static int opcode_dec(struct pt_ild *ild, uint8_t length)
+{
+	uint8_t b, m;
+
+	if (!ild)
+		return -pte_internal;
+
+	/*no need to check max_bytes - it was checked in previous scanners */
+	b = get_byte(ild, length);
+	if (b != 0x0F) {	/* 1B opcodes, map 0 */
+		ild->map = PTI_MAP_0;
+		ild->nominal_opcode = b;
+
+		return modrm_dec(ild, length + 1);
+	}
+
+	length++;		/* eat the 0x0F */
+
+	if (ild->max_bytes <= length)
+		return -pte_bad_insn;
+
+	/* 0x0F opcodes MAPS 1,2,3 */
+	m = get_byte(ild, length);
+	if (m == 0x38) {
+		ild->map = PTI_MAP_2;
+
+		return get_next_as_opcode(ild, length + 1);
+	} else if (m == 0x3A) {
+		ild->map = PTI_MAP_3;
+		ild->imm1_bytes = 1;
+
+		return get_next_as_opcode(ild, length + 1);
+	} else if (bits_match(m, 0xf8, 0x38)) {
+		ild->map = PTI_MAP_INVALID;
+
+		return get_next_as_opcode(ild, length + 1);
+	} else if (m == 0x0F) {	/* 3dNow */
+		ild->map = PTI_MAP_AMD3DNOW;
+		ild->imm1_bytes = 1;
+		/* real opcode is in immediate later on, but we need an
+		 * opcode now. */
+		ild->nominal_opcode = 0x0F;
+
+		return modrm_dec(ild, length + 1);
+	} else {	/* map 1 (simple two byte opcodes) */
+		ild->nominal_opcode = m;
+		ild->map = PTI_MAP_1;
+
+		return modrm_dec(ild, length + 1);
+	}
+}
+
+typedef int (*prefix_decoder)(struct pt_ild *ild, uint8_t length, uint8_t rex);
+
+static int prefix_osz(struct pt_ild *ild, uint8_t length, uint8_t rex);
+static int prefix_asz(struct pt_ild *ild, uint8_t length, uint8_t rex);
+static int prefix_lock(struct pt_ild *ild, uint8_t length, uint8_t rex);
+static int prefix_f2(struct pt_ild *ild, uint8_t length, uint8_t rex);
+static int prefix_f3(struct pt_ild *ild, uint8_t length, uint8_t rex);
+static int prefix_rex(struct pt_ild *ild, uint8_t length, uint8_t rex);
+static int prefix_vex_c4(struct pt_ild *ild, uint8_t length, uint8_t rex);
+static int prefix_vex_c5(struct pt_ild *ild, uint8_t length, uint8_t rex);
+static int prefix_evex(struct pt_ild *ild, uint8_t length, uint8_t rex);
+static int prefix_ignore(struct pt_ild *ild, uint8_t length, uint8_t rex);
+static int prefix_done(struct pt_ild *ild, uint8_t length, uint8_t rex);
+
+static const prefix_decoder prefix_table[256] = {
+	/* 00 = */ prefix_done,
+	/* 01 = */ prefix_done,
+	/* 02 = */ prefix_done,
+	/* 03 = */ prefix_done,
+	/* 04 = */ prefix_done,
+	/* 05 = */ prefix_done,
+	/* 06 = */ prefix_done,
+	/* 07 = */ prefix_done,
+	/* 08 = */ prefix_done,
+	/* 09 = */ prefix_done,
+	/* 0a = */ prefix_done,
+	/* 0b = */ prefix_done,
+	/* 0c = */ prefix_done,
+	/* 0d = */ prefix_done,
+	/* 0e = */ prefix_done,
+	/* 0f = */ prefix_done,
+
+	/* 10 = */ prefix_done,
+	/* 11 = */ prefix_done,
+	/* 12 = */ prefix_done,
+	/* 13 = */ prefix_done,
+	/* 14 = */ prefix_done,
+	/* 15 = */ prefix_done,
+	/* 16 = */ prefix_done,
+	/* 17 = */ prefix_done,
+	/* 18 = */ prefix_done,
+	/* 19 = */ prefix_done,
+	/* 1a = */ prefix_done,
+	/* 1b = */ prefix_done,
+	/* 1c = */ prefix_done,
+	/* 1d = */ prefix_done,
+	/* 1e = */ prefix_done,
+	/* 1f = */ prefix_done,
+
+	/* 20 = */ prefix_done,
+	/* 21 = */ prefix_done,
+	/* 22 = */ prefix_done,
+	/* 23 = */ prefix_done,
+	/* 24 = */ prefix_done,
+	/* 25 = */ prefix_done,
+	/* 26 = */ prefix_ignore,
+	/* 27 = */ prefix_done,
+	/* 28 = */ prefix_done,
+	/* 29 = */ prefix_done,
+	/* 2a = */ prefix_done,
+	/* 2b = */ prefix_done,
+	/* 2c = */ prefix_done,
+	/* 2d = */ prefix_done,
+	/* 2e = */ prefix_ignore,
+	/* 2f = */ prefix_done,
+
+	/* 30 = */ prefix_done,
+	/* 31 = */ prefix_done,
+	/* 32 = */ prefix_done,
+	/* 33 = */ prefix_done,
+	/* 34 = */ prefix_done,
+	/* 35 = */ prefix_done,
+	/* 36 = */ prefix_ignore,
+	/* 37 = */ prefix_done,
+	/* 38 = */ prefix_done,
+	/* 39 = */ prefix_done,
+	/* 3a = */ prefix_done,
+	/* 3b = */ prefix_done,
+	/* 3c = */ prefix_done,
+	/* 3d = */ prefix_done,
+	/* 3e = */ prefix_ignore,
+	/* 3f = */ prefix_done,
+
+	/* 40 = */ prefix_rex,
+	/* 41 = */ prefix_rex,
+	/* 42 = */ prefix_rex,
+	/* 43 = */ prefix_rex,
+	/* 44 = */ prefix_rex,
+	/* 45 = */ prefix_rex,
+	/* 46 = */ prefix_rex,
+	/* 47 = */ prefix_rex,
+	/* 48 = */ prefix_rex,
+	/* 49 = */ prefix_rex,
+	/* 4a = */ prefix_rex,
+	/* 4b = */ prefix_rex,
+	/* 4c = */ prefix_rex,
+	/* 4d = */ prefix_rex,
+	/* 4e = */ prefix_rex,
+	/* 4f = */ prefix_rex,
+
+	/* 50 = */ prefix_done,
+	/* 51 = */ prefix_done,
+	/* 52 = */ prefix_done,
+	/* 53 = */ prefix_done,
+	/* 54 = */ prefix_done,
+	/* 55 = */ prefix_done,
+	/* 56 = */ prefix_done,
+	/* 57 = */ prefix_done,
+	/* 58 = */ prefix_done,
+	/* 59 = */ prefix_done,
+	/* 5a = */ prefix_done,
+	/* 5b = */ prefix_done,
+	/* 5c = */ prefix_done,
+	/* 5d = */ prefix_done,
+	/* 5e = */ prefix_done,
+	/* 5f = */ prefix_done,
+
+	/* 60 = */ prefix_done,
+	/* 61 = */ prefix_done,
+	/* 62 = */ prefix_evex,
+	/* 63 = */ prefix_done,
+	/* 64 = */ prefix_ignore,
+	/* 65 = */ prefix_ignore,
+	/* 66 = */ prefix_osz,
+	/* 67 = */ prefix_asz,
+	/* 68 = */ prefix_done,
+	/* 69 = */ prefix_done,
+	/* 6a = */ prefix_done,
+	/* 6b = */ prefix_done,
+	/* 6c = */ prefix_done,
+	/* 6d = */ prefix_done,
+	/* 6e = */ prefix_done,
+	/* 6f = */ prefix_done,
+
+	/* 70 = */ prefix_done,
+	/* 71 = */ prefix_done,
+	/* 72 = */ prefix_done,
+	/* 73 = */ prefix_done,
+	/* 74 = */ prefix_done,
+	/* 75 = */ prefix_done,
+	/* 76 = */ prefix_done,
+	/* 77 = */ prefix_done,
+	/* 78 = */ prefix_done,
+	/* 79 = */ prefix_done,
+	/* 7a = */ prefix_done,
+	/* 7b = */ prefix_done,
+	/* 7c = */ prefix_done,
+	/* 7d = */ prefix_done,
+	/* 7e = */ prefix_done,
+	/* 7f = */ prefix_done,
+
+	/* 80 = */ prefix_done,
+	/* 81 = */ prefix_done,
+	/* 82 = */ prefix_done,
+	/* 83 = */ prefix_done,
+	/* 84 = */ prefix_done,
+	/* 85 = */ prefix_done,
+	/* 86 = */ prefix_done,
+	/* 87 = */ prefix_done,
+	/* 88 = */ prefix_done,
+	/* 89 = */ prefix_done,
+	/* 8a = */ prefix_done,
+	/* 8b = */ prefix_done,
+	/* 8c = */ prefix_done,
+	/* 8d = */ prefix_done,
+	/* 8e = */ prefix_done,
+	/* 8f = */ prefix_done,
+
+	/* 90 = */ prefix_done,
+	/* 91 = */ prefix_done,
+	/* 92 = */ prefix_done,
+	/* 93 = */ prefix_done,
+	/* 94 = */ prefix_done,
+	/* 95 = */ prefix_done,
+	/* 96 = */ prefix_done,
+	/* 97 = */ prefix_done,
+	/* 98 = */ prefix_done,
+	/* 99 = */ prefix_done,
+	/* 9a = */ prefix_done,
+	/* 9b = */ prefix_done,
+	/* 9c = */ prefix_done,
+	/* 9d = */ prefix_done,
+	/* 9e = */ prefix_done,
+	/* 9f = */ prefix_done,
+
+	/* a0 = */ prefix_done,
+	/* a1 = */ prefix_done,
+	/* a2 = */ prefix_done,
+	/* a3 = */ prefix_done,
+	/* a4 = */ prefix_done,
+	/* a5 = */ prefix_done,
+	/* a6 = */ prefix_done,
+	/* a7 = */ prefix_done,
+	/* a8 = */ prefix_done,
+	/* a9 = */ prefix_done,
+	/* aa = */ prefix_done,
+	/* ab = */ prefix_done,
+	/* ac = */ prefix_done,
+	/* ad = */ prefix_done,
+	/* ae = */ prefix_done,
+	/* af = */ prefix_done,
+
+	/* b0 = */ prefix_done,
+	/* b1 = */ prefix_done,
+	/* b2 = */ prefix_done,
+	/* b3 = */ prefix_done,
+	/* b4 = */ prefix_done,
+	/* b5 = */ prefix_done,
+	/* b6 = */ prefix_done,
+	/* b7 = */ prefix_done,
+	/* b8 = */ prefix_done,
+	/* b9 = */ prefix_done,
+	/* ba = */ prefix_done,
+	/* bb = */ prefix_done,
+	/* bc = */ prefix_done,
+	/* bd = */ prefix_done,
+	/* be = */ prefix_done,
+	/* bf = */ prefix_done,
+
+	/* c0 = */ prefix_done,
+	/* c1 = */ prefix_done,
+	/* c2 = */ prefix_done,
+	/* c3 = */ prefix_done,
+	/* c4 = */ prefix_vex_c4,
+	/* c5 = */ prefix_vex_c5,
+	/* c6 = */ prefix_done,
+	/* c7 = */ prefix_done,
+	/* c8 = */ prefix_done,
+	/* c9 = */ prefix_done,
+	/* ca = */ prefix_done,
+	/* cb = */ prefix_done,
+	/* cc = */ prefix_done,
+	/* cd = */ prefix_done,
+	/* ce = */ prefix_done,
+	/* cf = */ prefix_done,
+
+	/* d0 = */ prefix_done,
+	/* d1 = */ prefix_done,
+	/* d2 = */ prefix_done,
+	/* d3 = */ prefix_done,
+	/* d4 = */ prefix_done,
+	/* d5 = */ prefix_done,
+	/* d6 = */ prefix_done,
+	/* d7 = */ prefix_done,
+	/* d8 = */ prefix_done,
+	/* d9 = */ prefix_done,
+	/* da = */ prefix_done,
+	/* db = */ prefix_done,
+	/* dc = */ prefix_done,
+	/* dd = */ prefix_done,
+	/* de = */ prefix_done,
+	/* df = */ prefix_done,
+
+	/* e0 = */ prefix_done,
+	/* e1 = */ prefix_done,
+	/* e2 = */ prefix_done,
+	/* e3 = */ prefix_done,
+	/* e4 = */ prefix_done,
+	/* e5 = */ prefix_done,
+	/* e6 = */ prefix_done,
+	/* e7 = */ prefix_done,
+	/* e8 = */ prefix_done,
+	/* e9 = */ prefix_done,
+	/* ea = */ prefix_done,
+	/* eb = */ prefix_done,
+	/* ec = */ prefix_done,
+	/* ed = */ prefix_done,
+	/* ee = */ prefix_done,
+	/* ef = */ prefix_done,
+
+	/* f0 = */ prefix_lock,
+	/* f1 = */ prefix_done,
+	/* f2 = */ prefix_f2,
+	/* f3 = */ prefix_f3,
+	/* f4 = */ prefix_done,
+	/* f5 = */ prefix_done,
+	/* f6 = */ prefix_done,
+	/* f7 = */ prefix_done,
+	/* f8 = */ prefix_done,
+	/* f9 = */ prefix_done,
+	/* fa = */ prefix_done,
+	/* fb = */ prefix_done,
+	/* fc = */ prefix_done,
+	/* fd = */ prefix_done,
+	/* fe = */ prefix_done,
+	/* ff = */ prefix_done
+};
+
+static inline int prefix_decode(struct pt_ild *ild, uint8_t length, uint8_t rex)
+{
+	uint8_t byte;
+
+	if (!ild)
+		return -pte_internal;
+
+	if (ild->max_bytes <= length)
+		return -pte_bad_insn;
+
+	byte = get_byte(ild, length);
+
+	return prefix_table[byte](ild, length, rex);
+}
+
+static inline int prefix_next(struct pt_ild *ild, uint8_t length, uint8_t rex)
+{
+	return prefix_decode(ild, length + 1, rex);
+}
+
+static int prefix_osz(struct pt_ild *ild, uint8_t length, uint8_t rex)
+{
+	(void) rex;
+
+	if (!ild)
+		return -pte_internal;
+
+	ild->u.s.osz = 1;
+
+	return prefix_next(ild, length, 0);
+}
+
+static int prefix_asz(struct pt_ild *ild, uint8_t length, uint8_t rex)
+{
+	(void) rex;
+
+	if (!ild)
+		return -pte_internal;
+
+	ild->u.s.asz = 1;
+
+	return prefix_next(ild, length, 0);
+}
+
+static int prefix_lock(struct pt_ild *ild, uint8_t length, uint8_t rex)
+{
+	(void) rex;
+
+	if (!ild)
+		return -pte_internal;
+
+	ild->u.s.lock = 1;
+
+	return prefix_next(ild, length, 0);
+}
+
+static int prefix_f2(struct pt_ild *ild, uint8_t length, uint8_t rex)
+{
+	(void) rex;
+
+	if (!ild)
+		return -pte_internal;
+
+	ild->u.s.f2 = 1;
+	ild->u.s.last_f2f3 = 2;
+
+	return prefix_next(ild, length, 0);
+}
+
+static int prefix_f3(struct pt_ild *ild, uint8_t length, uint8_t rex)
+{
+	(void) rex;
+
+	if (!ild)
+		return -pte_internal;
+
+	ild->u.s.f3 = 1;
+	ild->u.s.last_f2f3 = 3;
+
+	return prefix_next(ild, length, 0);
+}
+
+static int prefix_ignore(struct pt_ild *ild, uint8_t length, uint8_t rex)
+{
+	(void) rex;
+
+	return prefix_next(ild, length, 0);
+}
+
+static int prefix_done(struct pt_ild *ild, uint8_t length, uint8_t rex)
+{
+	if (!ild)
+		return -pte_internal;
+
+	if (rex & 0x04)
+		ild->u.s.rex_r = 1;
+	if (rex & 0x08)
+		ild->u.s.rex_w = 1;
+
+	return opcode_dec(ild, length);
+}
+
+static int prefix_rex(struct pt_ild *ild, uint8_t length, uint8_t rex)
+{
+	(void) rex;
+
+	if (!ild)
+		return -pte_internal;
+
+	if (mode_64b(ild))
+		return prefix_next(ild, length, get_byte(ild, length));
+	else
+		return opcode_dec(ild, length);
+}
+
+static inline int prefix_vex_done(struct pt_ild *ild, uint8_t length)
+{
+	if (!ild)
+		return -pte_internal;
+
+	ild->nominal_opcode = get_byte(ild, length);
+
+	return modrm_dec(ild, length + 1);
+}
+
+static int prefix_vex_c5(struct pt_ild *ild, uint8_t length, uint8_t rex)
+{
+	uint8_t max_bytes;
+	uint8_t p1;
+
+	(void) rex;
+
+	if (!ild)
+		return -pte_internal;
+
+	max_bytes = ild->max_bytes;
+
+	/* Read the next byte to validate that this is indeed VEX. */
+	if (max_bytes <= (length + 1))
+		return -pte_bad_insn;
+
+	p1 = get_byte(ild, length + 1);
+
+	/* If p1[7:6] is not 11b in non-64-bit mode, this is LDS, not VEX. */
+	if (!mode_64b(ild) && !bits_match(p1, 0xc0, 0xc0))
+		return opcode_dec(ild, length);
+
+	/* We need at least 3 bytes
+	 * - 2 for the VEX prefix and payload and
+	 * - 1 for the opcode.
+	 */
+	if (max_bytes < (length + 3))
+		return -pte_bad_insn;
+
+	ild->u.s.vex = 1;
+	if (p1 & 0x80)
+		ild->u.s.rex_r = 1;
+
+	ild->map = PTI_MAP_1;
+
+	/* Eat the VEX. */
+	length += 2;
+	return prefix_vex_done(ild, length);
+}
+
+static int prefix_vex_c4(struct pt_ild *ild, uint8_t length, uint8_t rex)
+{
+	uint8_t max_bytes;
+	uint8_t p1, p2, map;
+
+	(void) rex;
+
+	if (!ild)
+		return -pte_internal;
+
+	max_bytes = ild->max_bytes;
+
+	/* Read the next byte to validate that this is indeed VEX. */
+	if (max_bytes <= (length + 1))
+		return -pte_bad_insn;
+
+	p1 = get_byte(ild, length + 1);
+
+	/* If p1[7:6] is not 11b in non-64-bit mode, this is LES, not VEX. */
+	if (!mode_64b(ild) && !bits_match(p1, 0xc0, 0xc0))
+		return opcode_dec(ild, length);
+
+	/* We need at least 4 bytes
+	 * - 3 for the VEX prefix and payload and
+	 * - 1 for the opcode.
+	 */
+	if (max_bytes < (length + 4))
+		return -pte_bad_insn;
+
+	p2 = get_byte(ild, length + 2);
+
+	ild->u.s.vex = 1;
+	if (p1 & 0x80)
+		ild->u.s.rex_r = 1;
+	if (p2 & 0x80)
+		ild->u.s.rex_w = 1;
+
+	map = p1 & 0x1f;
+	if (PTI_MAP_INVALID <= map)
+		return -pte_bad_insn;
+
+	ild->map = map;
+	if (map == PTI_MAP_3)
+		ild->imm1_bytes = 1;
+
+	/* Eat the VEX. */
+	length += 3;
+	return prefix_vex_done(ild, length);
+}
+
+static int prefix_evex(struct pt_ild *ild, uint8_t length, uint8_t rex)
+{
+	uint8_t max_bytes;
+	uint8_t p1, p2, map;
+
+	(void) rex;
+
+	if (!ild)
+		return -pte_internal;
+
+	max_bytes = ild->max_bytes;
+
+	/* Read the next byte to validate that this is indeed EVEX. */
+	if (max_bytes <= (length + 1))
+		return -pte_bad_insn;
+
+	p1 = get_byte(ild, length + 1);
+
+	/* If p1[7:6] is not 11b in non-64-bit mode, this is BOUND, not EVEX. */
+	if (!mode_64b(ild) && !bits_match(p1, 0xc0, 0xc0))
+		return opcode_dec(ild, length);
+
+	/* We need at least 5 bytes
+	 * - 4 for the EVEX prefix and payload and
+	 * - 1 for the opcode.
+	 */
+	if (max_bytes < (length + 5))
+		return -pte_bad_insn;
+
+	p2 = get_byte(ild, length + 2);
+
+	ild->u.s.vex = 1;
+	if (p1 & 0x80)
+		ild->u.s.rex_r = 1;
+	if (p2 & 0x80)
+		ild->u.s.rex_w = 1;
+
+	map = p1 & 0x03;
+	ild->map = map;
+
+	if (map == PTI_MAP_3)
+		ild->imm1_bytes = 1;
+
+	/* Eat the EVEX. */
+	length += 4;
+	return prefix_vex_done(ild, length);
+}
+
+static int decode(struct pt_ild *ild)
+{
+	return prefix_decode(ild, 0, 0);
+}
+
+static int set_branch_target(struct pt_insn_ext *iext, const struct pt_ild *ild)
+{
+	if (!iext || !ild)
+		return -pte_internal;
+
+	iext->variant.branch.is_direct = 1;
+
+	if (ild->disp_bytes == 1) {
+		const int8_t *b = (const int8_t *)
+			get_byte_ptr(ild, ild->disp_pos);
+
+		iext->variant.branch.displacement = *b;
+	} else if (ild->disp_bytes == 2) {
+		const int16_t *w = (const int16_t *)
+			get_byte_ptr(ild, ild->disp_pos);
+
+		iext->variant.branch.displacement = *w;
+	} else if (ild->disp_bytes == 4) {
+		const int32_t *d = (const int32_t *)
+			get_byte_ptr(ild, ild->disp_pos);
+
+		iext->variant.branch.displacement = *d;
+	} else
+		return -pte_bad_insn;
+
+	return 0;
+}
+
+static int pt_instruction_length_decode(struct pt_ild *ild)
+{
+	if (!ild)
+		return -pte_internal;
+
+	ild->u.i = 0;
+	ild->imm1_bytes = 0;
+	ild->imm2_bytes = 0;
+	ild->disp_bytes = 0;
+	ild->modrm_byte = 0;
+	ild->map = PTI_MAP_INVALID;
+
+	if (!ild->mode)
+		return -pte_bad_insn;
+
+	return decode(ild);
+}
+
+static int pt_instruction_decode(struct pt_insn *insn, struct pt_insn_ext *iext,
+				 const struct pt_ild *ild)
+{
+	uint8_t opcode, map;
+
+	if (!iext || !ild)
+		return -pte_internal;
+
+	iext->iclass = PTI_INST_INVALID;
+	memset(&iext->variant, 0, sizeof(iext->variant));
+
+	insn->iclass = ptic_other;
+
+	opcode = ild->nominal_opcode;
+	map = ild->map;
+
+	if (map > PTI_MAP_1)
+		return 0;	/* uninteresting */
+	if (ild->u.s.vex)
+		return 0;	/* uninteresting */
+
+	/* PTI_INST_JCC,   70...7F, 0F (0x80...0x8F) */
+	if (opcode >= 0x70 && opcode <= 0x7F) {
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_cond_jump;
+			iext->iclass = PTI_INST_JCC;
+
+			return set_branch_target(iext, ild);
+		}
+		return 0;
+	}
+	if (opcode >= 0x80 && opcode <= 0x8F) {
+		if (map == PTI_MAP_1) {
+			insn->iclass = ptic_cond_jump;
+			iext->iclass = PTI_INST_JCC;
+
+			return set_branch_target(iext, ild);
+		}
+		return 0;
+	}
+
+	switch (ild->nominal_opcode) {
+	case 0x9A:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_far_call;
+			iext->iclass = PTI_INST_CALL_9A;
+		}
+		return 0;
+
+	case 0xFF:
+		if (map == PTI_MAP_0) {
+			uint8_t reg = pti_get_modrm_reg(ild);
+
+			if (reg == 2) {
+				insn->iclass = ptic_call;
+				iext->iclass = PTI_INST_CALL_FFr2;
+			} else if (reg == 3) {
+				insn->iclass = ptic_far_call;
+				iext->iclass = PTI_INST_CALL_FFr3;
+			} else if (reg == 4) {
+				insn->iclass = ptic_jump;
+				iext->iclass = PTI_INST_JMP_FFr4;
+			} else if (reg == 5) {
+				insn->iclass = ptic_far_jump;
+				iext->iclass = PTI_INST_JMP_FFr5;
+			}
+		}
+		return 0;
+
+	case 0xE8:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_call;
+			iext->iclass = PTI_INST_CALL_E8;
+
+			return set_branch_target(iext, ild);
+		}
+		return 0;
+
+	case 0xCD:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_far_call;
+			iext->iclass = PTI_INST_INT;
+		}
+
+		return 0;
+
+	case 0xCC:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_far_call;
+			iext->iclass = PTI_INST_INT3;
+		}
+
+		return 0;
+
+	case 0xCE:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_far_call;
+			iext->iclass = PTI_INST_INTO;
+		}
+
+		return 0;
+
+	case 0xF1:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_far_call;
+			iext->iclass = PTI_INST_INT1;
+		}
+
+		return 0;
+
+	case 0xCF:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_far_return;
+			iext->iclass = PTI_INST_IRET;
+		}
+		return 0;
+
+	case 0xE9:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_jump;
+			iext->iclass = PTI_INST_JMP_E9;
+
+			return set_branch_target(iext, ild);
+		}
+		return 0;
+
+	case 0xEA:
+		if (map == PTI_MAP_0) {
+			/* Far jumps are treated as indirect jumps. */
+			insn->iclass = ptic_far_jump;
+			iext->iclass = PTI_INST_JMP_EA;
+		}
+		return 0;
+
+	case 0xEB:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_jump;
+			iext->iclass = PTI_INST_JMP_EB;
+
+			return set_branch_target(iext, ild);
+		}
+		return 0;
+
+	case 0xE3:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_cond_jump;
+			iext->iclass = PTI_INST_JrCXZ;
+
+			return set_branch_target(iext, ild);
+		}
+		return 0;
+
+	case 0xE0:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_cond_jump;
+			iext->iclass = PTI_INST_LOOPNE;
+
+			return set_branch_target(iext, ild);
+		}
+		return 0;
+
+	case 0xE1:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_cond_jump;
+			iext->iclass = PTI_INST_LOOPE;
+
+			return set_branch_target(iext, ild);
+		}
+		return 0;
+
+	case 0xE2:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_cond_jump;
+			iext->iclass = PTI_INST_LOOP;
+
+			return set_branch_target(iext, ild);
+		}
+		return 0;
+
+	case 0x22:
+		if (map == PTI_MAP_1)
+			if (pti_get_modrm_reg(ild) == 3)
+				if (!ild->u.s.rex_r)
+					iext->iclass = PTI_INST_MOV_CR3;
+
+		return 0;
+
+	case 0xC3:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_return;
+			iext->iclass = PTI_INST_RET_C3;
+		}
+		return 0;
+
+	case 0xC2:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_return;
+			iext->iclass = PTI_INST_RET_C2;
+		}
+		return 0;
+
+	case 0xCB:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_far_return;
+			iext->iclass = PTI_INST_RET_CB;
+		}
+		return 0;
+
+	case 0xCA:
+		if (map == PTI_MAP_0) {
+			insn->iclass = ptic_far_return;
+			iext->iclass = PTI_INST_RET_CA;
+		}
+		return 0;
+
+	case 0x05:
+		if (map == PTI_MAP_1) {
+			insn->iclass = ptic_far_call;
+			iext->iclass = PTI_INST_SYSCALL;
+		}
+		return 0;
+
+	case 0x34:
+		if (map == PTI_MAP_1) {
+			insn->iclass = ptic_far_call;
+			iext->iclass = PTI_INST_SYSENTER;
+		}
+		return 0;
+
+	case 0x35:
+		if (map == PTI_MAP_1) {
+			insn->iclass = ptic_far_return;
+			iext->iclass = PTI_INST_SYSEXIT;
+		}
+		return 0;
+
+	case 0x07:
+		if (map == PTI_MAP_1) {
+			insn->iclass = ptic_far_return;
+			iext->iclass = PTI_INST_SYSRET;
+		}
+		return 0;
+
+	case 0x01:
+		if (map == PTI_MAP_1) {
+			switch (ild->modrm_byte) {
+			case 0xc1:
+				insn->iclass = ptic_far_call;
+				iext->iclass = PTI_INST_VMCALL;
+				break;
+
+			case 0xc2:
+				insn->iclass = ptic_far_return;
+				iext->iclass = PTI_INST_VMLAUNCH;
+				break;
+
+			case 0xc3:
+				insn->iclass = ptic_far_return;
+				iext->iclass = PTI_INST_VMRESUME;
+				break;
+
+			default:
+				break;
+			}
+		}
+		return 0;
+
+	case 0xc7:
+		if (map == PTI_MAP_1 &&
+		    pti_get_modrm_mod(ild) != 3 &&
+		    pti_get_modrm_reg(ild) == 6)
+			iext->iclass = PTI_INST_VMPTRLD;
+
+		return 0;
+
+	case 0xae:
+		if (map == PTI_MAP_1 && ild->u.s.f3 && !ild->u.s.osz &&
+		    pti_get_modrm_reg(ild) == 4) {
+			insn->iclass = ptic_ptwrite;
+			iext->iclass = PTI_INST_PTWRITE;
+		}
+		return 0;
+
+	default:
+		return 0;
+	}
+}
+
+int pt_ild_decode(struct pt_insn *insn, struct pt_insn_ext *iext)
+{
+	struct pt_ild ild;
+	int size;
+
+	if (!insn || !iext)
+		return -pte_internal;
+
+	ild.mode = insn->mode;
+	ild.itext = insn->raw;
+	ild.max_bytes = insn->size;
+
+	size = pt_instruction_length_decode(&ild);
+	if (size < 0)
+		return size;
+
+	insn->size = (uint8_t) size;
+
+	return pt_instruction_decode(insn, iext, &ild);
+}
diff --git a/libipt/src/pt_image.c b/libipt/src/pt_image.c
new file mode 100644
index 000000000000..5ec82c6e1a4e
--- /dev/null
+++ b/libipt/src/pt_image.c
@@ -0,0 +1,723 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_image.h"
+#include "pt_section.h"
+#include "pt_asid.h"
+#include "pt_image_section_cache.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+
+static char *dupstr(const char *str)
+{
+	char *dup;
+	size_t len;
+
+	if (!str)
+		return NULL;
+
+	/* Silently truncate the name if it gets too big. */
+	len = strnlen(str, 4096ul);
+
+	dup = malloc(len + 1);
+	if (!dup)
+		return NULL;
+
+	dup[len] = 0;
+
+	return memcpy(dup, str, len);
+}
+
+static struct pt_section_list *pt_mk_section_list(struct pt_section *section,
+						  const struct pt_asid *asid,
+						  uint64_t vaddr,
+						  uint64_t offset,
+						  uint64_t size, int isid)
+{
+	struct pt_section_list *list;
+	int errcode;
+
+	list = malloc(sizeof(*list));
+	if (!list)
+		return NULL;
+
+	memset(list, 0, sizeof(*list));
+
+	errcode = pt_section_get(section);
+	if (errcode < 0)
+		goto out_mem;
+
+	pt_msec_init(&list->section, section, asid, vaddr, offset, size);
+	list->isid = isid;
+
+	return list;
+
+out_mem:
+	free(list);
+	return NULL;
+}
+
+static void pt_section_list_free(struct pt_section_list *list)
+{
+	if (!list)
+		return;
+
+	pt_section_put(list->section.section);
+	pt_msec_fini(&list->section);
+	free(list);
+}
+
+static void pt_section_list_free_tail(struct pt_section_list *list)
+{
+	while (list) {
+		struct pt_section_list *trash;
+
+		trash = list;
+		list = list->next;
+
+		pt_section_list_free(trash);
+	}
+}
+
+void pt_image_init(struct pt_image *image, const char *name)
+{
+	if (!image)
+		return;
+
+	memset(image, 0, sizeof(*image));
+
+	image->name = dupstr(name);
+}
+
+void pt_image_fini(struct pt_image *image)
+{
+	if (!image)
+		return;
+
+	pt_section_list_free_tail(image->sections);
+	free(image->name);
+
+	memset(image, 0, sizeof(*image));
+}
+
+struct pt_image *pt_image_alloc(const char *name)
+{
+	struct pt_image *image;
+
+	image = malloc(sizeof(*image));
+	if (image)
+		pt_image_init(image, name);
+
+	return image;
+}
+
+void pt_image_free(struct pt_image *image)
+{
+	pt_image_fini(image);
+	free(image);
+}
+
+const char *pt_image_name(const struct pt_image *image)
+{
+	if (!image)
+		return NULL;
+
+	return image->name;
+}
+
+int pt_image_add(struct pt_image *image, struct pt_section *section,
+		 const struct pt_asid *asid, uint64_t vaddr, int isid)
+{
+	struct pt_section_list **list, *next, *removed, *new;
+	uint64_t size, begin, end;
+	int errcode;
+
+	if (!image || !section)
+		return -pte_internal;
+
+	size = pt_section_size(section);
+	begin = vaddr;
+	end = begin + size;
+
+	next = pt_mk_section_list(section, asid, begin, 0ull, size, isid);
+	if (!next)
+		return -pte_nomem;
+
+	removed = NULL;
+	errcode = 0;
+
+	/* Check for overlaps while we move to the end of the list. */
+	list = &(image->sections);
+	while (*list) {
+		const struct pt_mapped_section *msec;
+		const struct pt_asid *masid;
+		struct pt_section_list *current;
+		struct pt_section *lsec;
+		uint64_t lbegin, lend, loff;
+
+		current = *list;
+		msec = &current->section;
+		masid = pt_msec_asid(msec);
+
+		errcode = pt_asid_match(masid, asid);
+		if (errcode < 0)
+			break;
+
+		if (!errcode) {
+			list = &((*list)->next);
+			continue;
+		}
+
+		lbegin = pt_msec_begin(msec);
+		lend = pt_msec_end(msec);
+
+		if ((end <= lbegin) || (lend <= begin)) {
+			list = &((*list)->next);
+			continue;
+		}
+
+		/* The new section overlaps with @msec's section. */
+		lsec = pt_msec_section(msec);
+		loff = pt_msec_offset(msec);
+
+		/* We remove @msec and insert new sections for the remaining
+		 * parts, if any.  Those new sections are not mapped initially
+		 * and need to be added to the end of the section list.
+		 */
+		*list = current->next;
+
+		/* Keep a list of removed sections so we can re-add them in case
+		 * of errors.
+		 */
+		current->next = removed;
+		removed = current;
+
+		/* Add a section covering the remaining bytes at the front. */
+		if (lbegin < begin) {
+			new = pt_mk_section_list(lsec, masid, lbegin, loff,
+						 begin - lbegin, current->isid);
+			if (!new) {
+				errcode = -pte_nomem;
+				break;
+			}
+
+			new->next = next;
+			next = new;
+		}
+
+		/* Add a section covering the remaining bytes at the back. */
+		if (end < lend) {
+			new = pt_mk_section_list(lsec, masid, end,
+						 loff + (end - lbegin),
+						 lend - end, current->isid);
+			if (!new) {
+				errcode = -pte_nomem;
+				break;
+			}
+
+			new->next = next;
+			next = new;
+		}
+	}
+
+	if (errcode < 0) {
+		pt_section_list_free_tail(next);
+
+		/* Re-add removed sections to the tail of the section list. */
+		for (; *list; list = &((*list)->next))
+			;
+
+		*list = removed;
+		return errcode;
+	}
+
+	pt_section_list_free_tail(removed);
+
+	*list = next;
+	return 0;
+}
+
+int pt_image_remove(struct pt_image *image, struct pt_section *section,
+		    const struct pt_asid *asid, uint64_t vaddr)
+{
+	struct pt_section_list **list;
+
+	if (!image || !section)
+		return -pte_internal;
+
+	for (list = &image->sections; *list; list = &((*list)->next)) {
+		struct pt_mapped_section *msec;
+		const struct pt_section *sec;
+		const struct pt_asid *masid;
+		struct pt_section_list *trash;
+		uint64_t begin;
+		int errcode;
+
+		trash = *list;
+		msec = &trash->section;
+		masid = pt_msec_asid(msec);
+
+		errcode = pt_asid_match(masid, asid);
+		if (errcode < 0)
+			return errcode;
+
+		if (!errcode)
+			continue;
+
+		begin = pt_msec_begin(msec);
+		sec = pt_msec_section(msec);
+		if (sec == section && begin == vaddr) {
+			*list = trash->next;
+			pt_section_list_free(trash);
+
+			return 0;
+		}
+	}
+
+	return -pte_bad_image;
+}
+
+int pt_image_add_file(struct pt_image *image, const char *filename,
+		      uint64_t offset, uint64_t size,
+		      const struct pt_asid *uasid, uint64_t vaddr)
+{
+	struct pt_section *section;
+	struct pt_asid asid;
+	int errcode;
+
+	if (!image || !filename)
+		return -pte_invalid;
+
+	errcode = pt_asid_from_user(&asid, uasid);
+	if (errcode < 0)
+		return errcode;
+
+	section = NULL;
+	errcode = pt_mk_section(&section, filename, offset, size);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = pt_image_add(image, section, &asid, vaddr, 0);
+	if (errcode < 0) {
+		(void) pt_section_put(section);
+		return errcode;
+	}
+
+	/* The image list got its own reference; let's drop ours. */
+	errcode = pt_section_put(section);
+	if (errcode < 0)
+		return errcode;
+
+	return 0;
+}
+
+int pt_image_copy(struct pt_image *image, const struct pt_image *src)
+{
+	struct pt_section_list *list;
+	int ignored;
+
+	if (!image || !src)
+		return -pte_invalid;
+
+	/* There is nothing to do if we copy an image to itself.
+	 *
+	 * Besides, pt_image_add() may move sections around, which would
+	 * interfere with our section iteration.
+	 */
+	if (image == src)
+		return 0;
+
+	ignored = 0;
+	for (list = src->sections; list; list = list->next) {
+		int errcode;
+
+		errcode = pt_image_add(image, list->section.section,
+				       &list->section.asid,
+				       list->section.vaddr,
+				       list->isid);
+		if (errcode < 0)
+			ignored += 1;
+	}
+
+	return ignored;
+}
+
+int pt_image_remove_by_filename(struct pt_image *image, const char *filename,
+				const struct pt_asid *uasid)
+{
+	struct pt_section_list **list;
+	struct pt_asid asid;
+	int errcode, removed;
+
+	if (!image || !filename)
+		return -pte_invalid;
+
+	errcode = pt_asid_from_user(&asid, uasid);
+	if (errcode < 0)
+		return errcode;
+
+	removed = 0;
+	for (list = &image->sections; *list;) {
+		struct pt_mapped_section *msec;
+		const struct pt_section *sec;
+		const struct pt_asid *masid;
+		struct pt_section_list *trash;
+		const char *tname;
+
+		trash = *list;
+		msec = &trash->section;
+		masid = pt_msec_asid(msec);
+
+		errcode = pt_asid_match(masid, &asid);
+		if (errcode < 0)
+			return errcode;
+
+		if (!errcode) {
+			list = &trash->next;
+			continue;
+		}
+
+		sec = pt_msec_section(msec);
+		tname = pt_section_filename(sec);
+
+		if (tname && (strcmp(tname, filename) == 0)) {
+			*list = trash->next;
+			pt_section_list_free(trash);
+
+			removed += 1;
+		} else
+			list = &trash->next;
+	}
+
+	return removed;
+}
+
+int pt_image_remove_by_asid(struct pt_image *image,
+			    const struct pt_asid *uasid)
+{
+	struct pt_section_list **list;
+	struct pt_asid asid;
+	int errcode, removed;
+
+	if (!image)
+		return -pte_invalid;
+
+	errcode = pt_asid_from_user(&asid, uasid);
+	if (errcode < 0)
+		return errcode;
+
+	removed = 0;
+	for (list = &image->sections; *list;) {
+		struct pt_mapped_section *msec;
+		const struct pt_asid *masid;
+		struct pt_section_list *trash;
+
+		trash = *list;
+		msec = &trash->section;
+		masid = pt_msec_asid(msec);
+
+		errcode = pt_asid_match(masid, &asid);
+		if (errcode < 0)
+			return errcode;
+
+		if (!errcode) {
+			list = &trash->next;
+			continue;
+		}
+
+		*list = trash->next;
+		pt_section_list_free(trash);
+
+		removed += 1;
+	}
+
+	return removed;
+}
+
+int pt_image_set_callback(struct pt_image *image,
+			  read_memory_callback_t *callback, void *context)
+{
+	if (!image)
+		return -pte_invalid;
+
+	image->readmem.callback = callback;
+	image->readmem.context = context;
+
+	return 0;
+}
+
+static int pt_image_read_callback(struct pt_image *image, int *isid,
+				  uint8_t *buffer, uint16_t size,
+				  const struct pt_asid *asid, uint64_t addr)
+{
+	read_memory_callback_t *callback;
+
+	if (!image || !isid)
+		return -pte_internal;
+
+	callback = image->readmem.callback;
+	if (!callback)
+		return -pte_nomap;
+
+	*isid = 0;
+
+	return callback(buffer, size, asid, addr, image->readmem.context);
+}
+
+/* Check whether a mapped section contains an address.
+ *
+ * Returns zero if @msec contains @vaddr.
+ * Returns a negative error code otherwise.
+ * Returns -pte_nomap if @msec does not contain @vaddr.
+ */
+static inline int pt_image_check_msec(const struct pt_mapped_section *msec,
+				      const struct pt_asid *asid,
+				      uint64_t vaddr)
+{
+	const struct pt_asid *masid;
+	uint64_t begin, end;
+	int errcode;
+
+	if (!msec)
+		return -pte_internal;
+
+	begin = pt_msec_begin(msec);
+	end = pt_msec_end(msec);
+	if (vaddr < begin || end <= vaddr)
+		return -pte_nomap;
+
+	masid = pt_msec_asid(msec);
+	errcode = pt_asid_match(masid, asid);
+	if (errcode <= 0) {
+		if (!errcode)
+			errcode = -pte_nomap;
+
+		return errcode;
+	}
+
+	return 0;
+}
+
+/* Find the section containing a given address in a given address space.
+ *
+ * On success, the found section is moved to the front of the section list.
+ * If caching is enabled, maps the section.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_image_fetch_section(struct pt_image *image,
+				  const struct pt_asid *asid, uint64_t vaddr)
+{
+	struct pt_section_list **start, **list;
+
+	if (!image)
+		return -pte_internal;
+
+	start = &image->sections;
+	for (list = start; *list;) {
+		struct pt_mapped_section *msec;
+		struct pt_section_list *elem;
+		int errcode;
+
+		elem = *list;
+		msec = &elem->section;
+
+		errcode = pt_image_check_msec(msec, asid, vaddr);
+		if (errcode < 0) {
+			if (errcode != -pte_nomap)
+				return errcode;
+
+			list = &elem->next;
+			continue;
+		}
+
+		/* Move the section to the front if it isn't already. */
+		if (list != start) {
+			*list = elem->next;
+			elem->next = *start;
+			*start = elem;
+		}
+
+		return 0;
+	}
+
+	return -pte_nomap;
+}
+
+int pt_image_read(struct pt_image *image, int *isid, uint8_t *buffer,
+		  uint16_t size, const struct pt_asid *asid, uint64_t addr)
+{
+	struct pt_mapped_section *msec;
+	struct pt_section_list *slist;
+	struct pt_section *section;
+	int errcode, status;
+
+	if (!image || !isid)
+		return -pte_internal;
+
+	errcode = pt_image_fetch_section(image, asid, addr);
+	if (errcode < 0) {
+		if (errcode != -pte_nomap)
+			return errcode;
+
+		return pt_image_read_callback(image, isid, buffer, size, asid,
+					      addr);
+	}
+
+	slist = image->sections;
+	if (!slist)
+		return -pte_internal;
+
+	*isid = slist->isid;
+	msec = &slist->section;
+
+	section = pt_msec_section(msec);
+
+	errcode = pt_section_map(section);
+	if (errcode < 0)
+		return errcode;
+
+	status = pt_msec_read(msec, buffer, size, addr);
+
+	errcode = pt_section_unmap(section);
+		if (errcode < 0)
+			return errcode;
+
+	if (status < 0) {
+		if (status != -pte_nomap)
+			return status;
+
+		return pt_image_read_callback(image, isid, buffer, size, asid,
+					      addr);
+	}
+
+	return status;
+}
+
+int pt_image_add_cached(struct pt_image *image,
+			struct pt_image_section_cache *iscache, int isid,
+			const struct pt_asid *uasid)
+{
+	struct pt_section *section;
+	struct pt_asid asid;
+	uint64_t vaddr;
+	int errcode, status;
+
+	if (!image || !iscache)
+		return -pte_invalid;
+
+	errcode = pt_iscache_lookup(iscache, &section, &vaddr, isid);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = pt_asid_from_user(&asid, uasid);
+	if (errcode < 0)
+		return errcode;
+
+	status = pt_image_add(image, section, &asid, vaddr, isid);
+
+	/* We grab a reference when we add the section.  Drop the one we
+	 * obtained from cache lookup.
+	 */
+	errcode = pt_section_put(section);
+	if (errcode < 0)
+		return errcode;
+
+	return status;
+}
+
+int pt_image_find(struct pt_image *image, struct pt_mapped_section *usec,
+		  const struct pt_asid *asid, uint64_t vaddr)
+{
+	struct pt_mapped_section *msec;
+	struct pt_section_list *slist;
+	struct pt_section *section;
+	int errcode;
+
+	if (!image || !usec)
+		return -pte_internal;
+
+	errcode = pt_image_fetch_section(image, asid, vaddr);
+	if (errcode < 0)
+		return errcode;
+
+	slist = image->sections;
+	if (!slist)
+		return -pte_internal;
+
+	msec = &slist->section;
+	section = pt_msec_section(msec);
+
+	errcode = pt_section_get(section);
+	if (errcode < 0)
+		return errcode;
+
+	*usec = *msec;
+
+	return slist->isid;
+}
+
+int pt_image_validate(const struct pt_image *image,
+		      const struct pt_mapped_section *usec, uint64_t vaddr,
+		      int isid)
+{
+	const struct pt_section_list *slist;
+	uint64_t begin, end;
+	int status;
+
+	if (!image || !usec)
+		return -pte_internal;
+
+	/* Check that @vaddr lies within @usec. */
+	begin = pt_msec_begin(usec);
+	end = pt_msec_end(usec);
+	if (vaddr < begin || end <= vaddr)
+		return -pte_nomap;
+
+	/* We assume that @usec is a copy of the top of our stack and accept
+	 * sporadic validation fails if it isn't, e.g. because it has moved
+	 * down.
+	 *
+	 * A failed validation requires decoders to re-fetch the section so it
+	 * only results in a (relatively small) performance loss.
+	 */
+	slist = image->sections;
+	if (!slist)
+		return -pte_nomap;
+
+	if (slist->isid != isid)
+		return -pte_nomap;
+
+	status = memcmp(&slist->section, usec, sizeof(*usec));
+	if (status)
+		return -pte_nomap;
+
+	return 0;
+}
diff --git a/libipt/src/pt_image_section_cache.c b/libipt/src/pt_image_section_cache.c
new file mode 100644
index 000000000000..c2fe64bf3ce7
--- /dev/null
+++ b/libipt/src/pt_image_section_cache.c
@@ -0,0 +1,1096 @@
+/*
+ * Copyright (c) 2016-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_image_section_cache.h"
+#include "pt_section.h"
+
+#include "intel-pt.h"
+
+#include <stdlib.h>
+
+
+static char *dupstr(const char *str)
+{
+	char *dup;
+	size_t len;
+
+	if (!str)
+		return NULL;
+
+	/* Silently truncate the name if it gets too big. */
+	len = strnlen(str, 4096ul);
+
+	dup = malloc(len + 1);
+	if (!dup)
+		return NULL;
+
+	dup[len] = 0;
+
+	return memcpy(dup, str, len);
+}
+
+int pt_iscache_init(struct pt_image_section_cache *iscache, const char *name)
+{
+	if (!iscache)
+		return -pte_internal;
+
+	memset(iscache, 0, sizeof(*iscache));
+	iscache->limit = UINT64_MAX;
+	if (name) {
+		iscache->name = dupstr(name);
+		if (!iscache->name)
+			return -pte_nomem;
+	}
+
+#if defined(FEATURE_THREADS)
+	{
+		int errcode;
+
+		errcode = mtx_init(&iscache->lock, mtx_plain);
+		if (errcode != thrd_success)
+			return -pte_bad_lock;
+	}
+#endif /* defined(FEATURE_THREADS) */
+
+	return 0;
+}
+
+void pt_iscache_fini(struct pt_image_section_cache *iscache)
+{
+	if (!iscache)
+		return;
+
+	(void) pt_iscache_clear(iscache);
+	free(iscache->name);
+
+#if defined(FEATURE_THREADS)
+
+	mtx_destroy(&iscache->lock);
+
+#endif /* defined(FEATURE_THREADS) */
+}
+
+static inline int pt_iscache_lock(struct pt_image_section_cache *iscache)
+{
+	if (!iscache)
+		return -pte_internal;
+
+#if defined(FEATURE_THREADS)
+	{
+		int errcode;
+
+		errcode = mtx_lock(&iscache->lock);
+		if (errcode != thrd_success)
+			return -pte_bad_lock;
+	}
+#endif /* defined(FEATURE_THREADS) */
+
+	return 0;
+}
+
+static inline int pt_iscache_unlock(struct pt_image_section_cache *iscache)
+{
+	if (!iscache)
+		return -pte_internal;
+
+#if defined(FEATURE_THREADS)
+	{
+		int errcode;
+
+		errcode = mtx_unlock(&iscache->lock);
+		if (errcode != thrd_success)
+			return -pte_bad_lock;
+	}
+#endif /* defined(FEATURE_THREADS) */
+
+	return 0;
+}
+
+static inline int isid_from_index(uint16_t index)
+{
+	return index + 1;
+}
+
+static int pt_iscache_expand(struct pt_image_section_cache *iscache)
+{
+	struct pt_iscache_entry *entries;
+	uint16_t capacity, target;
+
+	if (!iscache)
+		return -pte_internal;
+
+	capacity = iscache->capacity;
+	target = capacity + 8;
+
+	/* Check for overflows. */
+	if (target < capacity)
+		return -pte_nomem;
+
+	entries = realloc(iscache->entries, target * sizeof(*entries));
+	if (!entries)
+		return -pte_nomem;
+
+	iscache->capacity = target;
+	iscache->entries = entries;
+	return 0;
+}
+
+static int pt_iscache_find_locked(struct pt_image_section_cache *iscache,
+				  const char *filename, uint64_t offset,
+				  uint64_t size, uint64_t laddr)
+{
+	uint16_t idx, end;
+
+	if (!iscache || !filename)
+		return -pte_internal;
+
+	end = iscache->size;
+	for (idx = 0; idx < end; ++idx) {
+		const struct pt_iscache_entry *entry;
+		const struct pt_section *section;
+		const char *sec_filename;
+		uint64_t sec_offset, sec_size;
+
+		entry = &iscache->entries[idx];
+
+		/* We do not zero-initialize the array - a NULL check is
+		 * pointless.
+		 */
+		section = entry->section;
+		sec_filename = pt_section_filename(section);
+		sec_offset = pt_section_offset(section);
+		sec_size = pt_section_size(section);
+
+		if (entry->laddr != laddr)
+			continue;
+
+		if (sec_offset != offset)
+			continue;
+
+		if (sec_size != size)
+			continue;
+
+		/* We should not have a section without a filename. */
+		if (!sec_filename)
+			return -pte_internal;
+
+		if (strcmp(sec_filename, filename) != 0)
+			continue;
+
+		return isid_from_index(idx);
+	}
+
+	return 0;
+}
+
+static int pt_iscache_lru_free(struct pt_iscache_lru_entry *lru)
+{
+	while (lru) {
+		struct pt_iscache_lru_entry *trash;
+		int errcode;
+
+		trash = lru;
+		lru = lru->next;
+
+		errcode = pt_section_unmap(trash->section);
+		if (errcode < 0)
+			return errcode;
+
+		free(trash);
+	}
+
+	return 0;
+}
+
+static int pt_iscache_lru_prune(struct pt_image_section_cache *iscache,
+				struct pt_iscache_lru_entry **tail)
+{
+	struct pt_iscache_lru_entry *lru, **pnext;
+	uint64_t limit, used;
+
+	if (!iscache || !tail)
+		return -pte_internal;
+
+	limit = iscache->limit;
+	used = 0ull;
+
+	pnext = &iscache->lru;
+	for (lru = *pnext; lru; pnext = &lru->next, lru = *pnext) {
+
+		used += lru->size;
+		if (used <= limit)
+			continue;
+
+		/* The cache got too big; prune it starting from @lru. */
+		iscache->used = used - lru->size;
+		*pnext = NULL;
+		*tail = lru;
+
+		return 0;
+	}
+
+	/* We shouldn't prune the cache unnecessarily. */
+	return -pte_internal;
+}
+
+/* Add @section to the front of @iscache->lru.
+ *
+ * Returns a positive integer if we need to prune the cache.
+ * Returns zero if we don't need to prune the cache.
+ * Returns a negative pt_error_code otherwise.
+ */
+static int pt_isache_lru_new(struct pt_image_section_cache *iscache,
+			     struct pt_section *section)
+{
+	struct pt_iscache_lru_entry *lru;
+	uint64_t memsize, used, total, limit;
+	int errcode;
+
+	if (!iscache)
+		return -pte_internal;
+
+	errcode = pt_section_memsize(section, &memsize);
+	if (errcode < 0)
+		return errcode;
+
+	/* Don't try to add the section if it is too big.  We'd prune it again
+	 * together with all other sections in our cache.
+	 */
+	limit = iscache->limit;
+	if (limit < memsize)
+		return 0;
+
+	errcode = pt_section_map_share(section);
+	if (errcode < 0)
+		return errcode;
+
+	lru = malloc(sizeof(*lru));
+	if (!lru) {
+		(void) pt_section_unmap(section);
+		return -pte_nomem;
+	}
+
+	lru->section = section;
+	lru->size = memsize;
+
+	lru->next = iscache->lru;
+	iscache->lru = lru;
+
+	used = iscache->used;
+	total = used + memsize;
+	if (total < used || total < memsize)
+		return -pte_overflow;
+
+	iscache->used = total;
+
+	return (limit < total) ? 1 : 0;
+}
+
+/* Add or move @section to the front of @iscache->lru.
+ *
+ * Returns a positive integer if we need to prune the cache.
+ * Returns zero if we don't need to prune the cache.
+ * Returns a negative pt_error_code otherwise.
+ */
+static int pt_iscache_lru_add(struct pt_image_section_cache *iscache,
+			      struct pt_section *section)
+{
+	struct pt_iscache_lru_entry *lru, **pnext;
+
+	if (!iscache)
+		return -pte_internal;
+
+	pnext = &iscache->lru;
+	for (lru = *pnext; lru; pnext = &lru->next, lru = *pnext) {
+
+		if (lru->section != section)
+			continue;
+
+		/* We found it in the cache.  Move it to the front. */
+		*pnext = lru->next;
+		lru->next = iscache->lru;
+		iscache->lru = lru;
+
+		return 0;
+	}
+
+	/* We didn't find it in the cache.  Add it. */
+	return pt_isache_lru_new(iscache, section);
+}
+
+
+/* Remove @section from @iscache->lru.
+ *
+ * Returns zero on success, a negative pt_error_code otherwise.
+ */
+static int pt_iscache_lru_remove(struct pt_image_section_cache *iscache,
+				 const struct pt_section *section)
+{
+	struct pt_iscache_lru_entry *lru, **pnext;
+
+	if (!iscache)
+		return -pte_internal;
+
+	pnext = &iscache->lru;
+	for (lru = *pnext; lru; pnext = &lru->next, lru = *pnext) {
+
+		if (lru->section != section)
+			continue;
+
+		/* We found it in the cache.  Remove it. */
+		*pnext = lru->next;
+		lru->next = NULL;
+		break;
+	}
+
+	return pt_iscache_lru_free(lru);
+}
+
+
+/* Add or move @section to the front of @iscache->lru and update its size.
+ *
+ * Returns a positive integer if we need to prune the cache.
+ * Returns zero if we don't need to prune the cache.
+ * Returns a negative pt_error_code otherwise.
+ */
+static int pt_iscache_lru_resize(struct pt_image_section_cache *iscache,
+				 struct pt_section *section, uint64_t memsize)
+{
+	struct pt_iscache_lru_entry *lru;
+	uint64_t oldsize, used;
+	int status;
+
+	if (!iscache)
+		return -pte_internal;
+
+	status = pt_iscache_lru_add(iscache, section);
+	if (status < 0)
+		return status;
+
+	lru = iscache->lru;
+	if (!lru) {
+		if (status)
+			return -pte_internal;
+		return 0;
+	}
+
+	/* If @section is cached, it must be first.
+	 *
+	 * We may choose not to cache it, though, e.g. if it is too big.
+	 */
+	if (lru->section != section) {
+		if (iscache->limit < memsize)
+			return 0;
+
+		return -pte_internal;
+	}
+
+	oldsize = lru->size;
+	lru->size = memsize;
+
+	/* If we need to prune anyway, we're done. */
+	if (status)
+		return status;
+
+	used = iscache->used;
+	used -= oldsize;
+	used += memsize;
+
+	iscache->used = used;
+
+	return (iscache->limit < used) ? 1 : 0;
+}
+
+/* Clear @iscache->lru.
+ *
+ * Unlike other iscache_lru functions, the caller does not lock @iscache.
+ *
+ * Return zero on success, a negative pt_error_code otherwise.
+ */
+static int pt_iscache_lru_clear(struct pt_image_section_cache *iscache)
+{
+	struct pt_iscache_lru_entry *lru;
+	int errcode;
+
+	errcode = pt_iscache_lock(iscache);
+	if (errcode < 0)
+		return errcode;
+
+	lru = iscache->lru;
+	iscache->lru = NULL;
+	iscache->used = 0ull;
+
+	errcode = pt_iscache_unlock(iscache);
+	if (errcode < 0)
+		return errcode;
+
+	return pt_iscache_lru_free(lru);
+}
+
+/* Search @iscache for a partial or exact match of @section loaded at @laddr and
+ * return the corresponding index or @iscache->size if no match is found.
+ *
+ * The caller must lock @iscache.
+ *
+ * Returns a non-zero index on success, a negative pt_error_code otherwise.
+ */
+static int
+pt_iscache_find_section_locked(const struct pt_image_section_cache *iscache,
+			       const char *filename, uint64_t offset,
+			       uint64_t size, uint64_t laddr)
+{
+	const struct pt_section *section;
+	uint16_t idx, end;
+	int match;
+
+	if (!iscache || !filename)
+		return -pte_internal;
+
+	section = NULL;
+	match = end = iscache->size;
+	for (idx = 0; idx < end; ++idx) {
+		const struct pt_iscache_entry *entry;
+		const struct pt_section *sec;
+
+		entry = &iscache->entries[idx];
+
+		/* We do not zero-initialize the array - a NULL check is
+		 * pointless.
+		 */
+		sec = entry->section;
+
+		/* Avoid redundant match checks. */
+		if (sec != section) {
+			const char *sec_filename;
+
+			/* We don't have duplicates.  Skip the check. */
+			if (section)
+				continue;
+
+			if (offset != pt_section_offset(sec))
+				continue;
+
+			if (size != pt_section_size(sec))
+				continue;
+
+			sec_filename = pt_section_filename(sec);
+			if (!sec_filename)
+				return -pte_internal;
+
+			if (strcmp(filename, sec_filename) != 0)
+				continue;
+
+			/* Use the cached section instead. */
+			section = sec;
+			match = idx;
+		}
+
+		/* If we didn't continue, @section == @sec and we have a match.
+		 *
+		 * If we also find a matching load address, we're done.
+		 */
+		if (laddr == entry->laddr)
+			return idx;
+	}
+
+	return match;
+}
+
+int pt_iscache_add(struct pt_image_section_cache *iscache,
+		   struct pt_section *section, uint64_t laddr)
+{
+	const char *filename;
+	uint64_t offset, size;
+	uint16_t idx;
+	int errcode;
+
+	if (!iscache || !section)
+		return -pte_internal;
+
+	/* We must have a filename for @section. */
+	filename = pt_section_filename(section);
+	if (!filename)
+		return -pte_internal;
+
+	offset = pt_section_offset(section);
+	size = pt_section_size(section);
+
+	/* Adding a section is slightly complicated by a potential deadlock
+	 * scenario:
+	 *
+	 *   - in order to add a section, we need to attach to it, which
+	 *     requires taking the section's attach lock.
+	 *
+	 *   - if we are already attached to it, we may receive on-map
+	 *     notifications, which will be sent while holding the attach lock
+	 *     and require taking the iscache lock.
+	 *
+	 * Hence we can't attach to a section while holding the iscache lock.
+	 *
+	 *
+	 * We therefore attach to @section first and then lock @iscache.
+	 *
+	 * This opens a small window where an existing @section may be removed
+	 * from @iscache and replaced by a new matching section.  We would want
+	 * to share that new section rather than adding a duplicate @section.
+	 *
+	 * After locking @iscache, we therefore check for existing matching
+	 * sections and, if one is found, update @section.  This involves
+	 * detaching from @section and attaching to the existing section.
+	 *
+	 * And for this, we will have to temporarily unlock @iscache again.
+	 */
+	errcode = pt_section_get(section);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = pt_section_attach(section, iscache);
+	if (errcode < 0)
+		goto out_put;
+
+	errcode = pt_iscache_lock(iscache);
+	if (errcode < 0)
+		goto out_detach;
+
+	/* We may need to repeat this step.
+	 *
+	 * Typically we don't and this takes only a single iteration.  One
+	 * scenario where we do repeat this is when adding a section with an
+	 * out-of-bounds size.
+	 *
+	 * We will not find a matching section in pt_iscache_add_file() so we
+	 * create a new section.  This will have its size reduced to match the
+	 * actual file size.
+	 *
+	 * For this reduced size, we may now find an existing section, and we
+	 * will take another trip in the below loop.
+	 */
+	for (;;) {
+		const struct pt_iscache_entry *entry;
+		struct pt_section *sec;
+		int match;
+
+		/* Find an existing section matching @section that we'd share
+		 * rather than adding @section.
+		 */
+		match = pt_iscache_find_section_locked(iscache, filename,
+						       offset, size, laddr);
+		if (match < 0) {
+			errcode = match;
+			goto out_unlock_detach;
+		}
+
+		/* We're done if we have not found a matching section. */
+		if (iscache->size <= match)
+			break;
+
+		entry = &iscache->entries[match];
+
+		/* We're also done if we found the same section again.
+		 *
+		 * We further check for a perfect match.  In that case, we don't
+		 * need to insert anything, at all.
+		 */
+		sec = entry->section;
+		if (sec == section) {
+			if (entry->laddr == laddr) {
+				errcode = pt_iscache_unlock(iscache);
+				if (errcode < 0)
+					goto out_detach;
+
+				errcode = pt_section_detach(section, iscache);
+				if (errcode < 0)
+					goto out_lru;
+
+				errcode = pt_section_put(section);
+				if (errcode < 0)
+					return errcode;
+
+				return isid_from_index((uint16_t) match);
+			}
+
+			break;
+		}
+
+		/* We update @section to share the existing @sec.
+		 *
+		 * This requires detaching from @section, which, in turn,
+		 * requires temporarily unlocking @iscache.
+		 *
+		 * We further need to remove @section from @iscache->lru.
+		 */
+		errcode = pt_section_get(sec);
+		if (errcode < 0)
+			goto out_unlock_detach;
+
+		errcode = pt_iscache_unlock(iscache);
+		if (errcode < 0) {
+			(void) pt_section_put(sec);
+			goto out_detach;
+		}
+
+		errcode = pt_section_detach(section, iscache);
+		if (errcode < 0) {
+			(void) pt_section_put(sec);
+			goto out_lru;
+		}
+
+		errcode = pt_section_attach(sec, iscache);
+		if (errcode < 0) {
+			(void) pt_section_put(sec);
+			goto out_lru;
+		}
+
+		errcode = pt_iscache_lock(iscache);
+		if (errcode < 0) {
+			(void) pt_section_put(section);
+			/* Complete the swap for cleanup. */
+			section = sec;
+			goto out_detach;
+		}
+
+		/* We may have received on-map notifications for @section and we
+		 * may have added @section to @iscache->lru.
+		 *
+		 * Since we're still holding a reference to it, no harm has been
+		 * done.  But we need to remove it before we drop our reference.
+		 */
+		errcode = pt_iscache_lru_remove(iscache, section);
+		if (errcode < 0) {
+			(void) pt_section_put(section);
+			/* Complete the swap for cleanup. */
+			section = sec;
+			goto out_unlock_detach;
+		}
+
+		/* Drop the reference to @section. */
+		errcode = pt_section_put(section);
+		if (errcode < 0) {
+			/* Complete the swap for cleanup. */
+			section = sec;
+			goto out_unlock_detach;
+		}
+
+		/* Swap sections.
+		 *
+		 * We will try again in the next iteration.
+		 */
+		section = sec;
+	}
+
+	/* Expand the cache, if necessary. */
+	if (iscache->capacity <= iscache->size) {
+		/* We must never exceed the capacity. */
+		if (iscache->capacity < iscache->size) {
+			errcode = -pte_internal;
+			goto out_unlock_detach;
+		}
+
+		errcode = pt_iscache_expand(iscache);
+		if (errcode < 0)
+			goto out_unlock_detach;
+
+		/* Make sure it is big enough, now. */
+		if (iscache->capacity <= iscache->size) {
+			errcode = -pte_internal;
+			goto out_unlock_detach;
+		}
+	}
+
+	/* Insert a new entry for @section at @laddr.
+	 *
+	 * This hands both attach and reference over to @iscache.  We will
+	 * detach and drop the reference again when the entry is removed.
+	 */
+	idx = iscache->size++;
+
+	iscache->entries[idx].section = section;
+	iscache->entries[idx].laddr = laddr;
+
+	errcode = pt_iscache_unlock(iscache);
+	if (errcode < 0)
+		return errcode;
+
+	return isid_from_index(idx);
+
+ out_unlock_detach:
+	(void) pt_iscache_unlock(iscache);
+
+ out_detach:
+	(void) pt_section_detach(section, iscache);
+
+ out_lru:
+	(void) pt_iscache_lru_clear(iscache);
+
+ out_put:
+	(void) pt_section_put(section);
+
+	return errcode;
+}
+
+int pt_iscache_find(struct pt_image_section_cache *iscache,
+		    const char *filename, uint64_t offset, uint64_t size,
+		    uint64_t laddr)
+{
+	int errcode, isid;
+
+	errcode = pt_iscache_lock(iscache);
+	if (errcode < 0)
+		return errcode;
+
+	isid = pt_iscache_find_locked(iscache, filename, offset, size, laddr);
+
+	errcode = pt_iscache_unlock(iscache);
+	if (errcode < 0)
+		return errcode;
+
+	return isid;
+}
+
+int pt_iscache_lookup(struct pt_image_section_cache *iscache,
+		      struct pt_section **section, uint64_t *laddr, int isid)
+{
+	uint16_t index;
+	int errcode, status;
+
+	if (!iscache || !section || !laddr)
+		return -pte_internal;
+
+	if (isid <= 0)
+		return -pte_bad_image;
+
+	isid -= 1;
+	if (isid > UINT16_MAX)
+		return -pte_internal;
+
+	index = (uint16_t) isid;
+
+	errcode = pt_iscache_lock(iscache);
+	if (errcode < 0)
+		return errcode;
+
+	if (iscache->size <= index)
+		status = -pte_bad_image;
+	else {
+		const struct pt_iscache_entry *entry;
+
+		entry = &iscache->entries[index];
+		*section = entry->section;
+		*laddr = entry->laddr;
+
+		status = pt_section_get(*section);
+	}
+
+	errcode = pt_iscache_unlock(iscache);
+	if (errcode < 0)
+		return errcode;
+
+	return status;
+}
+
+int pt_iscache_clear(struct pt_image_section_cache *iscache)
+{
+	struct pt_iscache_lru_entry *lru;
+	struct pt_iscache_entry *entries;
+	uint16_t idx, end;
+	int errcode;
+
+	if (!iscache)
+		return -pte_internal;
+
+	errcode = pt_iscache_lock(iscache);
+	if (errcode < 0)
+		return errcode;
+
+	entries = iscache->entries;
+	end = iscache->size;
+	lru = iscache->lru;
+
+	iscache->entries = NULL;
+	iscache->capacity = 0;
+	iscache->size = 0;
+	iscache->lru = NULL;
+	iscache->used = 0ull;
+
+	errcode = pt_iscache_unlock(iscache);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = pt_iscache_lru_free(lru);
+	if (errcode < 0)
+		return errcode;
+
+	for (idx = 0; idx < end; ++idx) {
+		struct pt_section *section;
+
+		section = entries[idx].section;
+
+		/* We do not zero-initialize the array - a NULL check is
+		 * pointless.
+		 */
+		errcode = pt_section_detach(section, iscache);
+		if (errcode < 0)
+			return errcode;
+
+		errcode = pt_section_put(section);
+		if (errcode < 0)
+			return errcode;
+	}
+
+	free(entries);
+	return 0;
+}
+
+struct pt_image_section_cache *pt_iscache_alloc(const char *name)
+{
+	struct pt_image_section_cache *iscache;
+
+	iscache = malloc(sizeof(*iscache));
+	if (iscache)
+		pt_iscache_init(iscache, name);
+
+	return iscache;
+}
+
+void pt_iscache_free(struct pt_image_section_cache *iscache)
+{
+	if (!iscache)
+		return;
+
+	pt_iscache_fini(iscache);
+	free(iscache);
+}
+
+int pt_iscache_set_limit(struct pt_image_section_cache *iscache, uint64_t limit)
+{
+	struct pt_iscache_lru_entry *tail;
+	int errcode, status;
+
+	if (!iscache)
+		return -pte_invalid;
+
+	status = 0;
+	tail = NULL;
+
+	errcode = pt_iscache_lock(iscache);
+	if (errcode < 0)
+		return errcode;
+
+	iscache->limit = limit;
+	if (limit < iscache->used)
+		status = pt_iscache_lru_prune(iscache, &tail);
+
+	errcode = pt_iscache_unlock(iscache);
+
+	if (errcode < 0 || status < 0)
+		return (status < 0) ? status : errcode;
+
+	return pt_iscache_lru_free(tail);
+}
+
+const char *pt_iscache_name(const struct pt_image_section_cache *iscache)
+{
+	if (!iscache)
+		return NULL;
+
+	return iscache->name;
+}
+
+int pt_iscache_add_file(struct pt_image_section_cache *iscache,
+			const char *filename, uint64_t offset, uint64_t size,
+			uint64_t vaddr)
+{
+	struct pt_section *section;
+	int errcode, match, isid;
+
+	if (!iscache || !filename)
+		return -pte_invalid;
+
+	errcode = pt_iscache_lock(iscache);
+	if (errcode < 0)
+		return errcode;
+
+	match = pt_iscache_find_section_locked(iscache, filename, offset,
+					       size, vaddr);
+	if (match < 0) {
+		(void) pt_iscache_unlock(iscache);
+		return match;
+	}
+
+	/* If we found a perfect match, we will share the existing entry.
+	 *
+	 * If we found a section, we need to grab a reference before we unlock.
+	 *
+	 * If we didn't find a matching section, we create a new section, which
+	 * implicitly gives us a reference to it.
+	 */
+	if (match < iscache->size) {
+		const struct pt_iscache_entry *entry;
+
+		entry = &iscache->entries[match];
+		if (entry->laddr == vaddr) {
+			errcode = pt_iscache_unlock(iscache);
+			if (errcode < 0)
+				return errcode;
+
+			return isid_from_index((uint16_t) match);
+		}
+
+		section = entry->section;
+
+		errcode = pt_section_get(section);
+		if (errcode < 0) {
+			(void) pt_iscache_unlock(iscache);
+			return errcode;
+		}
+
+		errcode = pt_iscache_unlock(iscache);
+		if (errcode < 0) {
+			(void) pt_section_put(section);
+			return errcode;
+		}
+	} else {
+		errcode = pt_iscache_unlock(iscache);
+		if (errcode < 0)
+			return errcode;
+
+		section = NULL;
+		errcode = pt_mk_section(&section, filename, offset, size);
+		if (errcode < 0)
+			return errcode;
+	}
+
+	/* We unlocked @iscache and hold a reference to @section. */
+	isid = pt_iscache_add(iscache, section, vaddr);
+
+	/* We grab a reference when we add the section.  Drop the one we
+	 * obtained before.
+	 */
+	errcode = pt_section_put(section);
+	if (errcode < 0)
+		return errcode;
+
+	return isid;
+}
+
+
+int pt_iscache_read(struct pt_image_section_cache *iscache, uint8_t *buffer,
+		    uint64_t size, int isid, uint64_t vaddr)
+{
+	struct pt_section *section;
+	uint64_t laddr;
+	int errcode, status;
+
+	if (!iscache || !buffer || !size)
+		return -pte_invalid;
+
+	errcode = pt_iscache_lookup(iscache, &section, &laddr, isid);
+	if (errcode < 0)
+		return errcode;
+
+	if (vaddr < laddr) {
+		(void) pt_section_put(section);
+		return -pte_nomap;
+	}
+
+	vaddr -= laddr;
+
+	errcode = pt_section_map(section);
+	if (errcode < 0) {
+		(void) pt_section_put(section);
+		return errcode;
+	}
+
+	/* We truncate the read if it gets too big.  The user is expected to
+	 * issue further reads for the remaining part.
+	 */
+	if (UINT16_MAX < size)
+		size = UINT16_MAX;
+
+	status = pt_section_read(section, buffer, (uint16_t) size, vaddr);
+
+	errcode = pt_section_unmap(section);
+	if (errcode < 0) {
+		(void) pt_section_put(section);
+		return errcode;
+	}
+
+	errcode = pt_section_put(section);
+	if (errcode < 0)
+		return errcode;
+
+	return status;
+}
+
+int pt_iscache_notify_map(struct pt_image_section_cache *iscache,
+			  struct pt_section *section)
+{
+	struct pt_iscache_lru_entry *tail;
+	int errcode, status;
+
+	tail = NULL;
+
+	errcode = pt_iscache_lock(iscache);
+	if (errcode < 0)
+		return errcode;
+
+	status = pt_iscache_lru_add(iscache, section);
+	if (status > 0)
+		status = pt_iscache_lru_prune(iscache, &tail);
+
+	errcode = pt_iscache_unlock(iscache);
+
+	if (errcode < 0 || status < 0)
+		return (status < 0) ? status : errcode;
+
+	return pt_iscache_lru_free(tail);
+}
+
+int pt_iscache_notify_resize(struct pt_image_section_cache *iscache,
+			     struct pt_section *section, uint64_t memsize)
+{
+	struct pt_iscache_lru_entry *tail;
+	int errcode, status;
+
+	tail = NULL;
+
+	errcode = pt_iscache_lock(iscache);
+	if (errcode < 0)
+		return errcode;
+
+	status = pt_iscache_lru_resize(iscache, section, memsize);
+	if (status > 0)
+		status = pt_iscache_lru_prune(iscache, &tail);
+
+	errcode = pt_iscache_unlock(iscache);
+
+	if (errcode < 0 || status < 0)
+		return (status < 0) ? status : errcode;
+
+	return pt_iscache_lru_free(tail);
+}
diff --git a/libipt/src/pt_insn.c b/libipt/src/pt_insn.c
new file mode 100644
index 000000000000..0e8979159536
--- /dev/null
+++ b/libipt/src/pt_insn.c
@@ -0,0 +1,373 @@
+/*
+ * Copyright (c) 2016-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_insn.h"
+#include "pt_ild.h"
+#include "pt_image.h"
+#include "pt_compiler.h"
+
+#include "intel-pt.h"
+
+
+int pt_insn_changes_cpl(const struct pt_insn *insn,
+			const struct pt_insn_ext *iext)
+{
+	(void) insn;
+
+	if (!iext)
+		return 0;
+
+	switch (iext->iclass) {
+	default:
+		return 0;
+
+	case PTI_INST_INT:
+	case PTI_INST_INT3:
+	case PTI_INST_INT1:
+	case PTI_INST_INTO:
+	case PTI_INST_IRET:
+	case PTI_INST_SYSCALL:
+	case PTI_INST_SYSENTER:
+	case PTI_INST_SYSEXIT:
+	case PTI_INST_SYSRET:
+		return 1;
+	}
+}
+
+int pt_insn_changes_cr3(const struct pt_insn *insn,
+			const struct pt_insn_ext *iext)
+{
+	(void) insn;
+
+	if (!iext)
+		return 0;
+
+	switch (iext->iclass) {
+	default:
+		return 0;
+
+	case PTI_INST_MOV_CR3:
+		return 1;
+	}
+}
+
+int pt_insn_is_branch(const struct pt_insn *insn,
+		      const struct pt_insn_ext *iext)
+{
+	(void) iext;
+
+	if (!insn)
+		return 0;
+
+	switch (insn->iclass) {
+	default:
+		return 0;
+
+	case ptic_call:
+	case ptic_return:
+	case ptic_jump:
+	case ptic_cond_jump:
+	case ptic_far_call:
+	case ptic_far_return:
+	case ptic_far_jump:
+		return 1;
+	}
+}
+
+int pt_insn_is_far_branch(const struct pt_insn *insn,
+			  const struct pt_insn_ext *iext)
+{
+	(void) iext;
+
+	if (!insn)
+		return 0;
+
+	switch (insn->iclass) {
+	default:
+		return 0;
+
+	case ptic_far_call:
+	case ptic_far_return:
+	case ptic_far_jump:
+		return 1;
+	}
+}
+
+int pt_insn_binds_to_pip(const struct pt_insn *insn,
+			 const struct pt_insn_ext *iext)
+{
+	if (!iext)
+		return 0;
+
+	switch (iext->iclass) {
+	default:
+		return pt_insn_is_far_branch(insn, iext);
+
+	case PTI_INST_MOV_CR3:
+	case PTI_INST_VMLAUNCH:
+	case PTI_INST_VMRESUME:
+		return 1;
+	}
+}
+
+int pt_insn_binds_to_vmcs(const struct pt_insn *insn,
+			  const struct pt_insn_ext *iext)
+{
+	if (!iext)
+		return 0;
+
+	switch (iext->iclass) {
+	default:
+		return pt_insn_is_far_branch(insn, iext);
+
+	case PTI_INST_VMPTRLD:
+	case PTI_INST_VMLAUNCH:
+	case PTI_INST_VMRESUME:
+		return 1;
+	}
+}
+
+int pt_insn_is_ptwrite(const struct pt_insn *insn,
+		       const struct pt_insn_ext *iext)
+{
+	(void) iext;
+
+	if (!insn)
+		return 0;
+
+	switch (insn->iclass) {
+	default:
+		return 0;
+
+	case ptic_ptwrite:
+		return 1;
+	}
+}
+
+int pt_insn_next_ip(uint64_t *pip, const struct pt_insn *insn,
+		    const struct pt_insn_ext *iext)
+{
+	uint64_t ip;
+
+	if (!insn || !iext)
+		return -pte_internal;
+
+	ip = insn->ip + insn->size;
+
+	switch (insn->iclass) {
+	case ptic_ptwrite:
+	case ptic_other:
+		break;
+
+	case ptic_call:
+	case ptic_jump:
+		if (iext->variant.branch.is_direct) {
+			ip += (uint64_t) (int64_t)
+				iext->variant.branch.displacement;
+			break;
+		}
+
+		fallthrough;
+	default:
+		return -pte_bad_query;
+
+	case ptic_error:
+		return -pte_bad_insn;
+	}
+
+	if (pip)
+		*pip = ip;
+
+	return 0;
+}
+
+/* Retry decoding an instruction after a preceding decode error.
+ *
+ * Instruction length decode typically fails due to 'not enough
+ * bytes'.
+ *
+ * This may be caused by partial updates of text sections
+ * represented via new image sections overlapping the original
+ * text section's image section.  We stop reading memory at the
+ * end of the section so we do not read the full instruction if
+ * parts of it have been overwritten by the update.
+ *
+ * Try to read the remaining bytes and decode the instruction again.  If we
+ * succeed, set @insn->truncated to indicate that the instruction is truncated
+ * in @insn->isid.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ * Returns -pte_bad_insn if the instruction could not be decoded.
+ */
+static int pt_insn_decode_retry(struct pt_insn *insn, struct pt_insn_ext *iext,
+				struct pt_image *image,
+				const struct pt_asid *asid)
+{
+	int size, errcode, isid;
+	uint8_t isize, remaining;
+
+	if (!insn)
+		return -pte_internal;
+
+	isize = insn->size;
+	remaining = sizeof(insn->raw) - isize;
+
+	/* We failed for real if we already read the maximum number of bytes for
+	 * an instruction.
+	 */
+	if (!remaining)
+		return -pte_bad_insn;
+
+	/* Read the remaining bytes from the image. */
+	size = pt_image_read(image, &isid, &insn->raw[isize], remaining, asid,
+			     insn->ip + isize);
+	if (size <= 0) {
+		/* We should have gotten an error if we were not able to read at
+		 * least one byte.  Check this to guarantee termination.
+		 */
+		if (!size)
+			return -pte_internal;
+
+		/* Preserve the original error if there are no more bytes. */
+		if (size == -pte_nomap)
+			size = -pte_bad_insn;
+
+		return size;
+	}
+
+	/* Add the newly read bytes to the instruction's size. */
+	insn->size += (uint8_t) size;
+
+	/* Store the new size to avoid infinite recursion in case instruction
+	 * decode fails after length decode, which would set @insn->size to the
+	 * actual length.
+	 */
+	size = insn->size;
+
+	/* Try to decode the instruction again.
+	 *
+	 * If we fail again, we recursively retry again until we either fail to
+	 * read more bytes or reach the maximum number of bytes for an
+	 * instruction.
+	 */
+	errcode = pt_ild_decode(insn, iext);
+	if (errcode < 0) {
+		if (errcode != -pte_bad_insn)
+			return errcode;
+
+		/* If instruction length decode already determined the size,
+		 * there's no point in reading more bytes.
+		 */
+		if (insn->size != (uint8_t) size)
+			return errcode;
+
+		return pt_insn_decode_retry(insn, iext, image, asid);
+	}
+
+	/* We succeeded this time, so the instruction crosses image section
+	 * boundaries.
+	 *
+	 * This poses the question which isid to use for the instruction.
+	 *
+	 * To reconstruct exactly this instruction at a later time, we'd need to
+	 * store all isids involved together with the number of bytes read for
+	 * each isid.  Since @insn already provides the exact bytes for this
+	 * instruction, we assume that the isid will be used solely for source
+	 * correlation.  In this case, it should refer to the first byte of the
+	 * instruction - as it already does.
+	 */
+	insn->truncated = 1;
+
+	return errcode;
+}
+
+int pt_insn_decode(struct pt_insn *insn, struct pt_insn_ext *iext,
+		   struct pt_image *image, const struct pt_asid *asid)
+{
+	int size, errcode;
+
+	if (!insn)
+		return -pte_internal;
+
+	/* Read the memory at the current IP in the current address space. */
+	size = pt_image_read(image, &insn->isid, insn->raw, sizeof(insn->raw),
+			     asid, insn->ip);
+	if (size < 0)
+		return size;
+
+	/* We initialize @insn->size to the maximal possible size.  It will be
+	 * set to the actual size during instruction decode.
+	 */
+	insn->size = (uint8_t) size;
+
+	errcode = pt_ild_decode(insn, iext);
+	if (errcode < 0) {
+		if (errcode != -pte_bad_insn)
+			return errcode;
+
+		/* If instruction length decode already determined the size,
+		 * there's no point in reading more bytes.
+		 */
+		if (insn->size != (uint8_t) size)
+			return errcode;
+
+		return pt_insn_decode_retry(insn, iext, image, asid);
+	}
+
+	return errcode;
+}
+
+int pt_insn_range_is_contiguous(uint64_t begin, uint64_t end,
+				enum pt_exec_mode mode, struct pt_image *image,
+				const struct pt_asid *asid, size_t steps)
+{
+	struct pt_insn_ext iext;
+	struct pt_insn insn;
+
+	memset(&insn, 0, sizeof(insn));
+
+	insn.mode = mode;
+	insn.ip = begin;
+
+	while (insn.ip != end) {
+		int errcode;
+
+		if (!steps--)
+			return 0;
+
+		errcode = pt_insn_decode(&insn, &iext, image, asid);
+		if (errcode < 0)
+			return errcode;
+
+		errcode = pt_insn_next_ip(&insn.ip, &insn, &iext);
+		if (errcode < 0)
+			return errcode;
+	}
+
+	return 1;
+}
diff --git a/libipt/src/pt_insn_decoder.c b/libipt/src/pt_insn_decoder.c
new file mode 100644
index 000000000000..c6088d4a4731
--- /dev/null
+++ b/libipt/src/pt_insn_decoder.c
@@ -0,0 +1,1769 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_insn_decoder.h"
+#include "pt_insn.h"
+#include "pt_config.h"
+#include "pt_asid.h"
+#include "pt_compiler.h"
+
+#include "intel-pt.h"
+
+#include <string.h>
+#include <stdlib.h>
+
+
+static int pt_insn_check_ip_event(struct pt_insn_decoder *,
+				  const struct pt_insn *,
+				  const struct pt_insn_ext *);
+
+
+static void pt_insn_reset(struct pt_insn_decoder *decoder)
+{
+	if (!decoder)
+		return;
+
+	decoder->mode = ptem_unknown;
+	decoder->ip = 0ull;
+	decoder->status = 0;
+	decoder->enabled = 0;
+	decoder->process_event = 0;
+	decoder->speculative = 0;
+	decoder->process_insn = 0;
+	decoder->bound_paging = 0;
+	decoder->bound_vmcs = 0;
+	decoder->bound_ptwrite = 0;
+
+	pt_retstack_init(&decoder->retstack);
+	pt_asid_init(&decoder->asid);
+}
+
+static int pt_insn_status(const struct pt_insn_decoder *decoder, int flags)
+{
+	int status;
+
+	if (!decoder)
+		return -pte_internal;
+
+	status = decoder->status;
+
+	/* Indicate whether tracing is disabled or enabled.
+	 *
+	 * This duplicates the indication in struct pt_insn and covers the case
+	 * where we indicate the status after synchronizing.
+	 */
+	if (!decoder->enabled)
+		flags |= pts_ip_suppressed;
+
+	/* Forward end-of-trace indications.
+	 *
+	 * Postpone it as long as we're still processing events, though.
+	 */
+	if ((status & pts_eos) && !decoder->process_event)
+		flags |= pts_eos;
+
+	return flags;
+}
+
+/* Initialize the query decoder flags based on our flags. */
+
+static int pt_insn_init_qry_flags(struct pt_conf_flags *qflags,
+				  const struct pt_conf_flags *flags)
+{
+	if (!qflags || !flags)
+		return -pte_internal;
+
+	memset(qflags, 0, sizeof(*qflags));
+	qflags->variant.query.keep_tcal_on_ovf =
+		flags->variant.insn.keep_tcal_on_ovf;
+
+	return 0;
+}
+
+int pt_insn_decoder_init(struct pt_insn_decoder *decoder,
+			 const struct pt_config *uconfig)
+{
+	struct pt_config config;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	errcode = pt_config_from_user(&config, uconfig);
+	if (errcode < 0)
+		return errcode;
+
+	/* The user supplied decoder flags. */
+	decoder->flags = config.flags;
+
+	/* Set the flags we need for the query decoder we use. */
+	errcode = pt_insn_init_qry_flags(&config.flags, &decoder->flags);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = pt_qry_decoder_init(&decoder->query, &config);
+	if (errcode < 0)
+		return errcode;
+
+	pt_image_init(&decoder->default_image, NULL);
+	decoder->image = &decoder->default_image;
+
+	errcode = pt_msec_cache_init(&decoder->scache);
+	if (errcode < 0)
+		return errcode;
+
+	pt_insn_reset(decoder);
+
+	return 0;
+}
+
+void pt_insn_decoder_fini(struct pt_insn_decoder *decoder)
+{
+	if (!decoder)
+		return;
+
+	pt_msec_cache_fini(&decoder->scache);
+	pt_image_fini(&decoder->default_image);
+	pt_qry_decoder_fini(&decoder->query);
+}
+
+struct pt_insn_decoder *pt_insn_alloc_decoder(const struct pt_config *config)
+{
+	struct pt_insn_decoder *decoder;
+	int errcode;
+
+	decoder = malloc(sizeof(*decoder));
+	if (!decoder)
+		return NULL;
+
+	errcode = pt_insn_decoder_init(decoder, config);
+	if (errcode < 0) {
+		free(decoder);
+		return NULL;
+	}
+
+	return decoder;
+}
+
+void pt_insn_free_decoder(struct pt_insn_decoder *decoder)
+{
+	if (!decoder)
+		return;
+
+	pt_insn_decoder_fini(decoder);
+	free(decoder);
+}
+
+/* Maybe synthesize a tick event.
+ *
+ * If we're not already processing events, check the current time against the
+ * last event's time.  If it changed, synthesize a tick event with the new time.
+ *
+ * Returns zero if no tick event has been created.
+ * Returns a positive integer if a tick event has been created.
+ * Returns a negative error code otherwise.
+ */
+static int pt_insn_tick(struct pt_insn_decoder *decoder, uint64_t ip)
+{
+	struct pt_event *ev;
+	uint64_t tsc;
+	uint32_t lost_mtc, lost_cyc;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	/* We're not generating tick events if tracing is disabled. */
+	if (!decoder->enabled)
+		return -pte_internal;
+
+	/* Events already provide a timestamp so there is no need to synthesize
+	 * an artificial tick event.  There's no room, either, since this would
+	 * overwrite the in-progress event.
+	 *
+	 * In rare cases where we need to proceed to an event location using
+	 * trace this may cause us to miss a timing update if the event is not
+	 * forwarded to the user.
+	 *
+	 * The only case I can come up with at the moment is a MODE.EXEC binding
+	 * to the TIP IP of a far branch.
+	 */
+	if (decoder->process_event)
+		return 0;
+
+	errcode = pt_qry_time(&decoder->query, &tsc, &lost_mtc, &lost_cyc);
+	if (errcode < 0) {
+		/* If we don't have wall-clock time, we use relative time. */
+		if (errcode != -pte_no_time)
+			return errcode;
+	}
+
+	ev = &decoder->event;
+
+	/* We're done if time has not changed since the last event. */
+	if (tsc == ev->tsc)
+		return 0;
+
+	/* Time has changed so we create a new tick event. */
+	memset(ev, 0, sizeof(*ev));
+	ev->type = ptev_tick;
+	ev->variant.tick.ip = ip;
+
+	/* Indicate if we have wall-clock time or only relative time. */
+	if (errcode != -pte_no_time)
+		ev->has_tsc = 1;
+	ev->tsc = tsc;
+	ev->lost_mtc = lost_mtc;
+	ev->lost_cyc = lost_cyc;
+
+	/* We now have an event to process. */
+	decoder->process_event = 1;
+
+	return 1;
+}
+
+/* Query an indirect branch.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_insn_indirect_branch(struct pt_insn_decoder *decoder,
+				   uint64_t *ip)
+{
+	uint64_t evip;
+	int status, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	evip = decoder->ip;
+
+	status = pt_qry_indirect_branch(&decoder->query, ip);
+	if (status < 0)
+		return status;
+
+	if (decoder->flags.variant.insn.enable_tick_events) {
+		errcode = pt_insn_tick(decoder, evip);
+		if (errcode < 0)
+			return errcode;
+	}
+
+	return status;
+}
+
+/* Query a conditional branch.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_insn_cond_branch(struct pt_insn_decoder *decoder, int *taken)
+{
+	int status, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	status = pt_qry_cond_branch(&decoder->query, taken);
+	if (status < 0)
+		return status;
+
+	if (decoder->flags.variant.insn.enable_tick_events) {
+		errcode = pt_insn_tick(decoder, decoder->ip);
+		if (errcode < 0)
+			return errcode;
+	}
+
+	return status;
+}
+
+static int pt_insn_start(struct pt_insn_decoder *decoder, int status)
+{
+	if (!decoder)
+		return -pte_internal;
+
+	if (status < 0)
+		return status;
+
+	decoder->status = status;
+
+	if (!(status & pts_ip_suppressed))
+		decoder->enabled = 1;
+
+	/* Process any initial events.
+	 *
+	 * Some events are processed after proceeding to the next IP in order to
+	 * indicate things like tracing disable or trace stop in the preceding
+	 * instruction.  Those events will be processed without such an
+	 * indication before decoding the current instruction.
+	 *
+	 * We do this already here so we can indicate user-events that precede
+	 * the first instruction.
+	 */
+	return pt_insn_check_ip_event(decoder, NULL, NULL);
+}
+
+int pt_insn_sync_forward(struct pt_insn_decoder *decoder)
+{
+	int status;
+
+	if (!decoder)
+		return -pte_invalid;
+
+	pt_insn_reset(decoder);
+
+	status = pt_qry_sync_forward(&decoder->query, &decoder->ip);
+
+	return pt_insn_start(decoder, status);
+}
+
+int pt_insn_sync_backward(struct pt_insn_decoder *decoder)
+{
+	int status;
+
+	if (!decoder)
+		return -pte_invalid;
+
+	pt_insn_reset(decoder);
+
+	status = pt_qry_sync_backward(&decoder->query, &decoder->ip);
+
+	return pt_insn_start(decoder, status);
+}
+
+int pt_insn_sync_set(struct pt_insn_decoder *decoder, uint64_t offset)
+{
+	int status;
+
+	if (!decoder)
+		return -pte_invalid;
+
+	pt_insn_reset(decoder);
+
+	status = pt_qry_sync_set(&decoder->query, &decoder->ip, offset);
+
+	return pt_insn_start(decoder, status);
+}
+
+int pt_insn_get_offset(const struct pt_insn_decoder *decoder, uint64_t *offset)
+{
+	if (!decoder)
+		return -pte_invalid;
+
+	return pt_qry_get_offset(&decoder->query, offset);
+}
+
+int pt_insn_get_sync_offset(const struct pt_insn_decoder *decoder,
+			    uint64_t *offset)
+{
+	if (!decoder)
+		return -pte_invalid;
+
+	return pt_qry_get_sync_offset(&decoder->query, offset);
+}
+
+struct pt_image *pt_insn_get_image(struct pt_insn_decoder *decoder)
+{
+	if (!decoder)
+		return NULL;
+
+	return decoder->image;
+}
+
+int pt_insn_set_image(struct pt_insn_decoder *decoder,
+		      struct pt_image *image)
+{
+	if (!decoder)
+		return -pte_invalid;
+
+	if (!image)
+		image = &decoder->default_image;
+
+	decoder->image = image;
+	return 0;
+}
+
+const struct pt_config *
+pt_insn_get_config(const struct pt_insn_decoder *decoder)
+{
+	if (!decoder)
+		return NULL;
+
+	return pt_qry_get_config(&decoder->query);
+}
+
+int pt_insn_time(struct pt_insn_decoder *decoder, uint64_t *time,
+		 uint32_t *lost_mtc, uint32_t *lost_cyc)
+{
+	if (!decoder || !time)
+		return -pte_invalid;
+
+	return pt_qry_time(&decoder->query, time, lost_mtc, lost_cyc);
+}
+
+int pt_insn_core_bus_ratio(struct pt_insn_decoder *decoder, uint32_t *cbr)
+{
+	if (!decoder || !cbr)
+		return -pte_invalid;
+
+	return pt_qry_core_bus_ratio(&decoder->query, cbr);
+}
+
+int pt_insn_asid(const struct pt_insn_decoder *decoder, struct pt_asid *asid,
+		 size_t size)
+{
+	if (!decoder || !asid)
+		return -pte_invalid;
+
+	return pt_asid_to_user(asid, &decoder->asid, size);
+}
+
+static inline int event_pending(struct pt_insn_decoder *decoder)
+{
+	int status;
+
+	if (!decoder)
+		return -pte_invalid;
+
+	if (decoder->process_event)
+		return 1;
+
+	status = decoder->status;
+	if (!(status & pts_event_pending))
+		return 0;
+
+	status = pt_qry_event(&decoder->query, &decoder->event,
+			      sizeof(decoder->event));
+	if (status < 0)
+		return status;
+
+	decoder->process_event = 1;
+	decoder->status = status;
+	return 1;
+}
+
+static int check_erratum_skd022(struct pt_insn_decoder *decoder)
+{
+	struct pt_insn_ext iext;
+	struct pt_insn insn;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	insn.mode = decoder->mode;
+	insn.ip = decoder->ip;
+
+	errcode = pt_insn_decode(&insn, &iext, decoder->image, &decoder->asid);
+	if (errcode < 0)
+		return 0;
+
+	switch (iext.iclass) {
+	default:
+		return 0;
+
+	case PTI_INST_VMLAUNCH:
+	case PTI_INST_VMRESUME:
+		return 1;
+	}
+}
+
+static inline int handle_erratum_skd022(struct pt_insn_decoder *decoder)
+{
+	struct pt_event *ev;
+	uint64_t ip;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	errcode = check_erratum_skd022(decoder);
+	if (errcode <= 0)
+		return errcode;
+
+	/* We turn the async disable into a sync disable.  It will be processed
+	 * after decoding the instruction.
+	 */
+	ev = &decoder->event;
+
+	ip = ev->variant.async_disabled.ip;
+
+	ev->type = ptev_disabled;
+	ev->variant.disabled.ip = ip;
+
+	return 1;
+}
+
+static int pt_insn_proceed(struct pt_insn_decoder *decoder,
+			   const struct pt_insn *insn,
+			   const struct pt_insn_ext *iext)
+{
+	if (!decoder || !insn || !iext)
+		return -pte_internal;
+
+	/* Branch displacements apply to the next instruction. */
+	decoder->ip += insn->size;
+
+	/* We handle non-branches, non-taken conditional branches, and
+	 * compressed returns directly in the switch and do some pre-work for
+	 * calls.
+	 *
+	 * All kinds of branches are handled below the switch.
+	 */
+	switch (insn->iclass) {
+	case ptic_ptwrite:
+	case ptic_other:
+		return 0;
+
+	case ptic_cond_jump: {
+		int status, taken;
+
+		status = pt_insn_cond_branch(decoder, &taken);
+		if (status < 0)
+			return status;
+
+		decoder->status = status;
+		if (!taken)
+			return 0;
+
+		break;
+	}
+
+	case ptic_call:
+		/* Log the call for return compression.
+		 *
+		 * Unless this is a call to the next instruction as is used
+		 * for position independent code.
+		 */
+		if (iext->variant.branch.displacement ||
+		    !iext->variant.branch.is_direct)
+			pt_retstack_push(&decoder->retstack, decoder->ip);
+
+		break;
+
+	case ptic_return: {
+		int taken, status;
+
+		/* Check for a compressed return. */
+		status = pt_insn_cond_branch(decoder, &taken);
+		if (status >= 0) {
+			decoder->status = status;
+
+			/* A compressed return is indicated by a taken
+			 * conditional branch.
+			 */
+			if (!taken)
+				return -pte_bad_retcomp;
+
+			return pt_retstack_pop(&decoder->retstack,
+					       &decoder->ip);
+		}
+
+		break;
+	}
+
+	case ptic_jump:
+	case ptic_far_call:
+	case ptic_far_return:
+	case ptic_far_jump:
+		break;
+
+	case ptic_error:
+		return -pte_bad_insn;
+	}
+
+	/* Process a direct or indirect branch.
+	 *
+	 * This combines calls, uncompressed returns, taken conditional jumps,
+	 * and all flavors of far transfers.
+	 */
+	if (iext->variant.branch.is_direct)
+		decoder->ip += (uint64_t) (int64_t)
+			iext->variant.branch.displacement;
+	else {
+		int status;
+
+		status = pt_insn_indirect_branch(decoder, &decoder->ip);
+
+		if (status < 0)
+			return status;
+
+		decoder->status = status;
+
+		/* We do need an IP to proceed. */
+		if (status & pts_ip_suppressed)
+			return -pte_noip;
+	}
+
+	return 0;
+}
+
+static int pt_insn_at_skl014(const struct pt_event *ev,
+			     const struct pt_insn *insn,
+			     const struct pt_insn_ext *iext,
+			     const struct pt_config *config)
+{
+	uint64_t ip;
+	int status;
+
+	if (!ev || !insn || !iext || !config)
+		return -pte_internal;
+
+	if (!ev->ip_suppressed)
+		return 0;
+
+	switch (insn->iclass) {
+	case ptic_call:
+	case ptic_jump:
+		/* The erratum only applies to unconditional direct branches. */
+		if (!iext->variant.branch.is_direct)
+			break;
+
+		/* Check the filter against the branch target. */
+		ip = insn->ip;
+		ip += insn->size;
+		ip += (uint64_t) (int64_t) iext->variant.branch.displacement;
+
+		status = pt_filter_addr_check(&config->addr_filter, ip);
+		if (status <= 0) {
+			if (status < 0)
+				return status;
+
+			return 1;
+		}
+		break;
+
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int pt_insn_at_disabled_event(const struct pt_event *ev,
+				     const struct pt_insn *insn,
+				     const struct pt_insn_ext *iext,
+				     const struct pt_config *config)
+{
+	if (!ev || !insn || !iext || !config)
+		return -pte_internal;
+
+	if (ev->ip_suppressed) {
+		if (pt_insn_is_far_branch(insn, iext) ||
+		    pt_insn_changes_cpl(insn, iext) ||
+		    pt_insn_changes_cr3(insn, iext))
+			return 1;
+
+		/* If we don't have a filter configuration we assume that no
+		 * address filters were used and the erratum does not apply.
+		 *
+		 * We might otherwise disable tracing too early.
+		 */
+		if (config->addr_filter.config.addr_cfg &&
+		    config->errata.skl014 &&
+		    pt_insn_at_skl014(ev, insn, iext, config))
+			return 1;
+	} else {
+		switch (insn->iclass) {
+		case ptic_ptwrite:
+		case ptic_other:
+			break;
+
+		case ptic_call:
+		case ptic_jump:
+			/* If we got an IP with the disabled event, we may
+			 * ignore direct branches that go to a different IP.
+			 */
+			if (iext->variant.branch.is_direct) {
+				uint64_t ip;
+
+				ip = insn->ip;
+				ip += insn->size;
+				ip += (uint64_t) (int64_t)
+					iext->variant.branch.displacement;
+
+				if (ip != ev->variant.disabled.ip)
+					break;
+			}
+
+			fallthrough;
+		case ptic_return:
+		case ptic_far_call:
+		case ptic_far_return:
+		case ptic_far_jump:
+		case ptic_cond_jump:
+			return 1;
+
+		case ptic_error:
+			return -pte_bad_insn;
+		}
+	}
+
+	return 0;
+}
+
+/* Postpone proceeding past @insn/@iext and indicate a pending event.
+ *
+ * There may be further events pending on @insn/@iext.  Postpone proceeding past
+ * @insn/@iext until we processed all events that bind to it.
+ *
+ * Returns a non-negative pt_status_flag bit-vector indicating a pending event
+ * on success, a negative pt_error_code otherwise.
+ */
+static int pt_insn_postpone(struct pt_insn_decoder *decoder,
+			    const struct pt_insn *insn,
+			    const struct pt_insn_ext *iext)
+{
+	if (!decoder || !insn || !iext)
+		return -pte_internal;
+
+	if (!decoder->process_insn) {
+		decoder->process_insn = 1;
+		decoder->insn = *insn;
+		decoder->iext = *iext;
+	}
+
+	return pt_insn_status(decoder, pts_event_pending);
+}
+
+/* Remove any postponed instruction from @decoder.
+ *
+ * Returns zero on success, a negative pt_error_code otherwise.
+ */
+static int pt_insn_clear_postponed(struct pt_insn_decoder *decoder)
+{
+	if (!decoder)
+		return -pte_internal;
+
+	decoder->process_insn = 0;
+	decoder->bound_paging = 0;
+	decoder->bound_vmcs = 0;
+	decoder->bound_ptwrite = 0;
+
+	return 0;
+}
+
+/* Proceed past a postponed instruction.
+ *
+ * Returns zero on success, a negative pt_error_code otherwise.
+ */
+static int pt_insn_proceed_postponed(struct pt_insn_decoder *decoder)
+{
+	int status;
+
+	if (!decoder)
+		return -pte_internal;
+
+	if (!decoder->process_insn)
+		return -pte_internal;
+
+	/* There's nothing to do if tracing got disabled. */
+	if (!decoder->enabled)
+		return pt_insn_clear_postponed(decoder);
+
+	status = pt_insn_proceed(decoder, &decoder->insn, &decoder->iext);
+	if (status < 0)
+		return status;
+
+	return pt_insn_clear_postponed(decoder);
+}
+
+/* Check for events that bind to instruction.
+ *
+ * Check whether an event is pending that binds to @insn/@iext, and, if that is
+ * the case, proceed past @insn/@iext and indicate the event by setting
+ * pts_event_pending.
+ *
+ * If that is not the case, we return zero.  This is what pt_insn_status() would
+ * return since:
+ *
+ *   - we suppress pts_eos as long as we're processing events
+ *   - we do not set pts_ip_suppressed since tracing must be enabled
+ *
+ * Returns a non-negative pt_status_flag bit-vector on success, a negative error
+ * code otherwise.
+ */
+static int pt_insn_check_insn_event(struct pt_insn_decoder *decoder,
+				    const struct pt_insn *insn,
+				    const struct pt_insn_ext *iext)
+{
+	struct pt_event *ev;
+	int status;
+
+	if (!decoder)
+		return -pte_internal;
+
+	status = event_pending(decoder);
+	if (status <= 0)
+		return status;
+
+	ev = &decoder->event;
+	switch (ev->type) {
+	case ptev_enabled:
+	case ptev_overflow:
+	case ptev_async_paging:
+	case ptev_async_vmcs:
+	case ptev_async_disabled:
+	case ptev_async_branch:
+	case ptev_exec_mode:
+	case ptev_tsx:
+	case ptev_stop:
+	case ptev_exstop:
+	case ptev_mwait:
+	case ptev_pwre:
+	case ptev_pwrx:
+	case ptev_tick:
+	case ptev_cbr:
+	case ptev_mnt:
+		/* We're only interested in events that bind to instructions. */
+		return 0;
+
+	case ptev_disabled:
+		status = pt_insn_at_disabled_event(ev, insn, iext,
+						   &decoder->query.config);
+		if (status <= 0)
+			return status;
+
+		/* We're at a synchronous disable event location.
+		 *
+		 * Let's determine the IP at which we expect tracing to resume.
+		 */
+		status = pt_insn_next_ip(&decoder->ip, insn, iext);
+		if (status < 0) {
+			/* We don't know the IP on error. */
+			decoder->ip = 0ull;
+
+			/* For indirect calls, assume that we return to the next
+			 * instruction.
+			 *
+			 * We only check the instruction class, not the
+			 * is_direct property, since direct calls would have
+			 * been handled by pt_insn_nex_ip() or would have
+			 * provoked a different error.
+			 */
+			if (status != -pte_bad_query)
+				return status;
+
+			switch (insn->iclass) {
+			case ptic_call:
+			case ptic_far_call:
+				decoder->ip = insn->ip + insn->size;
+				break;
+
+			default:
+				break;
+			}
+		}
+
+		break;
+
+	case ptev_paging:
+		/* We bind at most one paging event to an instruction. */
+		if (decoder->bound_paging)
+			return 0;
+
+		if (!pt_insn_binds_to_pip(insn, iext))
+			return 0;
+
+		/* We bound a paging event.  Make sure we do not bind further
+		 * paging events to this instruction.
+		 */
+		decoder->bound_paging = 1;
+
+		return pt_insn_postpone(decoder, insn, iext);
+
+	case ptev_vmcs:
+		/* We bind at most one vmcs event to an instruction. */
+		if (decoder->bound_vmcs)
+			return 0;
+
+		if (!pt_insn_binds_to_vmcs(insn, iext))
+			return 0;
+
+		/* We bound a vmcs event.  Make sure we do not bind further vmcs
+		 * events to this instruction.
+		 */
+		decoder->bound_vmcs = 1;
+
+		return pt_insn_postpone(decoder, insn, iext);
+
+	case ptev_ptwrite:
+		/* We bind at most one ptwrite event to an instruction. */
+		if (decoder->bound_ptwrite)
+			return 0;
+
+		if (ev->ip_suppressed) {
+			if (!pt_insn_is_ptwrite(insn, iext))
+				return 0;
+
+			/* Fill in the event IP.  Our users will need them to
+			 * make sense of the PTWRITE payload.
+			 */
+			ev->variant.ptwrite.ip = decoder->ip;
+			ev->ip_suppressed = 0;
+		} else {
+			/* The ptwrite event contains the IP of the ptwrite
+			 * instruction (CLIP) unlike most events that contain
+			 * the IP of the first instruction that did not complete
+			 * (NLIP).
+			 *
+			 * It's easier to handle this case here, as well.
+			 */
+			if (decoder->ip != ev->variant.ptwrite.ip)
+				return 0;
+		}
+
+		/* We bound a ptwrite event.  Make sure we do not bind further
+		 * ptwrite events to this instruction.
+		 */
+		decoder->bound_ptwrite = 1;
+
+		return pt_insn_postpone(decoder, insn, iext);
+	}
+
+	return pt_insn_status(decoder, pts_event_pending);
+}
+
+enum {
+	/* The maximum number of steps to take when determining whether the
+	 * event location can be reached.
+	 */
+	bdm64_max_steps	= 0x100
+};
+
+/* Try to work around erratum BDM64.
+ *
+ * If we got a transaction abort immediately following a branch that produced
+ * trace, the trace for that branch might have been corrupted.
+ *
+ * Returns a positive integer if the erratum was handled.
+ * Returns zero if the erratum does not seem to apply.
+ * Returns a negative error code otherwise.
+ */
+static int handle_erratum_bdm64(struct pt_insn_decoder *decoder,
+				const struct pt_event *ev,
+				const struct pt_insn *insn,
+				const struct pt_insn_ext *iext)
+{
+	int status;
+
+	if (!decoder || !ev || !insn || !iext)
+		return -pte_internal;
+
+	/* This only affects aborts. */
+	if (!ev->variant.tsx.aborted)
+		return 0;
+
+	/* This only affects branches. */
+	if (!pt_insn_is_branch(insn, iext))
+		return 0;
+
+	/* Let's check if we can reach the event location from here.
+	 *
+	 * If we can, let's assume the erratum did not hit.  We might still be
+	 * wrong but we're not able to tell.
+	 */
+	status = pt_insn_range_is_contiguous(decoder->ip, ev->variant.tsx.ip,
+					     decoder->mode, decoder->image,
+					     &decoder->asid, bdm64_max_steps);
+	if (status > 0)
+		return 0;
+
+	/* We can't reach the event location.  This could either mean that we
+	 * stopped too early (and status is zero) or that the erratum hit.
+	 *
+	 * We assume the latter and pretend that the previous branch brought us
+	 * to the event location, instead.
+	 */
+	decoder->ip = ev->variant.tsx.ip;
+
+	return 1;
+}
+
+/* Check whether a peek TSX event should be postponed.
+ *
+ * This involves handling erratum BDM64.
+ *
+ * Returns a positive integer if the event is to be postponed.
+ * Returns zero if the event should be processed.
+ * Returns a negative error code otherwise.
+ */
+static inline int pt_insn_postpone_tsx(struct pt_insn_decoder *decoder,
+				       const struct pt_insn *insn,
+				       const struct pt_insn_ext *iext,
+				       const struct pt_event *ev)
+{
+	int status;
+
+	if (!decoder || !ev)
+		return -pte_internal;
+
+	if (ev->ip_suppressed)
+		return 0;
+
+	if (insn && iext && decoder->query.config.errata.bdm64) {
+		status = handle_erratum_bdm64(decoder, ev, insn, iext);
+		if (status < 0)
+			return status;
+	}
+
+	if (decoder->ip != ev->variant.tsx.ip)
+		return 1;
+
+	return 0;
+}
+
+/* Check for events that bind to an IP.
+ *
+ * Check whether an event is pending that binds to @decoder->ip, and, if that is
+ * the case, indicate the event by setting pt_pts_event_pending.
+ *
+ * Returns a non-negative pt_status_flag bit-vector on success, a negative error
+ * code otherwise.
+ */
+static int pt_insn_check_ip_event(struct pt_insn_decoder *decoder,
+				  const struct pt_insn *insn,
+				  const struct pt_insn_ext *iext)
+{
+	struct pt_event *ev;
+	int status;
+
+	if (!decoder)
+		return -pte_internal;
+
+	status = event_pending(decoder);
+	if (status <= 0) {
+		if (status < 0)
+			return status;
+
+		return pt_insn_status(decoder, 0);
+	}
+
+	ev = &decoder->event;
+	switch (ev->type) {
+	case ptev_disabled:
+		break;
+
+	case ptev_enabled:
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_async_disabled:
+		if (ev->variant.async_disabled.at != decoder->ip)
+			break;
+
+		if (decoder->query.config.errata.skd022) {
+			int errcode;
+
+			errcode = handle_erratum_skd022(decoder);
+			if (errcode != 0) {
+				if (errcode < 0)
+					return errcode;
+
+				/* If the erratum applies, we postpone the
+				 * modified event to the next call to
+				 * pt_insn_next().
+				 */
+				break;
+			}
+		}
+
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_tsx:
+		status = pt_insn_postpone_tsx(decoder, insn, iext, ev);
+		if (status != 0) {
+			if (status < 0)
+				return status;
+
+			break;
+		}
+
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_async_branch:
+		if (ev->variant.async_branch.from != decoder->ip)
+			break;
+
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_overflow:
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_exec_mode:
+		if (!ev->ip_suppressed &&
+		    ev->variant.exec_mode.ip != decoder->ip)
+			break;
+
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_paging:
+		if (decoder->enabled)
+			break;
+
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_async_paging:
+		if (!ev->ip_suppressed &&
+		    ev->variant.async_paging.ip != decoder->ip)
+			break;
+
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_vmcs:
+		if (decoder->enabled)
+			break;
+
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_async_vmcs:
+		if (!ev->ip_suppressed &&
+		    ev->variant.async_vmcs.ip != decoder->ip)
+			break;
+
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_stop:
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_exstop:
+		if (!ev->ip_suppressed && decoder->enabled &&
+		    decoder->ip != ev->variant.exstop.ip)
+			break;
+
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_mwait:
+		if (!ev->ip_suppressed && decoder->enabled &&
+		    decoder->ip != ev->variant.mwait.ip)
+			break;
+
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_pwre:
+	case ptev_pwrx:
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_ptwrite:
+		/* Any event binding to the current PTWRITE instruction is
+		 * handled in pt_insn_check_insn_event().
+		 *
+		 * Any subsequent ptwrite event binds to a different instruction
+		 * and must wait until the next iteration - as long as tracing
+		 * is enabled.
+		 *
+		 * When tracing is disabled, we forward all ptwrite events
+		 * immediately to the user.
+		 */
+		if (decoder->enabled)
+			break;
+
+		return pt_insn_status(decoder, pts_event_pending);
+
+	case ptev_tick:
+	case ptev_cbr:
+	case ptev_mnt:
+		return pt_insn_status(decoder, pts_event_pending);
+	}
+
+	return pt_insn_status(decoder, 0);
+}
+
+static inline int insn_to_user(struct pt_insn *uinsn, size_t size,
+			       const struct pt_insn *insn)
+{
+	if (!uinsn || !insn)
+		return -pte_internal;
+
+	if (uinsn == insn)
+		return 0;
+
+	/* Zero out any unknown bytes. */
+	if (sizeof(*insn) < size) {
+		memset(uinsn + sizeof(*insn), 0, size - sizeof(*insn));
+
+		size = sizeof(*insn);
+	}
+
+	memcpy(uinsn, insn, size);
+
+	return 0;
+}
+
+static int pt_insn_decode_cached(struct pt_insn_decoder *decoder,
+				 const struct pt_mapped_section *msec,
+				 struct pt_insn *insn, struct pt_insn_ext *iext)
+{
+	int status;
+
+	if (!decoder || !insn || !iext)
+		return -pte_internal;
+
+	/* Try reading the memory containing @insn from the cached section.  If
+	 * that fails, if we don't have a cached section, or if decode fails
+	 * later on, fall back to decoding @insn from @decoder->image.
+	 *
+	 * The latter will also handle truncated instructions that cross section
+	 * boundaries.
+	 */
+
+	if (!msec)
+		return pt_insn_decode(insn, iext, decoder->image,
+				      &decoder->asid);
+
+	status = pt_msec_read(msec, insn->raw, sizeof(insn->raw), insn->ip);
+	if (status < 0) {
+		if (status != -pte_nomap)
+			return status;
+
+		return pt_insn_decode(insn, iext, decoder->image,
+				      &decoder->asid);
+	}
+
+	/* We initialize @insn->size to the maximal possible size.  It will be
+	 * set to the actual size during instruction decode.
+	 */
+	insn->size = (uint8_t) status;
+
+	status = pt_ild_decode(insn, iext);
+	if (status < 0) {
+		if (status != -pte_bad_insn)
+			return status;
+
+		return pt_insn_decode(insn, iext, decoder->image,
+				      &decoder->asid);
+	}
+
+	return status;
+}
+
+static int pt_insn_msec_lookup(struct pt_insn_decoder *decoder,
+			       const struct pt_mapped_section **pmsec)
+{
+	struct pt_msec_cache *scache;
+	struct pt_image *image;
+	uint64_t ip;
+	int isid;
+
+	if (!decoder || !pmsec)
+		return -pte_internal;
+
+	scache = &decoder->scache;
+	image = decoder->image;
+	ip = decoder->ip;
+
+	isid = pt_msec_cache_read(scache, pmsec, image, ip);
+	if (isid < 0) {
+		if (isid != -pte_nomap)
+			return isid;
+
+		return pt_msec_cache_fill(scache, pmsec, image,
+					  &decoder->asid, ip);
+	}
+
+	return isid;
+}
+
+int pt_insn_next(struct pt_insn_decoder *decoder, struct pt_insn *uinsn,
+		 size_t size)
+{
+	const struct pt_mapped_section *msec;
+	struct pt_insn_ext iext;
+	struct pt_insn insn, *pinsn;
+	int status, isid;
+
+	if (!uinsn || !decoder)
+		return -pte_invalid;
+
+	/* Tracing must be enabled.
+	 *
+	 * If it isn't we should be processing events until we either run out of
+	 * trace or process a tracing enabled event.
+	 */
+	if (!decoder->enabled) {
+		if (decoder->status & pts_eos)
+			return -pte_eos;
+
+		return -pte_no_enable;
+	}
+
+	pinsn = size == sizeof(insn) ? uinsn : &insn;
+
+	/* Zero-initialize the instruction in case of error returns. */
+	memset(pinsn, 0, sizeof(*pinsn));
+
+	/* Fill in a few things from the current decode state.
+	 *
+	 * This reflects the state of the last pt_insn_next(), pt_insn_event()
+	 * or pt_insn_start() call.
+	 */
+	if (decoder->speculative)
+		pinsn->speculative = 1;
+	pinsn->ip = decoder->ip;
+	pinsn->mode = decoder->mode;
+
+	isid = pt_insn_msec_lookup(decoder, &msec);
+	if (isid < 0) {
+		if (isid != -pte_nomap)
+			return isid;
+
+		msec = NULL;
+	}
+
+	/* We set an incorrect isid if @msec is NULL.  This will be corrected
+	 * when we read the memory from the image later on.
+	 */
+	pinsn->isid = isid;
+
+	status = pt_insn_decode_cached(decoder, msec, pinsn, &iext);
+	if (status < 0) {
+		/* Provide the incomplete instruction - the IP and mode fields
+		 * are valid and may help diagnose the error.
+		 */
+		(void) insn_to_user(uinsn, size, pinsn);
+		return status;
+	}
+
+	/* Provide the decoded instruction to the user.  It won't change during
+	 * event processing.
+	 */
+	status = insn_to_user(uinsn, size, pinsn);
+	if (status < 0)
+		return status;
+
+	/* Check for events that bind to the current instruction.
+	 *
+	 * If an event is indicated, we're done.
+	 */
+	status = pt_insn_check_insn_event(decoder, pinsn, &iext);
+	if (status != 0) {
+		if (status < 0)
+			return status;
+
+		if (status & pts_event_pending)
+			return status;
+	}
+
+	/* Determine the next instruction's IP. */
+	status = pt_insn_proceed(decoder, pinsn, &iext);
+	if (status < 0)
+		return status;
+
+	/* Indicate events that bind to the new IP.
+	 *
+	 * Although we only look at the IP for binding events, we pass the
+	 * decoded instruction in order to handle errata.
+	 */
+	return pt_insn_check_ip_event(decoder, pinsn, &iext);
+}
+
+static int pt_insn_process_enabled(struct pt_insn_decoder *decoder)
+{
+	struct pt_event *ev;
+
+	if (!decoder)
+		return -pte_internal;
+
+	ev = &decoder->event;
+
+	/* This event can't be a status update. */
+	if (ev->status_update)
+		return -pte_bad_context;
+
+	/* We must have an IP in order to start decoding. */
+	if (ev->ip_suppressed)
+		return -pte_noip;
+
+	/* We must currently be disabled. */
+	if (decoder->enabled)
+		return -pte_bad_context;
+
+	decoder->ip = ev->variant.enabled.ip;
+	decoder->enabled = 1;
+
+	return 0;
+}
+
+static int pt_insn_process_disabled(struct pt_insn_decoder *decoder)
+{
+	struct pt_event *ev;
+
+	if (!decoder)
+		return -pte_internal;
+
+	ev = &decoder->event;
+
+	/* This event can't be a status update. */
+	if (ev->status_update)
+		return -pte_bad_context;
+
+	/* We must currently be enabled. */
+	if (!decoder->enabled)
+		return -pte_bad_context;
+
+	/* We preserve @decoder->ip.  This is where we expect tracing to resume
+	 * and we'll indicate that on the subsequent enabled event if tracing
+	 * actually does resume from there.
+	 */
+	decoder->enabled = 0;
+
+	return 0;
+}
+
+static int pt_insn_process_async_branch(struct pt_insn_decoder *decoder)
+{
+	struct pt_event *ev;
+
+	if (!decoder)
+		return -pte_internal;
+
+	ev = &decoder->event;
+
+	/* This event can't be a status update. */
+	if (ev->status_update)
+		return -pte_bad_context;
+
+	/* Tracing must be enabled in order to make sense of the event. */
+	if (!decoder->enabled)
+		return -pte_bad_context;
+
+	decoder->ip = ev->variant.async_branch.to;
+
+	return 0;
+}
+
+static int pt_insn_process_paging(struct pt_insn_decoder *decoder)
+{
+	uint64_t cr3;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	cr3 = decoder->event.variant.paging.cr3;
+	if (decoder->asid.cr3 != cr3) {
+		errcode = pt_msec_cache_invalidate(&decoder->scache);
+		if (errcode < 0)
+			return errcode;
+
+		decoder->asid.cr3 = cr3;
+	}
+
+	return 0;
+}
+
+static int pt_insn_process_overflow(struct pt_insn_decoder *decoder)
+{
+	struct pt_event *ev;
+
+	if (!decoder)
+		return -pte_internal;
+
+	ev = &decoder->event;
+
+	/* This event can't be a status update. */
+	if (ev->status_update)
+		return -pte_bad_context;
+
+	/* If the IP is suppressed, the overflow resolved while tracing was
+	 * disabled.  Otherwise it resolved while tracing was enabled.
+	 */
+	if (ev->ip_suppressed) {
+		/* Tracing is disabled.
+		 *
+		 * It doesn't make sense to preserve the previous IP.  This will
+		 * just be misleading.  Even if tracing had been disabled
+		 * before, as well, we might have missed the re-enable in the
+		 * overflow.
+		 */
+		decoder->enabled = 0;
+		decoder->ip = 0ull;
+	} else {
+		/* Tracing is enabled and we're at the IP at which the overflow
+		 * resolved.
+		 */
+		decoder->ip = ev->variant.overflow.ip;
+		decoder->enabled = 1;
+	}
+
+	/* We don't know the TSX state.  Let's assume we execute normally.
+	 *
+	 * We also don't know the execution mode.  Let's keep what we have
+	 * in case we don't get an update before we have to decode the next
+	 * instruction.
+	 */
+	decoder->speculative = 0;
+
+	return 0;
+}
+
+static int pt_insn_process_exec_mode(struct pt_insn_decoder *decoder)
+{
+	enum pt_exec_mode mode;
+	struct pt_event *ev;
+
+	if (!decoder)
+		return -pte_internal;
+
+	ev = &decoder->event;
+	mode = ev->variant.exec_mode.mode;
+
+	/* Use status update events to diagnose inconsistencies. */
+	if (ev->status_update && decoder->enabled &&
+	    decoder->mode != ptem_unknown && decoder->mode != mode)
+		return -pte_bad_status_update;
+
+	decoder->mode = mode;
+
+	return 0;
+}
+
+static int pt_insn_process_tsx(struct pt_insn_decoder *decoder)
+{
+	if (!decoder)
+		return -pte_internal;
+
+	decoder->speculative = decoder->event.variant.tsx.speculative;
+
+	return 0;
+}
+
+static int pt_insn_process_stop(struct pt_insn_decoder *decoder)
+{
+	struct pt_event *ev;
+
+	if (!decoder)
+		return -pte_internal;
+
+	ev = &decoder->event;
+
+	/* This event can't be a status update. */
+	if (ev->status_update)
+		return -pte_bad_context;
+
+	/* Tracing is always disabled before it is stopped. */
+	if (decoder->enabled)
+		return -pte_bad_context;
+
+	return 0;
+}
+
+static int pt_insn_process_vmcs(struct pt_insn_decoder *decoder)
+{
+	uint64_t vmcs;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	vmcs = decoder->event.variant.vmcs.base;
+	if (decoder->asid.vmcs != vmcs) {
+		errcode = pt_msec_cache_invalidate(&decoder->scache);
+		if (errcode < 0)
+			return errcode;
+
+		decoder->asid.vmcs = vmcs;
+	}
+
+	return 0;
+}
+
+int pt_insn_event(struct pt_insn_decoder *decoder, struct pt_event *uevent,
+		  size_t size)
+{
+	struct pt_event *ev;
+	int status;
+
+	if (!decoder || !uevent)
+		return -pte_invalid;
+
+	/* We must currently process an event. */
+	if (!decoder->process_event)
+		return -pte_bad_query;
+
+	ev = &decoder->event;
+	switch (ev->type) {
+	default:
+		/* This is not a user event.
+		 *
+		 * We either indicated it wrongly or the user called
+		 * pt_insn_event() without a pts_event_pending indication.
+		 */
+		return -pte_bad_query;
+
+	case ptev_enabled:
+		/* Indicate that tracing resumes from the IP at which tracing
+		 * had been disabled before (with some special treatment for
+		 * calls).
+		 */
+		if (decoder->ip == ev->variant.enabled.ip)
+			ev->variant.enabled.resumed = 1;
+
+		status = pt_insn_process_enabled(decoder);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_async_disabled:
+		if (!ev->ip_suppressed &&
+		    decoder->ip != ev->variant.async_disabled.at)
+			return -pte_bad_query;
+
+		fallthrough;
+	case ptev_disabled:
+		status = pt_insn_process_disabled(decoder);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_async_branch:
+		if (decoder->ip != ev->variant.async_branch.from)
+			return -pte_bad_query;
+
+		status = pt_insn_process_async_branch(decoder);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_async_paging:
+		if (!ev->ip_suppressed &&
+		    decoder->ip != ev->variant.async_paging.ip)
+			return -pte_bad_query;
+
+		fallthrough;
+	case ptev_paging:
+		status = pt_insn_process_paging(decoder);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_async_vmcs:
+		if (!ev->ip_suppressed &&
+		    decoder->ip != ev->variant.async_vmcs.ip)
+			return -pte_bad_query;
+
+		fallthrough;
+	case ptev_vmcs:
+		status = pt_insn_process_vmcs(decoder);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_overflow:
+		status = pt_insn_process_overflow(decoder);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_exec_mode:
+		status = pt_insn_process_exec_mode(decoder);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_tsx:
+		status = pt_insn_process_tsx(decoder);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_stop:
+		status = pt_insn_process_stop(decoder);
+		if (status < 0)
+			return status;
+
+		break;
+
+	case ptev_exstop:
+		if (!ev->ip_suppressed && decoder->enabled &&
+		    decoder->ip != ev->variant.exstop.ip)
+			return -pte_bad_query;
+
+		break;
+
+	case ptev_mwait:
+		if (!ev->ip_suppressed && decoder->enabled &&
+		    decoder->ip != ev->variant.mwait.ip)
+			return -pte_bad_query;
+
+		break;
+
+	case ptev_pwre:
+	case ptev_pwrx:
+	case ptev_ptwrite:
+	case ptev_tick:
+	case ptev_cbr:
+	case ptev_mnt:
+		break;
+	}
+
+	/* Copy the event to the user.  Make sure we're not writing beyond the
+	 * memory provided by the user.
+	 *
+	 * We might truncate details of an event but only for those events the
+	 * user can't know about, anyway.
+	 */
+	if (sizeof(*ev) < size)
+		size = sizeof(*ev);
+
+	memcpy(uevent, ev, size);
+
+	/* This completes processing of the current event. */
+	decoder->process_event = 0;
+
+	/* If we just handled an instruction event, check for further events
+	 * that bind to this instruction.
+	 *
+	 * If we don't have further events, proceed beyond the instruction so we
+	 * can check for IP events, as well.
+	 */
+	if (decoder->process_insn) {
+		status = pt_insn_check_insn_event(decoder, &decoder->insn,
+						  &decoder->iext);
+
+		if (status != 0) {
+			if (status < 0)
+				return status;
+
+			if (status & pts_event_pending)
+				return status;
+		}
+
+		/* Proceed to the next instruction. */
+		status = pt_insn_proceed_postponed(decoder);
+		if (status < 0)
+			return status;
+	}
+
+	/* Indicate further events that bind to the same IP. */
+	return pt_insn_check_ip_event(decoder, NULL, NULL);
+}
diff --git a/libipt/src/pt_last_ip.c b/libipt/src/pt_last_ip.c
new file mode 100644
index 000000000000..a6a82b7cbb4e
--- /dev/null
+++ b/libipt/src/pt_last_ip.c
@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_last_ip.h"
+
+#include "intel-pt.h"
+
+
+void pt_last_ip_init(struct pt_last_ip *last_ip)
+{
+	if (!last_ip)
+		return;
+
+	last_ip->ip = 0ull;
+	last_ip->have_ip = 0;
+	last_ip->suppressed = 0;
+}
+
+int pt_last_ip_query(uint64_t *ip, const struct pt_last_ip *last_ip)
+{
+	if (!last_ip)
+		return -pte_internal;
+
+	if (!last_ip->have_ip) {
+		if (ip)
+			*ip = 0ull;
+		return -pte_noip;
+	}
+
+	if (last_ip->suppressed) {
+		if (ip)
+			*ip = 0ull;
+		return -pte_ip_suppressed;
+	}
+
+	if (ip)
+		*ip = last_ip->ip;
+
+	return 0;
+}
+
+/* Sign-extend a uint64_t value. */
+static uint64_t sext(uint64_t val, uint8_t sign)
+{
+	uint64_t signbit, mask;
+
+	signbit = 1ull << (sign - 1);
+	mask = ~0ull << sign;
+
+	return val & signbit ? val | mask : val & ~mask;
+}
+
+int pt_last_ip_update_ip(struct pt_last_ip *last_ip,
+			 const struct pt_packet_ip *packet,
+			 const struct pt_config *config)
+{
+	(void) config;
+
+	if (!last_ip || !packet)
+		return -pte_internal;
+
+	switch (packet->ipc) {
+	case pt_ipc_suppressed:
+		last_ip->suppressed = 1;
+		return 0;
+
+	case pt_ipc_sext_48:
+		last_ip->ip = sext(packet->ip, 48);
+		last_ip->have_ip = 1;
+		last_ip->suppressed = 0;
+		return 0;
+
+	case pt_ipc_update_16:
+		last_ip->ip = (last_ip->ip & ~0xffffull)
+			| (packet->ip & 0xffffull);
+		last_ip->have_ip = 1;
+		last_ip->suppressed = 0;
+		return 0;
+
+	case pt_ipc_update_32:
+		last_ip->ip = (last_ip->ip & ~0xffffffffull)
+			| (packet->ip & 0xffffffffull);
+		last_ip->have_ip = 1;
+		last_ip->suppressed = 0;
+		return 0;
+
+	case pt_ipc_update_48:
+		last_ip->ip = (last_ip->ip & ~0xffffffffffffull)
+			| (packet->ip & 0xffffffffffffull);
+		last_ip->have_ip = 1;
+		last_ip->suppressed = 0;
+		return 0;
+
+	case pt_ipc_full:
+		last_ip->ip = packet->ip;
+		last_ip->have_ip = 1;
+		last_ip->suppressed = 0;
+		return 0;
+	}
+
+	return -pte_bad_packet;
+}
diff --git a/libipt/src/pt_msec_cache.c b/libipt/src/pt_msec_cache.c
new file mode 100644
index 000000000000..c5db101bedb8
--- /dev/null
+++ b/libipt/src/pt_msec_cache.c
@@ -0,0 +1,136 @@
+/*
+ * Copyright (c) 2017-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_msec_cache.h"
+#include "pt_section.h"
+#include "pt_image.h"
+
+#include <string.h>
+
+
+int pt_msec_cache_init(struct pt_msec_cache *cache)
+{
+	if (!cache)
+		return -pte_internal;
+
+	memset(cache, 0, sizeof(*cache));
+
+	return 0;
+}
+
+void pt_msec_cache_fini(struct pt_msec_cache *cache)
+{
+	if (!cache)
+		return;
+
+	(void) pt_msec_cache_invalidate(cache);
+	pt_msec_fini(&cache->msec);
+}
+
+int pt_msec_cache_invalidate(struct pt_msec_cache *cache)
+{
+	struct pt_section *section;
+	int errcode;
+
+	if (!cache)
+		return -pte_internal;
+
+	section = pt_msec_section(&cache->msec);
+	if (!section)
+		return 0;
+
+	errcode = pt_section_unmap(section);
+	if (errcode < 0)
+		return errcode;
+
+	cache->msec.section = NULL;
+
+	return pt_section_put(section);
+}
+
+int pt_msec_cache_read(struct pt_msec_cache *cache,
+		       const struct pt_mapped_section **pmsec,
+		       struct pt_image *image, uint64_t vaddr)
+{
+	struct pt_mapped_section *msec;
+	int isid, errcode;
+
+	if (!cache || !pmsec)
+		return -pte_internal;
+
+	msec = &cache->msec;
+	isid = cache->isid;
+
+	errcode = pt_image_validate(image, msec, vaddr, isid);
+	if (errcode < 0)
+		return errcode;
+
+	*pmsec = msec;
+
+	return isid;
+
+}
+
+int pt_msec_cache_fill(struct pt_msec_cache *cache,
+		       const struct pt_mapped_section **pmsec,
+		       struct pt_image *image, const struct pt_asid *asid,
+		       uint64_t vaddr)
+{
+	struct pt_mapped_section *msec;
+	struct pt_section *section;
+	int errcode, isid;
+
+	if (!cache || !pmsec)
+		return -pte_internal;
+
+	errcode = pt_msec_cache_invalidate(cache);
+	if (errcode < 0)
+		return errcode;
+
+	msec = &cache->msec;
+
+	isid = pt_image_find(image, msec, asid, vaddr);
+	if (isid < 0)
+		return isid;
+
+	section = pt_msec_section(msec);
+
+	errcode = pt_section_map(section);
+	if (errcode < 0) {
+		(void) pt_section_put(section);
+		msec->section = NULL;
+
+		return errcode;
+	}
+
+	*pmsec = msec;
+
+	cache->isid = isid;
+
+	return isid;
+}
diff --git a/libipt/src/pt_packet.c b/libipt/src/pt_packet.c
new file mode 100644
index 000000000000..8ec46f7db450
--- /dev/null
+++ b/libipt/src/pt_packet.c
@@ -0,0 +1,573 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_packet.h"
+#include "pt_opcodes.h"
+
+#include "intel-pt.h"
+
+#include <limits.h>
+
+
+static uint64_t pt_pkt_read_value(const uint8_t *pos, int size)
+{
+	uint64_t val;
+	int idx;
+
+	for (val = 0, idx = 0; idx < size; ++idx) {
+		uint64_t byte = *pos++;
+
+		byte <<= (idx * 8);
+		val |= byte;
+	}
+
+	return val;
+}
+
+int pt_pkt_read_unknown(struct pt_packet *packet, const uint8_t *pos,
+			const struct pt_config *config)
+{
+	int (*decode)(struct pt_packet_unknown *, const struct pt_config *,
+		      const uint8_t *, void *);
+	int size;
+
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	decode = config->decode.callback;
+	if (!decode)
+		return -pte_bad_opc;
+
+	/* Fill in some default values. */
+	packet->payload.unknown.packet = pos;
+	packet->payload.unknown.priv = NULL;
+
+	/* We accept a size of zero to allow the callback to modify the
+	 * trace buffer and resume normal decoding.
+	 */
+	size = (*decode)(&packet->payload.unknown, config, pos,
+			 config->decode.context);
+	if (size < 0)
+		return size;
+
+	if (size > UCHAR_MAX)
+		return -pte_invalid;
+
+	packet->type = ppt_unknown;
+	packet->size = (uint8_t) size;
+
+	if (config->end < pos + size)
+		return -pte_eos;
+
+	return size;
+}
+
+int pt_pkt_read_psb(const uint8_t *pos, const struct pt_config *config)
+{
+	int count;
+
+	if (!pos || !config)
+		return -pte_internal;
+
+	if (config->end < pos + ptps_psb)
+		return -pte_eos;
+
+	pos += pt_opcs_psb;
+
+	for (count = 0; count < pt_psb_repeat_count; ++count) {
+		if (*pos++ != pt_psb_hi)
+			return -pte_bad_packet;
+		if (*pos++ != pt_psb_lo)
+			return -pte_bad_packet;
+	}
+
+	return ptps_psb;
+}
+
+static int pt_pkt_ip_size(enum pt_ip_compression ipc)
+{
+	switch (ipc) {
+	case pt_ipc_suppressed:
+		return 0;
+
+	case pt_ipc_update_16:
+		return 2;
+
+	case pt_ipc_update_32:
+		return 4;
+
+	case pt_ipc_update_48:
+	case pt_ipc_sext_48:
+		return 6;
+
+	case pt_ipc_full:
+		return 8;
+	}
+
+	return -pte_bad_packet;
+}
+
+int pt_pkt_read_ip(struct pt_packet_ip *packet, const uint8_t *pos,
+		   const struct pt_config *config)
+{
+	uint64_t ip;
+	uint8_t ipc;
+	int ipsize;
+
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	ipc = (*pos++ >> pt_opm_ipc_shr) & pt_opm_ipc_shr_mask;
+
+	ip = 0ull;
+	ipsize = pt_pkt_ip_size((enum pt_ip_compression) ipc);
+	if (ipsize < 0)
+		return ipsize;
+
+	if (config->end < pos + ipsize)
+		return -pte_eos;
+
+	if (ipsize)
+		ip = pt_pkt_read_value(pos, ipsize);
+
+	packet->ipc = (enum pt_ip_compression) ipc;
+	packet->ip = ip;
+
+	return ipsize + 1;
+}
+
+static uint8_t pt_pkt_tnt_bit_size(uint64_t payload)
+{
+	uint8_t size;
+
+	/* The payload bit-size is the bit-index of the payload's stop-bit,
+	 * which itself is not part of the payload proper.
+	 */
+	for (size = 0; ; size += 1) {
+		payload >>= 1;
+		if (!payload)
+			break;
+	}
+
+	return size;
+}
+
+static int pt_pkt_read_tnt(struct pt_packet_tnt *packet, uint64_t payload)
+{
+	uint8_t bit_size;
+
+	if (!packet)
+		return -pte_internal;
+
+	bit_size = pt_pkt_tnt_bit_size(payload);
+	if (!bit_size)
+		return -pte_bad_packet;
+
+	/* Remove the stop bit from the payload. */
+	payload &= ~(1ull << bit_size);
+
+	packet->payload = payload;
+	packet->bit_size = bit_size;
+
+	return 0;
+}
+
+int pt_pkt_read_tnt_8(struct pt_packet_tnt *packet, const uint8_t *pos,
+		      const struct pt_config *config)
+{
+	int errcode;
+
+	(void) config;
+
+	if (!pos)
+		return -pte_internal;
+
+	errcode = pt_pkt_read_tnt(packet, pos[0] >> pt_opm_tnt_8_shr);
+	if (errcode < 0)
+		return errcode;
+
+	return ptps_tnt_8;
+}
+
+int pt_pkt_read_tnt_64(struct pt_packet_tnt *packet, const uint8_t *pos,
+		       const struct pt_config *config)
+{
+	uint64_t payload;
+	int errcode;
+
+	if (!pos || !config)
+		return -pte_internal;
+
+	if (config->end < pos + ptps_tnt_64)
+		return -pte_eos;
+
+	payload = pt_pkt_read_value(pos + pt_opcs_tnt_64, pt_pl_tnt_64_size);
+
+	errcode = pt_pkt_read_tnt(packet, payload);
+	if (errcode < 0)
+		return errcode;
+
+	return ptps_tnt_64;
+}
+
+int pt_pkt_read_pip(struct pt_packet_pip *packet, const uint8_t *pos,
+		    const struct pt_config *config)
+{
+	uint64_t payload;
+
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	if (config->end < pos + ptps_pip)
+		return -pte_eos;
+
+	/* Read the payload. */
+	payload = pt_pkt_read_value(pos + pt_opcs_pip, pt_pl_pip_size);
+
+	/* Extract the non-root information from the payload. */
+	packet->nr = payload & pt_pl_pip_nr;
+
+	/* Create the cr3 value. */
+	payload  >>= pt_pl_pip_shr;
+	payload  <<= pt_pl_pip_shl;
+	packet->cr3 = payload;
+
+	return ptps_pip;
+}
+
+static int pt_pkt_read_mode_exec(struct pt_packet_mode_exec *packet,
+				 uint8_t mode)
+{
+	if (!packet)
+		return -pte_internal;
+
+	packet->csl = (mode & pt_mob_exec_csl) != 0;
+	packet->csd = (mode & pt_mob_exec_csd) != 0;
+
+	return ptps_mode;
+}
+
+static int pt_pkt_read_mode_tsx(struct pt_packet_mode_tsx *packet,
+				uint8_t mode)
+{
+	if (!packet)
+		return -pte_internal;
+
+	packet->intx = (mode & pt_mob_tsx_intx) != 0;
+	packet->abrt = (mode & pt_mob_tsx_abrt) != 0;
+
+	return ptps_mode;
+}
+
+int pt_pkt_read_mode(struct pt_packet_mode *packet, const uint8_t *pos,
+		     const struct pt_config *config)
+{
+	uint8_t payload, mode, leaf;
+
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	if (config->end < pos + ptps_mode)
+		return -pte_eos;
+
+	payload = pos[pt_opcs_mode];
+	leaf = payload & pt_mom_leaf;
+	mode = payload & pt_mom_bits;
+
+	packet->leaf = (enum pt_mode_leaf) leaf;
+	switch (leaf) {
+	default:
+		return -pte_bad_packet;
+
+	case pt_mol_exec:
+		return pt_pkt_read_mode_exec(&packet->bits.exec, mode);
+
+	case pt_mol_tsx:
+		return pt_pkt_read_mode_tsx(&packet->bits.tsx, mode);
+	}
+}
+
+int pt_pkt_read_tsc(struct pt_packet_tsc *packet, const uint8_t *pos,
+		    const struct pt_config *config)
+{
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	if (config->end < pos + ptps_tsc)
+		return -pte_eos;
+
+	packet->tsc = pt_pkt_read_value(pos + pt_opcs_tsc, pt_pl_tsc_size);
+
+	return ptps_tsc;
+}
+
+int pt_pkt_read_cbr(struct pt_packet_cbr *packet, const uint8_t *pos,
+		    const struct pt_config *config)
+{
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	if (config->end < pos + ptps_cbr)
+		return -pte_eos;
+
+	packet->ratio = pos[2];
+
+	return ptps_cbr;
+}
+
+int pt_pkt_read_tma(struct pt_packet_tma *packet, const uint8_t *pos,
+		    const struct pt_config *config)
+{
+	uint16_t ctc, fc;
+
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	if (config->end < pos + ptps_tma)
+		return -pte_eos;
+
+	ctc = pos[pt_pl_tma_ctc_0];
+	ctc |= pos[pt_pl_tma_ctc_1] << 8;
+
+	fc = pos[pt_pl_tma_fc_0];
+	fc |= pos[pt_pl_tma_fc_1] << 8;
+
+	if (fc & ~pt_pl_tma_fc_mask)
+		return -pte_bad_packet;
+
+	packet->ctc = ctc;
+	packet->fc = fc;
+
+	return ptps_tma;
+}
+
+int pt_pkt_read_mtc(struct pt_packet_mtc *packet, const uint8_t *pos,
+		    const struct pt_config *config)
+{
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	if (config->end < pos + ptps_mtc)
+		return -pte_eos;
+
+	packet->ctc = pos[pt_opcs_mtc];
+
+	return ptps_mtc;
+}
+
+int pt_pkt_read_cyc(struct pt_packet_cyc *packet, const uint8_t *pos,
+		    const struct pt_config *config)
+{
+	const uint8_t *begin, *end;
+	uint64_t value;
+	uint8_t cyc, ext, shl;
+
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	begin = pos;
+	end = config->end;
+
+	/* The first byte contains the opcode and part of the payload.
+	 * We already checked that this first byte is within bounds.
+	 */
+	cyc = *pos++;
+
+	ext = cyc & pt_opm_cyc_ext;
+	cyc >>= pt_opm_cyc_shr;
+
+	value = cyc;
+	shl = (8 - pt_opm_cyc_shr);
+
+	while (ext) {
+		uint64_t bits;
+
+		if (end <= pos)
+			return -pte_eos;
+
+		bits = *pos++;
+		ext = bits & pt_opm_cycx_ext;
+
+		bits >>= pt_opm_cycx_shr;
+		bits <<= shl;
+
+		shl += (8 - pt_opm_cycx_shr);
+		if (sizeof(value) * 8 < shl)
+			return -pte_bad_packet;
+
+		value |= bits;
+	}
+
+	packet->value = value;
+
+	return (int) (pos - begin);
+}
+
+int pt_pkt_read_vmcs(struct pt_packet_vmcs *packet, const uint8_t *pos,
+		     const struct pt_config *config)
+{
+	uint64_t payload;
+
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	if (config->end < pos + ptps_vmcs)
+		return -pte_eos;
+
+	payload = pt_pkt_read_value(pos + pt_opcs_vmcs, pt_pl_vmcs_size);
+
+	packet->base = payload << pt_pl_vmcs_shl;
+
+	return ptps_vmcs;
+}
+
+int pt_pkt_read_mnt(struct pt_packet_mnt *packet, const uint8_t *pos,
+		    const struct pt_config *config)
+{
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	if (config->end < pos + ptps_mnt)
+		return -pte_eos;
+
+	packet->payload = pt_pkt_read_value(pos + pt_opcs_mnt, pt_pl_mnt_size);
+
+	return ptps_mnt;
+}
+
+int pt_pkt_read_exstop(struct pt_packet_exstop *packet, const uint8_t *pos,
+		       const struct pt_config *config)
+{
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	if (config->end < pos + ptps_exstop)
+		return -pte_eos;
+
+	packet->ip = pos[1] & pt_pl_exstop_ip_mask ? 1 : 0;
+
+	return ptps_exstop;
+}
+
+int pt_pkt_read_mwait(struct pt_packet_mwait *packet, const uint8_t *pos,
+		      const struct pt_config *config)
+{
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	if (config->end < pos + ptps_mwait)
+		return -pte_eos;
+
+	packet->hints = (uint32_t) pt_pkt_read_value(pos + pt_opcs_mwait,
+						     pt_pl_mwait_hints_size);
+	packet->ext = (uint32_t) pt_pkt_read_value(pos + pt_opcs_mwait +
+						   pt_pl_mwait_hints_size,
+						   pt_pl_mwait_ext_size);
+	return ptps_mwait;
+}
+
+int pt_pkt_read_pwre(struct pt_packet_pwre *packet, const uint8_t *pos,
+		     const struct pt_config *config)
+{
+	uint64_t payload;
+
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	if (config->end < pos + ptps_pwre)
+		return -pte_eos;
+
+	payload = pt_pkt_read_value(pos + pt_opcs_pwre, pt_pl_pwre_size);
+
+	memset(packet, 0, sizeof(*packet));
+	packet->state = (uint8_t) ((payload & pt_pl_pwre_state_mask) >>
+				   pt_pl_pwre_state_shr);
+	packet->sub_state = (uint8_t) ((payload & pt_pl_pwre_sub_state_mask) >>
+				       pt_pl_pwre_sub_state_shr);
+	if (payload & pt_pl_pwre_hw_mask)
+		packet->hw = 1;
+
+	return ptps_pwre;
+}
+
+int pt_pkt_read_pwrx(struct pt_packet_pwrx *packet, const uint8_t *pos,
+		     const struct pt_config *config)
+{
+	uint64_t payload;
+
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	if (config->end < pos + ptps_pwrx)
+		return -pte_eos;
+
+	payload = pt_pkt_read_value(pos + pt_opcs_pwrx, pt_pl_pwrx_size);
+
+	memset(packet, 0, sizeof(*packet));
+	packet->last = (uint8_t) ((payload & pt_pl_pwrx_last_mask) >>
+				  pt_pl_pwrx_last_shr);
+	packet->deepest = (uint8_t) ((payload & pt_pl_pwrx_deepest_mask) >>
+				     pt_pl_pwrx_deepest_shr);
+	if (payload & pt_pl_pwrx_wr_int)
+		packet->interrupt = 1;
+	if (payload & pt_pl_pwrx_wr_store)
+		packet->store = 1;
+	if (payload & pt_pl_pwrx_wr_hw)
+		packet->autonomous = 1;
+
+	return ptps_pwrx;
+}
+
+int pt_pkt_read_ptw(struct pt_packet_ptw *packet, const uint8_t *pos,
+		    const struct pt_config *config)
+{
+	uint8_t opc, plc;
+	int size;
+
+	if (!packet || !pos || !config)
+		return -pte_internal;
+
+	/* Skip the ext opcode. */
+	pos++;
+
+	opc = *pos++;
+	plc = (opc >> pt_opm_ptw_pb_shr) & pt_opm_ptw_pb_shr_mask;
+
+	size = pt_ptw_size(plc);
+	if (size < 0)
+		return size;
+
+	if (config->end < pos + size)
+		return -pte_eos;
+
+	packet->payload = pt_pkt_read_value(pos, size);
+	packet->plc = plc;
+	packet->ip = opc & pt_opm_ptw_ip ? 1 : 0;
+
+	return pt_opcs_ptw + size;
+}
diff --git a/libipt/src/pt_packet_decoder.c b/libipt/src/pt_packet_decoder.c
new file mode 100644
index 000000000000..685ae49c3fb1
--- /dev/null
+++ b/libipt/src/pt_packet_decoder.c
@@ -0,0 +1,738 @@
+/*
+ * Copyright (c) 2014-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_packet_decoder.h"
+#include "pt_decoder_function.h"
+#include "pt_packet.h"
+#include "pt_sync.h"
+#include "pt_config.h"
+#include "pt_opcodes.h"
+
+#include <string.h>
+#include <stdlib.h>
+#include <stddef.h>
+
+
+int pt_pkt_decoder_init(struct pt_packet_decoder *decoder,
+			const struct pt_config *config)
+{
+	int errcode;
+
+	if (!decoder || !config)
+		return -pte_invalid;
+
+	memset(decoder, 0, sizeof(*decoder));
+
+	errcode = pt_config_from_user(&decoder->config, config);
+	if (errcode < 0)
+		return errcode;
+
+	return 0;
+}
+
+struct pt_packet_decoder *pt_pkt_alloc_decoder(const struct pt_config *config)
+{
+	struct pt_packet_decoder *decoder;
+	int errcode;
+
+	decoder = malloc(sizeof(*decoder));
+	if (!decoder)
+		return NULL;
+
+	errcode = pt_pkt_decoder_init(decoder, config);
+	if (errcode < 0) {
+		free(decoder);
+		return NULL;
+	}
+
+	return decoder;
+}
+
+void pt_pkt_decoder_fini(struct pt_packet_decoder *decoder)
+{
+	(void) decoder;
+
+	/* Nothing to do. */
+}
+
+void pt_pkt_free_decoder(struct pt_packet_decoder *decoder)
+{
+	pt_pkt_decoder_fini(decoder);
+	free(decoder);
+}
+
+int pt_pkt_sync_forward(struct pt_packet_decoder *decoder)
+{
+	const uint8_t *pos, *sync, *begin;
+	ptrdiff_t space;
+	int errcode;
+
+	if (!decoder)
+		return -pte_invalid;
+
+	begin = decoder->config.begin;
+	sync = decoder->sync;
+	pos = decoder->pos;
+	if (!pos)
+		pos = begin;
+
+	if (pos == sync)
+		pos += ptps_psb;
+
+	if (pos < begin)
+		return -pte_internal;
+
+	/* Start a bit earlier so we find PSB that have been partially consumed
+	 * by a preceding packet.
+	 */
+	space = pos - begin;
+	if (ptps_psb <= space)
+		space = ptps_psb - 1;
+
+	pos -= space;
+
+	errcode = pt_sync_forward(&sync, pos, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->sync = sync;
+	decoder->pos = sync;
+
+	return 0;
+}
+
+int pt_pkt_sync_backward(struct pt_packet_decoder *decoder)
+{
+	const uint8_t *pos, *sync;
+	int errcode;
+
+	if (!decoder)
+		return -pte_invalid;
+
+	pos = decoder->pos;
+	if (!pos)
+		pos = decoder->config.end;
+
+	errcode = pt_sync_backward(&sync, pos, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->sync = sync;
+	decoder->pos = sync;
+
+	return 0;
+}
+
+int pt_pkt_sync_set(struct pt_packet_decoder *decoder, uint64_t offset)
+{
+	const uint8_t *begin, *end, *pos;
+
+	if (!decoder)
+		return -pte_invalid;
+
+	begin = decoder->config.begin;
+	end = decoder->config.end;
+	pos = begin + offset;
+
+	if (end < pos || pos < begin)
+		return -pte_eos;
+
+	decoder->sync = pos;
+	decoder->pos = pos;
+
+	return 0;
+}
+
+int pt_pkt_get_offset(const struct pt_packet_decoder *decoder, uint64_t *offset)
+{
+	const uint8_t *begin, *pos;
+
+	if (!decoder || !offset)
+		return -pte_invalid;
+
+	begin = decoder->config.begin;
+	pos = decoder->pos;
+
+	if (!pos)
+		return -pte_nosync;
+
+	*offset = (uint64_t) (int64_t) (pos - begin);
+	return 0;
+}
+
+int pt_pkt_get_sync_offset(const struct pt_packet_decoder *decoder,
+			   uint64_t *offset)
+{
+	const uint8_t *begin, *sync;
+
+	if (!decoder || !offset)
+		return -pte_invalid;
+
+	begin = decoder->config.begin;
+	sync = decoder->sync;
+
+	if (!sync)
+		return -pte_nosync;
+
+	*offset = (uint64_t) (int64_t) (sync - begin);
+	return 0;
+}
+
+const struct pt_config *
+pt_pkt_get_config(const struct pt_packet_decoder *decoder)
+{
+	if (!decoder)
+		return NULL;
+
+	return &decoder->config;
+}
+
+static inline int pkt_to_user(struct pt_packet *upkt, size_t size,
+			      const struct pt_packet *pkt)
+{
+	if (!upkt || !pkt)
+		return -pte_internal;
+
+	if (upkt == pkt)
+		return 0;
+
+	/* Zero out any unknown bytes. */
+	if (sizeof(*pkt) < size) {
+		memset(upkt + sizeof(*pkt), 0, size - sizeof(*pkt));
+
+		size = sizeof(*pkt);
+	}
+
+	memcpy(upkt, pkt, size);
+
+	return 0;
+}
+
+int pt_pkt_next(struct pt_packet_decoder *decoder, struct pt_packet *packet,
+		size_t psize)
+{
+	const struct pt_decoder_function *dfun;
+	struct pt_packet pkt, *ppkt;
+	int errcode, size;
+
+	if (!packet || !decoder)
+		return -pte_invalid;
+
+	ppkt = psize == sizeof(pkt) ? packet : &pkt;
+
+	errcode = pt_df_fetch(&dfun, decoder->pos, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	if (!dfun)
+		return -pte_internal;
+
+	if (!dfun->packet)
+		return -pte_internal;
+
+	size = dfun->packet(decoder, ppkt);
+	if (size < 0)
+		return size;
+
+	errcode = pkt_to_user(packet, psize, ppkt);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += size;
+
+	return size;
+}
+
+int pt_pkt_decode_unknown(struct pt_packet_decoder *decoder,
+			  struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_unknown(packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	return size;
+}
+
+int pt_pkt_decode_pad(struct pt_packet_decoder *decoder,
+		      struct pt_packet *packet)
+{
+	(void) decoder;
+
+	if (!packet)
+		return -pte_internal;
+
+	packet->type = ppt_pad;
+	packet->size = ptps_pad;
+
+	return ptps_pad;
+}
+
+int pt_pkt_decode_psb(struct pt_packet_decoder *decoder,
+		      struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_psb(decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_psb;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_tip(struct pt_packet_decoder *decoder,
+		      struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_ip(&packet->payload.ip, decoder->pos,
+			      &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_tip;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_tnt_8(struct pt_packet_decoder *decoder,
+			struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_tnt_8(&packet->payload.tnt, decoder->pos,
+				 &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_tnt_8;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_tnt_64(struct pt_packet_decoder *decoder,
+			 struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_tnt_64(&packet->payload.tnt, decoder->pos,
+				  &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_tnt_64;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_tip_pge(struct pt_packet_decoder *decoder,
+			  struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_ip(&packet->payload.ip, decoder->pos,
+			      &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_tip_pge;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_tip_pgd(struct pt_packet_decoder *decoder,
+			  struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_ip(&packet->payload.ip, decoder->pos,
+			      &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_tip_pgd;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_fup(struct pt_packet_decoder *decoder,
+		      struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_ip(&packet->payload.ip, decoder->pos,
+			      &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_fup;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_pip(struct pt_packet_decoder *decoder,
+		      struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_pip(&packet->payload.pip, decoder->pos,
+			       &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_pip;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_ovf(struct pt_packet_decoder *decoder,
+		      struct pt_packet *packet)
+{
+	(void) decoder;
+
+	if (!packet)
+		return -pte_internal;
+
+	packet->type = ppt_ovf;
+	packet->size = ptps_ovf;
+
+	return ptps_ovf;
+}
+
+int pt_pkt_decode_mode(struct pt_packet_decoder *decoder,
+		       struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_mode(&packet->payload.mode, decoder->pos,
+				&decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_mode;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_psbend(struct pt_packet_decoder *decoder,
+			 struct pt_packet *packet)
+{
+	(void) decoder;
+
+	if (!packet)
+		return -pte_internal;
+
+	packet->type = ppt_psbend;
+	packet->size = ptps_psbend;
+
+	return ptps_psbend;
+}
+
+int pt_pkt_decode_tsc(struct pt_packet_decoder *decoder,
+		      struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_tsc(&packet->payload.tsc, decoder->pos,
+			       &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_tsc;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_cbr(struct pt_packet_decoder *decoder,
+		      struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_cbr(&packet->payload.cbr, decoder->pos,
+			       &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_cbr;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_tma(struct pt_packet_decoder *decoder,
+		      struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_tma(&packet->payload.tma, decoder->pos,
+			       &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_tma;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_mtc(struct pt_packet_decoder *decoder,
+		      struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_mtc(&packet->payload.mtc, decoder->pos,
+			       &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_mtc;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_cyc(struct pt_packet_decoder *decoder,
+		      struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_cyc(&packet->payload.cyc, decoder->pos,
+			       &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_cyc;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_stop(struct pt_packet_decoder *decoder,
+		       struct pt_packet *packet)
+{
+	(void) decoder;
+
+	if (!packet)
+		return -pte_internal;
+
+	packet->type = ppt_stop;
+	packet->size = ptps_stop;
+
+	return ptps_stop;
+}
+
+int pt_pkt_decode_vmcs(struct pt_packet_decoder *decoder,
+		       struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_vmcs(&packet->payload.vmcs, decoder->pos,
+			       &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_vmcs;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_mnt(struct pt_packet_decoder *decoder,
+		      struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_mnt(&packet->payload.mnt, decoder->pos,
+			       &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_mnt;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_exstop(struct pt_packet_decoder *decoder,
+			 struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_exstop(&packet->payload.exstop, decoder->pos,
+				  &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_exstop;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_mwait(struct pt_packet_decoder *decoder,
+			struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_mwait(&packet->payload.mwait, decoder->pos,
+				 &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_mwait;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_pwre(struct pt_packet_decoder *decoder,
+		       struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_pwre(&packet->payload.pwre, decoder->pos,
+				&decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_pwre;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_pwrx(struct pt_packet_decoder *decoder,
+		       struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_pwrx(&packet->payload.pwrx, decoder->pos,
+				&decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_pwrx;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
+
+int pt_pkt_decode_ptw(struct pt_packet_decoder *decoder,
+		      struct pt_packet *packet)
+{
+	int size;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	size = pt_pkt_read_ptw(&packet->payload.ptw, decoder->pos,
+			       &decoder->config);
+	if (size < 0)
+		return size;
+
+	packet->type = ppt_ptw;
+	packet->size = (uint8_t) size;
+
+	return size;
+}
diff --git a/libipt/src/pt_query_decoder.c b/libipt/src/pt_query_decoder.c
new file mode 100644
index 000000000000..888972039d02
--- /dev/null
+++ b/libipt/src/pt_query_decoder.c
@@ -0,0 +1,3659 @@
+/*
+ * Copyright (c) 2014-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_query_decoder.h"
+#include "pt_sync.h"
+#include "pt_decoder_function.h"
+#include "pt_packet.h"
+#include "pt_packet_decoder.h"
+#include "pt_config.h"
+#include "pt_opcodes.h"
+#include "pt_compiler.h"
+
+#include "intel-pt.h"
+
+#include <string.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <limits.h>
+
+
+/* Find a FUP in a PSB+ header.
+ *
+ * The packet @decoder must be synchronized onto the trace stream at the
+ * beginning or somewhere inside a PSB+ header.
+ *
+ * It uses @packet to hold trace packets during its search.  If the search is
+ * successful, @packet will contain the first (and hopefully only) FUP packet in
+ * this PSB+.  Otherwise, @packet may contain anything.
+ *
+ * Returns one if a FUP packet is found (@packet will contain it).
+ * Returns zero if no FUP packet is found (@packet is undefined).
+ * Returns a negative error code otherwise.
+ */
+static int pt_qry_find_header_fup(struct pt_packet *packet,
+				  struct pt_packet_decoder *decoder)
+{
+	if (!packet || !decoder)
+		return -pte_internal;
+
+	for (;;) {
+		int errcode;
+
+		errcode = pt_pkt_next(decoder, packet, sizeof(*packet));
+		if (errcode < 0)
+			return errcode;
+
+		switch (packet->type) {
+		default:
+			/* Ignore the packet. */
+			break;
+
+		case ppt_psbend:
+			/* There's no FUP in here. */
+			return 0;
+
+		case ppt_fup:
+			/* Found it. */
+			return 1;
+		}
+	}
+}
+
+int pt_qry_decoder_init(struct pt_query_decoder *decoder,
+			const struct pt_config *config)
+{
+	int errcode;
+
+	if (!decoder)
+		return -pte_invalid;
+
+	memset(decoder, 0, sizeof(*decoder));
+
+	errcode = pt_config_from_user(&decoder->config, config);
+	if (errcode < 0)
+		return errcode;
+
+	pt_last_ip_init(&decoder->ip);
+	pt_tnt_cache_init(&decoder->tnt);
+	pt_time_init(&decoder->time);
+	pt_time_init(&decoder->last_time);
+	pt_tcal_init(&decoder->tcal);
+	pt_evq_init(&decoder->evq);
+
+	return 0;
+}
+
+struct pt_query_decoder *pt_qry_alloc_decoder(const struct pt_config *config)
+{
+	struct pt_query_decoder *decoder;
+	int errcode;
+
+	decoder = malloc(sizeof(*decoder));
+	if (!decoder)
+		return NULL;
+
+	errcode = pt_qry_decoder_init(decoder, config);
+	if (errcode < 0) {
+		free(decoder);
+		return NULL;
+	}
+
+	return decoder;
+}
+
+void pt_qry_decoder_fini(struct pt_query_decoder *decoder)
+{
+	(void) decoder;
+
+	/* Nothing to do. */
+}
+
+void pt_qry_free_decoder(struct pt_query_decoder *decoder)
+{
+	pt_qry_decoder_fini(decoder);
+	free(decoder);
+}
+
+static void pt_qry_reset(struct pt_query_decoder *decoder)
+{
+	if (!decoder)
+		return;
+
+	decoder->enabled = 0;
+	decoder->consume_packet = 0;
+	decoder->event = NULL;
+
+	pt_last_ip_init(&decoder->ip);
+	pt_tnt_cache_init(&decoder->tnt);
+	pt_time_init(&decoder->time);
+	pt_time_init(&decoder->last_time);
+	pt_tcal_init(&decoder->tcal);
+	pt_evq_init(&decoder->evq);
+}
+
+static int pt_qry_will_event(const struct pt_query_decoder *decoder)
+{
+	const struct pt_decoder_function *dfun;
+
+	if (!decoder)
+		return -pte_internal;
+
+	dfun = decoder->next;
+	if (!dfun)
+		return 0;
+
+	if (dfun->flags & pdff_event)
+		return 1;
+
+	if (dfun->flags & pdff_psbend)
+		return pt_evq_pending(&decoder->evq, evb_psbend);
+
+	if (dfun->flags & pdff_tip)
+		return pt_evq_pending(&decoder->evq, evb_tip);
+
+	if (dfun->flags & pdff_fup)
+		return pt_evq_pending(&decoder->evq, evb_fup);
+
+	return 0;
+}
+
+static int pt_qry_will_eos(const struct pt_query_decoder *decoder)
+{
+	const struct pt_decoder_function *dfun;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	dfun = decoder->next;
+	if (dfun)
+		return 0;
+
+	/* The decoding function may be NULL for two reasons:
+	 *
+	 *   - we ran out of trace
+	 *   - we ran into a fetch error such as -pte_bad_opc
+	 *
+	 * Let's fetch again.
+	 */
+	errcode = pt_df_fetch(&dfun, decoder->pos, &decoder->config);
+	return errcode == -pte_eos;
+}
+
+static int pt_qry_status_flags(const struct pt_query_decoder *decoder)
+{
+	int flags = 0;
+
+	if (!decoder)
+		return -pte_internal;
+
+	/* Some packets force out TNT and any deferred TIPs in order to
+	 * establish the correct context for the subsequent packet.
+	 *
+	 * Users are expected to first navigate to the correct code region
+	 * by using up the cached TNT bits before interpreting any subsequent
+	 * packets.
+	 *
+	 * We do need to read ahead in order to signal upcoming events.  We may
+	 * have already decoded those packets while our user has not navigated
+	 * to the correct code region, yet.
+	 *
+	 * In order to have our user use up the cached TNT bits first, we do
+	 * not indicate the next event until the TNT cache is empty.
+	 */
+	if (pt_tnt_cache_is_empty(&decoder->tnt)) {
+		if (pt_qry_will_event(decoder))
+			flags |= pts_event_pending;
+
+		if (pt_qry_will_eos(decoder))
+			flags |= pts_eos;
+	}
+
+	return flags;
+}
+
+static int pt_qry_provoke_fetch_error(const struct pt_query_decoder *decoder)
+{
+	const struct pt_decoder_function *dfun;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	/* Repeat the decoder fetch to reproduce the error. */
+	errcode = pt_df_fetch(&dfun, decoder->pos, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	/* We must get some error or something's wrong. */
+	return -pte_internal;
+}
+
+static int pt_qry_read_ahead(struct pt_query_decoder *decoder)
+{
+	if (!decoder)
+		return -pte_internal;
+
+	for (;;) {
+		const struct pt_decoder_function *dfun;
+		int errcode;
+
+		errcode = pt_df_fetch(&decoder->next, decoder->pos,
+				      &decoder->config);
+		if (errcode)
+			return errcode;
+
+		dfun = decoder->next;
+		if (!dfun)
+			return -pte_internal;
+
+		if (!dfun->decode)
+			return -pte_internal;
+
+		/* We're done once we reach
+		 *
+		 * - a branching related packet. */
+		if (dfun->flags & (pdff_tip | pdff_tnt))
+			return 0;
+
+		/* - an event related packet. */
+		if (pt_qry_will_event(decoder))
+			return 0;
+
+		/* Decode status update packets. */
+		errcode = dfun->decode(decoder);
+		if (errcode) {
+			/* Ignore truncated status packets at the end.
+			 *
+			 * Move beyond the packet and clear @decoder->next to
+			 * indicate that we were not able to fetch the next
+			 * packet.
+			 */
+			if (errcode == -pte_eos) {
+				decoder->pos = decoder->config.end;
+				decoder->next = NULL;
+			}
+
+			return errcode;
+		}
+	}
+}
+
+static int pt_qry_start(struct pt_query_decoder *decoder, const uint8_t *pos,
+			uint64_t *addr)
+{
+	const struct pt_decoder_function *dfun;
+	int status, errcode;
+
+	if (!decoder || !pos)
+		return -pte_invalid;
+
+	pt_qry_reset(decoder);
+
+	decoder->sync = pos;
+	decoder->pos = pos;
+
+	errcode = pt_df_fetch(&decoder->next, pos, &decoder->config);
+	if (errcode)
+		return errcode;
+
+	dfun = decoder->next;
+
+	/* We do need to start at a PSB in order to initialize the state. */
+	if (dfun != &pt_decode_psb)
+		return -pte_nosync;
+
+	/* Decode the PSB+ header to initialize the state. */
+	errcode = dfun->decode(decoder);
+	if (errcode < 0)
+		return errcode;
+
+	/* Fill in the start address.
+	 * We do this before reading ahead since the latter may read an
+	 * adjacent PSB+ that might change the decoder's IP, causing us
+	 * to skip code.
+	 */
+	if (addr) {
+		status = pt_last_ip_query(addr, &decoder->ip);
+
+		/* Make sure we don't clobber it later on. */
+		if (!status)
+			addr = NULL;
+	}
+
+	/* Read ahead until the first query-relevant packet. */
+	errcode = pt_qry_read_ahead(decoder);
+	if (errcode < 0)
+		return errcode;
+
+	/* We return the current decoder status. */
+	status = pt_qry_status_flags(decoder);
+	if (status < 0)
+		return status;
+
+	errcode = pt_last_ip_query(addr, &decoder->ip);
+	if (errcode < 0) {
+		/* Indicate the missing IP in the status. */
+		if (addr)
+			status |= pts_ip_suppressed;
+	}
+
+	return status;
+}
+
+static int pt_qry_apply_tsc(struct pt_time *time, struct pt_time_cal *tcal,
+			    const struct pt_packet_tsc *packet,
+			    const struct pt_config *config)
+{
+	int errcode;
+
+	/* We ignore configuration errors.  They will result in imprecise
+	 * calibration which will result in imprecise cycle-accurate timing.
+	 *
+	 * We currently do not track them.
+	 */
+	errcode = pt_tcal_update_tsc(tcal, packet, config);
+	if (errcode < 0 && (errcode != -pte_bad_config))
+		return errcode;
+
+	/* We ignore configuration errors.  They will result in imprecise
+	 * timing and are tracked as packet losses in struct pt_time.
+	 */
+	errcode = pt_time_update_tsc(time, packet, config);
+	if (errcode < 0 && (errcode != -pte_bad_config))
+		return errcode;
+
+	return 0;
+}
+
+static int pt_qry_apply_header_tsc(struct pt_time *time,
+				   struct pt_time_cal *tcal,
+				   const struct pt_packet_tsc *packet,
+				   const struct pt_config *config)
+{
+	int errcode;
+
+	/* We ignore configuration errors.  They will result in imprecise
+	 * calibration which will result in imprecise cycle-accurate timing.
+	 *
+	 * We currently do not track them.
+	 */
+	errcode = pt_tcal_header_tsc(tcal, packet, config);
+	if (errcode < 0 && (errcode != -pte_bad_config))
+		return errcode;
+
+	/* We ignore configuration errors.  They will result in imprecise
+	 * timing and are tracked as packet losses in struct pt_time.
+	 */
+	errcode = pt_time_update_tsc(time, packet, config);
+	if (errcode < 0 && (errcode != -pte_bad_config))
+		return errcode;
+
+	return 0;
+}
+
+static int pt_qry_apply_cbr(struct pt_time *time, struct pt_time_cal *tcal,
+			    const struct pt_packet_cbr *packet,
+			    const struct pt_config *config)
+{
+	int errcode;
+
+	/* We ignore configuration errors.  They will result in imprecise
+	 * calibration which will result in imprecise cycle-accurate timing.
+	 *
+	 * We currently do not track them.
+	 */
+	errcode = pt_tcal_update_cbr(tcal, packet, config);
+	if (errcode < 0 && (errcode != -pte_bad_config))
+		return errcode;
+
+	/* We ignore configuration errors.  They will result in imprecise
+	 * timing and are tracked as packet losses in struct pt_time.
+	 */
+	errcode = pt_time_update_cbr(time, packet, config);
+	if (errcode < 0 && (errcode != -pte_bad_config))
+		return errcode;
+
+	return 0;
+}
+
+static int pt_qry_apply_header_cbr(struct pt_time *time,
+				   struct pt_time_cal *tcal,
+				   const struct pt_packet_cbr *packet,
+				   const struct pt_config *config)
+{
+	int errcode;
+
+	/* We ignore configuration errors.  They will result in imprecise
+	 * calibration which will result in imprecise cycle-accurate timing.
+	 *
+	 * We currently do not track them.
+	 */
+	errcode = pt_tcal_header_cbr(tcal, packet, config);
+	if (errcode < 0 && (errcode != -pte_bad_config))
+		return errcode;
+
+	/* We ignore configuration errors.  They will result in imprecise
+	 * timing and are tracked as packet losses in struct pt_time.
+	 */
+	errcode = pt_time_update_cbr(time, packet, config);
+	if (errcode < 0 && (errcode != -pte_bad_config))
+		return errcode;
+
+	return 0;
+}
+
+static int pt_qry_apply_tma(struct pt_time *time, struct pt_time_cal *tcal,
+			    const struct pt_packet_tma *packet,
+			    const struct pt_config *config)
+{
+	int errcode;
+
+	/* We ignore configuration errors.  They will result in imprecise
+	 * calibration which will result in imprecise cycle-accurate timing.
+	 *
+	 * We currently do not track them.
+	 */
+	errcode = pt_tcal_update_tma(tcal, packet, config);
+	if (errcode < 0 && (errcode != -pte_bad_config))
+		return errcode;
+
+	/* We ignore configuration errors.  They will result in imprecise
+	 * timing and are tracked as packet losses in struct pt_time.
+	 */
+	errcode = pt_time_update_tma(time, packet, config);
+	if (errcode < 0 && (errcode != -pte_bad_config))
+		return errcode;
+
+	return 0;
+}
+
+static int pt_qry_apply_mtc(struct pt_time *time, struct pt_time_cal *tcal,
+			    const struct pt_packet_mtc *packet,
+			    const struct pt_config *config)
+{
+	int errcode;
+
+	/* We ignore configuration errors.  They will result in imprecise
+	 * calibration which will result in imprecise cycle-accurate timing.
+	 *
+	 * We currently do not track them.
+	 */
+	errcode = pt_tcal_update_mtc(tcal, packet, config);
+	if (errcode < 0 && (errcode != -pte_bad_config))
+		return errcode;
+
+	/* We ignore configuration errors.  They will result in imprecise
+	 * timing and are tracked as packet losses in struct pt_time.
+	 */
+	errcode = pt_time_update_mtc(time, packet, config);
+	if (errcode < 0 && (errcode != -pte_bad_config))
+		return errcode;
+
+	return 0;
+}
+
+static int pt_qry_apply_cyc(struct pt_time *time, struct pt_time_cal *tcal,
+			    const struct pt_packet_cyc *packet,
+			    const struct pt_config *config)
+{
+	uint64_t fcr;
+	int errcode;
+
+	/* We ignore configuration errors.  They will result in imprecise
+	 * calibration which will result in imprecise cycle-accurate timing.
+	 *
+	 * We currently do not track them.
+	 */
+	errcode = pt_tcal_update_cyc(tcal, packet, config);
+	if (errcode < 0 && (errcode != -pte_bad_config))
+		return errcode;
+
+	/* We need the FastCounter to Cycles ratio below.  Fall back to
+	 * an invalid ratio of 0 if calibration has not kicked in, yet.
+	 *
+	 * This will be tracked as packet loss in struct pt_time.
+	 */
+	errcode = pt_tcal_fcr(&fcr, tcal);
+	if (errcode < 0) {
+		if (errcode == -pte_no_time)
+			fcr = 0ull;
+		else
+			return errcode;
+	}
+
+	/* We ignore configuration errors.  They will result in imprecise
+	 * timing and are tracked as packet losses in struct pt_time.
+	 */
+	errcode = pt_time_update_cyc(time, packet, config, fcr);
+	if (errcode < 0 && (errcode != -pte_bad_config))
+		return errcode;
+
+	return 0;
+}
+
+int pt_qry_sync_forward(struct pt_query_decoder *decoder, uint64_t *ip)
+{
+	const uint8_t *pos, *sync, *begin;
+	ptrdiff_t space;
+	int errcode;
+
+	if (!decoder)
+		return -pte_invalid;
+
+	begin = decoder->config.begin;
+	sync = decoder->sync;
+	pos = decoder->pos;
+	if (!pos)
+		pos = begin;
+
+	if (pos == sync)
+		pos += ptps_psb;
+
+	if (pos < begin)
+		return -pte_internal;
+
+	/* Start a bit earlier so we find PSB that have been partially consumed
+	 * by a preceding packet.
+	 */
+	space = pos - begin;
+	if (ptps_psb <= space)
+		space = ptps_psb - 1;
+
+	pos -= space;
+
+	errcode = pt_sync_forward(&sync, pos, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	return pt_qry_start(decoder, sync, ip);
+}
+
+int pt_qry_sync_backward(struct pt_query_decoder *decoder, uint64_t *ip)
+{
+	const uint8_t *start, *sync;
+	int errcode;
+
+	if (!decoder)
+		return -pte_invalid;
+
+	start = decoder->pos;
+	if (!start)
+		start = decoder->config.end;
+
+	sync = start;
+	for (;;) {
+		errcode = pt_sync_backward(&sync, sync, &decoder->config);
+		if (errcode < 0)
+			return errcode;
+
+		errcode = pt_qry_start(decoder, sync, ip);
+		if (errcode < 0) {
+			/* Ignore incomplete trace segments at the end.  We need
+			 * a full PSB+ to start decoding.
+			 */
+			if (errcode == -pte_eos)
+				continue;
+
+			return errcode;
+		}
+
+		/* An empty trace segment in the middle of the trace might bring
+		 * us back to where we started.
+		 *
+		 * We're done when we reached a new position.
+		 */
+		if (decoder->pos != start)
+			break;
+	}
+
+	return 0;
+}
+
+int pt_qry_sync_set(struct pt_query_decoder *decoder, uint64_t *ip,
+		    uint64_t offset)
+{
+	const uint8_t *sync, *pos;
+	int errcode;
+
+	if (!decoder)
+		return -pte_invalid;
+
+	pos = decoder->config.begin + offset;
+
+	errcode = pt_sync_set(&sync, pos, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	return pt_qry_start(decoder, sync, ip);
+}
+
+int pt_qry_get_offset(const struct pt_query_decoder *decoder, uint64_t *offset)
+{
+	const uint8_t *begin, *pos;
+
+	if (!decoder || !offset)
+		return -pte_invalid;
+
+	begin = decoder->config.begin;
+	pos = decoder->pos;
+
+	if (!pos)
+		return -pte_nosync;
+
+	*offset = (uint64_t) (int64_t) (pos - begin);
+	return 0;
+}
+
+int pt_qry_get_sync_offset(const struct pt_query_decoder *decoder,
+			   uint64_t *offset)
+{
+	const uint8_t *begin, *sync;
+
+	if (!decoder || !offset)
+		return -pte_invalid;
+
+	begin = decoder->config.begin;
+	sync = decoder->sync;
+
+	if (!sync)
+		return -pte_nosync;
+
+	*offset = (uint64_t) (int64_t) (sync - begin);
+	return 0;
+}
+
+const struct pt_config *
+pt_qry_get_config(const struct pt_query_decoder *decoder)
+{
+	if (!decoder)
+		return NULL;
+
+	return &decoder->config;
+}
+
+static int pt_qry_cache_tnt(struct pt_query_decoder *decoder)
+{
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	for (;;) {
+		const struct pt_decoder_function *dfun;
+
+		dfun = decoder->next;
+		if (!dfun)
+			return pt_qry_provoke_fetch_error(decoder);
+
+		if (!dfun->decode)
+			return -pte_internal;
+
+		/* There's an event ahead of us. */
+		if (pt_qry_will_event(decoder))
+			return -pte_bad_query;
+
+		/* Diagnose a TIP that has not been part of an event. */
+		if (dfun->flags & pdff_tip)
+			return -pte_bad_query;
+
+		/* Clear the decoder's current event so we know when we
+		 * accidentally skipped an event.
+		 */
+		decoder->event = NULL;
+
+		/* Apply the decoder function. */
+		errcode = dfun->decode(decoder);
+		if (errcode)
+			return errcode;
+
+		/* If we skipped an event, we're in trouble. */
+		if (decoder->event)
+			return -pte_event_ignored;
+
+		/* We're done when we decoded a TNT packet. */
+		if (dfun->flags & pdff_tnt)
+			break;
+
+		/* Read ahead until the next query-relevant packet. */
+		errcode = pt_qry_read_ahead(decoder);
+		if (errcode)
+			return errcode;
+	}
+
+	/* Preserve the time at the TNT packet. */
+	decoder->last_time = decoder->time;
+
+	/* Read ahead until the next query-relevant packet. */
+	errcode = pt_qry_read_ahead(decoder);
+	if ((errcode < 0) && (errcode != -pte_eos))
+		return errcode;
+
+	return 0;
+}
+
+int pt_qry_cond_branch(struct pt_query_decoder *decoder, int *taken)
+{
+	int errcode, query;
+
+	if (!decoder || !taken)
+		return -pte_invalid;
+
+	/* We cache the latest tnt packet in the decoder. Let's re-fill the
+	 * cache in case it is empty.
+	 */
+	if (pt_tnt_cache_is_empty(&decoder->tnt)) {
+		errcode = pt_qry_cache_tnt(decoder);
+		if (errcode < 0)
+			return errcode;
+	}
+
+	query = pt_tnt_cache_query(&decoder->tnt);
+	if (query < 0)
+		return query;
+
+	*taken = query;
+
+	return pt_qry_status_flags(decoder);
+}
+
+int pt_qry_indirect_branch(struct pt_query_decoder *decoder, uint64_t *addr)
+{
+	int errcode, flags;
+
+	if (!decoder || !addr)
+		return -pte_invalid;
+
+	flags = 0;
+	for (;;) {
+		const struct pt_decoder_function *dfun;
+
+		dfun = decoder->next;
+		if (!dfun)
+			return pt_qry_provoke_fetch_error(decoder);
+
+		if (!dfun->decode)
+			return -pte_internal;
+
+		/* There's an event ahead of us. */
+		if (pt_qry_will_event(decoder))
+			return -pte_bad_query;
+
+		/* Clear the decoder's current event so we know when we
+		 * accidentally skipped an event.
+		 */
+		decoder->event = NULL;
+
+		/* We may see a single TNT packet if the current tnt is empty.
+		 *
+		 * If we see a TNT while the current tnt is not empty, it means
+		 * that our user got out of sync. Let's report no data and hope
+		 * that our user is able to re-sync.
+		 */
+		if ((dfun->flags & pdff_tnt) &&
+		    !pt_tnt_cache_is_empty(&decoder->tnt))
+			return -pte_bad_query;
+
+		/* Apply the decoder function. */
+		errcode = dfun->decode(decoder);
+		if (errcode)
+			return errcode;
+
+		/* If we skipped an event, we're in trouble. */
+		if (decoder->event)
+			return -pte_event_ignored;
+
+		/* We're done when we found a TIP packet that isn't part of an
+		 * event.
+		 */
+		if (dfun->flags & pdff_tip) {
+			uint64_t ip;
+
+			/* We already decoded it, so the branch destination
+			 * is stored in the decoder's last ip.
+			 */
+			errcode = pt_last_ip_query(&ip, &decoder->ip);
+			if (errcode < 0)
+				flags |= pts_ip_suppressed;
+			else
+				*addr = ip;
+
+			break;
+		}
+
+		/* Read ahead until the next query-relevant packet. */
+		errcode = pt_qry_read_ahead(decoder);
+		if (errcode)
+			return errcode;
+	}
+
+	/* Preserve the time at the TIP packet. */
+	decoder->last_time = decoder->time;
+
+	/* Read ahead until the next query-relevant packet. */
+	errcode = pt_qry_read_ahead(decoder);
+	if ((errcode < 0) && (errcode != -pte_eos))
+		return errcode;
+
+	flags |= pt_qry_status_flags(decoder);
+
+	return flags;
+}
+
+int pt_qry_event(struct pt_query_decoder *decoder, struct pt_event *event,
+		 size_t size)
+{
+	int errcode, flags;
+
+	if (!decoder || !event)
+		return -pte_invalid;
+
+	if (size < offsetof(struct pt_event, variant))
+		return -pte_invalid;
+
+	/* We do not allow querying for events while there are still TNT
+	 * bits to consume.
+	 */
+	if (!pt_tnt_cache_is_empty(&decoder->tnt))
+		return -pte_bad_query;
+
+	/* Do not provide more than we actually have. */
+	if (sizeof(*event) < size)
+		size = sizeof(*event);
+
+	flags = 0;
+	for (;;) {
+		const struct pt_decoder_function *dfun;
+
+		dfun = decoder->next;
+		if (!dfun)
+			return pt_qry_provoke_fetch_error(decoder);
+
+		if (!dfun->decode)
+			return -pte_internal;
+
+		/* We must not see a TIP or TNT packet unless it belongs
+		 * to an event.
+		 *
+		 * If we see one, it means that our user got out of sync.
+		 * Let's report no data and hope that our user is able
+		 * to re-sync.
+		 */
+		if ((dfun->flags & (pdff_tip | pdff_tnt)) &&
+		    !pt_qry_will_event(decoder))
+			return -pte_bad_query;
+
+		/* Clear the decoder's current event so we know when decoding
+		 * produces a new event.
+		 */
+		decoder->event = NULL;
+
+		/* Apply any other decoder function. */
+		errcode = dfun->decode(decoder);
+		if (errcode)
+			return errcode;
+
+		/* Check if there has been an event.
+		 *
+		 * Some packets may result in events in some but not in all
+		 * configurations.
+		 */
+		if (decoder->event) {
+			(void) memcpy(event, decoder->event, size);
+			break;
+		}
+
+		/* Read ahead until the next query-relevant packet. */
+		errcode = pt_qry_read_ahead(decoder);
+		if (errcode)
+			return errcode;
+	}
+
+	/* Preserve the time at the event. */
+	decoder->last_time = decoder->time;
+
+	/* Read ahead until the next query-relevant packet. */
+	errcode = pt_qry_read_ahead(decoder);
+	if ((errcode < 0) && (errcode != -pte_eos))
+		return errcode;
+
+	flags |= pt_qry_status_flags(decoder);
+
+	return flags;
+}
+
+int pt_qry_time(struct pt_query_decoder *decoder, uint64_t *time,
+		uint32_t *lost_mtc, uint32_t *lost_cyc)
+{
+	if (!decoder || !time)
+		return -pte_invalid;
+
+	return pt_time_query_tsc(time, lost_mtc, lost_cyc, &decoder->last_time);
+}
+
+int pt_qry_core_bus_ratio(struct pt_query_decoder *decoder, uint32_t *cbr)
+{
+	if (!decoder || !cbr)
+		return -pte_invalid;
+
+	return pt_time_query_cbr(cbr, &decoder->last_time);
+}
+
+static int pt_qry_event_time(struct pt_event *event,
+			     const struct pt_query_decoder *decoder)
+{
+	int errcode;
+
+	if (!event || !decoder)
+		return -pte_internal;
+
+	errcode = pt_time_query_tsc(&event->tsc, &event->lost_mtc,
+				    &event->lost_cyc, &decoder->time);
+	if (errcode < 0) {
+		if (errcode != -pte_no_time)
+			return errcode;
+	} else
+		event->has_tsc = 1;
+
+	return 0;
+}
+
+int pt_qry_decode_unknown(struct pt_query_decoder *decoder)
+{
+	struct pt_packet packet;
+	int size;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_unknown(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_decode_pad(struct pt_query_decoder *decoder)
+{
+	if (!decoder)
+		return -pte_internal;
+
+	decoder->pos += ptps_pad;
+
+	return 0;
+}
+
+static int pt_qry_read_psb_header(struct pt_query_decoder *decoder)
+{
+	if (!decoder)
+		return -pte_internal;
+
+	pt_last_ip_init(&decoder->ip);
+
+	for (;;) {
+		const struct pt_decoder_function *dfun;
+		int errcode;
+
+		errcode = pt_df_fetch(&decoder->next, decoder->pos,
+				      &decoder->config);
+		if (errcode)
+			return errcode;
+
+		dfun = decoder->next;
+		if (!dfun)
+			return -pte_internal;
+
+		/* We're done once we reach an psbend packet. */
+		if (dfun->flags & pdff_psbend)
+			return 0;
+
+		if (!dfun->header)
+			return -pte_bad_context;
+
+		errcode = dfun->header(decoder);
+		if (errcode)
+			return errcode;
+	}
+}
+
+int pt_qry_decode_psb(struct pt_query_decoder *decoder)
+{
+	const uint8_t *pos;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	pos = decoder->pos;
+
+	size = pt_pkt_read_psb(pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	errcode = pt_tcal_update_psb(&decoder->tcal, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += size;
+
+	errcode = pt_qry_read_psb_header(decoder);
+	if (errcode < 0) {
+		/* Move back to the PSB so we have a chance to recover and
+		 * continue decoding.
+		 */
+		decoder->pos = pos;
+
+		/* Clear any PSB+ events that have already been queued. */
+		(void) pt_evq_clear(&decoder->evq, evb_psbend);
+
+		/* Reset the decoder's decode function. */
+		decoder->next = &pt_decode_psb;
+
+		return errcode;
+	}
+
+	/* The next packet following the PSB header will be of type PSBEND.
+	 *
+	 * Decoding this packet will publish the PSB events what have been
+	 * accumulated while reading the PSB header.
+	 */
+	return 0;
+}
+
+static int pt_qry_event_ip(uint64_t *ip, struct pt_event *event,
+			   const struct pt_query_decoder *decoder)
+{
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	errcode = pt_last_ip_query(ip, &decoder->ip);
+	if (errcode < 0) {
+		switch (pt_errcode(errcode)) {
+		case pte_noip:
+		case pte_ip_suppressed:
+			event->ip_suppressed = 1;
+			break;
+
+		default:
+			return errcode;
+		}
+	}
+
+	return 0;
+}
+
+/* Decode a generic IP packet.
+ *
+ * Returns the number of bytes read, on success.
+ * Returns -pte_eos if the ip does not fit into the buffer.
+ * Returns -pte_bad_packet if the ip compression is not known.
+ */
+static int pt_qry_decode_ip(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_ip packet;
+	int errcode, size;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_ip(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	errcode = pt_last_ip_update_ip(&decoder->ip, &packet, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	/* We do not update the decoder's position, yet. */
+
+	return size;
+}
+
+static int pt_qry_consume_tip(struct pt_query_decoder *decoder, int size)
+{
+	if (!decoder)
+		return -pte_internal;
+
+	decoder->pos += size;
+	return 0;
+}
+
+static int pt_qry_event_tip(struct pt_event *ev,
+			    struct pt_query_decoder *decoder)
+{
+	if (!ev || !decoder)
+		return -pte_internal;
+
+	switch (ev->type) {
+	case ptev_async_branch:
+		decoder->consume_packet = 1;
+
+		return pt_qry_event_ip(&ev->variant.async_branch.to, ev,
+				       decoder);
+
+	case ptev_async_paging:
+		return pt_qry_event_ip(&ev->variant.async_paging.ip, ev,
+				       decoder);
+
+	case ptev_async_vmcs:
+		return pt_qry_event_ip(&ev->variant.async_vmcs.ip, ev,
+				       decoder);
+
+	case ptev_exec_mode:
+		return pt_qry_event_ip(&ev->variant.exec_mode.ip, ev,
+				       decoder);
+
+	default:
+		break;
+	}
+
+	return -pte_bad_context;
+}
+
+int pt_qry_decode_tip(struct pt_query_decoder *decoder)
+{
+	struct pt_event *ev;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_qry_decode_ip(decoder);
+	if (size < 0)
+		return size;
+
+	/* Process any pending events binding to TIP. */
+	ev = pt_evq_dequeue(&decoder->evq, evb_tip);
+	if (ev) {
+		errcode = pt_qry_event_tip(ev, decoder);
+		if (errcode < 0)
+			return errcode;
+
+		/* Publish the event. */
+		decoder->event = ev;
+
+		/* Process further pending events. */
+		if (pt_evq_pending(&decoder->evq, evb_tip))
+			return 0;
+
+		/* No further events.
+		 *
+		 * If none of the events consumed the packet, we're done.
+		 */
+		if (!decoder->consume_packet)
+			return 0;
+
+		/* We're done with this packet. Clear the flag we set previously
+		 * and consume it.
+		 */
+		decoder->consume_packet = 0;
+	}
+
+	return pt_qry_consume_tip(decoder, size);
+}
+
+int pt_qry_decode_tnt_8(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_tnt packet;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_tnt_8(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	errcode = pt_tnt_cache_update_tnt(&decoder->tnt, &packet,
+					  &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_decode_tnt_64(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_tnt packet;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_tnt_64(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	errcode = pt_tnt_cache_update_tnt(&decoder->tnt, &packet,
+					  &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += size;
+	return 0;
+}
+
+static int pt_qry_consume_tip_pge(struct pt_query_decoder *decoder, int size)
+{
+	if (!decoder)
+		return -pte_internal;
+
+	decoder->pos += size;
+	return 0;
+}
+
+static int pt_qry_event_tip_pge(struct pt_event *ev,
+				const struct pt_query_decoder *decoder)
+{
+	if (!ev)
+		return -pte_internal;
+
+	switch (ev->type) {
+	case ptev_exec_mode:
+		return pt_qry_event_ip(&ev->variant.exec_mode.ip, ev, decoder);
+
+	default:
+		break;
+	}
+
+	return -pte_bad_context;
+}
+
+int pt_qry_decode_tip_pge(struct pt_query_decoder *decoder)
+{
+	struct pt_event *ev;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_qry_decode_ip(decoder);
+	if (size < 0)
+		return size;
+
+	/* We send the enable event first. This is more convenient for our users
+	 * and does not require them to either store or blindly apply other
+	 * events that might be pending.
+	 *
+	 * We use the consume packet decoder flag to indicate this.
+	 */
+	if (!decoder->consume_packet) {
+		/* This packet signals a standalone enabled event. */
+		ev = pt_evq_standalone(&decoder->evq);
+		if (!ev)
+			return -pte_internal;
+
+		ev->type = ptev_enabled;
+
+		/* We can't afford having a suppressed IP here. */
+		errcode = pt_last_ip_query(&ev->variant.enabled.ip,
+					   &decoder->ip);
+		if (errcode < 0)
+			return -pte_bad_packet;
+
+		errcode = pt_qry_event_time(ev, decoder);
+		if (errcode < 0)
+			return errcode;
+
+		/* Discard any cached TNT bits.
+		 *
+		 * They should have been consumed at the corresponding disable
+		 * event. If they have not, for whatever reason, discard them
+		 * now so our user does not get out of sync.
+		 */
+		pt_tnt_cache_init(&decoder->tnt);
+
+		/* Process pending events next. */
+		decoder->consume_packet = 1;
+		decoder->enabled = 1;
+	} else {
+		/* Process any pending events binding to TIP. */
+		ev = pt_evq_dequeue(&decoder->evq, evb_tip);
+		if (ev) {
+			errcode = pt_qry_event_tip_pge(ev, decoder);
+			if (errcode < 0)
+				return errcode;
+		}
+	}
+
+	/* We must have an event. Either the initial enable event or one of the
+	 * queued events.
+	 */
+	if (!ev)
+		return -pte_internal;
+
+	/* Publish the event. */
+	decoder->event = ev;
+
+	/* Process further pending events. */
+	if (pt_evq_pending(&decoder->evq, evb_tip))
+		return 0;
+
+	/* We must consume the packet. */
+	if (!decoder->consume_packet)
+		return -pte_internal;
+
+	decoder->consume_packet = 0;
+
+	return pt_qry_consume_tip_pge(decoder, size);
+}
+
+static int pt_qry_consume_tip_pgd(struct pt_query_decoder *decoder, int size)
+{
+	if (!decoder)
+		return -pte_internal;
+
+	decoder->enabled = 0;
+	decoder->pos += size;
+	return 0;
+}
+
+static int pt_qry_event_tip_pgd(struct pt_event *ev,
+				const struct pt_query_decoder *decoder)
+{
+	if (!ev)
+		return -pte_internal;
+
+	switch (ev->type) {
+	case ptev_async_branch: {
+		uint64_t at;
+
+		/* Turn the async branch into an async disable. */
+		at = ev->variant.async_branch.from;
+
+		ev->type = ptev_async_disabled;
+		ev->variant.async_disabled.at = at;
+
+		return pt_qry_event_ip(&ev->variant.async_disabled.ip, ev,
+				       decoder);
+	}
+
+	case ptev_async_paging:
+	case ptev_async_vmcs:
+	case ptev_exec_mode:
+		/* These events are ordered after the async disable event.  It
+		 * is not quite clear what IP to give them.
+		 *
+		 * If we give them the async disable's source IP, we'd make an
+		 * error if the IP is updated when applying the async disable
+		 * event.
+		 *
+		 * If we give them the async disable's destination IP, we'd make
+		 * an error if the IP is not updated when applying the async
+		 * disable event.  That's what our decoders do since tracing is
+		 * likely to resume from there.
+		 *
+		 * In all cases, tracing will be disabled when those events are
+		 * applied, so we may as well suppress the IP.
+		 */
+		ev->ip_suppressed = 1;
+
+		return 0;
+
+	default:
+		break;
+	}
+
+	return -pte_bad_context;
+}
+
+int pt_qry_decode_tip_pgd(struct pt_query_decoder *decoder)
+{
+	struct pt_event *ev;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_qry_decode_ip(decoder);
+	if (size < 0)
+		return size;
+
+	/* Process any pending events binding to TIP. */
+	ev = pt_evq_dequeue(&decoder->evq, evb_tip);
+	if (ev) {
+		errcode = pt_qry_event_tip_pgd(ev, decoder);
+		if (errcode < 0)
+			return errcode;
+	} else {
+		/* This packet signals a standalone disabled event. */
+		ev = pt_evq_standalone(&decoder->evq);
+		if (!ev)
+			return -pte_internal;
+		ev->type = ptev_disabled;
+
+		errcode = pt_qry_event_ip(&ev->variant.disabled.ip, ev,
+					  decoder);
+		if (errcode < 0)
+			return errcode;
+
+		errcode = pt_qry_event_time(ev, decoder);
+		if (errcode < 0)
+			return errcode;
+	}
+
+	/* We must have an event. Either the initial enable event or one of the
+	 * queued events.
+	 */
+	if (!ev)
+		return -pte_internal;
+
+	/* Publish the event. */
+	decoder->event = ev;
+
+	/* Process further pending events. */
+	if (pt_evq_pending(&decoder->evq, evb_tip))
+		return 0;
+
+	return pt_qry_consume_tip_pgd(decoder, size);
+}
+
+static int pt_qry_consume_fup(struct pt_query_decoder *decoder, int size)
+{
+	if (!decoder)
+		return -pte_internal;
+
+	decoder->pos += size;
+	return 0;
+}
+
+static int scan_for_erratum_bdm70(struct pt_packet_decoder *decoder)
+{
+	for (;;) {
+		struct pt_packet packet;
+		int errcode;
+
+		errcode = pt_pkt_next(decoder, &packet, sizeof(packet));
+		if (errcode < 0) {
+			/* Running out of packets is not an error. */
+			if (errcode == -pte_eos)
+				errcode = 0;
+
+			return errcode;
+		}
+
+		switch (packet.type) {
+		default:
+			/* All other packets cancel our search.
+			 *
+			 * We do not enumerate those packets since we also
+			 * want to include new packets.
+			 */
+			return 0;
+
+		case ppt_tip_pge:
+			/* We found it - the erratum applies. */
+			return 1;
+
+		case ppt_pad:
+		case ppt_tsc:
+		case ppt_cbr:
+		case ppt_psbend:
+		case ppt_pip:
+		case ppt_mode:
+		case ppt_vmcs:
+		case ppt_tma:
+		case ppt_mtc:
+		case ppt_cyc:
+		case ppt_mnt:
+			/* Intentionally skip a few packets. */
+			continue;
+		}
+	}
+}
+
+static int check_erratum_bdm70(const uint8_t *pos,
+			       const struct pt_config *config)
+{
+	struct pt_packet_decoder decoder;
+	int errcode;
+
+	if (!pos || !config)
+		return -pte_internal;
+
+	errcode = pt_pkt_decoder_init(&decoder, config);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = pt_pkt_sync_set(&decoder, (uint64_t) (pos - config->begin));
+	if (errcode >= 0)
+		errcode = scan_for_erratum_bdm70(&decoder);
+
+	pt_pkt_decoder_fini(&decoder);
+	return errcode;
+}
+
+int pt_qry_header_fup(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_ip packet;
+	int errcode, size;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_ip(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	if (decoder->config.errata.bdm70 && !decoder->enabled) {
+		errcode = check_erratum_bdm70(decoder->pos + size,
+					      &decoder->config);
+		if (errcode < 0)
+			return errcode;
+
+		if (errcode)
+			return pt_qry_consume_fup(decoder, size);
+	}
+
+	errcode = pt_last_ip_update_ip(&decoder->ip, &packet, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	/* Tracing is enabled if we have an IP in the header. */
+	if (packet.ipc != pt_ipc_suppressed)
+		decoder->enabled = 1;
+
+	return pt_qry_consume_fup(decoder, size);
+}
+
+static int pt_qry_event_fup(struct pt_event *ev,
+			    struct pt_query_decoder *decoder)
+{
+	if (!ev || !decoder)
+		return -pte_internal;
+
+	switch (ev->type) {
+	case ptev_overflow:
+		decoder->consume_packet = 1;
+
+		/* We can't afford having a suppressed IP here. */
+		return pt_last_ip_query(&ev->variant.overflow.ip,
+					&decoder->ip);
+
+	case ptev_tsx:
+		if (!(ev->variant.tsx.aborted))
+			decoder->consume_packet = 1;
+
+		return pt_qry_event_ip(&ev->variant.tsx.ip, ev, decoder);
+
+	case ptev_exstop:
+		decoder->consume_packet = 1;
+
+		return pt_qry_event_ip(&ev->variant.exstop.ip, ev, decoder);
+
+	case ptev_mwait:
+		decoder->consume_packet = 1;
+
+		return pt_qry_event_ip(&ev->variant.mwait.ip, ev, decoder);
+
+	case ptev_ptwrite:
+		decoder->consume_packet = 1;
+
+		return pt_qry_event_ip(&ev->variant.ptwrite.ip, ev, decoder);
+
+	default:
+		break;
+	}
+
+	return -pte_internal;
+}
+
+int pt_qry_decode_fup(struct pt_query_decoder *decoder)
+{
+	struct pt_event *ev;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_qry_decode_ip(decoder);
+	if (size < 0)
+		return size;
+
+	/* Process any pending events binding to FUP. */
+	ev = pt_evq_dequeue(&decoder->evq, evb_fup);
+	if (ev) {
+		errcode = pt_qry_event_fup(ev, decoder);
+		if (errcode < 0)
+			return errcode;
+
+		/* Publish the event. */
+		decoder->event = ev;
+
+		/* Process further pending events. */
+		if (pt_evq_pending(&decoder->evq, evb_fup))
+			return 0;
+
+		/* No further events.
+		 *
+		 * If none of the events consumed the packet, we're done.
+		 */
+		if (!decoder->consume_packet)
+			return 0;
+
+		/* We're done with this packet. Clear the flag we set previously
+		 * and consume it.
+		 */
+		decoder->consume_packet = 0;
+	} else {
+		/* FUP indicates an async branch event; it binds to TIP.
+		 *
+		 * We do need an IP in this case.
+		 */
+		uint64_t ip;
+
+		errcode = pt_last_ip_query(&ip, &decoder->ip);
+		if (errcode < 0)
+			return errcode;
+
+		ev = pt_evq_enqueue(&decoder->evq, evb_tip);
+		if (!ev)
+			return -pte_nomem;
+
+		ev->type = ptev_async_branch;
+		ev->variant.async_branch.from = ip;
+
+		errcode = pt_qry_event_time(ev, decoder);
+		if (errcode < 0)
+			return errcode;
+	}
+
+	return pt_qry_consume_fup(decoder, size);
+}
+
+int pt_qry_decode_pip(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_pip packet;
+	struct pt_event *event;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_pip(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	/* Paging events are either standalone or bind to the same TIP packet
+	 * as an in-flight async branch event.
+	 */
+	event = pt_evq_find(&decoder->evq, evb_tip, ptev_async_branch);
+	if (!event) {
+		event = pt_evq_standalone(&decoder->evq);
+		if (!event)
+			return -pte_internal;
+		event->type = ptev_paging;
+		event->variant.paging.cr3 = packet.cr3;
+		event->variant.paging.non_root = packet.nr;
+
+		decoder->event = event;
+	} else {
+		event = pt_evq_enqueue(&decoder->evq, evb_tip);
+		if (!event)
+			return -pte_nomem;
+
+		event->type = ptev_async_paging;
+		event->variant.async_paging.cr3 = packet.cr3;
+		event->variant.async_paging.non_root = packet.nr;
+	}
+
+	errcode = pt_qry_event_time(event, decoder);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_header_pip(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_pip packet;
+	struct pt_event *event;
+	int size;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_pip(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	/* Paging events are reported at the end of the PSB. */
+	event = pt_evq_enqueue(&decoder->evq, evb_psbend);
+	if (!event)
+		return -pte_nomem;
+
+	event->type = ptev_async_paging;
+	event->variant.async_paging.cr3 = packet.cr3;
+	event->variant.async_paging.non_root = packet.nr;
+
+	decoder->pos += size;
+	return 0;
+}
+
+static int pt_qry_event_psbend(struct pt_event *ev,
+			       struct pt_query_decoder *decoder)
+{
+	int errcode;
+
+	if (!ev || !decoder)
+		return -pte_internal;
+
+	/* PSB+ events are status updates. */
+	ev->status_update = 1;
+
+	errcode = pt_qry_event_time(ev, decoder);
+	if (errcode < 0)
+		return errcode;
+
+	switch (ev->type) {
+	case ptev_async_paging:
+		return pt_qry_event_ip(&ev->variant.async_paging.ip, ev,
+				       decoder);
+
+	case ptev_exec_mode:
+		return pt_qry_event_ip(&ev->variant.exec_mode.ip, ev, decoder);
+
+	case ptev_tsx:
+		return pt_qry_event_ip(&ev->variant.tsx.ip, ev, decoder);
+
+	case ptev_async_vmcs:
+		return pt_qry_event_ip(&ev->variant.async_vmcs.ip, ev,
+				       decoder);
+
+	case ptev_cbr:
+		return 0;
+
+	case ptev_mnt:
+		/* Maintenance packets may appear anywhere.  Do not mark them as
+		 * status updates even if they appear in PSB+.
+		 */
+		ev->status_update = 0;
+		return 0;
+
+	default:
+		break;
+	}
+
+	return -pte_internal;
+}
+
+static int pt_qry_process_pending_psb_events(struct pt_query_decoder *decoder)
+{
+	struct pt_event *ev;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	ev = pt_evq_dequeue(&decoder->evq, evb_psbend);
+	if (!ev)
+		return 0;
+
+	errcode = pt_qry_event_psbend(ev, decoder);
+	if (errcode < 0)
+		return errcode;
+
+	/* Publish the event. */
+	decoder->event = ev;
+
+	/* Signal a pending event. */
+	return 1;
+}
+
+/* Create a standalone overflow event with tracing disabled.
+ *
+ * Creates and published the event and disables tracing in @decoder.
+ *
+ * Returns zero on success, a negative pt_error_code otherwise.
+ */
+static int pt_qry_event_ovf_disabled(struct pt_query_decoder *decoder)
+{
+	struct pt_event *ev;
+
+	if (!decoder)
+		return -pte_internal;
+
+	ev = pt_evq_standalone(&decoder->evq);
+	if (!ev)
+		return -pte_internal;
+
+	ev->type = ptev_overflow;
+
+	/* We suppress the IP to indicate that tracing has been disabled before
+	 * the overflow resolved.  There can be several events before tracing is
+	 * enabled again.
+	 */
+	ev->ip_suppressed = 1;
+
+	decoder->enabled = 0;
+	decoder->event = ev;
+
+	return pt_qry_event_time(ev, decoder);
+}
+
+/* Queues an overflow event with tracing enabled.
+ *
+ * Creates and enqueues the event and enables tracing in @decoder.
+ *
+ * Returns zero on success, a negative pt_error_code otherwise.
+ */
+static int pt_qry_event_ovf_enabled(struct pt_query_decoder *decoder)
+{
+	struct pt_event *ev;
+
+	if (!decoder)
+		return -pte_internal;
+
+	ev = pt_evq_enqueue(&decoder->evq, evb_fup);
+	if (!ev)
+		return -pte_internal;
+
+	ev->type = ptev_overflow;
+
+	decoder->enabled = 1;
+
+	return pt_qry_event_time(ev, decoder);
+}
+
+/* Recover from SKD010.
+ *
+ * Creates and publishes an overflow event at @packet's IP payload.
+ *
+ * Further updates @decoder as follows:
+ *
+ *   - set time tracking to @time and @tcal
+ *   - set the position to @offset
+ *   - set ip to @packet's IP payload
+ *   - set tracing to be enabled
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int skd010_recover(struct pt_query_decoder *decoder,
+			  const struct pt_packet_ip *packet,
+			  const struct pt_time_cal *tcal,
+			  const struct pt_time *time, uint64_t offset)
+{
+	struct pt_last_ip ip;
+	struct pt_event *ev;
+	int errcode;
+
+	if (!decoder || !packet || !tcal || !time)
+		return -pte_internal;
+
+	/* We use the decoder's IP.  It should be newly initialized. */
+	ip = decoder->ip;
+
+	/* Extract the IP payload from the packet. */
+	errcode = pt_last_ip_update_ip(&ip, packet, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	/* Synthesize the overflow event. */
+	ev = pt_evq_standalone(&decoder->evq);
+	if (!ev)
+		return -pte_internal;
+
+	ev->type = ptev_overflow;
+
+	/* We do need a full IP. */
+	errcode = pt_last_ip_query(&ev->variant.overflow.ip, &ip);
+	if (errcode < 0)
+		return -pte_bad_context;
+
+	/* We continue decoding at the given offset. */
+	decoder->pos = decoder->config.begin + offset;
+
+	/* Tracing is enabled. */
+	decoder->enabled = 1;
+	decoder->ip = ip;
+
+	decoder->time = *time;
+	decoder->tcal = *tcal;
+
+	/* Publish the event. */
+	decoder->event = ev;
+
+	return pt_qry_event_time(ev, decoder);
+}
+
+/* Recover from SKD010 with tracing disabled.
+ *
+ * Creates and publishes a standalone overflow event.
+ *
+ * Further updates @decoder as follows:
+ *
+ *   - set time tracking to @time and @tcal
+ *   - set the position to @offset
+ *   - set tracing to be disabled
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int skd010_recover_disabled(struct pt_query_decoder *decoder,
+				   const struct pt_time_cal *tcal,
+				   const struct pt_time *time, uint64_t offset)
+{
+	if (!decoder || !tcal || !time)
+		return -pte_internal;
+
+	decoder->time = *time;
+	decoder->tcal = *tcal;
+
+	/* We continue decoding at the given offset. */
+	decoder->pos = decoder->config.begin + offset;
+
+	return pt_qry_event_ovf_disabled(decoder);
+}
+
+/* Scan ahead for a packet at which to resume after an overflow.
+ *
+ * This function is called after an OVF without a corresponding FUP.  This
+ * normally means that the overflow resolved while tracing was disabled.
+ *
+ * With erratum SKD010 it might also mean that the FUP (or TIP.PGE) was dropped.
+ * The overflow thus resolved while tracing was enabled (or tracing was enabled
+ * after the overflow resolved).  Search for an indication whether tracing is
+ * enabled or disabled by scanning upcoming packets.
+ *
+ * If we can confirm that tracing is disabled, the erratum does not apply and we
+ * can continue normally.
+ *
+ * If we can confirm that tracing is enabled, the erratum applies and we try to
+ * recover by synchronizing at a later packet and a different IP.  If we can't
+ * recover, pretend the erratum didn't apply so we run into the error later.
+ * Since this assumes that tracing is disabled, no harm should be done, i.e. no
+ * bad trace should be generated.
+ *
+ * Returns zero if the overflow is handled.
+ * Returns a positive value if the overflow is not yet handled.
+ * Returns a negative error code otherwise.
+ */
+static int skd010_scan_for_ovf_resume(struct pt_packet_decoder *pkt,
+				      struct pt_query_decoder *decoder)
+{
+	struct pt_time_cal tcal;
+	struct pt_time time;
+	struct {
+		struct pt_time_cal tcal;
+		struct pt_time time;
+		uint64_t offset;
+	} mode_tsx;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	/* Keep track of time as we skip packets. */
+	time = decoder->time;
+	tcal = decoder->tcal;
+
+	/* Keep track of a potential recovery point at MODE.TSX. */
+	memset(&mode_tsx, 0, sizeof(mode_tsx));
+
+	for (;;) {
+		struct pt_packet packet;
+		uint64_t offset;
+
+		errcode = pt_pkt_get_offset(pkt, &offset);
+		if (errcode < 0)
+			return errcode;
+
+		errcode = pt_pkt_next(pkt, &packet, sizeof(packet));
+		if (errcode < 0) {
+			/* Let's assume the trace is correct if we run out
+			 * of packets.
+			 */
+			if (errcode == -pte_eos)
+				errcode = 1;
+
+			return errcode;
+		}
+
+		switch (packet.type) {
+		case ppt_tip_pge:
+			/* Everything is fine.  There is nothing to do. */
+			return 1;
+
+		case ppt_tip_pgd:
+			/* This is a clear indication that the erratum
+			 * applies.
+			 *
+			 * We synchronize after the disable.
+			 */
+			return skd010_recover_disabled(decoder, &tcal, &time,
+						       offset + packet.size);
+
+		case ppt_tnt_8:
+		case ppt_tnt_64:
+			/* This is a clear indication that the erratum
+			 * apllies.
+			 *
+			 * Yet, we can't recover from it as we wouldn't know how
+			 * many TNT bits will have been used when we eventually
+			 * find an IP packet at which to resume tracing.
+			 */
+			return 1;
+
+		case ppt_pip:
+		case ppt_vmcs:
+			/* We could track those changes and synthesize extra
+			 * events after the overflow event when recovering from
+			 * the erratum.  This requires infrastructure that we
+			 * don't currently have, though, so we're not going to
+			 * do it.
+			 *
+			 * Instead, we ignore those changes.  We already don't
+			 * know how many other changes were lost in the
+			 * overflow.
+			 */
+			break;
+
+		case ppt_mode:
+			switch (packet.payload.mode.leaf) {
+			case pt_mol_exec:
+				/* A MODE.EXEC packet binds to TIP, i.e.
+				 *
+				 *   TIP.PGE:  everything is fine
+				 *   TIP:      the erratum applies
+				 *
+				 * In the TIP.PGE case, we may just follow the
+				 * normal code flow.
+				 *
+				 * In the TIP case, we'd be able to re-sync at
+				 * the TIP IP but have to skip packets up to and
+				 * including the TIP.
+				 *
+				 * We'd need to synthesize the MODE.EXEC event
+				 * after the overflow event when recovering at
+				 * the TIP.  We lack the infrastructure for this
+				 * - it's getting too complicated.
+				 *
+				 * Instead, we ignore the execution mode change;
+				 * we already don't know how many more such
+				 * changes were lost in the overflow.
+				 */
+				break;
+
+			case pt_mol_tsx:
+				/* A MODE.TSX packet may be standalone or bind
+				 * to FUP.
+				 *
+				 * If this is the second MODE.TSX, we're sure
+				 * that tracing is disabled and everything is
+				 * fine.
+				 */
+				if (mode_tsx.offset)
+					return 1;
+
+				/* If we find the FUP this packet binds to, we
+				 * may recover at the FUP IP and restart
+				 * processing packets from here.  Remember the
+				 * current state.
+				 */
+				mode_tsx.offset = offset;
+				mode_tsx.time = time;
+				mode_tsx.tcal = tcal;
+
+				break;
+			}
+
+			break;
+
+		case ppt_fup:
+			/* This is a pretty good indication that tracing
+			 * is indeed enabled and the erratum applies.
+			 */
+
+			/* If we got a MODE.TSX packet before, we synchronize at
+			 * the FUP IP but continue decoding packets starting
+			 * from the MODE.TSX.
+			 */
+			if (mode_tsx.offset)
+				return skd010_recover(decoder,
+						      &packet.payload.ip,
+						      &mode_tsx.tcal,
+						      &mode_tsx.time,
+						      mode_tsx.offset);
+
+			/* Without a preceding MODE.TSX, this FUP is the start
+			 * of an async branch or disable.  We synchronize at the
+			 * FUP IP and continue decoding packets from here.
+			 */
+			return skd010_recover(decoder, &packet.payload.ip,
+					      &tcal, &time, offset);
+
+		case ppt_tip:
+			/* We syhchronize at the TIP IP and continue decoding
+			 * packets after the TIP packet.
+			 */
+			return skd010_recover(decoder, &packet.payload.ip,
+					      &tcal, &time,
+					      offset + packet.size);
+
+		case ppt_psb:
+			/* We reached a synchronization point.  Tracing is
+			 * enabled if and only if the PSB+ contains a FUP.
+			 */
+			errcode = pt_qry_find_header_fup(&packet, pkt);
+			if (errcode < 0) {
+				/* If we ran out of packets, we can't tell.
+				 * Let's assume the trace is correct.
+				 */
+				if (errcode == -pte_eos)
+					errcode = 1;
+
+				return errcode;
+			}
+
+			/* If there is no FUP, tracing is disabled and
+			 * everything is fine.
+			 */
+			if (!errcode)
+				return 1;
+
+			/* We should have a FUP. */
+			if (packet.type != ppt_fup)
+				return -pte_internal;
+
+			/* Otherwise, we may synchronize at the FUP IP and
+			 * continue decoding packets at the PSB.
+			 */
+			return skd010_recover(decoder, &packet.payload.ip,
+					      &tcal, &time, offset);
+
+		case ppt_psbend:
+			/* We shouldn't see this. */
+			return -pte_bad_context;
+
+		case ppt_ovf:
+		case ppt_stop:
+			/* It doesn't matter if it had been enabled or disabled
+			 * before.  We may resume normally.
+			 */
+			return 1;
+
+		case ppt_unknown:
+		case ppt_invalid:
+			/* We can't skip this packet. */
+			return 1;
+
+		case ppt_pad:
+		case ppt_mnt:
+		case ppt_pwre:
+		case ppt_pwrx:
+			/* Ignore this packet. */
+			break;
+
+		case ppt_exstop:
+			/* We may skip a stand-alone EXSTOP. */
+			if (!packet.payload.exstop.ip)
+				break;
+
+			fallthrough;
+		case ppt_mwait:
+			/* To skip this packet, we'd need to take care of the
+			 * FUP it binds to.  This is getting complicated.
+			 */
+			return 1;
+
+		case ppt_ptw:
+			/* We may skip a stand-alone PTW. */
+			if (!packet.payload.ptw.ip)
+				break;
+
+			/* To skip this packet, we'd need to take care of the
+			 * FUP it binds to.  This is getting complicated.
+			 */
+			return 1;
+
+		case ppt_tsc:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_tsc(&time, &tcal,
+						   &packet.payload.tsc,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+
+		case ppt_cbr:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_cbr(&time, &tcal,
+						   &packet.payload.cbr,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+
+		case ppt_tma:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_tma(&time, &tcal,
+						   &packet.payload.tma,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+
+		case ppt_mtc:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_mtc(&time, &tcal,
+						   &packet.payload.mtc,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+
+		case ppt_cyc:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_cyc(&time, &tcal,
+						   &packet.payload.cyc,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+		}
+	}
+}
+
+static int pt_qry_handle_skd010(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_decoder pkt;
+	uint64_t offset;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	errcode = pt_qry_get_offset(decoder, &offset);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = pt_pkt_decoder_init(&pkt, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = pt_pkt_sync_set(&pkt, offset);
+	if (errcode >= 0)
+		errcode = skd010_scan_for_ovf_resume(&pkt, decoder);
+
+	pt_pkt_decoder_fini(&pkt);
+	return errcode;
+}
+
+/* Scan ahead for an indication whether tracing is enabled or disabled.
+ *
+ * Returns zero if tracing is clearly disabled.
+ * Returns a positive integer if tracing is enabled or if we can't tell.
+ * Returns a negative error code otherwise.
+ */
+static int apl12_tracing_is_disabled(struct pt_packet_decoder *decoder)
+{
+	if (!decoder)
+		return -pte_internal;
+
+	for (;;) {
+		struct pt_packet packet;
+		int status;
+
+		status = pt_pkt_next(decoder, &packet, sizeof(packet));
+		if (status < 0) {
+			/* Running out of packets is not an error. */
+			if (status == -pte_eos)
+				status = 1;
+
+			return status;
+		}
+
+		switch (packet.type) {
+		default:
+			/* Skip other packets. */
+			break;
+
+		case ppt_stop:
+			/* Tracing is disabled before a stop. */
+			return 0;
+
+		case ppt_tip_pge:
+			/* Tracing gets enabled - it must have been disabled. */
+			return 0;
+
+		case ppt_tnt_8:
+		case ppt_tnt_64:
+		case ppt_tip:
+		case ppt_tip_pgd:
+			/* Those packets are only generated when tracing is
+			 * enabled.  We're done.
+			 */
+			return 1;
+
+		case ppt_psb:
+			/* We reached a synchronization point.  Tracing is
+			 * enabled if and only if the PSB+ contains a FUP.
+			 */
+			status = pt_qry_find_header_fup(&packet, decoder);
+
+			/* If we ran out of packets, we can't tell. */
+			if (status == -pte_eos)
+				status = 1;
+
+			return status;
+
+		case ppt_psbend:
+			/* We shouldn't see this. */
+			return -pte_bad_context;
+
+		case ppt_ovf:
+			/* It doesn't matter - we run into the next overflow. */
+			return 1;
+
+		case ppt_unknown:
+		case ppt_invalid:
+			/* We can't skip this packet. */
+			return 1;
+		}
+	}
+}
+
+/* Apply workaround for erratum APL12.
+ *
+ * We resume from @offset (relative to @decoder->pos) with tracing disabled.  On
+ * our way to the resume location we process packets to update our state.
+ *
+ * Any event will be dropped.
+ *
+ * Returns zero on success, a negative pt_error_code otherwise.
+ */
+static int apl12_resume_disabled(struct pt_query_decoder *decoder,
+				 struct pt_packet_decoder *pkt,
+				 unsigned int offset)
+{
+	uint64_t begin, end;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	errcode = pt_qry_get_offset(decoder, &begin);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = pt_pkt_sync_set(pkt, begin);
+	if (errcode < 0)
+		return errcode;
+
+	end = begin + offset;
+	for (;;) {
+		struct pt_packet packet;
+		uint64_t next;
+
+		errcode = pt_pkt_next(pkt, &packet, sizeof(packet));
+		if (errcode < 0) {
+			/* Running out of packets is not an error. */
+			if (errcode == -pte_eos)
+				errcode = 0;
+
+			return errcode;
+		}
+
+		/* The offset is the start of the next packet. */
+		errcode = pt_pkt_get_offset(pkt, &next);
+		if (errcode < 0)
+			return errcode;
+
+		/* We're done when we reach @offset.
+		 *
+		 * The current @packet will be the FUP after which we started
+		 * our search.  We skip it.
+		 *
+		 * Check that we're not accidentally proceeding past @offset.
+		 */
+		if (end <= next) {
+			if (end < next)
+				return -pte_internal;
+
+			break;
+		}
+
+		switch (packet.type) {
+		default:
+			/* Skip other packets. */
+			break;
+
+		case ppt_mode:
+		case ppt_pip:
+		case ppt_vmcs:
+			/* We should not encounter those.
+			 *
+			 * We should not encounter a lot of packets but those
+			 * are state-relevant; let's check them explicitly.
+			 */
+			return -pte_internal;
+
+		case ppt_tsc:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_tsc(&decoder->time,
+						   &decoder->tcal,
+						   &packet.payload.tsc,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+
+		case ppt_cbr:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_cbr(&decoder->time,
+						   &decoder->tcal,
+						   &packet.payload.cbr,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+
+		case ppt_tma:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_tma(&decoder->time,
+						   &decoder->tcal,
+						   &packet.payload.tma,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+
+		case ppt_mtc:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_mtc(&decoder->time,
+						   &decoder->tcal,
+						   &packet.payload.mtc,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+
+		case ppt_cyc:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_cyc(&decoder->time,
+						   &decoder->tcal,
+						   &packet.payload.cyc,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+		}
+	}
+
+	decoder->pos += offset;
+
+	return pt_qry_event_ovf_disabled(decoder);
+}
+
+/* Handle erratum APL12.
+ *
+ * This function is called when a FUP is found after an OVF.  The @offset
+ * argument gives the relative offset from @decoder->pos to after the FUP.
+ *
+ * A FUP after OVF normally indicates that the overflow resolved while tracing
+ * is enabled.  Due to erratum APL12, however, the overflow may have resolved
+ * while tracing is disabled and still generate a FUP.
+ *
+ * We scan ahead for an indication whether tracing is actually disabled.  If we
+ * find one, the erratum applies and we proceed from after the FUP packet.
+ *
+ * This will drop any CBR or MTC events.  We will update @decoder's timing state
+ * on CBR but drop the event.
+ *
+ * Returns zero if the erratum was handled.
+ * Returns a positive integer if the erratum was not handled.
+ * Returns a negative pt_error_code otherwise.
+ */
+static int pt_qry_handle_apl12(struct pt_query_decoder *decoder,
+			       unsigned int offset)
+{
+	struct pt_packet_decoder pkt;
+	uint64_t here;
+	int status;
+
+	if (!decoder)
+		return -pte_internal;
+
+	status = pt_qry_get_offset(decoder, &here);
+	if (status < 0)
+		return status;
+
+	status = pt_pkt_decoder_init(&pkt, &decoder->config);
+	if (status < 0)
+		return status;
+
+	status = pt_pkt_sync_set(&pkt, here + offset);
+	if (status >= 0) {
+		status = apl12_tracing_is_disabled(&pkt);
+		if (!status)
+			status = apl12_resume_disabled(decoder, &pkt, offset);
+	}
+
+	pt_pkt_decoder_fini(&pkt);
+	return status;
+}
+
+/* Apply workaround for erratum APL11.
+ *
+ * We search for a TIP.PGD and, if we found one, resume from after that packet
+ * with tracing disabled.  On our way to the resume location we process packets
+ * to update our state.
+ *
+ * If we don't find a TIP.PGD but instead some other packet that indicates that
+ * tracing is disabled, indicate that the erratum does not apply.
+ *
+ * Any event will be dropped.
+ *
+ * Returns zero if the erratum was handled.
+ * Returns a positive integer if the erratum was not handled.
+ * Returns a negative pt_error_code otherwise.
+ */
+static int apl11_apply(struct pt_query_decoder *decoder,
+		       struct pt_packet_decoder *pkt)
+{
+	struct pt_time_cal tcal;
+	struct pt_time time;
+
+	if (!decoder)
+		return -pte_internal;
+
+	time = decoder->time;
+	tcal = decoder->tcal;
+	for (;;) {
+		struct pt_packet packet;
+		int errcode;
+
+		errcode = pt_pkt_next(pkt, &packet, sizeof(packet));
+		if (errcode < 0)
+			return errcode;
+
+		switch (packet.type) {
+		case ppt_tip_pgd: {
+			uint64_t offset;
+
+			/* We found a TIP.PGD.  The erratum applies.
+			 *
+			 * Resume from here with tracing disabled.
+			 */
+			errcode = pt_pkt_get_offset(pkt, &offset);
+			if (errcode < 0)
+				return errcode;
+
+			decoder->time = time;
+			decoder->tcal = tcal;
+			decoder->pos = decoder->config.begin + offset;
+
+			return pt_qry_event_ovf_disabled(decoder);
+		}
+
+		case ppt_invalid:
+			return -pte_bad_opc;
+
+		case ppt_fup:
+		case ppt_psb:
+		case ppt_tip_pge:
+		case ppt_stop:
+		case ppt_ovf:
+		case ppt_mode:
+		case ppt_pip:
+		case ppt_vmcs:
+		case ppt_exstop:
+		case ppt_mwait:
+		case ppt_pwre:
+		case ppt_pwrx:
+		case ppt_ptw:
+			/* The erratum does not apply. */
+			return 1;
+
+		case ppt_unknown:
+		case ppt_pad:
+		case ppt_mnt:
+			/* Skip those packets. */
+			break;
+
+		case ppt_psbend:
+		case ppt_tip:
+		case ppt_tnt_8:
+		case ppt_tnt_64:
+			return -pte_bad_context;
+
+
+		case ppt_tsc:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_tsc(&time, &tcal,
+						   &packet.payload.tsc,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+
+		case ppt_cbr:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_cbr(&time, &tcal,
+						   &packet.payload.cbr,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+
+		case ppt_tma:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_tma(&time, &tcal,
+						   &packet.payload.tma,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+
+		case ppt_mtc:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_mtc(&time, &tcal,
+						   &packet.payload.mtc,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+
+		case ppt_cyc:
+			/* Keep track of time. */
+			errcode = pt_qry_apply_cyc(&time, &tcal,
+						   &packet.payload.cyc,
+						   &decoder->config);
+			if (errcode < 0)
+				return errcode;
+
+			break;
+		}
+	}
+}
+
+/* Handle erratum APL11.
+ *
+ * This function is called when we diagnose a bad packet while searching for a
+ * FUP after an OVF.
+ *
+ * Due to erratum APL11 we may get an extra TIP.PGD after the OVF.  Find that
+ * TIP.PGD and resume from there with tracing disabled.
+ *
+ * This will drop any CBR or MTC events.  We will update @decoder's timing state
+ * on CBR but drop the event.
+ *
+ * Returns zero if the erratum was handled.
+ * Returns a positive integer if the erratum was not handled.
+ * Returns a negative pt_error_code otherwise.
+ */
+static int pt_qry_handle_apl11(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_decoder pkt;
+	uint64_t offset;
+	int status;
+
+	if (!decoder)
+		return -pte_internal;
+
+	status = pt_qry_get_offset(decoder, &offset);
+	if (status < 0)
+		return status;
+
+	status = pt_pkt_decoder_init(&pkt, &decoder->config);
+	if (status < 0)
+		return status;
+
+	status = pt_pkt_sync_set(&pkt, offset);
+	if (status >= 0)
+		status = apl11_apply(decoder, &pkt);
+
+	pt_pkt_decoder_fini(&pkt);
+	return status;
+}
+
+static int pt_pkt_find_ovf_fup(struct pt_packet_decoder *decoder)
+{
+	for (;;) {
+		struct pt_packet packet;
+		int errcode;
+
+		errcode = pt_pkt_next(decoder, &packet, sizeof(packet));
+		if (errcode < 0)
+			return errcode;
+
+		switch (packet.type) {
+		case ppt_fup:
+			return 1;
+
+		case ppt_invalid:
+			return -pte_bad_opc;
+
+		case ppt_unknown:
+		case ppt_pad:
+		case ppt_mnt:
+		case ppt_cbr:
+		case ppt_tsc:
+		case ppt_tma:
+		case ppt_mtc:
+		case ppt_cyc:
+			continue;
+
+		case ppt_psb:
+		case ppt_tip_pge:
+		case ppt_mode:
+		case ppt_pip:
+		case ppt_vmcs:
+		case ppt_stop:
+		case ppt_ovf:
+		case ppt_exstop:
+		case ppt_mwait:
+		case ppt_pwre:
+		case ppt_pwrx:
+		case ppt_ptw:
+			return 0;
+
+		case ppt_psbend:
+		case ppt_tip:
+		case ppt_tip_pgd:
+		case ppt_tnt_8:
+		case ppt_tnt_64:
+			return -pte_bad_context;
+		}
+	}
+}
+
+/* Find a FUP to which the current OVF may bind.
+ *
+ * Scans the trace for a FUP or for a packet that indicates that tracing is
+ * disabled.
+ *
+ * Return the relative offset of the packet following the found FUP on success.
+ * Returns zero if no FUP is found and tracing is assumed to be disabled.
+ * Returns a negative pt_error_code otherwise.
+ */
+static int pt_qry_find_ovf_fup(const struct pt_query_decoder *decoder)
+{
+	struct pt_packet_decoder pkt;
+	uint64_t begin, end, offset;
+	int status;
+
+	if (!decoder)
+		return -pte_internal;
+
+	status = pt_qry_get_offset(decoder, &begin);
+	if (status < 0)
+		return status;
+
+	status = pt_pkt_decoder_init(&pkt, &decoder->config);
+	if (status < 0)
+		return status;
+
+	status = pt_pkt_sync_set(&pkt, begin);
+	if (status >= 0) {
+		status = pt_pkt_find_ovf_fup(&pkt);
+		if (status > 0) {
+			status = pt_pkt_get_offset(&pkt, &end);
+			if (status < 0)
+				return status;
+
+			if (end <= begin)
+				return -pte_overflow;
+
+			offset = end - begin;
+			if (INT_MAX < offset)
+				return -pte_overflow;
+
+			status = (int) offset;
+		}
+	}
+
+	pt_pkt_decoder_fini(&pkt);
+	return status;
+}
+
+int pt_qry_decode_ovf(struct pt_query_decoder *decoder)
+{
+	struct pt_time_cal tcal;
+	struct pt_time time;
+	int status, offset;
+
+	if (!decoder)
+		return -pte_internal;
+
+	status = pt_qry_process_pending_psb_events(decoder);
+	if (status < 0)
+		return status;
+
+	/* If we have any pending psbend events, we're done for now. */
+	if (status)
+		return 0;
+
+	/* Reset the decoder state but preserve timing. */
+	time = decoder->time;
+	tcal = decoder->tcal;
+
+	pt_qry_reset(decoder);
+
+	decoder->time = time;
+	if (decoder->config.flags.variant.query.keep_tcal_on_ovf) {
+		status = pt_tcal_update_ovf(&tcal, &decoder->config);
+		if (status < 0)
+			return status;
+
+		decoder->tcal = tcal;
+	}
+
+	/* We must consume the OVF before we search for the binding packet. */
+	decoder->pos += ptps_ovf;
+
+	/* Overflow binds to either FUP or TIP.PGE.
+	 *
+	 * If the overflow can be resolved while PacketEn=1 it binds to FUP.  We
+	 * can see timing packets between OVF anf FUP but that's it.
+	 *
+	 * Otherwise, PacketEn will be zero when the overflow resolves and OVF
+	 * binds to TIP.PGE.  There can be packets between OVF and TIP.PGE that
+	 * do not depend on PacketEn.
+	 *
+	 * We don't need to decode everything until TIP.PGE, however.  As soon
+	 * as we see a non-timing non-FUP packet, we know that tracing has been
+	 * disabled before the overflow resolves.
+	 */
+	offset = pt_qry_find_ovf_fup(decoder);
+	if (offset <= 0) {
+		/* Check for erratum SKD010.
+		 *
+		 * The FUP may have been dropped.  If we can figure out that
+		 * tracing is enabled and hence the FUP is missing, we resume
+		 * at a later packet and a different IP.
+		 */
+		if (decoder->config.errata.skd010) {
+			status = pt_qry_handle_skd010(decoder);
+			if (status <= 0)
+				return status;
+		}
+
+		/* Check for erratum APL11.
+		 *
+		 * We may have gotten an extra TIP.PGD, which should be
+		 * diagnosed by our search for a subsequent FUP.
+		 */
+		if (decoder->config.errata.apl11 &&
+		    (offset == -pte_bad_context)) {
+			status = pt_qry_handle_apl11(decoder);
+			if (status <= 0)
+				return status;
+		}
+
+		/* Report the original error from searching for the FUP packet
+		 * if we were not able to fix the trace.
+		 *
+		 * We treat an overflow at the end of the trace as standalone.
+		 */
+		if (offset < 0 && offset != -pte_eos)
+			return offset;
+
+		return pt_qry_event_ovf_disabled(decoder);
+	} else {
+		/* Check for erratum APL12.
+		 *
+		 * We may get an extra FUP even though the overflow resolved
+		 * with tracing disabled.
+		 */
+		if (decoder->config.errata.apl12) {
+			status = pt_qry_handle_apl12(decoder,
+						     (unsigned int) offset);
+			if (status <= 0)
+				return status;
+		}
+
+		return pt_qry_event_ovf_enabled(decoder);
+	}
+}
+
+static int pt_qry_decode_mode_exec(struct pt_query_decoder *decoder,
+				   const struct pt_packet_mode_exec *packet)
+{
+	struct pt_event *event;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	/* MODE.EXEC binds to TIP. */
+	event = pt_evq_enqueue(&decoder->evq, evb_tip);
+	if (!event)
+		return -pte_nomem;
+
+	event->type = ptev_exec_mode;
+	event->variant.exec_mode.mode = pt_get_exec_mode(packet);
+
+	return pt_qry_event_time(event, decoder);
+}
+
+static int pt_qry_decode_mode_tsx(struct pt_query_decoder *decoder,
+				  const struct pt_packet_mode_tsx *packet)
+{
+	struct pt_event *event;
+
+	if (!decoder || !packet)
+		return -pte_internal;
+
+	/* MODE.TSX is standalone if tracing is disabled. */
+	if (!decoder->enabled) {
+		event = pt_evq_standalone(&decoder->evq);
+		if (!event)
+			return -pte_internal;
+
+		/* We don't have an IP in this case. */
+		event->variant.tsx.ip = 0;
+		event->ip_suppressed = 1;
+
+		/* Publish the event. */
+		decoder->event = event;
+	} else {
+		/* MODE.TSX binds to FUP. */
+		event = pt_evq_enqueue(&decoder->evq, evb_fup);
+		if (!event)
+			return -pte_nomem;
+	}
+
+	event->type = ptev_tsx;
+	event->variant.tsx.speculative = packet->intx;
+	event->variant.tsx.aborted = packet->abrt;
+
+	return pt_qry_event_time(event, decoder);
+}
+
+int pt_qry_decode_mode(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_mode packet;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_mode(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	errcode = 0;
+	switch (packet.leaf) {
+	case pt_mol_exec:
+		errcode = pt_qry_decode_mode_exec(decoder, &packet.bits.exec);
+		break;
+
+	case pt_mol_tsx:
+		errcode = pt_qry_decode_mode_tsx(decoder, &packet.bits.tsx);
+		break;
+	}
+
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_header_mode(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_mode packet;
+	struct pt_event *event;
+	int size;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_mode(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	/* Inside the header, events are reported at the end. */
+	event = pt_evq_enqueue(&decoder->evq, evb_psbend);
+	if (!event)
+		return -pte_nomem;
+
+	switch (packet.leaf) {
+	case pt_mol_exec:
+		event->type = ptev_exec_mode;
+		event->variant.exec_mode.mode =
+			pt_get_exec_mode(&packet.bits.exec);
+		break;
+
+	case pt_mol_tsx:
+		event->type = ptev_tsx;
+		event->variant.tsx.speculative = packet.bits.tsx.intx;
+		event->variant.tsx.aborted = packet.bits.tsx.abrt;
+		break;
+	}
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_decode_psbend(struct pt_query_decoder *decoder)
+{
+	int status;
+
+	if (!decoder)
+		return -pte_internal;
+
+	status = pt_qry_process_pending_psb_events(decoder);
+	if (status < 0)
+		return status;
+
+	/* If we had any psb events, we're done for now. */
+	if (status)
+		return 0;
+
+	/* Skip the psbend extended opcode that we fetched before if no more
+	 * psbend events are pending.
+	 */
+	decoder->pos += ptps_psbend;
+	return 0;
+}
+
+int pt_qry_decode_tsc(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_tsc packet;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_tsc(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	errcode = pt_qry_apply_tsc(&decoder->time, &decoder->tcal,
+				   &packet, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_header_tsc(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_tsc packet;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_tsc(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	errcode = pt_qry_apply_header_tsc(&decoder->time, &decoder->tcal,
+					  &packet, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_decode_cbr(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_cbr packet;
+	struct pt_event *event;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_cbr(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	errcode = pt_qry_apply_cbr(&decoder->time, &decoder->tcal,
+				   &packet, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	event = pt_evq_standalone(&decoder->evq);
+	if (!event)
+		return -pte_internal;
+
+	event->type = ptev_cbr;
+	event->variant.cbr.ratio = packet.ratio;
+
+	decoder->event = event;
+
+	errcode = pt_qry_event_time(event, decoder);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_header_cbr(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_cbr packet;
+	struct pt_event *event;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_cbr(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	errcode = pt_qry_apply_header_cbr(&decoder->time, &decoder->tcal,
+					  &packet, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	event = pt_evq_enqueue(&decoder->evq, evb_psbend);
+	if (!event)
+		return -pte_nomem;
+
+	event->type = ptev_cbr;
+	event->variant.cbr.ratio = packet.ratio;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_decode_tma(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_tma packet;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_tma(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	errcode = pt_qry_apply_tma(&decoder->time, &decoder->tcal,
+				   &packet, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_decode_mtc(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_mtc packet;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_mtc(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	errcode = pt_qry_apply_mtc(&decoder->time, &decoder->tcal,
+				   &packet, &decoder->config);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += size;
+	return 0;
+}
+
+static int check_erratum_skd007(struct pt_query_decoder *decoder,
+				const struct pt_packet_cyc *packet, int size)
+{
+	const uint8_t *pos;
+	uint16_t payload;
+
+	if (!decoder || !packet || size < 0)
+		return -pte_internal;
+
+	/* It must be a 2-byte CYC. */
+	if (size != 2)
+		return 0;
+
+	payload = (uint16_t) packet->value;
+
+	/* The 2nd byte of the CYC payload must look like an ext opcode. */
+	if ((payload & ~0x1f) != 0x20)
+		return 0;
+
+	/* Skip this CYC packet. */
+	pos = decoder->pos + size;
+	if (decoder->config.end <= pos)
+		return 0;
+
+	/* See if we got a second CYC that looks like an OVF ext opcode. */
+	if (*pos != pt_ext_ovf)
+		return 0;
+
+	/* We shouldn't get back-to-back CYCs unless they are sent when the
+	 * counter wraps around.  In this case, we'd expect a full payload.
+	 *
+	 * Since we got two non-full CYC packets, we assume the erratum hit.
+	 */
+
+	return 1;
+}
+
+int pt_qry_decode_cyc(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_cyc packet;
+	struct pt_config *config;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	config = &decoder->config;
+
+	size = pt_pkt_read_cyc(&packet, decoder->pos, config);
+	if (size < 0)
+		return size;
+
+	if (config->errata.skd007) {
+		errcode = check_erratum_skd007(decoder, &packet, size);
+		if (errcode < 0)
+			return errcode;
+
+		/* If the erratum hits, we ignore the partial CYC and instead
+		 * process the OVF following/overlapping it.
+		 */
+		if (errcode) {
+			/* We skip the first byte of the CYC, which brings us
+			 * to the beginning of the OVF packet.
+			 */
+			decoder->pos += 1;
+			return 0;
+		}
+	}
+
+	errcode = pt_qry_apply_cyc(&decoder->time, &decoder->tcal,
+				   &packet, config);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_decode_stop(struct pt_query_decoder *decoder)
+{
+	struct pt_event *event;
+	int errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	/* Stop events are reported immediately. */
+	event = pt_evq_standalone(&decoder->evq);
+	if (!event)
+		return -pte_internal;
+
+	event->type = ptev_stop;
+
+	decoder->event = event;
+
+	errcode = pt_qry_event_time(event, decoder);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += ptps_stop;
+	return 0;
+}
+
+int pt_qry_header_vmcs(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_vmcs packet;
+	struct pt_event *event;
+	int size;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_vmcs(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	event = pt_evq_enqueue(&decoder->evq, evb_psbend);
+	if (!event)
+		return -pte_nomem;
+
+	event->type = ptev_async_vmcs;
+	event->variant.async_vmcs.base = packet.base;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_decode_vmcs(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_vmcs packet;
+	struct pt_event *event;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_vmcs(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	/* VMCS events bind to the same IP as an in-flight async paging event.
+	 *
+	 * In that case, the VMCS event should be applied first.  We reorder
+	 * events here to simplify the life of higher layers.
+	 */
+	event = pt_evq_find(&decoder->evq, evb_tip, ptev_async_paging);
+	if (event) {
+		struct pt_event *paging;
+
+		paging = pt_evq_enqueue(&decoder->evq, evb_tip);
+		if (!paging)
+			return -pte_nomem;
+
+		*paging = *event;
+
+		event->type = ptev_async_vmcs;
+		event->variant.async_vmcs.base = packet.base;
+
+		decoder->pos += size;
+		return 0;
+	}
+
+	/* VMCS events bind to the same TIP packet as an in-flight async
+	 * branch event.
+	 */
+	event = pt_evq_find(&decoder->evq, evb_tip, ptev_async_branch);
+	if (event) {
+		event = pt_evq_enqueue(&decoder->evq, evb_tip);
+		if (!event)
+			return -pte_nomem;
+
+		event->type = ptev_async_vmcs;
+		event->variant.async_vmcs.base = packet.base;
+
+		decoder->pos += size;
+		return 0;
+	}
+
+	/* VMCS events that do not bind to an in-flight async event are
+	 * stand-alone.
+	 */
+	event = pt_evq_standalone(&decoder->evq);
+	if (!event)
+		return -pte_internal;
+
+	event->type = ptev_vmcs;
+	event->variant.vmcs.base = packet.base;
+
+	decoder->event = event;
+
+	errcode = pt_qry_event_time(event, decoder);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_decode_mnt(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_mnt packet;
+	struct pt_event *event;
+	int size, errcode;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_mnt(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	event = pt_evq_standalone(&decoder->evq);
+	if (!event)
+		return -pte_internal;
+
+	event->type = ptev_mnt;
+	event->variant.mnt.payload = packet.payload;
+
+	decoder->event = event;
+
+	errcode = pt_qry_event_time(event, decoder);
+	if (errcode < 0)
+		return errcode;
+
+	decoder->pos += size;
+
+	return 0;
+}
+
+int pt_qry_header_mnt(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_mnt packet;
+	struct pt_event *event;
+	int size;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_mnt(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	event = pt_evq_enqueue(&decoder->evq, evb_psbend);
+	if (!event)
+		return -pte_nomem;
+
+	event->type = ptev_mnt;
+	event->variant.mnt.payload = packet.payload;
+
+	decoder->pos += size;
+
+	return 0;
+}
+
+int pt_qry_decode_exstop(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_exstop packet;
+	struct pt_event *event;
+	int size;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_exstop(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	if (packet.ip) {
+		event = pt_evq_enqueue(&decoder->evq, evb_fup);
+		if (!event)
+			return -pte_internal;
+
+		event->type = ptev_exstop;
+	} else {
+		event = pt_evq_standalone(&decoder->evq);
+		if (!event)
+			return -pte_internal;
+
+		event->type = ptev_exstop;
+
+		event->ip_suppressed = 1;
+		event->variant.exstop.ip = 0ull;
+
+		decoder->event = event;
+	}
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_decode_mwait(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_mwait packet;
+	struct pt_event *event;
+	int size;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_mwait(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	event = pt_evq_enqueue(&decoder->evq, evb_fup);
+	if (!event)
+		return -pte_internal;
+
+	event->type = ptev_mwait;
+	event->variant.mwait.hints = packet.hints;
+	event->variant.mwait.ext = packet.ext;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_decode_pwre(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_pwre packet;
+	struct pt_event *event;
+	int size;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_pwre(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	event = pt_evq_standalone(&decoder->evq);
+	if (!event)
+		return -pte_internal;
+
+	event->type = ptev_pwre;
+	event->variant.pwre.state = packet.state;
+	event->variant.pwre.sub_state = packet.sub_state;
+
+	if (packet.hw)
+		event->variant.pwre.hw = 1;
+
+	decoder->event = event;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_decode_pwrx(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_pwrx packet;
+	struct pt_event *event;
+	int size;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_pwrx(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	event = pt_evq_standalone(&decoder->evq);
+	if (!event)
+		return -pte_internal;
+
+	event->type = ptev_pwrx;
+	event->variant.pwrx.last = packet.last;
+	event->variant.pwrx.deepest = packet.deepest;
+
+	if (packet.interrupt)
+		event->variant.pwrx.interrupt = 1;
+	if (packet.store)
+		event->variant.pwrx.store = 1;
+	if (packet.autonomous)
+		event->variant.pwrx.autonomous = 1;
+
+	decoder->event = event;
+
+	decoder->pos += size;
+	return 0;
+}
+
+int pt_qry_decode_ptw(struct pt_query_decoder *decoder)
+{
+	struct pt_packet_ptw packet;
+	struct pt_event *event;
+	int size, pls;
+
+	if (!decoder)
+		return -pte_internal;
+
+	size = pt_pkt_read_ptw(&packet, decoder->pos, &decoder->config);
+	if (size < 0)
+		return size;
+
+	pls = pt_ptw_size(packet.plc);
+	if (pls < 0)
+		return pls;
+
+	if (packet.ip) {
+		event = pt_evq_enqueue(&decoder->evq, evb_fup);
+		if (!event)
+			return -pte_internal;
+	} else {
+		event = pt_evq_standalone(&decoder->evq);
+		if (!event)
+			return -pte_internal;
+
+		event->ip_suppressed = 1;
+
+		decoder->event = event;
+	}
+
+	event->type = ptev_ptwrite;
+	event->variant.ptwrite.size = (uint8_t) pls;
+	event->variant.ptwrite.payload = packet.payload;
+
+	decoder->pos += size;
+	return 0;
+}
diff --git a/libipt/src/pt_retstack.c b/libipt/src/pt_retstack.c
new file mode 100644
index 000000000000..34733f25e869
--- /dev/null
+++ b/libipt/src/pt_retstack.c
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_retstack.h"
+
+#include "intel-pt.h"
+
+
+void pt_retstack_init(struct pt_retstack *retstack)
+{
+	if (!retstack)
+		return;
+
+	retstack->top = 0;
+	retstack->bottom = 0;
+}
+
+int pt_retstack_is_empty(const struct pt_retstack *retstack)
+{
+	if (!retstack)
+		return -pte_invalid;
+
+	return (retstack->top == retstack->bottom ? 1 : 0);
+}
+
+int pt_retstack_pop(struct pt_retstack *retstack, uint64_t *ip)
+{
+	uint8_t top;
+
+	if (!retstack)
+		return -pte_invalid;
+
+	top = retstack->top;
+
+	if (top == retstack->bottom)
+		return -pte_retstack_empty;
+
+	top = (!top ? pt_retstack_size : top - 1);
+
+	retstack->top = top;
+
+	if (ip)
+		*ip = retstack->stack[top];
+
+	return 0;
+}
+
+int pt_retstack_push(struct pt_retstack *retstack, uint64_t ip)
+{
+	uint8_t top, bottom;
+
+	if (!retstack)
+		return -pte_invalid;
+
+	top = retstack->top;
+	bottom = retstack->bottom;
+
+	retstack->stack[top] = ip;
+
+	top = (top == pt_retstack_size ? 0 : top + 1);
+
+	if (bottom == top)
+		bottom = (bottom == pt_retstack_size ? 0 : bottom + 1);
+
+	retstack->top = top;
+	retstack->bottom = bottom;
+
+	return 0;
+}
diff --git a/libipt/src/pt_section.c b/libipt/src/pt_section.c
new file mode 100644
index 000000000000..05f8ee4f7467
--- /dev/null
+++ b/libipt/src/pt_section.c
@@ -0,0 +1,654 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_section.h"
+#include "pt_block_cache.h"
+#include "pt_image_section_cache.h"
+
+#include "intel-pt.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+
+int pt_mk_section(struct pt_section **psection, const char *filename,
+		  uint64_t offset, uint64_t size)
+{
+	struct pt_section *section;
+	uint64_t fsize;
+	size_t flen;
+	void *status;
+	char *fname;
+	int errcode;
+
+	if (!psection)
+		return -pte_internal;
+
+	flen = strnlen(filename, FILENAME_MAX);
+	if (FILENAME_MAX <= flen)
+		return -pte_invalid;
+
+	flen += 1;
+
+	fname = malloc(flen);
+	if (!fname)
+		return -pte_nomem;
+
+	memcpy(fname, filename, flen);
+
+	errcode = pt_section_mk_status(&status, &fsize, fname);
+	if (errcode < 0)
+		goto out_fname;
+
+	/* Fail if the requested @offset lies beyond the end of @file. */
+	if (fsize <= offset) {
+		errcode = -pte_invalid;
+		goto out_status;
+	}
+
+	/* Truncate @size so the entire range lies within @file. */
+	fsize -= offset;
+	if (fsize < size)
+		size = fsize;
+
+	section = malloc(sizeof(*section));
+	if (!section) {
+		errcode = -pte_nomem;
+		goto out_status;
+	}
+
+	memset(section, 0, sizeof(*section));
+
+	section->filename = fname;
+	section->status = status;
+	section->offset = offset;
+	section->size = size;
+	section->ucount = 1;
+
+#if defined(FEATURE_THREADS)
+
+	errcode = mtx_init(&section->lock, mtx_plain);
+	if (errcode != thrd_success) {
+		free(section);
+
+		errcode = -pte_bad_lock;
+		goto out_status;
+	}
+
+	errcode = mtx_init(&section->alock, mtx_plain);
+	if (errcode != thrd_success) {
+		mtx_destroy(&section->lock);
+		free(section);
+
+		errcode = -pte_bad_lock;
+		goto out_status;
+	}
+
+#endif /* defined(FEATURE_THREADS) */
+
+	*psection = section;
+	return 0;
+
+out_status:
+	free(status);
+
+out_fname:
+	free(fname);
+	return errcode;
+}
+
+int pt_section_lock(struct pt_section *section)
+{
+	if (!section)
+		return -pte_internal;
+
+#if defined(FEATURE_THREADS)
+	{
+		int errcode;
+
+		errcode = mtx_lock(&section->lock);
+		if (errcode != thrd_success)
+			return -pte_bad_lock;
+	}
+#endif /* defined(FEATURE_THREADS) */
+
+	return 0;
+}
+
+int pt_section_unlock(struct pt_section *section)
+{
+	if (!section)
+		return -pte_internal;
+
+#if defined(FEATURE_THREADS)
+	{
+		int errcode;
+
+		errcode = mtx_unlock(&section->lock);
+		if (errcode != thrd_success)
+			return -pte_bad_lock;
+	}
+#endif /* defined(FEATURE_THREADS) */
+
+	return 0;
+}
+
+static void pt_section_free(struct pt_section *section)
+{
+	if (!section)
+		return;
+
+#if defined(FEATURE_THREADS)
+
+	mtx_destroy(&section->alock);
+	mtx_destroy(&section->lock);
+
+#endif /* defined(FEATURE_THREADS) */
+
+	free(section->filename);
+	free(section->status);
+	free(section);
+}
+
+int pt_section_get(struct pt_section *section)
+{
+	uint16_t ucount;
+	int errcode;
+
+	if (!section)
+		return -pte_internal;
+
+	errcode = pt_section_lock(section);
+	if (errcode < 0)
+		return errcode;
+
+	ucount = section->ucount + 1;
+	if (!ucount) {
+		(void) pt_section_unlock(section);
+		return -pte_overflow;
+	}
+
+	section->ucount = ucount;
+
+	return pt_section_unlock(section);
+}
+
+int pt_section_put(struct pt_section *section)
+{
+	uint16_t ucount, mcount;
+	int errcode;
+
+	if (!section)
+		return -pte_internal;
+
+	errcode = pt_section_lock(section);
+	if (errcode < 0)
+		return errcode;
+
+	mcount = section->mcount;
+	ucount = section->ucount;
+	if (ucount > 1) {
+		section->ucount = ucount - 1;
+		return pt_section_unlock(section);
+	}
+
+	errcode = pt_section_unlock(section);
+	if (errcode < 0)
+		return errcode;
+
+	if (!ucount || mcount)
+		return -pte_internal;
+
+	pt_section_free(section);
+	return 0;
+}
+
+static int pt_section_lock_attach(struct pt_section *section)
+{
+	if (!section)
+		return -pte_internal;
+
+#if defined(FEATURE_THREADS)
+	{
+		int errcode;
+
+		errcode = mtx_lock(&section->alock);
+		if (errcode != thrd_success)
+			return -pte_bad_lock;
+	}
+#endif /* defined(FEATURE_THREADS) */
+
+	return 0;
+}
+
+static int pt_section_unlock_attach(struct pt_section *section)
+{
+	if (!section)
+		return -pte_internal;
+
+#if defined(FEATURE_THREADS)
+	{
+		int errcode;
+
+		errcode = mtx_unlock(&section->alock);
+		if (errcode != thrd_success)
+			return -pte_bad_lock;
+	}
+#endif /* defined(FEATURE_THREADS) */
+
+	return 0;
+}
+
+int pt_section_attach(struct pt_section *section,
+		      struct pt_image_section_cache *iscache)
+{
+	uint16_t acount, ucount;
+	int errcode;
+
+	if (!section || !iscache)
+		return -pte_internal;
+
+	errcode = pt_section_lock_attach(section);
+	if (errcode < 0)
+		return errcode;
+
+	ucount = section->ucount;
+	acount = section->acount;
+	if (!acount) {
+		if (section->iscache || !ucount)
+			goto out_unlock;
+
+		section->iscache = iscache;
+		section->acount = 1;
+
+		return pt_section_unlock_attach(section);
+	}
+
+	acount += 1;
+	if (!acount) {
+		(void) pt_section_unlock_attach(section);
+		return -pte_overflow;
+	}
+
+	if (ucount < acount)
+		goto out_unlock;
+
+	if (section->iscache != iscache)
+		goto out_unlock;
+
+	section->acount = acount;
+
+	return pt_section_unlock_attach(section);
+
+ out_unlock:
+	(void) pt_section_unlock_attach(section);
+	return -pte_internal;
+}
+
+int pt_section_detach(struct pt_section *section,
+		      struct pt_image_section_cache *iscache)
+{
+	uint16_t acount, ucount;
+	int errcode;
+
+	if (!section || !iscache)
+		return -pte_internal;
+
+	errcode = pt_section_lock_attach(section);
+	if (errcode < 0)
+		return errcode;
+
+	if (section->iscache != iscache)
+		goto out_unlock;
+
+	acount = section->acount;
+	if (!acount)
+		goto out_unlock;
+
+	acount -= 1;
+	ucount = section->ucount;
+	if (ucount < acount)
+		goto out_unlock;
+
+	section->acount = acount;
+	if (!acount)
+		section->iscache = NULL;
+
+	return pt_section_unlock_attach(section);
+
+ out_unlock:
+	(void) pt_section_unlock_attach(section);
+	return -pte_internal;
+}
+
+const char *pt_section_filename(const struct pt_section *section)
+{
+	if (!section)
+		return NULL;
+
+	return section->filename;
+}
+
+uint64_t pt_section_size(const struct pt_section *section)
+{
+	if (!section)
+		return 0ull;
+
+	return section->size;
+}
+
+static int pt_section_bcache_memsize(const struct pt_section *section,
+				     uint64_t *psize)
+{
+	struct pt_block_cache *bcache;
+
+	if (!section || !psize)
+		return -pte_internal;
+
+	bcache = section->bcache;
+	if (!bcache) {
+		*psize = 0ull;
+		return 0;
+	}
+
+	*psize = sizeof(*bcache) +
+		(bcache->nentries * sizeof(struct pt_bcache_entry));
+
+	return 0;
+}
+
+static int pt_section_memsize_locked(const struct pt_section *section,
+				     uint64_t *psize)
+{
+	uint64_t msize, bcsize;
+	int (*memsize)(const struct pt_section *section, uint64_t *size);
+	int errcode;
+
+	if (!section || !psize)
+		return -pte_internal;
+
+	memsize = section->memsize;
+	if (!memsize) {
+		if (section->mcount)
+			return -pte_internal;
+
+		*psize = 0ull;
+		return 0;
+	}
+
+	errcode = memsize(section, &msize);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = pt_section_bcache_memsize(section, &bcsize);
+	if (errcode < 0)
+		return errcode;
+
+	*psize = msize + bcsize;
+
+	return 0;
+}
+
+int pt_section_memsize(struct pt_section *section, uint64_t *size)
+{
+	int errcode, status;
+
+	errcode = pt_section_lock(section);
+	if (errcode < 0)
+		return errcode;
+
+	status = pt_section_memsize_locked(section, size);
+
+	errcode = pt_section_unlock(section);
+	if (errcode < 0)
+		return errcode;
+
+	return status;
+}
+
+uint64_t pt_section_offset(const struct pt_section *section)
+{
+	if (!section)
+		return 0ull;
+
+	return section->offset;
+}
+
+int pt_section_alloc_bcache(struct pt_section *section)
+{
+	struct pt_image_section_cache *iscache;
+	struct pt_block_cache *bcache;
+	uint64_t ssize, memsize;
+	uint32_t csize;
+	int errcode;
+
+	if (!section)
+		return -pte_internal;
+
+	if (!section->mcount)
+		return -pte_internal;
+
+	ssize = pt_section_size(section);
+	csize = (uint32_t) ssize;
+
+	if (csize != ssize)
+		return -pte_not_supported;
+
+	memsize = 0ull;
+
+	/* We need to take both the attach and the section lock in order to pair
+	 * the block cache allocation and the resize notification.
+	 *
+	 * This allows map notifications in between but they only change the
+	 * order of sections in the cache.
+	 *
+	 * The attach lock needs to be taken first.
+	 */
+	errcode = pt_section_lock_attach(section);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = pt_section_lock(section);
+	if (errcode < 0)
+		goto out_alock;
+
+	bcache = pt_section_bcache(section);
+	if (bcache) {
+		errcode = 0;
+		goto out_lock;
+	}
+
+	bcache = pt_bcache_alloc(csize);
+	if (!bcache) {
+		errcode = -pte_nomem;
+		goto out_lock;
+	}
+
+	/* Install the block cache.  It will become visible and may be used
+	 * immediately.
+	 *
+	 * If we fail later on, we leave the block cache and report the error to
+	 * the allocating decoder thread.
+	 */
+	section->bcache = bcache;
+
+	errcode = pt_section_memsize_locked(section, &memsize);
+	if (errcode < 0)
+		goto out_lock;
+
+	errcode = pt_section_unlock(section);
+	if (errcode < 0)
+		goto out_alock;
+
+	if (memsize) {
+		iscache = section->iscache;
+		if (iscache) {
+			errcode = pt_iscache_notify_resize(iscache, section,
+							  memsize);
+			if (errcode < 0)
+				goto out_alock;
+		}
+	}
+
+	return pt_section_unlock_attach(section);
+
+
+out_lock:
+	(void) pt_section_unlock(section);
+
+out_alock:
+	(void) pt_section_unlock_attach(section);
+	return errcode;
+}
+
+int pt_section_on_map_lock(struct pt_section *section)
+{
+	struct pt_image_section_cache *iscache;
+	int errcode, status;
+
+	if (!section)
+		return -pte_internal;
+
+	errcode = pt_section_lock_attach(section);
+	if (errcode < 0)
+		return errcode;
+
+	iscache = section->iscache;
+	if (!iscache)
+		return pt_section_unlock_attach(section);
+
+	/* There is a potential deadlock when @section was unmapped again and
+	 * @iscache tries to map it.  This would cause this function to be
+	 * re-entered while we're still holding the attach lock.
+	 *
+	 * This scenario is very unlikely, though, since our caller does not yet
+	 * know whether pt_section_map() succeeded.
+	 */
+	status = pt_iscache_notify_map(iscache, section);
+
+	errcode = pt_section_unlock_attach(section);
+	if (errcode < 0)
+		return errcode;
+
+	return status;
+}
+
+int pt_section_map_share(struct pt_section *section)
+{
+	uint16_t mcount;
+	int errcode;
+
+	if (!section)
+		return -pte_internal;
+
+	errcode = pt_section_lock(section);
+	if (errcode < 0)
+		return errcode;
+
+	mcount = section->mcount;
+	if (!mcount) {
+		(void) pt_section_unlock(section);
+		return -pte_internal;
+	}
+
+	mcount += 1;
+	if (!mcount) {
+		(void) pt_section_unlock(section);
+		return -pte_overflow;
+	}
+
+	section->mcount = mcount;
+
+	return pt_section_unlock(section);
+}
+
+int pt_section_unmap(struct pt_section *section)
+{
+	uint16_t mcount;
+	int errcode, status;
+
+	if (!section)
+		return -pte_internal;
+
+	errcode = pt_section_lock(section);
+	if (errcode < 0)
+		return errcode;
+
+	mcount = section->mcount;
+
+	errcode = -pte_nomap;
+	if (!mcount)
+		goto out_unlock;
+
+	section->mcount = mcount -= 1;
+	if (mcount)
+		return pt_section_unlock(section);
+
+	errcode = -pte_internal;
+	if (!section->unmap)
+		goto out_unlock;
+
+	status = section->unmap(section);
+
+	pt_bcache_free(section->bcache);
+	section->bcache = NULL;
+
+	errcode = pt_section_unlock(section);
+	if (errcode < 0)
+		return errcode;
+
+	return status;
+
+out_unlock:
+	(void) pt_section_unlock(section);
+	return errcode;
+}
+
+int pt_section_read(const struct pt_section *section, uint8_t *buffer,
+		    uint16_t size, uint64_t offset)
+{
+	uint64_t limit, space;
+
+	if (!section)
+		return -pte_internal;
+
+	if (!section->read)
+		return -pte_nomap;
+
+	limit = section->size;
+	if (limit <= offset)
+		return -pte_nomap;
+
+	/* Truncate if we try to read past the end of the section. */
+	space = limit - offset;
+	if (space < size)
+		size = (uint16_t) space;
+
+	return section->read(section, buffer, size, offset);
+}
diff --git a/libipt/src/pt_section_file.c b/libipt/src/pt_section_file.c
new file mode 100644
index 000000000000..d8a898131aa0
--- /dev/null
+++ b/libipt/src/pt_section_file.c
@@ -0,0 +1,255 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_section.h"
+#include "pt_section_file.h"
+
+#include "intel-pt.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+
+static int fmap_init(struct pt_sec_file_mapping *mapping)
+{
+	if (!mapping)
+		return -pte_internal;
+
+	memset(mapping, 0, sizeof(*mapping));
+
+#if defined(FEATURE_THREADS)
+	{
+		int errcode;
+
+		errcode = mtx_init(&mapping->lock, mtx_plain);
+		if (errcode != thrd_success)
+			return -pte_bad_lock;
+	}
+#endif /* defined(FEATURE_THREADS) */
+
+	return 0;
+}
+
+static void fmap_fini(struct pt_sec_file_mapping *mapping)
+{
+	if (!mapping)
+		return;
+
+	fclose(mapping->file);
+
+#if defined(FEATURE_THREADS)
+
+	mtx_destroy(&mapping->lock);
+
+#endif /* defined(FEATURE_THREADS) */
+}
+
+static int fmap_lock(struct pt_sec_file_mapping *mapping)
+{
+	if (!mapping)
+		return -pte_internal;
+
+#if defined(FEATURE_THREADS)
+	{
+		int errcode;
+
+		errcode = mtx_lock(&mapping->lock);
+		if (errcode != thrd_success)
+			return -pte_bad_lock;
+	}
+#endif /* defined(FEATURE_THREADS) */
+
+	return 0;
+}
+
+static int fmap_unlock(struct pt_sec_file_mapping *mapping)
+{
+	if (!mapping)
+		return -pte_internal;
+
+#if defined(FEATURE_THREADS)
+	{
+		int errcode;
+
+		errcode = mtx_unlock(&mapping->lock);
+		if (errcode != thrd_success)
+			return -pte_bad_lock;
+	}
+#endif /* defined(FEATURE_THREADS) */
+
+	return 0;
+}
+
+int pt_sec_file_map(struct pt_section *section, FILE *file)
+{
+	struct pt_sec_file_mapping *mapping;
+	uint64_t offset, size;
+	long begin, end, fsize;
+	int errcode;
+
+	if (!section)
+		return -pte_internal;
+
+	mapping = section->mapping;
+	if (mapping)
+		return -pte_internal;
+
+	offset = section->offset;
+	size = section->size;
+
+	begin = (long) offset;
+	end = begin + (long) size;
+
+	/* Check for overflows. */
+	if ((uint64_t) begin != offset)
+		return -pte_bad_image;
+
+	if ((uint64_t) end != (offset + size))
+		return -pte_bad_image;
+
+	if (end < begin)
+		return -pte_bad_image;
+
+	/* Validate that the section lies within the file. */
+	errcode = fseek(file, 0, SEEK_END);
+	if (errcode)
+		return -pte_bad_image;
+
+	fsize = ftell(file);
+	if (fsize < 0)
+		return -pte_bad_image;
+
+	if (fsize < end)
+		return -pte_bad_image;
+
+	mapping = malloc(sizeof(*mapping));
+	if (!mapping)
+		return -pte_nomem;
+
+	errcode = fmap_init(mapping);
+	if (errcode < 0)
+		goto out_mem;
+
+	mapping->file = file;
+	mapping->begin = begin;
+	mapping->end = end;
+
+	section->mapping = mapping;
+	section->unmap = pt_sec_file_unmap;
+	section->read = pt_sec_file_read;
+	section->memsize = pt_sec_file_memsize;
+
+	return 0;
+
+out_mem:
+	free(mapping);
+	return errcode;
+}
+
+int pt_sec_file_unmap(struct pt_section *section)
+{
+	struct pt_sec_file_mapping *mapping;
+
+	if (!section)
+		return -pte_internal;
+
+	mapping = section->mapping;
+
+	if (!mapping || !section->unmap || !section->read || !section->memsize)
+		return -pte_internal;
+
+	section->mapping = NULL;
+	section->unmap = NULL;
+	section->read = NULL;
+	section->memsize = NULL;
+
+	fmap_fini(mapping);
+	free(mapping);
+
+	return 0;
+}
+
+int pt_sec_file_read(const struct pt_section *section, uint8_t *buffer,
+		     uint16_t size, uint64_t offset)
+{
+	struct pt_sec_file_mapping *mapping;
+	FILE *file;
+	long begin;
+	size_t read;
+	int errcode;
+
+	if (!buffer || !section)
+		return -pte_internal;
+
+	mapping = section->mapping;
+	if (!mapping)
+		return -pte_internal;
+
+	file = mapping->file;
+
+	/* We already checked in pt_section_read() that the requested memory
+	 * lies within the section's boundaries.
+	 *
+	 * And we checked that the file covers the entire section in
+	 * pt_sec_file_map().  There's no need to check for overflows, again.
+	 */
+	begin = mapping->begin + (long) offset;
+
+	errcode = fmap_lock(mapping);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = fseek(file, begin, SEEK_SET);
+	if (errcode)
+		goto out_unlock;
+
+	read = fread(buffer, 1, size, file);
+
+	errcode = fmap_unlock(mapping);
+	if (errcode < 0)
+		return errcode;
+
+	return (int) read;
+
+out_unlock:
+	(void) fmap_unlock(mapping);
+	return -pte_nomap;
+}
+
+int pt_sec_file_memsize(const struct pt_section *section, uint64_t *size)
+{
+	if (!section || !size)
+		return -pte_internal;
+
+	if (!section->mapping)
+		return -pte_internal;
+
+	*size = 0ull;
+
+	return 0;
+}
diff --git a/libipt/src/pt_sync.c b/libipt/src/pt_sync.c
new file mode 100644
index 000000000000..53d36e74d529
--- /dev/null
+++ b/libipt/src/pt_sync.c
@@ -0,0 +1,248 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_sync.h"
+#include "pt_packet.h"
+#include "pt_opcodes.h"
+
+#include "intel-pt.h"
+
+
+/* A psb packet contains a unique 2-byte repeating pattern.
+ *
+ * There are only two ways to fill up a 64bit work with such a pattern.
+ */
+static const uint64_t psb_pattern[] = {
+	((uint64_t) pt_psb_lohi		| (uint64_t) pt_psb_lohi << 16 |
+	 (uint64_t) pt_psb_lohi << 32	| (uint64_t) pt_psb_lohi << 48),
+	((uint64_t) pt_psb_hilo		| (uint64_t) pt_psb_hilo << 16 |
+	 (uint64_t) pt_psb_hilo << 32	| (uint64_t) pt_psb_hilo << 48)
+};
+
+static const uint8_t *truncate(const uint8_t *pointer, size_t alignment)
+{
+	uintptr_t raw = (uintptr_t) pointer;
+
+	raw /= alignment;
+	raw *= alignment;
+
+	return (const uint8_t *) raw;
+}
+
+static const uint8_t *align(const uint8_t *pointer, size_t alignment)
+{
+	return truncate(pointer + alignment - 1, alignment);
+}
+
+/* Find a psb packet given a position somewhere in the payload.
+ *
+ * Return the position of the psb packet.
+ * Return NULL, if this is not a psb packet.
+ */
+static const uint8_t *pt_find_psb(const uint8_t *pos,
+				  const struct pt_config *config)
+{
+	const uint8_t *begin, *end;
+	int errcode;
+
+	if (!pos || !config)
+		return NULL;
+
+	begin = config->begin;
+	end = config->end;
+
+	/* Navigate to the end of the psb payload pattern.
+	 *
+	 * Beware that PSB is an extended opcode. We must not confuse the extend
+	 * opcode of the following packet as belonging to the PSB.
+	 */
+	if (*pos != pt_psb_hi)
+		pos++;
+
+	for (; (pos + 1) < end; pos += 2) {
+		uint8_t hi, lo;
+
+		hi = pos[0];
+		lo = pos[1];
+
+		if (hi != pt_psb_hi)
+			break;
+
+		if (lo != pt_psb_lo)
+			break;
+	}
+	/*
+	 * We're right after the psb payload and within the buffer.
+	 * Navigate to the expected beginning of the psb packet.
+	 */
+	pos -= ptps_psb;
+
+	/* Check if we're still inside the buffer. */
+	if (pos < begin)
+		return NULL;
+
+	/* Check that this is indeed a psb packet we're at. */
+	if (pos[0] != pt_opc_psb || pos[1] != pt_ext_psb)
+		return NULL;
+
+	errcode = pt_pkt_read_psb(pos, config);
+	if (errcode < 0)
+		return NULL;
+
+	return pos;
+}
+
+static int pt_sync_within_bounds(const uint8_t *pos, const uint8_t *begin,
+				 const uint8_t *end)
+{
+	/* We allow @pos == @end representing the very end of the trace.
+	 *
+	 * This will result in -pte_eos when we actually try to read from @pos.
+	 */
+	return (begin <= pos) && (pos <= end);
+}
+
+int pt_sync_set(const uint8_t **sync, const uint8_t *pos,
+		const struct pt_config *config)
+{
+	const uint8_t *begin, *end;
+	int errcode;
+
+	if (!sync || !pos || !config)
+		return -pte_internal;
+
+	begin = config->begin;
+	end = config->end;
+
+	if (!pt_sync_within_bounds(pos, begin, end))
+		return -pte_eos;
+
+	if (end < pos + 2)
+		return -pte_eos;
+
+	/* Check that this is indeed a psb packet we're at. */
+	if (pos[0] != pt_opc_psb || pos[1] != pt_ext_psb)
+		return -pte_nosync;
+
+	errcode = pt_pkt_read_psb(pos, config);
+	if (errcode < 0)
+		return errcode;
+
+	*sync = pos;
+
+	return 0;
+}
+
+int pt_sync_forward(const uint8_t **sync, const uint8_t *pos,
+		    const struct pt_config *config)
+{
+	const uint8_t *begin, *end, *start;
+
+	if (!sync || !pos || !config)
+		return -pte_internal;
+
+	start = pos;
+	begin = config->begin;
+	end = config->end;
+
+	if (!pt_sync_within_bounds(pos, begin, end))
+		return -pte_internal;
+
+	/* We search for a full 64bit word. It's OK to skip the current one. */
+	pos = align(pos, sizeof(*psb_pattern));
+
+	/* Search for the psb payload pattern in the buffer. */
+	for (;;) {
+		const uint8_t *current = pos;
+		uint64_t val;
+
+		pos += sizeof(uint64_t);
+		if (end < pos)
+			return -pte_eos;
+
+		val = * (const uint64_t *) current;
+
+		if ((val != psb_pattern[0]) && (val != psb_pattern[1]))
+			continue;
+
+		/* We found a 64bit word's worth of psb payload pattern. */
+		current = pt_find_psb(pos, config);
+		if (!current)
+			continue;
+
+		/* If @start points inside a PSB, we may find that one.  Ignore
+		 * it unless @start points to its beginning.
+		 */
+		if (current < start)
+			continue;
+
+		*sync = current;
+		return 0;
+	}
+}
+
+int pt_sync_backward(const uint8_t **sync, const uint8_t *pos,
+		    const struct pt_config *config)
+{
+	const uint8_t *begin, *end;
+
+	if (!sync || !pos || !config)
+		return -pte_internal;
+
+	begin = config->begin;
+	end = config->end;
+
+	if (!pt_sync_within_bounds(pos, begin, end))
+		return -pte_internal;
+
+	/* We search for a full 64bit word. It's OK to skip the current one. */
+	pos = truncate(pos, sizeof(*psb_pattern));
+
+	/* Search for the psb payload pattern in the buffer. */
+	for (;;) {
+		const uint8_t *next = pos;
+		uint64_t val;
+
+		pos -= sizeof(uint64_t);
+		if (pos < begin)
+			return -pte_eos;
+
+		val = * (const uint64_t *) pos;
+
+		if ((val != psb_pattern[0]) && (val != psb_pattern[1]))
+			continue;
+
+		/* We found a 64bit word's worth of psb payload pattern. */
+		next = pt_find_psb(next, config);
+		if (!next)
+			continue;
+
+		*sync = next;
+		return 0;
+	}
+}
diff --git a/libipt/src/pt_time.c b/libipt/src/pt_time.c
new file mode 100644
index 000000000000..d1698147b3d1
--- /dev/null
+++ b/libipt/src/pt_time.c
@@ -0,0 +1,760 @@
+/*
+ * Copyright (c) 2014-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_time.h"
+#include "pt_opcodes.h"
+
+#include "intel-pt.h"
+
+#include <string.h>
+#include <limits.h>
+
+
+void pt_time_init(struct pt_time *time)
+{
+	if (!time)
+		return;
+
+	memset(time, 0, sizeof(*time));
+}
+
+int pt_time_query_tsc(uint64_t *tsc, uint32_t *lost_mtc,
+		      uint32_t *lost_cyc, const struct pt_time *time)
+{
+	if (!tsc || !time)
+		return -pte_internal;
+
+	*tsc = time->tsc;
+
+	if (lost_mtc)
+		*lost_mtc = time->lost_mtc;
+	if (lost_cyc)
+		*lost_cyc = time->lost_cyc;
+
+	if (!time->have_tsc)
+		return -pte_no_time;
+
+	return 0;
+}
+
+int pt_time_query_cbr(uint32_t *cbr, const struct pt_time *time)
+{
+	if (!cbr || !time)
+		return -pte_internal;
+
+	if (!time->have_cbr)
+		return -pte_no_cbr;
+
+	*cbr = time->cbr;
+
+	return 0;
+}
+
+/* Compute the distance between two CTC sources.
+ *
+ * We adjust a single wrap-around but fail if the distance is bigger than that.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_time_ctc_delta(uint32_t *ctc_delta, uint32_t ctc,
+			     uint32_t last_ctc, const struct pt_config *config)
+{
+	if (!config || !ctc_delta)
+		return -pte_internal;
+
+	/* Correct a single wrap-around.  If we lost enough MTCs to wrap
+	 * around twice, timing will be wrong until the next TSC.
+	 */
+	if (ctc < last_ctc) {
+		ctc += 1u << (config->mtc_freq + pt_pl_mtc_bit_size);
+
+		/* Since we only store the CTC between TMA/MTC or MTC/TMC a
+		 * single correction should suffice.
+		 */
+		if (ctc < last_ctc)
+			return -pte_bad_packet;
+	}
+
+	*ctc_delta = ctc - last_ctc;
+	return 0;
+}
+
+/* Translate CTC into the same unit as the FastCounter by multiplying with P.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_time_ctc_fc(uint64_t *fc, uint64_t ctc,
+			  const struct pt_config *config)
+{
+	uint32_t eax, ebx;
+
+	if (!fc || !config)
+		return -pte_internal;
+
+	eax = config->cpuid_0x15_eax;
+	ebx = config->cpuid_0x15_ebx;
+
+	/* Neither multiply nor divide by zero. */
+	if (!eax || !ebx)
+		return -pte_bad_config;
+
+	*fc = (ctc * ebx) / eax;
+	return 0;
+}
+
+int pt_time_update_tsc(struct pt_time *time,
+		       const struct pt_packet_tsc *packet,
+		       const struct pt_config *config)
+{
+	(void) config;
+
+	if (!time || !packet)
+		return -pte_internal;
+
+	time->have_tsc = 1;
+	time->have_tma = 0;
+	time->have_mtc = 0;
+	time->tsc = time->base = packet->tsc;
+	time->ctc = 0;
+	time->fc = 0ull;
+
+	/* We got the full time; we recover from previous losses. */
+	time->lost_mtc = 0;
+	time->lost_cyc = 0;
+
+	return 0;
+}
+
+int pt_time_update_cbr(struct pt_time *time,
+		       const struct pt_packet_cbr *packet,
+		       const struct pt_config *config)
+{
+	uint8_t cbr;
+
+	(void) config;
+
+	if (!time || !packet)
+		return -pte_internal;
+
+	cbr = packet->ratio;
+	if (!cbr)
+		return -pte_bad_packet;
+
+	time->have_cbr = 1;
+	time->cbr = cbr;
+
+	return 0;
+}
+
+int pt_time_update_tma(struct pt_time *time,
+		       const struct pt_packet_tma *packet,
+		       const struct pt_config *config)
+{
+	uint32_t ctc, mtc_freq, mtc_hi, ctc_mask;
+	uint64_t fc;
+
+	if (!time || !packet || !config)
+		return -pte_internal;
+
+	/* Without a TSC something is seriously wrong. */
+	if (!time->have_tsc)
+		return -pte_bad_context;
+
+	/* We shouldn't have more than one TMA per TSC. */
+	if (time->have_tma)
+		return -pte_bad_context;
+
+	/* We're ignoring MTC between TSC and TMA. */
+	if (time->have_mtc)
+		return -pte_internal;
+
+	ctc = packet->ctc;
+	fc = packet->fc;
+
+	mtc_freq = config->mtc_freq;
+	mtc_hi = mtc_freq + pt_pl_mtc_bit_size;
+
+	/* A mask for the relevant CTC bits ignoring high-order bits that are
+	 * not provided by MTC.
+	 */
+	ctc_mask = (1u << mtc_hi) - 1u;
+
+	time->have_tma = 1;
+	time->base -= fc;
+	time->fc += fc;
+
+	/* If the MTC frequency is low enough that TMA provides the full CTC
+	 * value, we can use the TMA as an MTC.
+	 *
+	 * If it isn't, we will estimate the preceding MTC based on the CTC bits
+	 * the TMA provides at the next MTC.  We forget about the previous MTC
+	 * in this case.
+	 *
+	 * If no MTC packets are dropped around TMA, we will estimate the
+	 * forgotten value again at the next MTC.
+	 *
+	 * If MTC packets are dropped, we can't really tell where in this
+	 * extended MTC period the TSC occurred.  The estimation will place it
+	 * right before the next MTC.
+	 */
+	if (mtc_hi <= pt_pl_tma_ctc_bit_size)
+		time->have_mtc = 1;
+
+	/* In both cases, we store the TMA's CTC bits until the next MTC. */
+	time->ctc = time->ctc_cyc = ctc & ctc_mask;
+
+	return 0;
+}
+
+int pt_time_update_mtc(struct pt_time *time,
+		       const struct pt_packet_mtc *packet,
+		       const struct pt_config *config)
+{
+	uint32_t last_ctc, ctc, ctc_delta;
+	uint64_t tsc, base;
+	uint8_t mtc_freq;
+	int errcode, have_tsc, have_tma, have_mtc;
+
+	if (!time || !packet || !config)
+		return -pte_internal;
+
+	have_tsc = time->have_tsc;
+	have_tma = time->have_tma;
+	have_mtc = time->have_mtc;
+
+	/* We ignore MTCs between TSC and TMA to avoid apparent CTC overflows.
+	 *
+	 * Later MTCs will ensure that no time is lost - provided TMA provides
+	 * enough bits.  If TMA doesn't provide any of the MTC bits we may place
+	 * the TSC into the wrong MTC period.
+	 */
+	if (have_tsc && !have_tma)
+		return 0;
+
+	base = time->base;
+	last_ctc = time->ctc;
+	mtc_freq = config->mtc_freq;
+
+	ctc = (uint32_t) packet->ctc << mtc_freq;
+
+	/* Store our CTC value if we have or would have reset FC. */
+	if (time->fc || time->lost_cyc || !have_mtc)
+		time->ctc_cyc = ctc;
+
+	/* Prepare for the next packet in case we error out below. */
+	time->have_mtc = 1;
+	time->fc = 0ull;
+	time->ctc = ctc;
+
+	/* We recover from previous CYC losses. */
+	time->lost_cyc = 0;
+
+	/* Avoid a big jump when we see the first MTC with an arbitrary CTC
+	 * payload.
+	 */
+	if (!have_mtc) {
+		uint32_t ctc_lo, ctc_hi;
+
+		/* If we have not seen a TMA, we ignore this first MTC.
+		 *
+		 * We have no idea where in this MTC period tracing started.
+		 * We could lose an entire MTC period or just a tiny fraction.
+		 *
+		 * On the other hand, if we assumed a previous MTC value, we
+		 * might make just the same error.
+		 */
+		if (!have_tma)
+			return 0;
+
+		/* The TMA's CTC value didn't provide enough bits - otherwise,
+		 * we would have treated the TMA as an MTC.
+		 */
+		if (last_ctc & ~(uint32_t) pt_pl_tma_ctc_mask)
+			return -pte_internal;
+
+		/* Split this MTC's CTC value into low and high parts with
+		 * respect to the bits provided by TMA.
+		 */
+		ctc_lo = ctc & (uint32_t) pt_pl_tma_ctc_mask;
+		ctc_hi = ctc & ~(uint32_t) pt_pl_tma_ctc_mask;
+
+		/* We estimate the high-order CTC bits that are not provided by
+		 * TMA based on the CTC bits provided by this MTC.
+		 *
+		 * We assume that no MTC packets were dropped around TMA.  If
+		 * there are, we might place the TSC into the wrong MTC period
+		 * depending on how many CTC bits TMA provides and how many MTC
+		 * packets were dropped.
+		 *
+		 * Note that we may underflow which results in more bits to be
+		 * set than MTC packets may provide.  Drop those extra bits.
+		 */
+		if (ctc_lo < last_ctc) {
+			ctc_hi -= 1u << pt_pl_tma_ctc_bit_size;
+			ctc_hi &= (uint32_t) pt_pl_mtc_mask << mtc_freq;
+		}
+
+		last_ctc |= ctc_hi;
+	}
+
+	errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
+	if (errcode < 0) {
+		time->lost_mtc += 1;
+		return errcode;
+	}
+
+	errcode = pt_time_ctc_fc(&tsc, ctc_delta, config);
+	if (errcode < 0)
+		return errcode;
+
+	base += tsc;
+	time->tsc = time->base = base;
+
+	return 0;
+}
+
+/* Adjust a CYC packet's payload spanning multiple MTC periods.
+ *
+ * CYC packets measure the Fast Counter since the last CYC(-eligible) packet.
+ * Depending on the CYC threshold, we may not get a CYC for each MTC, so a CYC
+ * period may overlap with or even span multiple MTC periods.
+ *
+ * We can't do much about the overlap case without examining all packets in
+ * the respective periods.  We leave this as expected imprecision.
+ *
+ * If we find a CYC packet to span multiple MTC packets, though, we try to
+ * approximate the portion for the current MTC period by subtracting the
+ * estimated portion for previous MTC periods using calibration information.
+ *
+ * We only consider MTC.  For the first CYC after TSC, the corresponding TMA
+ * will contain the Fast Counter at TSC.
+ *
+ * Returns zero on success, a negative error code otherwise.
+ */
+static int pt_time_adjust_cyc(uint64_t *cyc, const struct pt_time *time,
+			      const struct pt_config *config, uint64_t fcr)
+{
+	uint32_t last_ctc, ctc, ctc_delta;
+	uint64_t fc, total_cyc, old_cyc;
+	int errcode;
+
+	if (!time || !config || !fcr)
+		return -pte_internal;
+
+	last_ctc = time->ctc_cyc;
+	ctc = time->ctc;
+
+	/* There is nothing to do if this is the current MTC period. */
+	if (ctc == last_ctc)
+		return 0;
+
+	/* Calibration computes
+	 *
+	 *   fc  = (ctc_delta * cpuid[0x15].ebx) / cpuid[0x15].eax.
+	 *   fcr = (fc << pt_tcal_fcr_shr) / cyc
+	 *
+	 * So cyc = (fc << pt_tcal_fcr_shr) / fcr.
+	 */
+
+	errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = pt_time_ctc_fc(&fc, ctc_delta, config);
+	if (errcode < 0)
+		return errcode;
+
+	old_cyc = (fc << pt_tcal_fcr_shr) / fcr;
+	total_cyc = *cyc;
+
+	/* Make sure we don't wrap around.  If we would, attribute the entire
+	 * CYC payload to any previous MTC period.
+	 *
+	 * We lost an unknown portion of the CYC payload for the current MTC
+	 * period, but it's usually better to run too slow than too fast.
+	 */
+	if (total_cyc < old_cyc)
+		total_cyc = old_cyc;
+
+	*cyc = total_cyc - old_cyc;
+	return 0;
+}
+
+int pt_time_update_cyc(struct pt_time *time,
+		       const struct pt_packet_cyc *packet,
+		       const struct pt_config *config, uint64_t fcr)
+{
+	uint64_t cyc, fc;
+
+	if (!time || !packet || !config)
+		return -pte_internal;
+
+	if (!fcr) {
+		time->lost_cyc += 1;
+		return 0;
+	}
+
+	cyc = packet->value;
+	fc = time->fc;
+	if (!fc) {
+		int errcode;
+
+		errcode = pt_time_adjust_cyc(&cyc, time, config, fcr);
+		if (errcode < 0)
+			return errcode;
+	}
+
+	fc += (cyc * fcr) >> pt_tcal_fcr_shr;
+
+	time->fc = fc;
+	time->tsc = time->base + fc;
+
+	return 0;
+}
+
+void pt_tcal_init(struct pt_time_cal *tcal)
+{
+	if (!tcal)
+		return;
+
+	memset(tcal, 0, sizeof(*tcal));
+
+	tcal->min_fcr = UINT64_MAX;
+}
+
+static int pt_tcal_have_fcr(const struct pt_time_cal *tcal)
+{
+	if (!tcal)
+		return 0;
+
+	return (tcal->min_fcr <= tcal->max_fcr);
+}
+
+int pt_tcal_fcr(uint64_t *fcr, const struct pt_time_cal *tcal)
+{
+	if (!fcr || !tcal)
+		return -pte_internal;
+
+	if (!pt_tcal_have_fcr(tcal))
+		return -pte_no_time;
+
+	*fcr = tcal->fcr;
+
+	return 0;
+}
+
+int pt_tcal_set_fcr(struct pt_time_cal *tcal, uint64_t fcr)
+{
+	if (!tcal)
+		return -pte_internal;
+
+	tcal->fcr = fcr;
+
+	if (fcr < tcal->min_fcr)
+		tcal->min_fcr = fcr;
+
+	if (fcr > tcal->max_fcr)
+		tcal->max_fcr = fcr;
+
+	return 0;
+}
+
+int pt_tcal_update_psb(struct pt_time_cal *tcal,
+		       const struct pt_config *config)
+{
+	if (!tcal || !config)
+		return -pte_internal;
+
+	if (config->errata.skl168)
+		tcal->check_skl168 = 1;
+
+	return 0;
+}
+
+int pt_tcal_update_tsc(struct pt_time_cal *tcal,
+		      const struct pt_packet_tsc *packet,
+		      const struct pt_config *config)
+{
+	(void) config;
+
+	if (!tcal || !packet)
+		return -pte_internal;
+
+	/* A TSC outside of PSB+ may indicate loss of time.  We do not use it
+	 * for calibration.  We store the TSC value for calibration at the next
+	 * TSC in PSB+, though.
+	 */
+	tcal->tsc = packet->tsc;
+	tcal->cyc_tsc = 0ull;
+
+	return 0;
+}
+
+int pt_tcal_header_tsc(struct pt_time_cal *tcal,
+		      const struct pt_packet_tsc *packet,
+		      const struct pt_config *config)
+{
+	uint64_t tsc, last_tsc, tsc_delta, cyc, fcr;
+
+	(void) config;
+
+	if (!tcal || !packet)
+		return -pte_internal;
+
+	last_tsc = tcal->tsc;
+	cyc = tcal->cyc_tsc;
+
+	tsc = packet->tsc;
+
+	tcal->tsc = tsc;
+	tcal->cyc_tsc = 0ull;
+
+	if (!last_tsc || !cyc)
+		return 0;
+
+	/* Prefer MTC over TSC for calibration. */
+	if (tcal->have_mtc)
+		return 0;
+
+	/* Correct a single wrap-around. */
+	if (tsc < last_tsc) {
+		tsc += 1ull << pt_pl_tsc_bit_size;
+
+		if (tsc < last_tsc)
+			return -pte_bad_packet;
+	}
+
+	tsc_delta = tsc - last_tsc;
+
+	/* We shift the nominator to improve rounding precision.
+	 *
+	 * Since we're only collecting the CYCs between two TSC, we shouldn't
+	 * overflow.  Let's rather fail than overflow.
+	 */
+	if (tsc_delta & ~(~0ull >> pt_tcal_fcr_shr))
+		return -pte_internal;
+
+	fcr = (tsc_delta << pt_tcal_fcr_shr) / cyc;
+
+	return pt_tcal_set_fcr(tcal, fcr);
+}
+
+int pt_tcal_update_cbr(struct pt_time_cal *tcal,
+		      const struct pt_packet_cbr *packet,
+		      const struct pt_config *config)
+{
+	/* A CBR outside of PSB+ indicates a frequency change.  Reset our
+	 * calibration state.
+	 */
+	pt_tcal_init(tcal);
+
+	return pt_tcal_header_cbr(tcal, packet, config);
+}
+
+int pt_tcal_header_cbr(struct pt_time_cal *tcal,
+		      const struct pt_packet_cbr *packet,
+		      const struct pt_config *config)
+{
+	uint64_t cbr, p1, fcr;
+
+	if (!tcal || !packet || !config)
+		return -pte_internal;
+
+	p1 = config->nom_freq;
+	if (!p1)
+		return 0;
+
+	/* If we know the nominal frequency, we can use it for calibration. */
+	cbr = packet->ratio;
+	if (!cbr)
+		return -pte_bad_packet;
+
+	fcr = (p1 << pt_tcal_fcr_shr) / cbr;
+
+	return pt_tcal_set_fcr(tcal, fcr);
+}
+
+int pt_tcal_update_tma(struct pt_time_cal *tcal,
+		      const struct pt_packet_tma *packet,
+		      const struct pt_config *config)
+{
+	(void) tcal;
+	(void) packet;
+	(void) config;
+
+	/* Nothing to do. */
+	return 0;
+}
+
+int pt_tcal_update_mtc(struct pt_time_cal *tcal,
+		      const struct pt_packet_mtc *packet,
+		      const struct pt_config *config)
+{
+	uint32_t last_ctc, ctc, ctc_delta, have_mtc, check_skl168;
+	uint64_t cyc, fc, fcr;
+	int errcode;
+
+	if (!tcal || !packet || !config)
+		return -pte_internal;
+
+	last_ctc = tcal->ctc;
+	have_mtc = tcal->have_mtc;
+	cyc = tcal->cyc_mtc;
+	check_skl168 = tcal->check_skl168;
+
+	/* This only affects the first MTC after PSB. */
+	tcal->check_skl168 = 0;
+
+	ctc = (uint32_t) packet->ctc << config->mtc_freq;
+
+	/* We need at least two MTC (including this). */
+	if (!have_mtc) {
+		tcal->cyc_mtc = 0ull;
+		tcal->ctc = ctc;
+		tcal->have_mtc = 1;
+
+		return 0;
+	}
+
+	/* Without any cycles, we can't calibrate.  Try again at the next
+	 * MTC and distribute the cycles over the combined MTC period.
+	 */
+	if (!cyc)
+		return 0;
+
+	/* Prepare for the next packet in case we error out below. */
+	tcal->have_mtc = 1;
+	tcal->cyc_mtc = 0ull;
+	tcal->ctc = ctc;
+
+	/* Let's pretend we will fail.  We'll correct it at the end. */
+	tcal->lost_mtc += 1;
+
+	errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);
+	if (errcode < 0)
+		return errcode;
+
+	errcode = pt_time_ctc_fc(&fc, ctc_delta, config);
+	if (errcode < 0)
+		return errcode;
+
+	/* We shift the nominator to improve rounding precision.
+	 *
+	 * Since we're only collecting the CYCs between two MTC, we shouldn't
+	 * overflow.  Let's rather fail than overflow.
+	 */
+	if (fc & ~(~0ull >> pt_tcal_fcr_shr))
+		return -pte_internal;
+
+	fcr = (fc << pt_tcal_fcr_shr) / cyc;
+
+	/* SKL168: Intel(R) PT CYC Packets Can be Dropped When Immediately
+	 * Preceding PSB.
+	 *
+	 * We skip this MTC if we lost one or more MTC since the last PSB or if
+	 * it looks like we lost a wrap CYC packet.
+	 *
+	 * This is not an error but we count that MTC as lost.
+	 */
+	if (check_skl168) {
+		/* If we lost one or more MTC, the case is clear. */
+		if ((1u << config->mtc_freq) < ctc_delta)
+			return 0;
+
+		/* The case is less clear for a lost wrap CYC packet since we do
+		 * have some variation in the number of cycles.
+		 *
+		 * The CYC counter wraps on the affected processors every 4096
+		 * cycles.  For low MTC frequencies (high values), losing one
+		 * may not be noticeable.
+		 *
+		 * We restrict the workaround to higher MTC frequencies (lower
+		 * values).
+		 *
+		 * We also need a previous FCR so we know how many cycles to
+		 * expect.
+		 */
+		if ((config->mtc_freq < 10) && pt_tcal_have_fcr(tcal)) {
+			uint64_t dfc;
+
+			/* We choose a slightly lower adjustment to account for
+			 * some normal variation.
+			 */
+			dfc = (tcal->fcr * (cyc + 0xf00)) >> pt_tcal_fcr_shr;
+
+			/* If we didn't drop a wrap CYC, @dfc should be way
+			 * bigger than @fc.  If it isn't, we assume that the
+			 * erratum applied.
+			 */
+			if (dfc < fc)
+				return 0;
+		}
+	}
+
+	errcode = pt_tcal_set_fcr(tcal, fcr);
+	if (errcode < 0)
+		return errcode;
+
+	/* We updated the FCR.  This recovers from previous MTC losses. */
+	tcal->lost_mtc = 0;
+
+	return 0;
+}
+
+int pt_tcal_update_cyc(struct pt_time_cal *tcal,
+		      const struct pt_packet_cyc *packet,
+		      const struct pt_config *config)
+{
+	uint64_t cyc;
+
+	(void) config;
+
+	if (!tcal || !packet)
+		return -pte_internal;
+
+	cyc = packet->value;
+	tcal->cyc_mtc += cyc;
+	tcal->cyc_tsc += cyc;
+
+	return 0;
+}
+
+int pt_tcal_update_ovf(struct pt_time_cal *tcal,
+		       const struct pt_config *config)
+{
+	if (!tcal || !config)
+		return -pte_internal;
+
+	tcal->tsc = 0ull;
+	tcal->cyc_tsc = 0ull;
+	tcal->cyc_mtc = 0ull;
+	tcal->ctc = 0;
+	tcal->have_mtc = 0;
+
+	return 0;
+}
diff --git a/libipt/src/pt_tnt_cache.c b/libipt/src/pt_tnt_cache.c
new file mode 100644
index 000000000000..74a0988c7b20
--- /dev/null
+++ b/libipt/src/pt_tnt_cache.c
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_tnt_cache.h"
+
+#include "intel-pt.h"
+
+
+void pt_tnt_cache_init(struct pt_tnt_cache *cache)
+{
+	if (!cache)
+		return;
+
+	cache->tnt = 0ull;
+	cache->index = 0ull;
+}
+
+int pt_tnt_cache_is_empty(const struct pt_tnt_cache *cache)
+{
+	if (!cache)
+		return -pte_invalid;
+
+	return cache->index == 0;
+}
+
+int pt_tnt_cache_query(struct pt_tnt_cache *cache)
+{
+	int taken;
+
+	if (!cache)
+		return -pte_invalid;
+
+	if (!cache->index)
+		return -pte_bad_query;
+
+	taken = (cache->tnt & cache->index) != 0;
+	cache->index >>= 1;
+
+	return taken;
+}
+
+int pt_tnt_cache_update_tnt(struct pt_tnt_cache *cache,
+			    const struct pt_packet_tnt *packet,
+			    const struct pt_config *config)
+{
+	uint8_t bit_size;
+
+	(void) config;
+
+	if (!cache || !packet)
+		return -pte_invalid;
+
+	if (cache->index)
+		return -pte_bad_context;
+
+	bit_size = packet->bit_size;
+	if (!bit_size)
+		return -pte_bad_packet;
+
+	cache->tnt = packet->payload;
+	cache->index = 1ull << (bit_size - 1);
+
+	return 0;
+}
diff --git a/libipt/src/pt_version.c b/libipt/src/pt_version.c
new file mode 100644
index 000000000000..203d3d1f0941
--- /dev/null
+++ b/libipt/src/pt_version.c
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "intel-pt.h"
+
+
+struct pt_version pt_library_version(void)
+{
+	struct pt_version v = {
+		/* .major = */ PT_VERSION_MAJOR,
+		/* .minor = */ PT_VERSION_MINOR,
+		/* .patch = */ PT_VERSION_PATCH,
+		/* .build = */ PT_VERSION_BUILD,
+		/* .ext = */ PT_VERSION_EXT
+	};
+
+	return v;
+}
diff --git a/libipt/src/windows/pt_cpuid.c b/libipt/src/windows/pt_cpuid.c
new file mode 100644
index 000000000000..4a2460abfd0e
--- /dev/null
+++ b/libipt/src/windows/pt_cpuid.c
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2013-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_cpuid.h"
+
+#include <intrin.h>
+
+extern void pt_cpuid(uint32_t leaf, uint32_t *eax, uint32_t *ebx,
+		     uint32_t *ecx, uint32_t *edx)
+{
+	int cpu_info[4];
+
+	__cpuid(cpu_info, leaf);
+	*eax = cpu_info[0];
+	*ebx = cpu_info[1];
+	*ecx = cpu_info[2];
+	*edx = cpu_info[3];
+}
diff --git a/libipt/src/windows/pt_section_windows.c b/libipt/src/windows/pt_section_windows.c
new file mode 100644
index 000000000000..b6229a06c9fa
--- /dev/null
+++ b/libipt/src/windows/pt_section_windows.c
@@ -0,0 +1,396 @@
+/*
+ * Copyright (c) 2015-2019, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pt_section.h"
+#include "pt_section_windows.h"
+#include "pt_section_file.h"
+
+#include "intel-pt.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <io.h>
+
+
+static int pt_sec_windows_fstat(const char *filename, struct _stat *stat)
+{
+	int fd, errcode;
+
+	if (!filename || !stat)
+		return -pte_internal;
+
+	fd = _open(filename, _O_RDONLY);
+	if (fd == -1)
+		return -pte_bad_file;
+
+	errcode = _fstat(fd, stat);
+
+	_close(fd);
+
+	if (errcode)
+		return -pte_bad_file;
+
+	return 0;
+}
+
+int pt_section_mk_status(void **pstatus, uint64_t *psize, const char *filename)
+{
+	struct pt_sec_windows_status *status;
+	struct _stat stat;
+	int errcode;
+
+	if (!pstatus || !psize)
+		return -pte_internal;
+
+	errcode = pt_sec_windows_fstat(filename, &stat);
+	if (errcode < 0)
+		return errcode;
+
+	if (stat.st_size < 0)
+		return -pte_bad_image;
+
+	status = malloc(sizeof(*status));
+	if (!status)
+		return -pte_nomem;
+
+	status->stat = stat;
+
+	*pstatus = status;
+	*psize = stat.st_size;
+
+	return 0;
+}
+
+static int check_file_status(struct pt_section *section, int fd)
+{
+	struct pt_sec_windows_status *status;
+	struct _stat stat;
+	int errcode;
+
+	if (!section)
+		return -pte_internal;
+
+	errcode = _fstat(fd, &stat);
+	if (errcode)
+		return -pte_bad_file;
+
+	status = section->status;
+	if (!status)
+		return -pte_internal;
+
+	if (stat.st_size != status->stat.st_size)
+		return -pte_bad_image;
+
+	if (stat.st_mtime != status->stat.st_mtime)
+		return -pte_bad_image;
+
+	return 0;
+}
+
+static DWORD granularity(void)
+{
+	struct _SYSTEM_INFO sysinfo;
+
+	GetSystemInfo(&sysinfo);
+
+	return sysinfo.dwAllocationGranularity;
+}
+
+int pt_sec_windows_map(struct pt_section *section, int fd)
+{
+	struct pt_sec_windows_mapping *mapping;
+	uint64_t offset, size, adjustment;
+	HANDLE fh, mh;
+	DWORD dsize;
+	uint8_t *base;
+	int errcode;
+
+	if (!section)
+		return -pte_internal;
+
+	offset = section->offset;
+	size = section->size;
+
+	adjustment = offset % granularity();
+
+	offset -= adjustment;
+	size += adjustment;
+
+	/* The section is supposed to fit into the file so we shouldn't
+	 * see any overflows, here.
+	 */
+	if (size < section->size)
+		return -pte_internal;
+
+	dsize = (DWORD) size;
+	if ((uint64_t) dsize != size)
+		return -pte_internal;
+
+	fh = (HANDLE) _get_osfhandle(fd);
+
+	mh = CreateFileMapping(fh, NULL, PAGE_READONLY, 0, 0, NULL);
+	if (!mh)
+		return -pte_bad_image;
+
+	base = MapViewOfFile(mh, FILE_MAP_READ, (DWORD) (offset >> 32),
+			     (DWORD) (uint32_t) offset, dsize);
+	if (!base) {
+		errcode = -pte_bad_image;
+		goto out_mh;
+	}
+
+	mapping = malloc(sizeof(*mapping));
+	if (!mapping) {
+		errcode = -pte_nomem;
+		goto out_map;
+	}
+
+	mapping->fd = fd;
+	mapping->mh = mh;
+	mapping->base = base;
+	mapping->begin = base + adjustment;
+	mapping->end = base + size;
+
+	section->mapping = mapping;
+	section->unmap = pt_sec_windows_unmap;
+	section->read = pt_sec_windows_read;
+	section->memsize = pt_sec_windows_memsize;
+
+	return 0;
+
+out_map:
+	UnmapViewOfFile(base);
+
+out_mh:
+	CloseHandle(mh);
+	return errcode;
+}
+
+static int pt_sec_windows_map_success(struct pt_section *section)
+{
+	uint16_t mcount;
+	int errcode, status;
+
+	if (!section)
+		return -pte_internal;
+
+	mcount = section->mcount + 1;
+	if (!mcount) {
+		(void) pt_section_unlock(section);
+		return -pte_overflow;
+	}
+
+	section->mcount = mcount;
+
+	errcode = pt_section_unlock(section);
+	if (errcode < 0)
+		return errcode;
+
+	status = pt_section_on_map(section);
+	if (status < 0) {
+		/* We had to release the section lock for pt_section_on_map() so
+		 * @section may have meanwhile been mapped by other threads.
+		 *
+		 * We still want to return the error so we release our mapping.
+		 * Our caller does not yet know whether pt_section_map()
+		 * succeeded.
+		 */
+		(void) pt_section_unmap(section);
+		return status;
+	}
+
+	return 0;
+}
+
+int pt_section_map(struct pt_section *section)
+{
+	const char *filename;
+	HANDLE fh;
+	FILE *file;
+	int fd, errcode;
+
+	if (!section)
+		return -pte_internal;
+
+	errcode = pt_section_lock(section);
+	if (errcode < 0)
+		return errcode;
+
+	if (section->mcount)
+		return pt_sec_windows_map_success(section);
+
+	if (section->mapping) {
+		errcode = -pte_internal;
+		goto out_unlock;
+	}
+
+	filename = section->filename;
+	if (!filename) {
+		errcode = -pte_internal;
+		goto out_unlock;
+	}
+
+	fh = CreateFileA(filename, GENERIC_READ, FILE_SHARE_READ, NULL,
+			 OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+	if (fh == INVALID_HANDLE_VALUE) {
+		/* We failed to open the file read-only.  Let's try to open it
+		 * read-write; maybe our user has the file open for writing.
+		 *
+		 * We will detect changes to the file via fstat().
+		 */
+
+		fh = CreateFileA(filename, GENERIC_READ, FILE_SHARE_WRITE,
+				 NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
+				 NULL);
+		if (fh == INVALID_HANDLE_VALUE) {
+			errcode = -pte_bad_file;
+			goto out_unlock;
+		}
+	}
+
+	fd = _open_osfhandle((intptr_t) fh, _O_RDONLY);
+	if (fd == -1) {
+		errcode = -pte_bad_file;
+		goto out_fh;
+	}
+
+	errcode = check_file_status(section, fd);
+	if (errcode < 0)
+		goto out_fd;
+
+	/* We leave the file open on success.  It will be closed when the
+	 * section is unmapped.
+	 */
+	errcode = pt_sec_windows_map(section, fd);
+	if (!errcode)
+		return pt_sec_windows_map_success(section);
+
+	/* Fall back to file based sections - report the original error
+	 * if we fail to convert the file descriptor.
+	 */
+	file = _fdopen(fd, "rb");
+	if (!file) {
+		errcode = -pte_bad_file;
+		goto out_fd;
+	}
+
+	/* We need to keep the file open on success.  It will be closed when
+	 * the section is unmapped.
+	 */
+	errcode = pt_sec_file_map(section, file);
+	if (!errcode)
+		return pt_sec_windows_map_success(section);
+
+	fclose(file);
+	goto out_unlock;
+
+out_fd:
+	_close(fd);
+	return errcode;
+
+out_fh:
+	CloseHandle(fh);
+
+out_unlock:
+	(void) pt_section_unlock(section);
+	return errcode;
+}
+
+int pt_sec_windows_unmap(struct pt_section *section)
+{
+	struct pt_sec_windows_mapping *mapping;
+
+	if (!section)
+		return -pte_internal;
+
+	mapping = section->mapping;
+	if (!mapping || !section->unmap || !section->read || !section->memsize)
+		return -pte_internal;
+
+	section->mapping = NULL;
+	section->unmap = NULL;
+	section->read = NULL;
+	section->memsize = NULL;
+
+	UnmapViewOfFile(mapping->begin);
+	CloseHandle(mapping->mh);
+	_close(mapping->fd);
+	free(mapping);
+
+	return 0;
+}
+
+int pt_sec_windows_read(const struct pt_section *section, uint8_t *buffer,
+		      uint16_t size, uint64_t offset)
+{
+	struct pt_sec_windows_mapping *mapping;
+	const uint8_t *begin;
+
+	if (!buffer || !section)
+		return -pte_internal;
+
+	mapping = section->mapping;
+	if (!mapping)
+		return -pte_internal;
+
+	/* We already checked in pt_section_read() that the requested memory
+	 * lies within the section's boundaries.
+	 *
+	 * And we checked that the entire section was mapped.  There's no need
+	 * to check for overflows, again.
+	 */
+	begin = mapping->begin + offset;
+
+	memcpy(buffer, begin, size);
+	return (int) size;
+}
+
+
+int pt_sec_windows_memsize(const struct pt_section *section, uint64_t *size)
+{
+	struct pt_sec_windows_mapping *mapping;
+	const uint8_t *begin, *end;
+
+	if (!section || !size)
+		return -pte_internal;
+
+	mapping = section->mapping;
+	if (!mapping)
+		return -pte_internal;
+
+	begin = mapping->base;
+	end =  mapping->end;
+
+	if (!begin || !end || end < begin)
+		return -pte_internal;
+
+	*size = (uint64_t) (end - begin);
+
+	return 0;
+}