6 files changed, 760 insertions, 643 deletions

diff --git a/sys/amd64/amd64/bpf_jit_machdep.c b/sys/amd64/amd64/bpf_jit_machdep.c
index b8c51909b223..2ccb6ac507c8 100644
--- a/sys/amd64/amd64/bpf_jit_machdep.c
+++ b/sys/amd64/amd64/bpf_jit_machdep.c
@@ -1,6 +1,6 @@
 /*-
- * Copyright (c) 2002 - 2003 NetGroup, Politecnico di Torino (Italy)
- * Copyright (c) 2005 Jung-uk Kim <jkim@FreeBSD.org>
+ * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy)
+ * Copyright (C) 2005-2008 Jung-uk Kim <jkim@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -32,16 +32,20 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#ifdef _KERNEL
 #include "opt_bpf.h"
-
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
-#include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/malloc.h>
-
 #include <net/if.h>
+#else
+#include <stdlib.h>
+#endif
+
+#include <sys/types.h>
+
 #include <net/bpf.h>
 #include <net/bpf_jitter.h>
 
@@ -53,7 +57,7 @@ bpf_filter_func	bpf_jit_compile(struct bpf_insn *, u_int, int *);
  * emit routine to update the jump table
  */
 static void
-emit_length(bpf_bin_stream *stream, u_int value, u_int len)
+emit_length(bpf_bin_stream *stream, __unused u_int value, u_int len)
 {
 
 	(stream->refs)[stream->bpf_pc] += len;
@@ -105,13 +109,17 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 
 	/* Do not compile an empty filter. */
 	if (nins == 0)
-		return NULL;
+		return (NULL);
 
 	/* Allocate the reference table for the jumps */
+#ifdef _KERNEL
 	stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int),
 	    M_BPFJIT, M_NOWAIT);
+#else
+	stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int));
+#endif
 	if (stream.refs == NULL)
-		return NULL;
+		return (NULL);
 
 	/* Reset the reference table */
 	for (i = 0; i < nins + 1; i++)
@@ -131,145 +139,167 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 		ins = prog;
 
 		/* create the procedure header */
-		PUSH(RBP);
-		MOVrq(RBP, RSP);
-		MOVoqd(RBP, -8, ESI);
-		MOVoqd(RBP, -12, EDX);
-		PUSH(RBX);
-		MOVrq(RBX, RDI);
+		MOVrq2(RBX, R8);
+		MOVrq(RDI, RBX);
+		MOVrd2(ESI, R9D);
+		MOVrd(EDX, EDI);
 
 		for (i = 0; i < nins; i++) {
 			stream.bpf_pc++;
 
 			switch (ins->code) {
 			default:
-				return NULL;
+#ifdef _KERNEL
+				return (NULL);
+#else
+				abort();
+#endif
 
 			case BPF_RET|BPF_K:
-				MOVid(EAX, ins->k);
-				POP(RBX);
-				LEAVE_RET();
+				MOVid(ins->k, EAX);
+				MOVrq3(R8, RBX);
+				RET();
 				break;
 
 			case BPF_RET|BPF_A:
-				POP(RBX);
-				LEAVE_RET();
+				MOVrq3(R8, RBX);
+				RET();
 				break;
 
 			case BPF_LD|BPF_W|BPF_ABS:
-				MOVid(ECX, ins->k);
-				MOVrd(ESI, ECX);
-				ADDib(ECX, sizeof(int));
-				CMPodd(ECX, RBP, -12);
-				JLEb(5);
-				ZERO_EAX();
-				POP(RBX);
-				LEAVE_RET();
-				MOVobd(EAX, RBX, RSI);
+				MOVid(ins->k, ESI);
+				CMPrd(EDI, ESI);
+				JAb(12);
+				MOVrd(EDI, ECX);
+				SUBrd(ESI, ECX);
+				CMPid(sizeof(int32_t), ECX);
+				JAEb(6);
+				ZEROrd(EAX);
+				MOVrq3(R8, RBX);
+				RET();
+				MOVobd(RBX, RSI, EAX);
 				BSWAP(EAX);
 				break;
 
 			case BPF_LD|BPF_H|BPF_ABS:
-				ZERO_EAX();
-				MOVid(ECX, ins->k);
-				MOVrd(ESI, ECX);
-				ADDib(ECX, sizeof(short));
-				CMPodd(ECX, RBP, -12);
-				JLEb(3);
-				POP(RBX);
-				LEAVE_RET();
-				MOVobw(AX, RBX, RSI);
+				ZEROrd(EAX);
+				MOVid(ins->k, ESI);
+				CMPrd(EDI, ESI);
+				JAb(12);
+				MOVrd(EDI, ECX);
+				SUBrd(ESI, ECX);
+				CMPid(sizeof(int16_t), ECX);
+				JAEb(4);
+				MOVrq3(R8, RBX);
+				RET();
+				MOVobw(RBX, RSI, AX);
 				SWAP_AX();
 				break;
 
 			case BPF_LD|BPF_B|BPF_ABS:
-				ZERO_EAX();
-				MOVid(ECX, ins->k);
-				CMPodd(ECX, RBP, -12);
-				JLEb(3);
-				POP(RBX);
-				LEAVE_RET();
-				MOVobb(AL, RBX, RCX);
+				ZEROrd(EAX);
+				MOVid(ins->k, ESI);
+				CMPrd(EDI, ESI);
+				JBb(4);
+				MOVrq3(R8, RBX);
+				RET();
+				MOVobb(RBX, RSI, AL);
 				break;
 
 			case BPF_LD|BPF_W|BPF_LEN:
-				MOVodd(EAX, RBP, -8);
+				MOVrd3(R9D, EAX);
 				break;
 
 			case BPF_LDX|BPF_W|BPF_LEN:
-				MOVodd(EDX, RBP, -8);
+				MOVrd3(R9D, EDX);
 				break;
 
 			case BPF_LD|BPF_W|BPF_IND:
-				MOVid(ECX, ins->k);
-				ADDrd(ECX, EDX);
-				MOVrd(ESI, ECX);
-				ADDib(ECX, sizeof(int));
-				CMPodd(ECX, RBP, -12);
-				JLEb(5);
-				ZERO_EAX();
-				POP(RBX);
-				LEAVE_RET();
-				MOVobd(EAX, RBX, RSI);
+				CMPrd(EDI, EDX);
+				JAb(27);
+				MOVid(ins->k, ESI);
+				MOVrd(EDI, ECX);
+				SUBrd(EDX, ECX);
+				CMPrd(ESI, ECX);
+				JBb(14);
+				ADDrd(EDX, ESI);
+				MOVrd(EDI, ECX);
+				SUBrd(ESI, ECX);
+				CMPid(sizeof(int32_t), ECX);
+				JAEb(6);
+				ZEROrd(EAX);
+				MOVrq3(R8, RBX);
+				RET();
+				MOVobd(RBX, RSI, EAX);
 				BSWAP(EAX);
 				break;
 
 			case BPF_LD|BPF_H|BPF_IND:
-				ZERO_EAX();
-				MOVid(ECX, ins->k);
-				ADDrd(ECX, EDX);
-				MOVrd(ESI, ECX);
-				ADDib(ECX, sizeof(short));
-				CMPodd(ECX, RBP, -12);
-				JLEb(3);
-				POP(RBX);
-				LEAVE_RET();
-				MOVobw(AX, RBX, RSI);
+				ZEROrd(EAX);
+				CMPrd(EDI, EDX);
+				JAb(27);
+				MOVid(ins->k, ESI);
+				MOVrd(EDI, ECX);
+				SUBrd(EDX, ECX);
+				CMPrd(ESI, ECX);
+				JBb(14);
+				ADDrd(EDX, ESI);
+				MOVrd(EDI, ECX);
+				SUBrd(ESI, ECX);
+				CMPid(sizeof(int16_t), ECX);
+				JAEb(4);
+				MOVrq3(R8, RBX);
+				RET();
+				MOVobw(RBX, RSI, AX);
 				SWAP_AX();
 				break;
 
 			case BPF_LD|BPF_B|BPF_IND:
-				ZERO_EAX();
-				MOVid(ECX, ins->k);
-				ADDrd(ECX, EDX);
-				CMPodd(ECX, RBP, -12);
-				JLEb(3);
-				POP(RBX);
-				LEAVE_RET();
-				MOVobb(AL, RBX, RCX);
+				ZEROrd(EAX);
+				CMPrd(EDI, EDX);
+				JAEb(13);
+				MOVid(ins->k, ESI);
+				MOVrd(EDI, ECX);
+				SUBrd(EDX, ECX);
+				CMPrd(ESI, ECX);
+				JAb(4);
+				MOVrq3(R8, RBX);
+				RET();
+				ADDrd(EDX, ESI);
+				MOVobb(RBX, RSI, AL);
 				break;
 
 			case BPF_LDX|BPF_MSH|BPF_B:
-				MOVid(ECX, ins->k);
-				CMPodd(ECX, RBP, -12);
-				JLEb(5);
-				ZERO_EAX();
-				POP(RBX);
-				LEAVE_RET();
-				ZERO_EDX();
-				MOVobb(DL, RBX, RCX);
-				ANDib(DL, 0xf);
-				SHLib(EDX, 2);
+				MOVid(ins->k, ESI);
+				CMPrd(EDI, ESI);
+				JBb(6);
+				ZEROrd(EAX);
+				MOVrq3(R8, RBX);
+				RET();
+				ZEROrd(EDX);
+				MOVobb(RBX, RSI, DL);
+				ANDib(0x0f, DL);
+				SHLib(2, EDX);
 				break;
 
 			case BPF_LD|BPF_IMM:
-				MOVid(EAX, ins->k);
+				MOVid(ins->k, EAX);
 				break;
 
 			case BPF_LDX|BPF_IMM:
-				MOVid(EDX, ins->k);
+				MOVid(ins->k, EDX);
 				break;
 
 			case BPF_LD|BPF_MEM:
-				MOViq(RCX, (uintptr_t)mem);
-				MOVid(ESI, ins->k * 4);
-				MOVobd(EAX, RCX, RSI);
+				MOViq((uintptr_t)mem, RCX);
+				MOVid(ins->k * 4, ESI);
+				MOVobd(RCX, RSI, EAX);
 				break;
 
 			case BPF_LDX|BPF_MEM:
-				MOViq(RCX, (uintptr_t)mem);
-				MOVid(ESI, ins->k * 4);
-				MOVobd(EDX, RCX, RSI);
+				MOViq((uintptr_t)mem, RCX);
+				MOVid(ins->k * 4, ESI);
+				MOVobd(RCX, RSI, EDX);
 				break;
 
 			case BPF_ST:
@@ -278,15 +308,15 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				 * be optimized if the previous instruction
 				 * was already of this type
 				 */
-				MOViq(RCX, (uintptr_t)mem);
-				MOVid(ESI, ins->k * 4);
-				MOVomd(RCX, RSI, EAX);
+				MOViq((uintptr_t)mem, RCX);
+				MOVid(ins->k * 4, ESI);
+				MOVomd(EAX, RCX, RSI);
 				break;
 
 			case BPF_STX:
-				MOViq(RCX, (uintptr_t)mem);
-				MOVid(ESI, ins->k * 4);
-				MOVomd(RCX, RSI, EDX);
+				MOViq((uintptr_t)mem, RCX);
+				MOVid(ins->k * 4, ESI);
+				MOVomd(EDX, RCX, RSI);
 				break;
 
 			case BPF_JMP|BPF_JA:
@@ -295,113 +325,102 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				break;
 
 			case BPF_JMP|BPF_JGT|BPF_K:
-				CMPid(EAX, ins->k);
-				/* 5 is the size of the following JMP */
-				JG(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5 );
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				CMPid(ins->k, EAX);
+				JCC(JA, JBE);
 				break;
 
 			case BPF_JMP|BPF_JGE|BPF_K:
-				CMPid(EAX, ins->k);
-				JGE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				CMPid(ins->k, EAX);
+				JCC(JAE, JB);
 				break;
 
 			case BPF_JMP|BPF_JEQ|BPF_K:
-				CMPid(EAX, ins->k);
-				JE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				CMPid(ins->k, EAX);
+				JCC(JE, JNE);
 				break;
 
 			case BPF_JMP|BPF_JSET|BPF_K:
-				MOVrd(ECX, EAX);
-				ANDid(ECX, ins->k);
-				JE(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				TESTid(ins->k, EAX);
+				JCC(JNE, JE);
 				break;
 
 			case BPF_JMP|BPF_JGT|BPF_X:
-				CMPrd(EAX, EDX);
-				JA(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				CMPrd(EDX, EAX);
+				JCC(JA, JBE);
 				break;
 
 			case BPF_JMP|BPF_JGE|BPF_X:
-				CMPrd(EAX, EDX);
-				JAE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				CMPrd(EDX, EAX);
+				JCC(JAE, JB);
 				break;
 
 			case BPF_JMP|BPF_JEQ|BPF_X:
-				CMPrd(EAX, EDX);
-				JE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				CMPrd(EDX, EAX);
+				JCC(JE, JNE);
 				break;
 
 			case BPF_JMP|BPF_JSET|BPF_X:
-				MOVrd(ECX, EAX);
-				ANDrd(ECX, EDX);
-				JE(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				TESTrd(EDX, EAX);
+				JCC(JNE, JE);
 				break;
 
 			case BPF_ALU|BPF_ADD|BPF_X:
-				ADDrd(EAX, EDX);
+				ADDrd(EDX, EAX);
 				break;
 
 			case BPF_ALU|BPF_SUB|BPF_X:
-				SUBrd(EAX, EDX);
+				SUBrd(EDX, EAX);
 				break;
 
 			case BPF_ALU|BPF_MUL|BPF_X:
-				MOVrd(ECX, EDX);
-				MULrd(EDX);
 				MOVrd(EDX, ECX);
+				MULrd(EDX);
+				MOVrd(ECX, EDX);
 				break;
 
 			case BPF_ALU|BPF_DIV|BPF_X:
-				CMPid(EDX, 0);
-				JNEb(5);
-				ZERO_EAX();
-				POP(RBX);
-				LEAVE_RET();
-				MOVrd(ECX, EDX);
-				ZERO_EDX();
-				DIVrd(ECX);
+				TESTrd(EDX, EDX);
+				JNEb(6);
+				ZEROrd(EAX);
+				MOVrq3(R8, RBX);
+				RET();
 				MOVrd(EDX, ECX);
+				ZEROrd(EDX);
+				DIVrd(ECX);
+				MOVrd(ECX, EDX);
 				break;
 
 			case BPF_ALU|BPF_AND|BPF_X:
-				ANDrd(EAX, EDX);
+				ANDrd(EDX, EAX);
 				break;
 
 			case BPF_ALU|BPF_OR|BPF_X:
-				ORrd(EAX, EDX);
+				ORrd(EDX, EAX);
 				break;
 
 			case BPF_ALU|BPF_LSH|BPF_X:
-				MOVrd(ECX, EDX);
+				MOVrd(EDX, ECX);
 				SHL_CLrb(EAX);
 				break;
 
 			case BPF_ALU|BPF_RSH|BPF_X:
-				MOVrd(ECX, EDX);
+				MOVrd(EDX, ECX);
 				SHR_CLrb(EAX);
 				break;
 
@@ -414,34 +433,34 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				break;
 
 			case BPF_ALU|BPF_MUL|BPF_K:
-				MOVrd(ECX, EDX);
-				MOVid(EDX, ins->k);
-				MULrd(EDX);
 				MOVrd(EDX, ECX);
+				MOVid(ins->k, EDX);
+				MULrd(EDX);
+				MOVrd(ECX, EDX);
 				break;
 
 			case BPF_ALU|BPF_DIV|BPF_K:
-				MOVrd(ECX, EDX);
-				ZERO_EDX();
-				MOVid(ESI, ins->k);
-				DIVrd(ESI);
 				MOVrd(EDX, ECX);
+				ZEROrd(EDX);
+				MOVid(ins->k, ESI);
+				DIVrd(ESI);
+				MOVrd(ECX, EDX);
 				break;
 
 			case BPF_ALU|BPF_AND|BPF_K:
-				ANDid(EAX, ins->k);
+				ANDid(ins->k, EAX);
 				break;
 
 			case BPF_ALU|BPF_OR|BPF_K:
-				ORid(EAX, ins->k);
+				ORid(ins->k, EAX);
 				break;
 
 			case BPF_ALU|BPF_LSH|BPF_K:
-				SHLib(EAX, (ins->k) & 255);
+				SHLib((ins->k) & 0xff, EAX);
 				break;
 
 			case BPF_ALU|BPF_RSH|BPF_K:
-				SHRib(EAX, (ins->k) & 255);
+				SHRib((ins->k) & 0xff, EAX);
 				break;
 
 			case BPF_ALU|BPF_NEG:
@@ -449,11 +468,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				break;
 
 			case BPF_MISC|BPF_TAX:
-				MOVrd(EDX, EAX);
+				MOVrd(EAX, EDX);
 				break;
 
 			case BPF_MISC|BPF_TXA:
-				MOVrd(EAX, EDX);
+				MOVrd(EDX, EAX);
 				break;
 			}
 			ins++;
@@ -463,11 +482,19 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 		if (pass == 2)
 			break;
 
+#ifdef _KERNEL
 		stream.ibuf = (char *)malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT);
 		if (stream.ibuf == NULL) {
 			free(stream.refs, M_BPFJIT);
-			return NULL;
+			return (NULL);
+		}
+#else
+		stream.ibuf = (char *)malloc(stream.cur_ip);
+		if (stream.ibuf == NULL) {
+			free(stream.refs);
+			return (NULL);
 		}
+#endif
 
 		/*
 		 * modify the reference table to contain the offsets and
@@ -488,7 +515,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 	 * the reference table is needed only during compilation,
 	 * now we can free it
 	 */
+#ifdef _KERNEL
 	free(stream.refs, M_BPFJIT);
+#else
+	free(stream.refs);
+#endif
 
-	return (bpf_filter_func)stream.ibuf;
+	return ((bpf_filter_func)stream.ibuf);
 }
diff --git a/sys/amd64/amd64/bpf_jit_machdep.h b/sys/amd64/amd64/bpf_jit_machdep.h
index b6a12f114da9..901dad6d7d7f 100644
--- a/sys/amd64/amd64/bpf_jit_machdep.h
+++ b/sys/amd64/amd64/bpf_jit_machdep.h
@@ -1,6 +1,6 @@
 /*-
- * Copyright (c) 2002 - 2003 NetGroup, Politecnico di Torino (Italy)
- * Copyright (c) 2005 Jung-uk Kim <jkim@FreeBSD.org>
+ * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy)
+ * Copyright (C) 2005-2008 Jung-uk Kim <jkim@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -45,6 +45,14 @@
 #define RBP	5
 #define RSI	6
 #define RDI	7
+#define R8	0
+#define R9	1
+#define R10	2
+#define R11	3
+#define R12	4
+#define R13	5
+#define R14	6
+#define R15	7
 
 #define EAX	0
 #define ECX	1
@@ -54,6 +62,14 @@
 #define EBP	5
 #define ESI	6
 #define EDI	7
+#define R8D	0
+#define R9D	1
+#define R10D	2
+#define R11D	3
+#define R12D	4
+#define R13D	5
+#define R14D	6
+#define R15D	7
 
 #define AX	0
 #define CX	1
@@ -104,254 +120,257 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
  * native Instruction Macros
  */
 
-/* mov r32,i32 */
-#define MOVid(r32, i32) do {						\
+/* movl i32,r32 */
+#define MOVid(i32, r32) do {						\
 	emitm(&stream, (11 << 4) | (1 << 3) | (r32 & 0x7), 1);		\
 	emitm(&stream, i32, 4);						\
 } while (0)
 
-/* mov r64,i64 */
-#define MOViq(r64, i64) do {						\
+/* movq i64,r64 */
+#define MOViq(i64, r64) do {						\
 	emitm(&stream, 0x48, 1);					\
 	emitm(&stream, (11 << 4) | (1 << 3) | (r64 & 0x7), 1);		\
 	emitm(&stream, i64, 4);						\
 	emitm(&stream, (i64 >> 32), 4);					\
 } while (0)
 
-/* mov dr32,sr32 */
-#define MOVrd(dr32, sr32) do {						\
-	emitm(&stream, (8 << 4) | 3 | (1 << 3), 1);			\
+/* movl sr32,dr32 */
+#define MOVrd(sr32, dr32) do {						\
+	emitm(&stream, 0x89, 1);					\
 	emitm(&stream,							\
-	    (3 << 6) | ((dr32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
-/* mov dr64,sr64 */
-#define MOVrq(dr64, sr64) do {						\
-	emitm(&stream, 0x48, 1);					\
-	emitm(&stream, (8 << 4) | 3 | (1 << 3), 1);			\
+/* movl sr32,dr32 (dr32 = %r8-15d) */
+#define MOVrd2(sr32, dr32) do {						\
+	emitm(&stream, 0x8941, 2);					\
+	emitm(&stream,							\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
+} while (0)
+
+/* movl sr32,dr32 (sr32 = %r8-15d) */
+#define MOVrd3(sr32, dr32) do {						\
+	emitm(&stream, 0x8944, 2);					\
+	emitm(&stream,							\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
+} while (0)
+
+/* movq sr64,dr64 */
+#define MOVrq(sr64, dr64) do {						\
+	emitm(&stream, 0x8948, 2);					\
 	emitm(&stream,							\
-	    (3 << 6) | ((dr64 & 0x7) << 3) | (sr64 & 0x7), 1);		\
+	    (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);		\
 } while (0)
 
-/* mov dr32,sr64[off] */
-#define MOVodd(dr32, sr64, off) do {					\
-	emitm(&stream, (8 << 4) | 3 | (1 << 3), 1);			\
+/* movq sr64,dr64 (dr64 = %r8-15) */
+#define MOVrq2(sr64, dr64) do {						\
+	emitm(&stream, 0x8949, 2);					\
 	emitm(&stream,							\
-	    (1 << 6) | ((dr32 & 0x7) << 3) | (sr64 & 0x7), 1);		\
-	emitm(&stream, off, 1);						\
+	    (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);		\
 } while (0)
 
-/* mov dr64[off],sr32 */
-#define MOVoqd(dr64, off, sr32) do {					\
-	emitm(&stream, (8 << 4) | 1 | (1 << 3), 1);			\
+/* movq sr64,dr64 (sr64 = %r8-15) */
+#define MOVrq3(sr64, dr64) do {						\
+	emitm(&stream, 0x894c, 2);					\
 	emitm(&stream,							\
-	    (1 << 6) | ((sr32 & 0x7) << 3) | (dr64 & 0x7), 1);		\
-	emitm(&stream, off, 1);						\
+	    (3 << 6) | ((sr64 & 0x7) << 3) | (dr64 & 0x7), 1);		\
 } while (0)
 
-/* mov dr32,sr64[or64] */
-#define MOVobd(dr32, sr64, or64) do {					\
-	emitm(&stream, (8 << 4) | 3 | (1 << 3), 1);			\
+/* movl (sr64,or64,1),dr32 */
+#define MOVobd(sr64, or64, dr32) do {					\
+	emitm(&stream, 0x8b, 1);					\
 	emitm(&stream, ((dr32 & 0x7) << 3) | 4, 1);			\
 	emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);		\
 } while (0)
 
-/* mov dr16,sr64[or64] */
-#define MOVobw(dr32, sr64, or64) do {					\
-	emitm(&stream, 0x66, 1);					\
-	emitm(&stream, (8 << 4) | 3 | (1 << 3), 1);			\
-	emitm(&stream, ((dr32 & 0x7) << 3) | 4, 1);			\
+/* movw (sr64,or64,1),dr16 */
+#define MOVobw(sr64, or64, dr16) do {					\
+	emitm(&stream, 0x8b66, 2);					\
+	emitm(&stream, ((dr16 & 0x7) << 3) | 4, 1);			\
 	emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);		\
 } while (0)
 
-/* mov dr8,sr64[or64] */
-#define MOVobb(dr8, sr64, or64) do {					\
+/* movb (sr64,or64,1),dr8 */
+#define MOVobb(sr64, or64, dr8) do {					\
 	emitm(&stream, 0x8a, 1);					\
 	emitm(&stream, ((dr8 & 0x7) << 3) | 4, 1);			\
 	emitm(&stream, ((or64 & 0x7) << 3) | (sr64 & 0x7), 1);		\
 } while (0)
 
-/* mov [dr64][or64],sr32 */
-#define MOVomd(dr64, or64, sr32) do {					\
+/* movl sr32,(dr64,or64,1) */
+#define MOVomd(sr32, dr64, or64) do {					\
 	emitm(&stream, 0x89, 1);					\
 	emitm(&stream, ((sr32 & 0x7) << 3) | 4, 1);			\
 	emitm(&stream, ((or64 & 0x7) << 3) | (dr64 & 0x7), 1);		\
 } while (0)
 
-/* bswap dr32 */
+/* bswapl dr32 */
 #define BSWAP(dr32) do {						\
 	emitm(&stream, 0xf, 1);						\
 	emitm(&stream, (0x19 << 3) | dr32, 1);				\
 } while (0)
 
-/* xchg al,ah */
+/* xchgb %al,%ah */
 #define SWAP_AX() do {							\
-	emitm(&stream, 0x86, 1);					\
-	emitm(&stream, 0xc4, 1);					\
-} while (0)
-
-/* push r64 */
-#define PUSH(r64) do {							\
-	emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1);		\
-} while (0)
-
-/* pop r64 */
-#define POP(r64) do {							\
-	emitm(&stream, (5 << 4) | (1 << 3) | (r64 & 0x7), 1);		\
+	emitm(&stream, 0xc486, 2);					\
 } while (0)
 
-/* leave/ret */
-#define LEAVE_RET() do {						\
-	emitm(&stream, 0xc9, 1);					\
+/* ret */
+#define RET() do {						\
 	emitm(&stream, 0xc3, 1);					\
 } while (0)
 
-/* add dr32,sr32 */
-#define ADDrd(dr32, sr32) do {						\
-	emitm(&stream, 0x03, 1);					\
+/* addl sr32,dr32 */
+#define ADDrd(sr32, dr32) do {						\
+	emitm(&stream, 0x01, 1);					\
 	emitm(&stream,							\
-	    (3 << 6) | ((dr32 & 0x7) << 3) | (sr32 & 0x7), 1);	\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
-/* add eax,i32 */
+/* addl i32,%eax */
 #define ADD_EAXi(i32) do {						\
 	emitm(&stream, 0x05, 1);					\
 	emitm(&stream, i32, 4);						\
 } while (0)
 
-/* add r32,i32 */
-#define ADDid(r32, i32) do {						\
-	emitm(&stream, 0x81, 1);					\
-	emitm(&stream, (24 << 3) | r32, 1);				\
-	emitm(&stream, i32, 4);						\
-} while (0)
-
-/* add r32,i8 */
-#define ADDib(r32, i8) do {						\
+/* addl i8,r32 */
+#define ADDib(i8, r32) do {						\
 	emitm(&stream, 0x83, 1);					\
 	emitm(&stream, (24 << 3) | r32, 1);				\
 	emitm(&stream, i8, 1);						\
 } while (0)
 
-/* sub dr32,sr32 */
-#define SUBrd(dr32, sr32) do {						\
-	emitm(&stream, 0x2b, 1);					\
+/* subl sr32,dr32 */
+#define SUBrd(sr32, dr32) do {						\
+	emitm(&stream, 0x29, 1);					\
 	emitm(&stream,							\
-	    (3 << 6) | ((dr32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
-/* sub eax,i32 */
+/* subl i32,%eax */
 #define SUB_EAXi(i32) do {						\
 	emitm(&stream, 0x2d, 1);					\
 	emitm(&stream, i32, 4);						\
 } while (0)
 
-/* mul r32 */
+/* mull r32 */
 #define MULrd(r32) do {							\
 	emitm(&stream, 0xf7, 1);					\
 	emitm(&stream, (7 << 5) | (r32 & 0x7), 1);			\
 } while (0)
 
-/* div r32 */
+/* divl r32 */
 #define DIVrd(r32) do {							\
 	emitm(&stream, 0xf7, 1);					\
 	emitm(&stream, (15 << 4) | (r32 & 0x7), 1);			\
 } while (0)
 
-/* and r8,i8 */
-#define ANDib(r8, i8) do {						\
-	emitm(&stream, 0x80, 1);					\
-	emitm(&stream, (7 << 5) | r8, 1);				\
+/* andb i8,r8 */
+#define ANDib(i8, r8) do {						\
+	if (r8 == AL) {							\
+		emitm(&stream, 0x24, 1);				\
+	} else {							\
+		emitm(&stream, 0x80, 1);				\
+		emitm(&stream, (7 << 5) | r8, 1);			\
+	}								\
 	emitm(&stream, i8, 1);						\
 } while (0)
 
-/* and r32,i32 */
-#define ANDid(r32, i32) do {						\
+/* andl i32,r32 */
+#define ANDid(i32, r32) do {						\
 	if (r32 == EAX) {						\
 		emitm(&stream, 0x25, 1);				\
-		emitm(&stream, i32, 4);					\
 	} else {							\
 		emitm(&stream, 0x81, 1);				\
 		emitm(&stream, (7 << 5) | r32, 1);			\
-		emitm(&stream, i32, 4);					\
 	}								\
+	emitm(&stream, i32, 4);						\
+} while (0)
+
+/* andl sr32,dr32 */
+#define ANDrd(sr32, dr32) do {						\
+	emitm(&stream, 0x21, 1);					\
+	emitm(&stream,							\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
+} while (0)
+
+/* testl i32,r32 */
+#define TESTid(i32, r32) do {						\
+	if (r32 == EAX) {						\
+		emitm(&stream, 0xa9, 1);				\
+	} else {							\
+		emitm(&stream, 0xf7, 1);				\
+		emitm(&stream, (3 << 6) | r32, 1);			\
+	}								\
+	emitm(&stream, i32, 4);						\
 } while (0)
 
-/* and dr32,sr32 */
-#define ANDrd(dr32, sr32) do {						\
-	emitm(&stream, 0x23, 1);					\
+/* testl sr32,dr32 */
+#define TESTrd(sr32, dr32) do {						\
+	emitm(&stream, 0x85, 1);					\
 	emitm(&stream,							\
-	    (3 << 6) | ((dr32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
-/* or dr32,sr32 */
-#define ORrd(dr32, sr32) do {						\
-	emitm(&stream, 0x0b, 1);					\
+/* orl sr32,dr32 */
+#define ORrd(sr32, dr32) do {						\
+	emitm(&stream, 0x09, 1);					\
 	emitm(&stream,							\
-	    (3 << 6) | ((dr32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
-/* or r32,i32 */
-#define ORid(r32, i32) do {						\
+/* orl i32,r32 */
+#define ORid(i32, r32) do {						\
 	if (r32 == EAX) {						\
 		emitm(&stream, 0x0d, 1);				\
-		emitm(&stream, i32, 4);					\
 	} else {							\
 		emitm(&stream, 0x81, 1);				\
 		emitm(&stream, (25 << 3) | r32, 1);			\
-		emitm(&stream, i32, 4);					\
 	}								\
+	emitm(&stream, i32, 4);						\
 } while (0)
 
-/* shl r32,i8 */
-#define SHLib(r32, i8) do {						\
+/* shll i8,r32 */
+#define SHLib(i8, r32) do {						\
 	emitm(&stream, 0xc1, 1);					\
 	emitm(&stream, (7 << 5) | (r32 & 0x7), 1);			\
 	emitm(&stream, i8, 1);						\
 } while (0)
 
-/* shl dr32,cl */
+/* shll %cl,dr32 */
 #define SHL_CLrb(dr32) do {						\
 	emitm(&stream, 0xd3, 1);					\
 	emitm(&stream, (7 << 5) | (dr32 & 0x7), 1);			\
 } while (0)
 
-/* shr r32,i8 */
-#define SHRib(r32, i8) do {						\
+/* shrl i8,r32 */
+#define SHRib(i8, r32) do {						\
 	emitm(&stream, 0xc1, 1);					\
 	emitm(&stream, (29 << 3) | (r32 & 0x7), 1);			\
 	emitm(&stream, i8, 1);						\
 } while (0)
 
-/* shr dr32,cl */
+/* shrl %cl,dr32 */
 #define SHR_CLrb(dr32) do {						\
 	emitm(&stream, 0xd3, 1);					\
 	emitm(&stream, (29 << 3) | (dr32 & 0x7), 1);			\
 } while (0)
 
-/* neg r32 */
+/* negl r32 */
 #define NEGd(r32) do {							\
 	emitm(&stream, 0xf7, 1);					\
 	emitm(&stream, (27 << 3) | (r32 & 0x7), 1);			\
 } while (0)
 
-/* cmp dr32,sr64[off] */
-#define CMPodd(dr32, sr64, off) do {					\
-	emitm(&stream, (3 << 4) | 3 | (1 << 3), 1);			\
-	emitm(&stream,							\
-	    (1 << 6) | ((dr32 & 0x7) << 3) | (sr64 & 0x7), 1);		\
-	emitm(&stream, off, 1);						\
-} while (0)
-
-/* cmp dr32,sr32 */
-#define CMPrd(dr32, sr32) do {						\
-	emitm(&stream, 0x3b, 1);					\
+/* cmpl sr32,dr32 */
+#define CMPrd(sr32, dr32) do {						\
+	emitm(&stream, 0x39, 1);					\
 	emitm(&stream,							\
-	    (3 << 6) | ((dr32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
-/* cmp dr32,i32 */
-#define CMPid(dr32, i32) do {						\
+/* cmpl i32,dr32 */
+#define CMPid(i32, dr32) do {						\
 	if (dr32 == EAX){						\
 		emitm(&stream, 0x3d, 1);				\
 		emitm(&stream, i32, 4);					\
@@ -362,58 +381,28 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 	}								\
 } while (0)
 
-/* jne off32 */
-#define JNEb(off8) do {							\
-	emitm(&stream, 0x75, 1);					\
+/* jb off8 */
+#define JBb(off8) do {							\
+	emitm(&stream, 0x72, 1);					\
 	emitm(&stream, off8, 1);					\
 } while (0)
 
-/* je off32 */
-#define JE(off32) do {							\
-	emitm(&stream, 0x0f, 1);					\
-	emitm(&stream, 0x84, 1);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jle off32 */
-#define JLE(off32) do {							\
-	emitm(&stream, 0x0f, 1);					\
-	emitm(&stream, 0x8e, 1);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jle off8 */
-#define JLEb(off8) do {							\
-	emitm(&stream, 0x7e, 1);					\
+/* jae off8 */
+#define JAEb(off8) do {							\
+	emitm(&stream, 0x73, 1);					\
 	emitm(&stream, off8, 1);					\
 } while (0)
 
-/* ja off32 */
-#define JA(off32) do {							\
-	emitm(&stream, 0x0f, 1);					\
-	emitm(&stream, 0x87, 1);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jae off32 */
-#define JAE(off32) do {							\
-	emitm(&stream, 0x0f, 1);					\
-	emitm(&stream, 0x83, 1);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jg off32 */
-#define JG(off32) do {							\
-	emitm(&stream, 0x0f, 1);					\
-	emitm(&stream, 0x8f, 1);					\
-	emitm(&stream, off32, 4);					\
+/* jne off8 */
+#define JNEb(off8) do {							\
+	emitm(&stream, 0x75, 1);					\
+	emitm(&stream, off8, 1);					\
 } while (0)
 
-/* jge off32 */
-#define JGE(off32) do {							\
-	emitm(&stream, 0x0f, 1);					\
-	emitm(&stream, 0x8d, 1);					\
-	emitm(&stream, off32, 4);					\
+/* ja off8 */
+#define JAb(off8) do {							\
+	emitm(&stream, 0x77, 1);					\
+	emitm(&stream, off8, 1);					\
 } while (0)
 
 /* jmp off32 */
@@ -422,16 +411,39 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 	emitm(&stream, off32, 4);					\
 } while (0)
 
-/* xor eax,eax */
-#define ZERO_EAX() do {							\
+/* xorl r32,r32 */
+#define ZEROrd(r32) do {						\
 	emitm(&stream, 0x31, 1);					\
-	emitm(&stream, 0xc0, 1);					\
+	emitm(&stream, (3 << 6) | ((r32 & 0x7) << 3) | (r32 & 0x7), 1);	\
 } while (0)
 
-/* xor edx,edx */
-#define ZERO_EDX() do {							\
-	emitm(&stream, 0x31, 1);					\
-	emitm(&stream, 0xd2, 1);					\
+/*
+ * Conditional long jumps
+ */
+#define	JB	0x82
+#define	JAE	0x83
+#define	JE	0x84
+#define	JNE	0x85
+#define	JBE	0x86
+#define	JA	0x87
+
+#define	JCC(t, f) do {							\
+	if (ins->jt != 0 && ins->jf != 0) {				\
+		/* 5 is the size of the following jmp */		\
+		emitm(&stream, ((t) << 8) | 0x0f, 2);			\
+		emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -	\
+		    stream.refs[stream.bpf_pc] + 5, 4);			\
+		JMP(stream.refs[stream.bpf_pc + ins->jf] -		\
+		    stream.refs[stream.bpf_pc]);			\
+	} else if (ins->jt != 0) {					\
+		emitm(&stream, ((t) << 8) | 0x0f, 2);			\
+		emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -	\
+		    stream.refs[stream.bpf_pc], 4);			\
+	} else {							\
+		emitm(&stream, ((f) << 8) | 0x0f, 2);			\
+		emitm(&stream, stream.refs[stream.bpf_pc + ins->jf] -	\
+		    stream.refs[stream.bpf_pc], 4);			\
+	}								\
 } while (0)
 
 #endif	/* _BPF_JIT_MACHDEP_H_ */
diff --git a/sys/i386/i386/bpf_jit_machdep.c b/sys/i386/i386/bpf_jit_machdep.c
index 362569ed900a..78b585256120 100644
--- a/sys/i386/i386/bpf_jit_machdep.c
+++ b/sys/i386/i386/bpf_jit_machdep.c
@@ -1,6 +1,6 @@
 /*-
- * Copyright (c) 2002 - 2003 NetGroup, Politecnico di Torino (Italy)
- * Copyright (c) 2005 Jung-uk Kim <jkim@FreeBSD.org>
+ * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy)
+ * Copyright (C) 2005-2008 Jung-uk Kim <jkim@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -32,16 +32,20 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#ifdef _KERNEL
 #include "opt_bpf.h"
-
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
-#include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/malloc.h>
-
 #include <net/if.h>
+#else
+#include <stdlib.h>
+#endif
+
+#include <sys/types.h>
+
 #include <net/bpf.h>
 #include <net/bpf_jitter.h>
 
@@ -53,7 +57,7 @@ bpf_filter_func	bpf_jit_compile(struct bpf_insn *, u_int, int *);
  * emit routine to update the jump table
  */
 static void
-emit_length(bpf_bin_stream *stream, u_int value, u_int len)
+emit_length(bpf_bin_stream *stream, __unused u_int value, u_int len)
 {
 
 	(stream->refs)[stream->bpf_pc] += len;
@@ -105,13 +109,17 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 
 	/* Do not compile an empty filter. */
 	if (nins == 0)
-		return NULL;
+		return (NULL);
 
 	/* Allocate the reference table for the jumps */
+#ifdef _KERNEL
 	stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int),
 	    M_BPFJIT, M_NOWAIT);
+#else
+	stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int));
+#endif
 	if (stream.refs == NULL)
-		return NULL;
+		return (NULL);
 
 	/* Reset the reference table */
 	for (i = 0; i < nins + 1; i++)
@@ -132,21 +140,26 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 
 		/* create the procedure header */
 		PUSH(EBP);
-		MOVrd(EBP, ESP);
+		MOVrd(ESP, EBP);
 		PUSH(EDI);
 		PUSH(ESI);
 		PUSH(EBX);
-		MOVodd(EBX, EBP, 8);
+		MOVodd(8, EBP, EBX);
+		MOVodd(16, EBP, EDI);
 
 		for (i = 0; i < nins; i++) {
 			stream.bpf_pc++;
 
 			switch (ins->code) {
 			default:
-				return NULL;
+#ifdef _KERNEL
+				return (NULL);
+#else
+				abort();
+#endif
 
 			case BPF_RET|BPF_K:
-				MOVid(EAX, ins->k);
+				MOVid(ins->k, EAX);
 				POP(EBX);
 				POP(ESI);
 				POP(EDI);
@@ -161,133 +174,153 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				break;
 
 			case BPF_LD|BPF_W|BPF_ABS:
-				MOVid(ECX, ins->k);
-				MOVrd(ESI, ECX);
-				ADDib(ECX, sizeof(int));
-				CMPodd(ECX, EBP, 0x10);
-				JLEb(7);
-				ZERO_EAX();
+				MOVid(ins->k, ESI);
+				CMPrd(EDI, ESI);
+				JAb(12);
+				MOVrd(EDI, ECX);
+				SUBrd(ESI, ECX);
+				CMPid(sizeof(int32_t), ECX);
+				JAEb(7);
+				ZEROrd(EAX);
 				POP(EBX);
 				POP(ESI);
 				POP(EDI);
 				LEAVE_RET();
-				MOVobd(EAX, EBX, ESI);
+				MOVobd(EBX, ESI, EAX);
 				BSWAP(EAX);
 				break;
 
 			case BPF_LD|BPF_H|BPF_ABS:
-				ZERO_EAX();
-				MOVid(ECX, ins->k);
-				MOVrd(ESI, ECX);
-				ADDib(ECX, sizeof(short));
-				CMPodd(ECX, EBP, 0x10);
-				JLEb(5);
+				ZEROrd(EAX);
+				MOVid(ins->k, ESI);
+				CMPrd(EDI, ESI);
+				JAb(12);
+				MOVrd(EDI, ECX);
+				SUBrd(ESI, ECX);
+				CMPid(sizeof(int16_t), ECX);
+				JAEb(5);
 				POP(EBX);
 				POP(ESI);
 				POP(EDI);
 				LEAVE_RET();
-				MOVobw(AX, EBX, ESI);
+				MOVobw(EBX, ESI, AX);
 				SWAP_AX();
 				break;
 
 			case BPF_LD|BPF_B|BPF_ABS:
-				ZERO_EAX();
-				MOVid(ECX, ins->k);
-				CMPodd(ECX, EBP, 0x10);
-				JLEb(5);
+				ZEROrd(EAX);
+				MOVid(ins->k, ESI);
+				CMPrd(EDI, ESI);
+				JBb(5);
 				POP(EBX);
 				POP(ESI);
 				POP(EDI);
 				LEAVE_RET();
-				MOVobb(AL, EBX, ECX);
+				MOVobb(EBX, ESI, AL);
 				break;
 
 			case BPF_LD|BPF_W|BPF_LEN:
-				MOVodd(EAX, EBP, 0xc);
+				MOVodd(12, EBP, EAX);
 				break;
 
 			case BPF_LDX|BPF_W|BPF_LEN:
-				MOVodd(EDX, EBP, 0xc);
+				MOVodd(12, EBP, EDX);
 				break;
 
 			case BPF_LD|BPF_W|BPF_IND:
-				MOVid(ECX, ins->k);
-				ADDrd(ECX, EDX);
-				MOVrd(ESI, ECX);
-				ADDib(ECX, sizeof(int));
-				CMPodd(ECX, EBP, 0x10);
-				JLEb(7);
-				ZERO_EAX();
+				CMPrd(EDI, EDX);
+				JAb(27);
+				MOVid(ins->k, ESI);
+				MOVrd(EDI, ECX);
+				SUBrd(EDX, ECX);
+				CMPrd(ESI, ECX);
+				JBb(14);
+				ADDrd(EDX, ESI);
+				MOVrd(EDI, ECX);
+				SUBrd(ESI, ECX);
+				CMPid(sizeof(int32_t), ECX);
+				JAEb(7);
+				ZEROrd(EAX);
 				POP(EBX);
 				POP(ESI);
 				POP(EDI);
 				LEAVE_RET();
-				MOVobd(EAX, EBX, ESI);
+				MOVobd(EBX, ESI, EAX);
 				BSWAP(EAX);
 				break;
 
 			case BPF_LD|BPF_H|BPF_IND:
-				ZERO_EAX();
-				MOVid(ECX, ins->k);
-				ADDrd(ECX, EDX);
-				MOVrd(ESI, ECX);
-				ADDib(ECX, sizeof(short));
-				CMPodd(ECX, EBP, 0x10);
-				JLEb(5);
+				ZEROrd(EAX);
+				CMPrd(EDI, EDX);
+				JAb(27);
+				MOVid(ins->k, ESI);
+				MOVrd(EDI, ECX);
+				SUBrd(EDX, ECX);
+				CMPrd(ESI, ECX);
+				JBb(14);
+				ADDrd(EDX, ESI);
+				MOVrd(EDI, ECX);
+				SUBrd(ESI, ECX);
+				CMPid(sizeof(int16_t), ECX);
+				JAEb(5);
 				POP(EBX);
 				POP(ESI);
 				POP(EDI);
 				LEAVE_RET();
-				MOVobw(AX, EBX, ESI);
+				MOVobw(EBX, ESI, AX);
 				SWAP_AX();
 				break;
 
 			case BPF_LD|BPF_B|BPF_IND:
-				ZERO_EAX();
-				MOVid(ECX, ins->k);
-				ADDrd(ECX, EDX);
-				CMPodd(ECX, EBP, 0x10);
-				JLEb(5);
+				ZEROrd(EAX);
+				CMPrd(EDI, EDX);
+				JAEb(13);
+				MOVid(ins->k, ESI);
+				MOVrd(EDI, ECX);
+				SUBrd(EDX, ECX);
+				CMPrd(ESI, ECX);
+				JAb(5);
 				POP(EBX);
 				POP(ESI);
 				POP(EDI);
 				LEAVE_RET();
-				MOVobb(AL, EBX, ECX);
+				ADDrd(EDX, ESI);
+				MOVobb(EBX, ESI, AL);
 				break;
 
 			case BPF_LDX|BPF_MSH|BPF_B:
-				MOVid(ECX, ins->k);
-				CMPodd(ECX, EBP, 0x10);
-				JLEb(7);
-				ZERO_EAX();
+				MOVid(ins->k, ESI);
+				CMPrd(EDI, ESI);
+				JBb(7);
+				ZEROrd(EAX);
 				POP(EBX);
 				POP(ESI);
 				POP(EDI);
 				LEAVE_RET();
-				ZERO_EDX();
-				MOVobb(DL, EBX, ECX);
-				ANDib(DL, 0xf);
-				SHLib(EDX, 2);
+				ZEROrd(EDX);
+				MOVobb(EBX, ESI, DL);
+				ANDib(0x0f, DL);
+				SHLib(2, EDX);
 				break;
 
 			case BPF_LD|BPF_IMM:
-				MOVid(EAX, ins->k);
+				MOVid(ins->k, EAX);
 				break;
 
 			case BPF_LDX|BPF_IMM:
-				MOVid(EDX, ins->k);
+				MOVid(ins->k, EDX);
 				break;
 
 			case BPF_LD|BPF_MEM:
-				MOVid(ECX, (uintptr_t)mem);
-				MOVid(ESI, ins->k * 4);
-				MOVobd(EAX, ECX, ESI);
+				MOVid((uintptr_t)mem, ECX);
+				MOVid(ins->k * 4, ESI);
+				MOVobd(ECX, ESI, EAX);
 				break;
 
 			case BPF_LDX|BPF_MEM:
-				MOVid(ECX, (uintptr_t)mem);
-				MOVid(ESI, ins->k * 4);
-				MOVobd(EDX, ECX, ESI);
+				MOVid((uintptr_t)mem, ECX);
+				MOVid(ins->k * 4, ESI);
+				MOVobd(ECX, ESI, EDX);
 				break;
 
 			case BPF_ST:
@@ -296,15 +329,15 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				 * be optimized if the previous instruction
 				 * was already of this type
 				 */
-				MOVid(ECX, (uintptr_t)mem);
-				MOVid(ESI, ins->k * 4);
-				MOVomd(ECX, ESI, EAX);
+				MOVid((uintptr_t)mem, ECX);
+				MOVid(ins->k * 4, ESI);
+				MOVomd(EAX, ECX, ESI);
 				break;
 
 			case BPF_STX:
-				MOVid(ECX, (uintptr_t)mem);
-				MOVid(ESI, ins->k * 4);
-				MOVomd(ECX, ESI, EDX);
+				MOVid((uintptr_t)mem, ECX);
+				MOVid(ins->k * 4, ESI);
+				MOVomd(EDX, ECX, ESI);
 				break;
 
 			case BPF_JMP|BPF_JA:
@@ -313,115 +346,104 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				break;
 
 			case BPF_JMP|BPF_JGT|BPF_K:
-				CMPid(EAX, ins->k);
-				/* 5 is the size of the following JMP */
-				JG(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5 );
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				CMPid(ins->k, EAX);
+				JCC(JA, JBE);
 				break;
 
 			case BPF_JMP|BPF_JGE|BPF_K:
-				CMPid(EAX, ins->k);
-				JGE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				CMPid(ins->k, EAX);
+				JCC(JAE, JB);
 				break;
 
 			case BPF_JMP|BPF_JEQ|BPF_K:
-				CMPid(EAX, ins->k);
-				JE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				CMPid(ins->k, EAX);
+				JCC(JE, JNE);
 				break;
 
 			case BPF_JMP|BPF_JSET|BPF_K:
-				MOVrd(ECX, EAX);
-				ANDid(ECX, ins->k);
-				JE(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				TESTid(ins->k, EAX);
+				JCC(JNE, JE);
 				break;
 
 			case BPF_JMP|BPF_JGT|BPF_X:
-				CMPrd(EAX, EDX);
-				JA(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				CMPrd(EDX, EAX);
+				JCC(JA, JBE);
 				break;
 
 			case BPF_JMP|BPF_JGE|BPF_X:
-				CMPrd(EAX, EDX);
-				JAE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				CMPrd(EDX, EAX);
+				JCC(JAE, JB);
 				break;
 
 			case BPF_JMP|BPF_JEQ|BPF_X:
-				CMPrd(EAX, EDX);
-				JE(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				CMPrd(EDX, EAX);
+				JCC(JE, JNE);
 				break;
 
 			case BPF_JMP|BPF_JSET|BPF_X:
-				MOVrd(ECX, EAX);
-				ANDrd(ECX, EDX);
-				JE(stream.refs[stream.bpf_pc + ins->jf] -
-				    stream.refs[stream.bpf_pc] + 5);
-				JMP(stream.refs[stream.bpf_pc + ins->jt] -
-				    stream.refs[stream.bpf_pc]);
+				if (ins->jt == 0 && ins->jf == 0)
+					break;
+				TESTrd(EDX, EAX);
+				JCC(JNE, JE);
 				break;
 
 			case BPF_ALU|BPF_ADD|BPF_X:
-				ADDrd(EAX, EDX);
+				ADDrd(EDX, EAX);
 				break;
 
 			case BPF_ALU|BPF_SUB|BPF_X:
-				SUBrd(EAX, EDX);
+				SUBrd(EDX, EAX);
 				break;
 
 			case BPF_ALU|BPF_MUL|BPF_X:
-				MOVrd(ECX, EDX);
-				MULrd(EDX);
 				MOVrd(EDX, ECX);
+				MULrd(EDX);
+				MOVrd(ECX, EDX);
 				break;
 
 			case BPF_ALU|BPF_DIV|BPF_X:
-				CMPid(EDX, 0);
+				TESTrd(EDX, EDX);
 				JNEb(7);
-				ZERO_EAX();
+				ZEROrd(EAX);
 				POP(EBX);
 				POP(ESI);
 				POP(EDI);
 				LEAVE_RET();
-				MOVrd(ECX, EDX);
-				ZERO_EDX();
-				DIVrd(ECX);
 				MOVrd(EDX, ECX);
+				ZEROrd(EDX);
+				DIVrd(ECX);
+				MOVrd(ECX, EDX);
 				break;
 
 			case BPF_ALU|BPF_AND|BPF_X:
-				ANDrd(EAX, EDX);
+				ANDrd(EDX, EAX);
 				break;
 
 			case BPF_ALU|BPF_OR|BPF_X:
-				ORrd(EAX, EDX);
+				ORrd(EDX, EAX);
 				break;
 
 			case BPF_ALU|BPF_LSH|BPF_X:
-				MOVrd(ECX, EDX);
+				MOVrd(EDX, ECX);
 				SHL_CLrb(EAX);
 				break;
 
 			case BPF_ALU|BPF_RSH|BPF_X:
-				MOVrd(ECX, EDX);
+				MOVrd(EDX, ECX);
 				SHR_CLrb(EAX);
 				break;
 
@@ -434,34 +456,34 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				break;
 
 			case BPF_ALU|BPF_MUL|BPF_K:
-				MOVrd(ECX, EDX);
-				MOVid(EDX, ins->k);
-				MULrd(EDX);
 				MOVrd(EDX, ECX);
+				MOVid(ins->k, EDX);
+				MULrd(EDX);
+				MOVrd(ECX, EDX);
 				break;
 
 			case BPF_ALU|BPF_DIV|BPF_K:
-				MOVrd(ECX, EDX);
-				ZERO_EDX();
-				MOVid(ESI, ins->k);
-				DIVrd(ESI);
 				MOVrd(EDX, ECX);
+				ZEROrd(EDX);
+				MOVid(ins->k, ESI);
+				DIVrd(ESI);
+				MOVrd(ECX, EDX);
 				break;
 
 			case BPF_ALU|BPF_AND|BPF_K:
-				ANDid(EAX, ins->k);
+				ANDid(ins->k, EAX);
 				break;
 
 			case BPF_ALU|BPF_OR|BPF_K:
-				ORid(EAX, ins->k);
+				ORid(ins->k, EAX);
 				break;
 
 			case BPF_ALU|BPF_LSH|BPF_K:
-				SHLib(EAX, (ins->k) & 255);
+				SHLib((ins->k) & 0xff, EAX);
 				break;
 
 			case BPF_ALU|BPF_RSH|BPF_K:
-				SHRib(EAX, (ins->k) & 255);
+				SHRib((ins->k) & 0xff, EAX);
 				break;
 
 			case BPF_ALU|BPF_NEG:
@@ -469,11 +491,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 				break;
 
 			case BPF_MISC|BPF_TAX:
-				MOVrd(EDX, EAX);
+				MOVrd(EAX, EDX);
 				break;
 
 			case BPF_MISC|BPF_TXA:
-				MOVrd(EAX, EDX);
+				MOVrd(EDX, EAX);
 				break;
 			}
 			ins++;
@@ -483,11 +505,19 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 		if (pass == 2)
 			break;
 
+#ifdef _KERNEL
 		stream.ibuf = (char *)malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT);
 		if (stream.ibuf == NULL) {
 			free(stream.refs, M_BPFJIT);
-			return NULL;
+			return (NULL);
+		}
+#else
+		stream.ibuf = (char *)malloc(stream.cur_ip);
+		if (stream.ibuf == NULL) {
+			free(stream.refs);
+			return (NULL);
 		}
+#endif
 
 		/*
 		 * modify the reference table to contain the offsets and
@@ -508,7 +538,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, int *mem)
 	 * the reference table is needed only during compilation,
 	 * now we can free it
 	 */
+#ifdef _KERNEL
 	free(stream.refs, M_BPFJIT);
+#else
+	free(stream.refs);
+#endif
 
-	return (bpf_filter_func)stream.ibuf;
+	return ((bpf_filter_func)stream.ibuf);
 }
diff --git a/sys/i386/i386/bpf_jit_machdep.h b/sys/i386/i386/bpf_jit_machdep.h
index d0d4222ba5f1..6a0d785b17ab 100644
--- a/sys/i386/i386/bpf_jit_machdep.h
+++ b/sys/i386/i386/bpf_jit_machdep.h
@@ -1,6 +1,6 @@
 /*-
- * Copyright (c) 2002 - 2003 NetGroup, Politecnico di Torino (Italy)
- * Copyright (c) 2005 Jung-uk Kim <jkim@FreeBSD.org>
+ * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy)
+ * Copyright (C) 2005-2008 Jung-uk Kim <jkim@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -95,230 +95,232 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
  * native Instruction Macros
  */
 
-/* mov r32,i32 */
-#define MOVid(r32, i32) do {						\
+/* movl i32,r32 */
+#define MOVid(i32, r32) do {						\
 	emitm(&stream, (11 << 4) | (1 << 3) | (r32 & 0x7), 1);		\
 	emitm(&stream, i32, 4);						\
 } while (0)
 
-/* mov dr32,sr32 */
-#define MOVrd(dr32, sr32) do {						\
-	emitm(&stream, (8 << 4) | 3 | (1 << 3), 1);			\
+/* movl sr32,dr32 */
+#define MOVrd(sr32, dr32) do {						\
+	emitm(&stream, 0x89, 1);					\
 	emitm(&stream,							\
-	    (3 << 6) | ((dr32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
-/* mov dr32,sr32[off] */
-#define MOVodd(dr32, sr32, off) do {					\
-	emitm(&stream, (8 << 4) | 3 | (1 << 3), 1);			\
+/* movl off(sr32),dr32 */
+#define MOVodd(off, sr32, dr32) do {					\
+	emitm(&stream, 0x8b, 1);					\
 	emitm(&stream,							\
 	    (1 << 6) | ((dr32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
 	emitm(&stream, off, 1);						\
 } while (0)
 
-/* mov dr32,sr32[or32] */
-#define MOVobd(dr32, sr32, or32) do {					\
-	emitm(&stream, (8 << 4) | 3 | (1 << 3), 1);			\
+/* movl (sr32,or32,1),dr32 */
+#define MOVobd(sr32, or32, dr32) do {					\
+	emitm(&stream, 0x8b, 1);					\
 	emitm(&stream, ((dr32 & 0x7) << 3) | 4, 1);			\
 	emitm(&stream, ((or32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
 } while (0)
 
-/* mov dr16,sr32[or32] */
-#define MOVobw(dr32, sr32, or32) do {					\
-	emitm(&stream, 0x66, 1);					\
-	emitm(&stream, (8 << 4) | 3 | (1 << 3), 1);			\
-	emitm(&stream, ((dr32 & 0x7) << 3) | 4, 1);			\
+/* movw (sr32,or32,1),dr16 */
+#define MOVobw(sr32, or32, dr16) do {					\
+	emitm(&stream, 0x8b66, 2);					\
+	emitm(&stream, ((dr16 & 0x7) << 3) | 4, 1);			\
 	emitm(&stream, ((or32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
 } while (0)
 
-/* mov dr8,sr32[or32] */
-#define MOVobb(dr8, sr32, or32) do {					\
+/* movb (sr32,or32,1),dr8 */
+#define MOVobb(sr32, or32, dr8) do {					\
 	emitm(&stream, 0x8a, 1);					\
 	emitm(&stream, ((dr8 & 0x7) << 3) | 4, 1);			\
 	emitm(&stream, ((or32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
 } while (0)
 
-/* mov [dr32][or32],sr32 */
-#define MOVomd(dr32, or32, sr32) do {					\
+/* movl sr32,(dr32,or32,1) */
+#define MOVomd(sr32, dr32, or32) do {					\
 	emitm(&stream, 0x89, 1);					\
 	emitm(&stream, ((sr32 & 0x7) << 3) | 4, 1);			\
 	emitm(&stream, ((or32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
-/* bswap dr32 */
+/* bswapl dr32 */
 #define BSWAP(dr32) do {						\
 	emitm(&stream, 0xf, 1);						\
 	emitm(&stream, (0x19 << 3) | dr32, 1);				\
 } while (0)
 
-/* xchg al,ah */
+/* xchgb %al,%ah */
 #define SWAP_AX() do {							\
-	emitm(&stream, 0x86, 1);					\
-	emitm(&stream, 0xc4, 1);					\
+	emitm(&stream, 0xc486, 2);					\
 } while (0)
 
-/* push r32 */
+/* pushl r32 */
 #define PUSH(r32) do {							\
 	emitm(&stream, (5 << 4) | (0 << 3) | (r32 & 0x7), 1);		\
 } while (0)
 
-/* pop r32 */
+/* popl r32 */
 #define POP(r32) do {							\
 	emitm(&stream, (5 << 4) | (1 << 3) | (r32 & 0x7), 1);		\
 } while (0)
 
 /* leave/ret */
 #define LEAVE_RET() do {						\
-	emitm(&stream, 0xc9, 1);					\
-	emitm(&stream, 0xc3, 1);					\
+	emitm(&stream, 0xc3c9, 2);					\
 } while (0)
 
-/* add dr32,sr32 */
-#define ADDrd(dr32, sr32) do {						\
-	emitm(&stream, 0x03, 1);					\
+/* addl sr32,dr32 */
+#define ADDrd(sr32, dr32) do {						\
+	emitm(&stream, 0x01, 1);					\
 	emitm(&stream,							\
-	    (3 << 6) | ((dr32 & 0x7) << 3) | (sr32 & 0x7), 1);	\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
-/* add eax,i32 */
+/* addl i32,%eax */
 #define ADD_EAXi(i32) do {						\
 	emitm(&stream, 0x05, 1);					\
 	emitm(&stream, i32, 4);						\
 } while (0)
 
-/* add r32,i32 */
-#define ADDid(r32, i32) do {						\
-	emitm(&stream, 0x81, 1);					\
-	emitm(&stream, (24 << 3) | r32, 1);				\
-	emitm(&stream, i32, 4);						\
-} while (0)
-
-/* add r32,i8 */
-#define ADDib(r32, i8) do {						\
+/* addl i8,r32 */
+#define ADDib(i8, r32) do {						\
 	emitm(&stream, 0x83, 1);					\
 	emitm(&stream, (24 << 3) | r32, 1);				\
 	emitm(&stream, i8, 1);						\
 } while (0)
 
-/* sub dr32,sr32 */
-#define SUBrd(dr32, sr32) do {						\
-	emitm(&stream, 0x2b, 1);					\
+/* subl sr32,dr32 */
+#define SUBrd(sr32, dr32) do {						\
+	emitm(&stream, 0x29, 1);					\
 	emitm(&stream,							\
-	    (3 << 6) | ((dr32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
-/* sub eax,i32 */
+/* subl i32,%eax */
 #define SUB_EAXi(i32) do {						\
 	emitm(&stream, 0x2d, 1);					\
 	emitm(&stream, i32, 4);						\
 } while (0)
 
-/* mul r32 */
+/* mull r32 */
 #define MULrd(r32) do {							\
 	emitm(&stream, 0xf7, 1);					\
 	emitm(&stream, (7 << 5) | (r32 & 0x7), 1);			\
 } while (0)
 
-/* div r32 */
+/* divl r32 */
 #define DIVrd(r32) do {							\
 	emitm(&stream, 0xf7, 1);					\
 	emitm(&stream, (15 << 4) | (r32 & 0x7), 1);			\
 } while (0)
 
-/* and r8,i8 */
-#define ANDib(r8, i8) do {						\
-	emitm(&stream, 0x80, 1);					\
-	emitm(&stream, (7 << 5) | r8, 1);				\
+/* andb i8,r8 */
+#define ANDib(i8, r8) do {						\
+	if (r8 == AL) {							\
+		emitm(&stream, 0x24, 1);				\
+	} else {							\
+		emitm(&stream, 0x80, 1);				\
+		emitm(&stream, (7 << 5) | r8, 1);			\
+	}								\
 	emitm(&stream, i8, 1);						\
 } while (0)
 
-/* and r32,i32 */
-#define ANDid(r32, i32) do {						\
+/* andl i32,r32 */
+#define ANDid(i32, r32) do {						\
 	if (r32 == EAX) {						\
 		emitm(&stream, 0x25, 1);				\
-		emitm(&stream, i32, 4);					\
 	} else {							\
 		emitm(&stream, 0x81, 1);				\
 		emitm(&stream, (7 << 5) | r32, 1);			\
-		emitm(&stream, i32, 4);					\
 	}								\
+	emitm(&stream, i32, 4);						\
 } while (0)
 
-/* and dr32,sr32 */
-#define ANDrd(dr32, sr32) do {						\
-	emitm(&stream, 0x23, 1);					\
+/* andl sr32,dr32 */
+#define ANDrd(sr32, dr32) do {						\
+	emitm(&stream, 0x21, 1);					\
 	emitm(&stream,							\
-	    (3 << 6) | ((dr32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
-/* or dr32,sr32 */
-#define ORrd(dr32, sr32) do {						\
-	emitm(&stream, 0x0b, 1);					\
+/* testl i32,r32 */
+#define TESTid(i32, r32) do {						\
+	if (r32 == EAX) {						\
+		emitm(&stream, 0xa9, 1);				\
+	} else {							\
+		emitm(&stream, 0xf7, 1);				\
+		emitm(&stream, (3 << 6) | r32, 1);			\
+	}								\
+	emitm(&stream, i32, 4);						\
+} while (0)
+
+/* testl sr32,dr32 */
+#define TESTrd(sr32, dr32) do {						\
+	emitm(&stream, 0x85, 1);					\
 	emitm(&stream,							\
-	    (3 << 6) | ((dr32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
-/* or r32,i32 */
-#define ORid(r32, i32) do {						\
+/* orl sr32,dr32 */
+#define ORrd(sr32, dr32) do {						\
+	emitm(&stream, 0x09, 1);					\
+	emitm(&stream,							\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
+} while (0)
+
+/* orl i32,r32 */
+#define ORid(i32, r32) do {						\
 	if (r32 == EAX) {						\
 		emitm(&stream, 0x0d, 1);				\
-		emitm(&stream, i32, 4);					\
 	} else {							\
 		emitm(&stream, 0x81, 1);				\
 		emitm(&stream, (25 << 3) | r32, 1);			\
-		emitm(&stream, i32, 4);					\
 	}								\
+	emitm(&stream, i32, 4);						\
 } while (0)
 
-/* shl r32,i8 */
-#define SHLib(r32, i8) do {						\
+/* shll i8,r32 */
+#define SHLib(i8, r32) do {						\
 	emitm(&stream, 0xc1, 1);					\
 	emitm(&stream, (7 << 5) | (r32 & 0x7), 1);			\
 	emitm(&stream, i8, 1);						\
 } while (0)
 
-/* shl dr32,cl */
+/* shll %cl,dr32 */
 #define SHL_CLrb(dr32) do {						\
 	emitm(&stream, 0xd3, 1);					\
 	emitm(&stream, (7 << 5) | (dr32 & 0x7), 1);			\
 } while (0)
 
-/* shr r32,i8 */
-#define SHRib(r32, i8) do {						\
+/* shrl i8,r32 */
+#define SHRib(i8, r32) do {						\
 	emitm(&stream, 0xc1, 1);					\
 	emitm(&stream, (29 << 3) | (r32 & 0x7), 1);			\
 	emitm(&stream, i8, 1);						\
 } while (0)
 
-/* shr dr32,cl */
+/* shrl %cl,dr32 */
 #define SHR_CLrb(dr32) do {						\
 	emitm(&stream, 0xd3, 1);					\
 	emitm(&stream, (29 << 3) | (dr32 & 0x7), 1);			\
 } while (0)
 
-/* neg r32 */
+/* negl r32 */
 #define NEGd(r32) do {							\
 	emitm(&stream, 0xf7, 1);					\
 	emitm(&stream, (27 << 3) | (r32 & 0x7), 1);			\
 } while (0)
 
-/* cmp dr32,sr32[off] */
-#define CMPodd(dr32, sr32, off) do {					\
-	emitm(&stream, (3 << 4) | 3 | (1 << 3), 1);			\
-	emitm(&stream,							\
-	    (1 << 6) | ((dr32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
-	emitm(&stream, off, 1);						\
-} while (0)
-
-/* cmp dr32,sr32 */
-#define CMPrd(dr32, sr32) do {						\
-	emitm(&stream, 0x3b, 1);					\
+/* cmpl sr32,dr32 */
+#define CMPrd(sr32, dr32) do {						\
+	emitm(&stream, 0x39, 1);					\
 	emitm(&stream,							\
-	    (3 << 6) | ((dr32 & 0x7) << 3) | (sr32 & 0x7), 1);		\
+	    (3 << 6) | ((sr32 & 0x7) << 3) | (dr32 & 0x7), 1);		\
 } while (0)
 
-/* cmp dr32,i32 */
-#define CMPid(dr32, i32) do {						\
+/* cmpl i32,dr32 */
+#define CMPid(i32, dr32) do {						\
 	if (dr32 == EAX){						\
 		emitm(&stream, 0x3d, 1);				\
 		emitm(&stream, i32, 4);					\
@@ -329,58 +331,28 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 	}								\
 } while (0)
 
-/* jne off32 */
-#define JNEb(off8) do {							\
-	emitm(&stream, 0x75, 1);					\
+/* jb off8 */
+#define JBb(off8) do {							\
+	emitm(&stream, 0x72, 1);					\
 	emitm(&stream, off8, 1);					\
 } while (0)
 
-/* je off32 */
-#define JE(off32) do {							\
-	emitm(&stream, 0x0f, 1);					\
-	emitm(&stream, 0x84, 1);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jle off32 */
-#define JLE(off32) do {							\
-	emitm(&stream, 0x0f, 1);					\
-	emitm(&stream, 0x8e, 1);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jle off8 */
-#define JLEb(off8) do {							\
-	emitm(&stream, 0x7e, 1);					\
+/* jae off8 */
+#define JAEb(off8) do {							\
+	emitm(&stream, 0x73, 1);					\
 	emitm(&stream, off8, 1);					\
 } while (0)
 
-/* ja off32 */
-#define JA(off32) do {							\
-	emitm(&stream, 0x0f, 1);					\
-	emitm(&stream, 0x87, 1);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jae off32 */
-#define JAE(off32) do {							\
-	emitm(&stream, 0x0f, 1);					\
-	emitm(&stream, 0x83, 1);					\
-	emitm(&stream, off32, 4);					\
-} while (0)
-
-/* jg off32 */
-#define JG(off32) do {							\
-	emitm(&stream, 0x0f, 1);					\
-	emitm(&stream, 0x8f, 1);					\
-	emitm(&stream, off32, 4);					\
+/* jne off8 */
+#define JNEb(off8) do {							\
+	emitm(&stream, 0x75, 1);					\
+	emitm(&stream, off8, 1);					\
 } while (0)
 
-/* jge off32 */
-#define JGE(off32) do {							\
-	emitm(&stream, 0x0f, 1);					\
-	emitm(&stream, 0x8d, 1);					\
-	emitm(&stream, off32, 4);					\
+/* ja off8 */
+#define JAb(off8) do {							\
+	emitm(&stream, 0x77, 1);					\
+	emitm(&stream, off8, 1);					\
 } while (0)
 
 /* jmp off32 */
@@ -389,16 +361,39 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
 	emitm(&stream, off32, 4);					\
 } while (0)
 
-/* xor eax,eax */
-#define ZERO_EAX() do {							\
+/* xorl r32,r32 */
+#define ZEROrd(r32) do {						\
 	emitm(&stream, 0x31, 1);					\
-	emitm(&stream, 0xc0, 1);					\
+	emitm(&stream, (3 << 6) | ((r32 & 0x7) << 3) | (r32 & 0x7), 1);	\
 } while (0)
 
-/* xor edx,edx */
-#define ZERO_EDX() do {							\
-	emitm(&stream, 0x31, 1);					\
-	emitm(&stream, 0xd2, 1);					\
+/*
+ * Conditional long jumps
+ */
+#define	JB	0x82
+#define	JAE	0x83
+#define	JE	0x84
+#define	JNE	0x85
+#define	JBE	0x86
+#define	JA	0x87
+
+#define	JCC(t, f) do {							\
+	if (ins->jt != 0 && ins->jf != 0) {				\
+		/* 5 is the size of the following jmp */		\
+		emitm(&stream, ((t) << 8) | 0x0f, 2);			\
+		emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -	\
+		    stream.refs[stream.bpf_pc] + 5, 4);			\
+		JMP(stream.refs[stream.bpf_pc + ins->jf] -		\
+		    stream.refs[stream.bpf_pc]);			\
+	} else if (ins->jt != 0) {					\
+		emitm(&stream, ((t) << 8) | 0x0f, 2);			\
+		emitm(&stream, stream.refs[stream.bpf_pc + ins->jt] -	\
+		    stream.refs[stream.bpf_pc], 4);			\
+	} else {							\
+		emitm(&stream, ((f) << 8) | 0x0f, 2);			\
+		emitm(&stream, stream.refs[stream.bpf_pc + ins->jf] -	\
+		    stream.refs[stream.bpf_pc], 4);			\
+	}								\
 } while (0)
 
 #endif	/* _BPF_JIT_MACHDEP_H_ */
diff --git a/sys/net/bpf_jitter.c b/sys/net/bpf_jitter.c
index b1a61c5ebb3c..69b2511f2a2e 100644
--- a/sys/net/bpf_jitter.c
+++ b/sys/net/bpf_jitter.c
@@ -1,6 +1,6 @@
 /*-
- * Copyright (c) 2002 - 2003 NetGroup, Politecnico di Torino (Italy)
- * Copyright (c) 2005 Jung-uk Kim <jkim@FreeBSD.org>
+ * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy)
+ * Copyright (C) 2005-2008 Jung-uk Kim <jkim@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -32,6 +32,7 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#ifdef _KERNEL
 #include "opt_bpf.h"
 
 #include <sys/param.h>
@@ -39,14 +40,19 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/sysctl.h>
+#else
+#include <stdlib.h>
+#include <sys/types.h>
+#endif
 
 #include <net/bpf.h>
 #include <net/bpf_jitter.h>
 
-MALLOC_DEFINE(M_BPFJIT, "BPF_JIT", "BPF JIT compiler");
-
 bpf_filter_func	bpf_jit_compile(struct bpf_insn *, u_int, int *);
 
+#ifdef _KERNEL
+MALLOC_DEFINE(M_BPFJIT, "BPF_JIT", "BPF JIT compiler");
+
 SYSCTL_NODE(_net, OID_AUTO, bpf_jitter, CTLFLAG_RW, 0, "BPF JIT compiler");
 int bpf_jitter_enable = 1;
 SYSCTL_INT(_net_bpf_jitter, OID_AUTO, enable, CTLFLAG_RW,
@@ -58,27 +64,27 @@ bpf_jitter(struct bpf_insn *fp, int nins)
 	bpf_jit_filter *filter;
 
 	/* Allocate the filter structure */
-	filter = (struct bpf_jit_filter *)malloc(sizeof(struct bpf_jit_filter),
+	filter = (struct bpf_jit_filter *)malloc(sizeof(*filter),
 	    M_BPFJIT, M_NOWAIT);
 	if (filter == NULL)
-		return NULL;
+		return (NULL);
 
 	/* Allocate the filter's memory */
 	filter->mem = (int *)malloc(BPF_MEMWORDS * sizeof(int),
 	    M_BPFJIT, M_NOWAIT);
 	if (filter->mem == NULL) {
 		free(filter, M_BPFJIT);
-		return NULL;
+		return (NULL);
 	}
 
 	/* Create the binary */
 	if ((filter->func = bpf_jit_compile(fp, nins, filter->mem)) == NULL) {
 		free(filter->mem, M_BPFJIT);
 		free(filter, M_BPFJIT);
-		return NULL;
+		return (NULL);
 	}
 
-	return filter;
+	return (filter);
 }
 
 void
@@ -89,3 +95,40 @@ bpf_destroy_jit_filter(bpf_jit_filter *filter)
 	free(filter->func, M_BPFJIT);
 	free(filter, M_BPFJIT);
 }
+#else
+bpf_jit_filter *
+bpf_jitter(struct bpf_insn *fp, int nins)
+{
+	bpf_jit_filter *filter;
+
+	/* Allocate the filter structure */
+	filter = (struct bpf_jit_filter *)malloc(sizeof(*filter));
+	if (filter == NULL)
+		return (NULL);
+
+	/* Allocate the filter's memory */
+	filter->mem = (int *)malloc(BPF_MEMWORDS * sizeof(int));
+	if (filter->mem == NULL) {
+		free(filter);
+		return (NULL);
+	}
+
+	/* Create the binary */
+	if ((filter->func = bpf_jit_compile(fp, nins, filter->mem)) == NULL) {
+		free(filter->mem);
+		free(filter);
+		return (NULL);
+	}
+
+	return (filter);
+}
+
+void
+bpf_destroy_jit_filter(bpf_jit_filter *filter)
+{
+
+	free(filter->mem);
+	free(filter->func);
+	free(filter);
+}
+#endif
diff --git a/sys/net/bpf_jitter.h b/sys/net/bpf_jitter.h
index 9eed2c92ddb9..bba33a4574eb 100644
--- a/sys/net/bpf_jitter.h
+++ b/sys/net/bpf_jitter.h
@@ -1,6 +1,6 @@
 /*-
- * Copyright (c) 2002 - 2003 NetGroup, Politecnico di Torino (Italy)
- * Copyright (c) 2005 Jung-uk Kim <jkim@FreeBSD.org>
+ * Copyright (C) 2002-2003 NetGroup, Politecnico di Torino (Italy)
+ * Copyright (C) 2005-2008 Jung-uk Kim <jkim@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -34,7 +34,9 @@
 #ifndef _NET_BPF_JITTER_H_
 #define _NET_BPF_JITTER_H_
 
+#ifdef _KERNEL
 MALLOC_DECLARE(M_BPFJIT);
+#endif
 
 extern int bpf_jitter_enable;