5 files changed, 55 insertions, 0 deletions

diff --git a/textproc/Makefile b/textproc/Makefile
index 7a67bc110dfc..1ea6fc168399 100644
--- a/textproc/Makefile
+++ b/textproc/Makefile
@@ -405,6 +405,7 @@
     SUBDIR += p5-String-ShellQuote
     SUBDIR += p5-String-ShowDiff
     SUBDIR += p5-String-Strip
+    SUBDIR += p5-String-Tokenizer
     SUBDIR += p5-Syntax-Highlight-Perl-Improved
     SUBDIR += p5-TeX-Hyphen
     SUBDIR += p5-Template-Extract
diff --git a/textproc/p5-String-Tokenizer/Makefile b/textproc/p5-String-Tokenizer/Makefile
new file mode 100644
index 000000000000..71bd48b94b15
--- /dev/null
+++ b/textproc/p5-String-Tokenizer/Makefile
@@ -0,0 +1,28 @@
+# New ports collection makefile for:	String-Tokenizer
+# Date created:				19 Feb 2006
+# Whom:					Aaron Dalton <aaron@FreeBSD.org>
+#
+# $FreeBSD$
+#
+
+PORTNAME=	String-Tokenizer
+PORTVERSION=	0.05
+CATEGORIES=	textproc perl5
+MASTER_SITES=	${MASTER_SITE_PERL_CPAN}
+MASTER_SITE_SUBDIR=	String
+PKGNAMEPREFIX=	p5-
+
+MAINTAINER=	aaron@FreeBSD.org
+COMMENT=	A simple string tokenizer
+
+MAN3=	String::Tokenizer.3
+
+PERL_CONFIGURE=	yes
+
+.include <bsd.port.pre.mk>
+
+.if ${PERL_LEVEL} < 500600
+IGNORE=	requires at least Perl5.6
+.endif
+
+.include <bsd.port.post.mk>
diff --git a/textproc/p5-String-Tokenizer/distinfo b/textproc/p5-String-Tokenizer/distinfo
new file mode 100644
index 000000000000..d2e7bb6cc810
--- /dev/null
+++ b/textproc/p5-String-Tokenizer/distinfo
@@ -0,0 +1,3 @@
+MD5 (String-Tokenizer-0.05.tar.gz) = 650bfc09bc0c7b671a67329b2bb901f8
+SHA256 (String-Tokenizer-0.05.tar.gz) = 576de65386eed4dc2e40dd12f5a5da2cdc7b2daebec6934c85d7636cfce94a93
+SIZE (String-Tokenizer-0.05.tar.gz) = 8591
diff --git a/textproc/p5-String-Tokenizer/pkg-descr b/textproc/p5-String-Tokenizer/pkg-descr
new file mode 100644
index 000000000000..4987dd069305
--- /dev/null
+++ b/textproc/p5-String-Tokenizer/pkg-descr
@@ -0,0 +1,18 @@
+A simple string tokenizer which takes a string and splits it on 
+whitespace. It also optionally takes a string of characters to use as 
+delimiters, and returns them with the token set as well. This allows for 
+splitting the string in many different ways.
+
+This is a very basic tokenizer, so more complex needs should be either 
+addressed with a custom written tokenizer or post-processing of the output 
+generated by this module. Basically, this will not fill everyones needs, 
+but it spans a gap between simple split / /, $string and the other options 
+that involve much larger and complex modules.
+
+Also note that this is not a lexical analyser. Many people confuse 
+tokenization with lexical analysis. A tokenizer mearly splits its input 
+into specific chunks, a lexical analyzer classifies those chunks. 
+Sometimes these two steps are combined, but not here.
+
+WWW: http://search.cpan.org/dist/String-Tokenizer
+Author: stevan little <stevan@iinteractive.com>
diff --git a/textproc/p5-String-Tokenizer/pkg-plist b/textproc/p5-String-Tokenizer/pkg-plist
new file mode 100644
index 000000000000..96d19bf71056
--- /dev/null
+++ b/textproc/p5-String-Tokenizer/pkg-plist
@@ -0,0 +1,5 @@
+%%SITE_PERL%%/String/Tokenizer.pm
+%%SITE_PERL%%/%%PERL_ARCH%%/auto/String/Tokenizer/.packlist
+@dirrm %%SITE_PERL%%/%%PERL_ARCH%%/auto/String/Tokenizer
+@dirrmtry %%SITE_PERL%%/%%PERL_ARCH%%/auto/String
+@dirrmtry %%SITE_PERL%%/String