simon/milkwood-java
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

General tidy up, removal of dead code, and bring back to Java 7 level.

Browse source
This commit is contained in:
simon 2013-10-31 13:38:04 +00:00
parent a876cb6d1b
commit 27efd21089
17 changed files with 699 additions and 832 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
/build/
/dist/

77
nbproject/build-impl.xml
View file

@ -54,43 +54,6 @@ is divided into following sections:
<property file="nbproject/project.properties"/> <property file="nbproject/project.properties"/>
</target> </target>
<target depends="-pre-init,-init-private,-init-user,-init-project,-init-macrodef-property" name="-do-init"> <target depends="-pre-init,-init-private,-init-user,-init-project,-init-macrodef-property" name="-do-init">
<j2seproject1:property name="platform.home" value="platforms.${platform.active}.home"/>
<j2seproject1:property name="platform.bootcp" value="platforms.${platform.active}.bootclasspath"/>
<j2seproject1:property name="platform.compiler" value="platforms.${platform.active}.compile"/>
<j2seproject1:property name="platform.javac.tmp" value="platforms.${platform.active}.javac"/>
<condition property="platform.javac" value="${platform.home}/bin/javac">
<equals arg1="${platform.javac.tmp}" arg2="$${platforms.${platform.active}.javac}"/>
</condition>
<property name="platform.javac" value="${platform.javac.tmp}"/>
<j2seproject1:property name="platform.java.tmp" value="platforms.${platform.active}.java"/>
<condition property="platform.java" value="${platform.home}/bin/java">
<equals arg1="${platform.java.tmp}" arg2="$${platforms.${platform.active}.java}"/>
</condition>
<property name="platform.java" value="${platform.java.tmp}"/>
<j2seproject1:property name="platform.javadoc.tmp" value="platforms.${platform.active}.javadoc"/>
<condition property="platform.javadoc" value="${platform.home}/bin/javadoc">
<equals arg1="${platform.javadoc.tmp}" arg2="$${platforms.${platform.active}.javadoc}"/>
</condition>
<property name="platform.javadoc" value="${platform.javadoc.tmp}"/>
<condition property="platform.invalid" value="true">
<or>
<contains string="${platform.javac}" substring="$${platforms."/>
<contains string="${platform.java}" substring="$${platforms."/>
<contains string="${platform.javadoc}" substring="$${platforms."/>
</or>
</condition>
<fail unless="platform.home">Must set platform.home</fail>
<fail unless="platform.bootcp">Must set platform.bootcp</fail>
<fail unless="platform.java">Must set platform.java</fail>
<fail unless="platform.javac">Must set platform.javac</fail>
<fail if="platform.invalid">
The J2SE Platform is not correctly set up.
Your active platform is: ${platform.active}, but the corresponding property "platforms.${platform.active}.home" is not found in the project's properties files.
Either open the project in the IDE and setup the Platform with the same name or add it manually.
For example like this:
ant -Duser.properties.file=&lt;path_to_property_file&gt; jar (where you put the property "platforms.${platform.active}.home" in a .properties file)
or ant -Dplatforms.${platform.active}.home=&lt;path_to_JDK_home&gt; jar (where no properties file is used)
</fail>
<available file="${manifest.file}" property="manifest.available"/> <available file="${manifest.file}" property="manifest.available"/>
<condition property="splashscreen.available"> <condition property="splashscreen.available">
<and> <and>
@ -225,6 +188,15 @@ is divided into following sections:
<condition else="" property="endorsed.classpath.cmd.line.arg" value="-Xbootclasspath/p:'${toString:endorsed.classpath.path}'"> <condition else="" property="endorsed.classpath.cmd.line.arg" value="-Xbootclasspath/p:'${toString:endorsed.classpath.path}'">
<length length="0" string="${endorsed.classpath}" when="greater"/> <length length="0" string="${endorsed.classpath}" when="greater"/>
</condition> </condition>
<condition else="false" property="jdkBug6558476">
<and>
<matches pattern="1\.[56]" string="${java.specification.version}"/>
<not>
<os family="unix"/>
</not>
</and>
</condition>
<property name="javac.fork" value="${jdkBug6558476}"/>
<property name="jar.index" value="false"/> <property name="jar.index" value="false"/>
<property name="jar.index.metainf" value="${jar.index}"/> <property name="jar.index.metainf" value="${jar.index}"/>
<property name="copylibs.rebase" value="true"/> <property name="copylibs.rebase" value="true"/>
@ -293,7 +265,7 @@ is divided into following sections:
<property location="${build.dir}/empty" name="empty.dir"/> <property location="${build.dir}/empty" name="empty.dir"/>
<mkdir dir="${empty.dir}"/> <mkdir dir="${empty.dir}"/>
<mkdir dir="@{apgeneratedsrcdir}"/> <mkdir dir="@{apgeneratedsrcdir}"/>
<javac debug="@{debug}" deprecation="${javac.deprecation}" destdir="@{destdir}" encoding="${source.encoding}" excludes="@{excludes}" executable="${platform.javac}" fork="yes" includeantruntime="false" includes="@{includes}" source="${javac.source}" sourcepath="@{sourcepath}" srcdir="@{srcdir}" target="${javac.target}" tempdir="${java.io.tmpdir}"> <javac debug="@{debug}" deprecation="${javac.deprecation}" destdir="@{destdir}" encoding="${source.encoding}" excludes="@{excludes}" fork="${javac.fork}" includeantruntime="false" includes="@{includes}" source="${javac.source}" sourcepath="@{sourcepath}" srcdir="@{srcdir}" target="${javac.target}" tempdir="${java.io.tmpdir}">
<src> <src>
<dirset dir="@{gensrcdir}" erroronmissingdir="false"> <dirset dir="@{gensrcdir}" erroronmissingdir="false">
<include name="*"/> <include name="*"/>
@ -332,7 +304,7 @@ is divided into following sections:
<sequential> <sequential>
<property location="${build.dir}/empty" name="empty.dir"/> <property location="${build.dir}/empty" name="empty.dir"/>
<mkdir dir="${empty.dir}"/> <mkdir dir="${empty.dir}"/>
<javac debug="@{debug}" deprecation="${javac.deprecation}" destdir="@{destdir}" encoding="${source.encoding}" excludes="@{excludes}" executable="${platform.javac}" fork="yes" includeantruntime="false" includes="@{includes}" source="${javac.source}" sourcepath="@{sourcepath}" srcdir="@{srcdir}" target="${javac.target}" tempdir="${java.io.tmpdir}"> <javac debug="@{debug}" deprecation="${javac.deprecation}" destdir="@{destdir}" encoding="${source.encoding}" excludes="@{excludes}" fork="${javac.fork}" includeantruntime="false" includes="@{includes}" source="${javac.source}" sourcepath="@{sourcepath}" srcdir="@{srcdir}" target="${javac.target}" tempdir="${java.io.tmpdir}">
<src> <src>
<dirset dir="@{gensrcdir}" erroronmissingdir="false"> <dirset dir="@{gensrcdir}" erroronmissingdir="false">
<include name="*"/> <include name="*"/>
@ -412,7 +384,7 @@ is divided into following sections:
<element name="customize" optional="true"/> <element name="customize" optional="true"/>
<sequential> <sequential>
<property name="junit.forkmode" value="perTest"/> <property name="junit.forkmode" value="perTest"/>
<junit dir="${work.dir}" errorproperty="tests.failed" failureproperty="tests.failed" fork="true" forkmode="${junit.forkmode}" jvm="${platform.java}" showoutput="true" tempdir="${build.dir}"> <junit dir="${work.dir}" errorproperty="tests.failed" failureproperty="tests.failed" fork="true" forkmode="${junit.forkmode}" showoutput="true" tempdir="${build.dir}">
<test methods="@{testmethods}" name="@{testincludes}" todir="${build.test.results.dir}"/> <test methods="@{testmethods}" name="@{testincludes}" todir="${build.test.results.dir}"/>
<syspropertyset> <syspropertyset>
<propertyref prefix="test-sys-prop."/> <propertyref prefix="test-sys-prop."/>
@ -435,7 +407,7 @@ is divided into following sections:
<element name="customize" optional="true"/> <element name="customize" optional="true"/>
<sequential> <sequential>
<property name="junit.forkmode" value="perTest"/> <property name="junit.forkmode" value="perTest"/>
<junit dir="${work.dir}" errorproperty="tests.failed" failureproperty="tests.failed" fork="true" forkmode="${junit.forkmode}" jvm="${platform.java}" showoutput="true" tempdir="${build.dir}"> <junit dir="${work.dir}" errorproperty="tests.failed" failureproperty="tests.failed" fork="true" forkmode="${junit.forkmode}" showoutput="true" tempdir="${build.dir}">
<batchtest todir="${build.test.results.dir}"> <batchtest todir="${build.test.results.dir}">
<fileset dir="${test.src.dir}" excludes="@{excludes},${excludes}" includes="@{includes}"> <fileset dir="${test.src.dir}" excludes="@{excludes},${excludes}" includes="@{includes}">
<filename name="@{testincludes}"/> <filename name="@{testincludes}"/>
@ -474,7 +446,7 @@ is divided into following sections:
</fileset> </fileset>
</union> </union>
<taskdef classname="org.testng.TestNGAntTask" classpath="${run.test.classpath}" name="testng"/> <taskdef classname="org.testng.TestNGAntTask" classpath="${run.test.classpath}" name="testng"/>
<testng classfilesetref="test.set" failureProperty="tests.failed" jvm="${platform.java}" methods="${testng.methods.arg}" mode="${testng.mode}" outputdir="${build.test.results.dir}" suitename="milkwood" testname="TestNG tests" workingDir="${work.dir}"> <testng classfilesetref="test.set" failureProperty="tests.failed" methods="${testng.methods.arg}" mode="${testng.mode}" outputdir="${build.test.results.dir}" suitename="milkwood" testname="TestNG tests" workingDir="${work.dir}">
<xmlfileset dir="${build.test.classes.dir}" includes="@{testincludes}"/> <xmlfileset dir="${build.test.classes.dir}" includes="@{testincludes}"/>
<propertyset> <propertyset>
<propertyref prefix="test-sys-prop."/> <propertyref prefix="test-sys-prop."/>
@ -554,7 +526,7 @@ is divided into following sections:
<element name="customize" optional="true"/> <element name="customize" optional="true"/>
<sequential> <sequential>
<property name="junit.forkmode" value="perTest"/> <property name="junit.forkmode" value="perTest"/>
<junit dir="${work.dir}" errorproperty="tests.failed" failureproperty="tests.failed" fork="true" forkmode="${junit.forkmode}" jvm="${platform.java}" showoutput="true" tempdir="${build.dir}"> <junit dir="${work.dir}" errorproperty="tests.failed" failureproperty="tests.failed" fork="true" forkmode="${junit.forkmode}" showoutput="true" tempdir="${build.dir}">
<test methods="@{testmethods}" name="@{testincludes}" todir="${build.test.results.dir}"/> <test methods="@{testmethods}" name="@{testincludes}" todir="${build.test.results.dir}"/>
<syspropertyset> <syspropertyset>
<propertyref prefix="test-sys-prop."/> <propertyref prefix="test-sys-prop."/>
@ -579,7 +551,7 @@ is divided into following sections:
<element name="customize" optional="true"/> <element name="customize" optional="true"/>
<sequential> <sequential>
<property name="junit.forkmode" value="perTest"/> <property name="junit.forkmode" value="perTest"/>
<junit dir="${work.dir}" errorproperty="tests.failed" failureproperty="tests.failed" fork="true" forkmode="${junit.forkmode}" jvm="${platform.java}" showoutput="true" tempdir="${build.dir}"> <junit dir="${work.dir}" errorproperty="tests.failed" failureproperty="tests.failed" fork="true" forkmode="${junit.forkmode}" showoutput="true" tempdir="${build.dir}">
<batchtest todir="${build.test.results.dir}"> <batchtest todir="${build.test.results.dir}">
<fileset dir="${test.src.dir}" excludes="@{excludes},${excludes}" includes="@{includes}"> <fileset dir="${test.src.dir}" excludes="@{excludes},${excludes}" includes="@{includes}">
<filename name="@{testincludes}"/> <filename name="@{testincludes}"/>
@ -759,9 +731,6 @@ is divided into following sections:
<classpath> <classpath>
<path path="@{classpath}"/> <path path="@{classpath}"/>
</classpath> </classpath>
<bootclasspath>
<path path="${platform.bootcp}"/>
</bootclasspath>
</nbjpdastart> </nbjpdastart>
</sequential> </sequential>
</macrodef> </macrodef>
@ -777,9 +746,7 @@ is divided into following sections:
</macrodef> </macrodef>
</target> </target>
<target name="-init-debug-args"> <target name="-init-debug-args">
<exec executable="${platform.java}" outputproperty="version-output"> <property name="version-output" value="java version &quot;${ant.java.version}"/>
<arg value="-version"/>
</exec>
<condition property="have-jdk-older-than-1.4"> <condition property="have-jdk-older-than-1.4">
<or> <or>
<contains string="${version-output}" substring="java version &quot;1.0"/> <contains string="${version-output}" substring="java version &quot;1.0"/>
@ -804,7 +771,7 @@ is divided into following sections:
<attribute default="${debug.classpath}" name="classpath"/> <attribute default="${debug.classpath}" name="classpath"/>
<element name="customize" optional="true"/> <element name="customize" optional="true"/>
<sequential> <sequential>
<java classname="@{classname}" dir="${work.dir}" fork="true" jvm="${platform.java}"> <java classname="@{classname}" dir="${work.dir}" fork="true">
<jvmarg line="${endorsed.classpath.cmd.line.arg}"/> <jvmarg line="${endorsed.classpath.cmd.line.arg}"/>
<jvmarg line="${debug-args-line}"/> <jvmarg line="${debug-args-line}"/>
<jvmarg value="-Xrunjdwp:transport=${debug-transport},address=${jpda.address}"/> <jvmarg value="-Xrunjdwp:transport=${debug-transport},address=${jpda.address}"/>
@ -831,7 +798,7 @@ is divided into following sections:
<attribute default="jvm" name="jvm"/> <attribute default="jvm" name="jvm"/>
<element name="customize" optional="true"/> <element name="customize" optional="true"/>
<sequential> <sequential>
<java classname="@{classname}" dir="${work.dir}" fork="true" jvm="${platform.java}"> <java classname="@{classname}" dir="${work.dir}" fork="true">
<jvmarg line="${endorsed.classpath.cmd.line.arg}"/> <jvmarg line="${endorsed.classpath.cmd.line.arg}"/>
<jvmarg value="-Dfile.encoding=${runtime.encoding}"/> <jvmarg value="-Dfile.encoding=${runtime.encoding}"/>
<redirector errorencoding="${runtime.encoding}" inputencoding="${runtime.encoding}" outputencoding="${runtime.encoding}"/> <redirector errorencoding="${runtime.encoding}" inputencoding="${runtime.encoding}" outputencoding="${runtime.encoding}"/>
@ -1018,7 +985,7 @@ is divided into following sections:
<path path="${run.classpath}"/> <path path="${run.classpath}"/>
<map from="${build.classes.dir.resolved}" to="${dist.jar.resolved}"/> <map from="${build.classes.dir.resolved}" to="${dist.jar.resolved}"/>
</pathconvert> </pathconvert>
<echo level="info">${platform.java} -cp "${run.classpath.with.dist.jar}" ${main.class}</echo> <echo level="info">java -cp "${run.classpath.with.dist.jar}" ${main.class}</echo>
</target> </target>
<target depends="init" if="do.archive" name="-do-jar-with-libraries-create-manifest" unless="manifest.available"> <target depends="init" if="do.archive" name="-do-jar-with-libraries-create-manifest" unless="manifest.available">
<tempfile deleteonexit="true" destdir="${build.dir}" property="tmp.manifest.file"/> <tempfile deleteonexit="true" destdir="${build.dir}" property="tmp.manifest.file"/>
@ -1045,7 +1012,7 @@ is divided into following sections:
<j2seproject3:copylibs manifest="${tmp.manifest.file}"/> <j2seproject3:copylibs manifest="${tmp.manifest.file}"/>
<echo level="info">To run this application from the command line without Ant, try:</echo> <echo level="info">To run this application from the command line without Ant, try:</echo>
<property location="${dist.jar}" name="dist.jar.resolved"/> <property location="${dist.jar}" name="dist.jar.resolved"/>
<echo level="info">${platform.java} -jar "${dist.jar.resolved}"</echo> <echo level="info">java -jar "${dist.jar.resolved}"</echo>
</target> </target>
<target depends="-do-jar-with-libraries-pack" if="do.archive" name="-do-jar-with-libraries-delete-manifest"> <target depends="-do-jar-with-libraries-pack" if="do.archive" name="-do-jar-with-libraries-delete-manifest">
<delete> <delete>
@ -1236,7 +1203,7 @@ is divided into following sections:
</not> </not>
</and> </and>
</condition> </condition>
<javadoc additionalparam="${javadoc.additionalparam}" author="${javadoc.author}" charset="UTF-8" destdir="${dist.javadoc.dir}" docencoding="UTF-8" encoding="${javadoc.encoding.used}" executable="${platform.javadoc}" failonerror="true" noindex="${javadoc.noindex}" nonavbar="${javadoc.nonavbar}" notree="${javadoc.notree}" private="${javadoc.private}" source="${javac.source}" splitindex="${javadoc.splitindex}" use="${javadoc.use}" useexternalfile="true" version="${javadoc.version}" windowtitle="${javadoc.windowtitle}"> <javadoc additionalparam="${javadoc.additionalparam}" author="${javadoc.author}" charset="UTF-8" destdir="${dist.javadoc.dir}" docencoding="UTF-8" encoding="${javadoc.encoding.used}" failonerror="true" noindex="${javadoc.noindex}" nonavbar="${javadoc.nonavbar}" notree="${javadoc.notree}" private="${javadoc.private}" source="${javac.source}" splitindex="${javadoc.splitindex}" use="${javadoc.use}" useexternalfile="true" version="${javadoc.version}" windowtitle="${javadoc.windowtitle}">
<classpath> <classpath>
<path path="${javac.classpath}"/> <path path="${javac.classpath}"/>
</classpath> </classpath>

6
nbproject/genfiles.properties
View file

@ -1,8 +1,8 @@
build.xml.data.CRC32=d35b316e build.xml.data.CRC32=31018a52
build.xml.script.CRC32=cd5c02b3 build.xml.script.CRC32=cd5c02b3
build.xml.stylesheet.CRC32=28e38971@1.56.1.46 build.xml.stylesheet.CRC32=28e38971@1.56.1.46
# This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml. # This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml.
# Do not edit this file. You may delete it but then the IDE will never regenerate such files for you. # Do not edit this file. You may delete it but then the IDE will never regenerate such files for you.
nbproject/build-impl.xml.data.CRC32=d35b316e nbproject/build-impl.xml.data.CRC32=31018a52
nbproject/build-impl.xml.script.CRC32=0441a68e nbproject/build-impl.xml.script.CRC32=fe6e4d15
nbproject/build-impl.xml.stylesheet.CRC32=c6d2a60f@1.56.1.46 nbproject/build-impl.xml.stylesheet.CRC32=c6d2a60f@1.56.1.46

4
nbproject/private/private.properties
View file

@ -1,2 +1,6 @@
compile.on.save=true compile.on.save=true
do.depend=false
do.jar=true
javac.debug=true
javadoc.preview=true
user.properties.file=/home/simon/.jmonkeyplatform/3.0/build.properties user.properties.file=/home/simon/.jmonkeyplatform/3.0/build.properties

11
nbproject/project.properties
View file

@ -1,9 +1,10 @@
annotation.processing.enabled=true annotation.processing.enabled=true
annotation.processing.enabled.in.editor=false annotation.processing.enabled.in.editor=false
annotation.processing.processor.options=
annotation.processing.processors.list= annotation.processing.processors.list=
annotation.processing.run.all.processors=true annotation.processing.run.all.processors=true
annotation.processing.source.output=${build.generated.sources.dir}/ap-source-output annotation.processing.source.output=${build.generated.sources.dir}/ap-source-output
application.title=milkwood
application.vendor=simon
build.classes.dir=${build.dir}/classes build.classes.dir=${build.dir}/classes
build.classes.excludes=**/*.java,**/*.form build.classes.excludes=**/*.java,**/*.form
# This directory is removed when the project is cleaned: # This directory is removed when the project is cleaned:
@ -24,6 +25,7 @@ debug.test.classpath=\
dist.dir=dist dist.dir=dist
dist.jar=${dist.dir}/milkwood.jar dist.jar=${dist.dir}/milkwood.jar
dist.javadoc.dir=${dist.dir}/javadoc dist.javadoc.dir=${dist.dir}/javadoc
endorsed.classpath=
excludes= excludes=
includes=** includes=**
jar.compress=false jar.compress=false
@ -33,8 +35,8 @@ javac.compilerargs=
javac.deprecation=false javac.deprecation=false
javac.processorpath=\ javac.processorpath=\
${javac.classpath} ${javac.classpath}
javac.source=1.6 javac.source=1.7
javac.target=1.6 javac.target=1.7
javac.test.classpath=\ javac.test.classpath=\
${javac.classpath}:\ ${javac.classpath}:\
${build.classes.dir} ${build.classes.dir}
@ -55,7 +57,8 @@ main.class=cc.journeyman.milkwood.Milkwood
manifest.file=manifest.mf manifest.file=manifest.mf
meta.inf.dir=${src.dir}/META-INF meta.inf.dir=${src.dir}/META-INF
mkdist.disabled=false mkdist.disabled=false
platform.active=JDK_1.6 obfuscate.options=-keep public class * extends com.jme3.app.Application{public *;}\n-keep public class * extends com.jme3.system.JmeSystemDelegate{public *;}\n-keep public class * implements com.jme3.renderer.Renderer{public *;}\n-keep public class * implements com.jme3.asset.AssetLoader{public *;}\n-keep public class * implements com.jme3.asset.AssetLocator{public *;}\n-keep public class * implements de.lessvoid.nifty.screen.ScreenController{public *;}\n-dontwarn\n-dontnote\n
platform.active=default_platform
platforms.JDK_1.6.home=/usr/lib/jvm/java-6-openjdk-amd64/ platforms.JDK_1.6.home=/usr/lib/jvm/java-6-openjdk-amd64/
run.classpath=\ run.classpath=\
${javac.classpath}:\ ${javac.classpath}:\

1
nbproject/project.xml
View file

@ -4,7 +4,6 @@
<configuration> <configuration>
<data xmlns="http://www.netbeans.org/ns/j2se-project/3"> <data xmlns="http://www.netbeans.org/ns/j2se-project/3">
<name>milkwood</name> <name>milkwood</name>
<explicit-platform explicit-source-supported="true"/>
<source-roots> <source-roots>
<root id="src.dir"/> <root id="src.dir"/>
</source-roots> </source-roots>

215
src/cc/journeyman/milkwood/Composer.java
View file

@ -1,7 +1,6 @@
package cc.journeyman.milkwood; package cc.journeyman.milkwood;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
/** /**
* Composes text output based on a rule tree. * Composes text output based on a rule tree.
@ -10,130 +9,126 @@ import java.util.Collections;
* *
*/ */
public class Composer { public class Composer {
/**
* Whether or not I am in debugging mode.
*/
private final boolean debug;
/** /**
* * Whether or not I am in debugging mode.
* @param debug */
* Whether or not I am in debugging mode. private final boolean debug;
*/
public Composer(boolean debug) {
this.debug = debug;
}
/** /**
* Recursive, backtracking, output generator. *
* * @param debug Whether or not I am in debugging mode.
* @param rules the rule set we're working to. */
* @param length the number of tokens still to be output. public Composer(boolean debug) {
* @return if a successful path forward is found, that path, else null. this.debug = debug;
*/ }
public WordSequence compose(RuleTreeNode rules, int length) {
WordStack preamble = composePreamble(rules);
WordSequence result = new WordSequence();
// composing the preamble will have ended with *ROOT* on top of the /**
// stack; * Recursive, backtracking, output generator.
// get rid of it. *
preamble.pop(); * @param rules the rule set we're working to.
* @param length the number of tokens still to be output.
* @return if a successful path forward is found, that path, else null.
*/
protected WordSequence compose(RuleTreeNode rules, int length) {
Window preamble = composePreamble(rules);
WordSequence result = new WordSequence();
if (debug) { // composing the preamble will have ended with *ROOT* on top of the
System.err.println( "Preamble: " + preamble); // stack;
} // get rid of it.
preamble.pop();
result.addAll(preamble); if (debug) {
System.err.println("Preamble: " + preamble);
}
WordStack body = this.compose(preamble, rules, length); result.addAll(preamble);
Collections.reverse(body);
result.addAll(body);
return result; result.addAll(this.compose(preamble, rules, length));
}
/** return result;
* Recursively attempt to find sequences in the ruleset to append to what's }
* been composed so far.
*
* @param glanceBack the last few words output.
* @param rules the rule set we're working to.
* @param length the number of tokens still to be output.
* @return if a successful path forward is found, that path, else null.
*/
private WordStack compose(WordStack glanceBack, RuleTreeNode rules,
int length) {
final WordStack result;
if ( debug) { /**
System.err.println( String.format( "%d: %s", length, glanceBack)); * Recursively attempt to find sequences in the ruleset to append to what's
} * been composed so far.
*
* @param glanceBack the last few words output.
* @param rules the rule set we're working to.
* @param length the number of tokens still to be output.
* @return if a successful path forward is found, that path, else null.
*/
private WordSequence compose(Window glanceBack, RuleTreeNode rules,
int length) {
final WordSequence result;
/* are we there yet? */ if (debug) {
if (length == 0) { System.err.println(String.format("%d: %s", length, glanceBack));
result = new WordStack(); }
} else {
/*
* are there any rules in this ruleset which matches the current
* sliding window? if so, then recurse; if not, then fail.
*/
Collection<String> words = rules.match(glanceBack.duplicate());
if (words.isEmpty()) { /* are we there yet? */
/* backtrack */ if (length == 0) {
result = null; result = new WordSequence();
} else { } else {
result = tryOptions(words, glanceBack, rules, length); /*
} * are there any rules in this ruleset which matches the current
} * sliding window? if so, then recurse; if not, then fail.
return result; */
} Collection<String> words = rules.match(glanceBack.duplicate());
/** if (words.isEmpty()) {
* Try each of these candidates in turn, attempting to recurse. /* backtrack */
* @param candidates words which could potentially be added to the output. result = null;
* @param glanceBack the last few words output. } else {
* @param allRules the rule set we're working to. result = tryOptions(words, glanceBack, rules, length);
* @param length the number of tokens still to be output. }
* @return if a successful path forward is found, that path, else null. }
*/ return result;
private WordStack tryOptions(Collection<String> candidates, }
WordStack glanceBack, RuleTreeNode allRules, int length) {
WordStack result = null;
for ( String candidate : candidates) { /**
result = compose( new WordStack(glanceBack, candidate), allRules, length - 1); * Try each of these candidates in turn, attempting to recurse.
if ( result != null) { *
/* by Jove, I think she's got it! */ * @param candidates words which could potentially be added to the output.
result.push(candidate); * @param glanceBack the last few words output.
break; * @param allRules the rule set we're working to.
} * @param length the number of tokens still to be output.
} * @return if a successful path forward is found, that path, else null.
*/
private WordSequence tryOptions(Collection<String> candidates,
Window glanceBack, RuleTreeNode allRules, int length) {
WordSequence result = null;
return result; for (String candidate : candidates) {
} result = compose(new Window(glanceBack, candidate), allRules, length - 1);
if (result != null) {
/* by Jove, I think she's got it! */
result.push(candidate);
break;
}
}
return result;
}
/** /**
* Random walk of the rule tree to extract (from the root) a legal sequence * Random walk of the rule tree to extract (from the root) a legal sequence
* of words the length of our tuple. * of words the length of our tuple.
* *
* @param rules * @param rules the rule tree (fragment) to walk.
* the rule tree (fragment) to walk. * @return a sequence of words.
* @return a sequence of words. */
*/ private Window composePreamble(RuleTreeNode rules) {
private WordStack composePreamble(RuleTreeNode rules) { final Window result;
final WordStack result; final RuleTreeNode successor = rules.getRule();
final RuleTreeNode successor = rules.getRule();
if (successor == null) {
result = new WordStack();
} else {
result = this.composePreamble(successor);
result.push(rules.getWord());
}
return result;
}
if (successor == null) {
result = new Window();
} else {
result = this.composePreamble(successor);
result.push(rules.getWord());
}
return result;
}
} }

64
src/cc/journeyman/milkwood/Digester.java
View file

@ -20,41 +20,39 @@ import java.util.Queue;
* *
*/ */
public class Digester { public class Digester {
/**
* Read tokens from the input stream, and compile them into the rule tree
* below this root.
*
* @param in
* the input stream from which I read.
* @param tupleLength
* the length of the tuples I read.
* @param root
* the ruleset to which I shall add.
* @return the number of tokens read.
* @throws IOException if can't read from file system.
*/
protected int read(final InputStream in, final int tupleLength,
final RuleTreeNode root) throws IOException {
int result = 0;
final Queue<WordSequence> openTuples = new LinkedList<WordSequence>();
final Tokeniser tok = new Tokeniser(in);
for (int type = tok.nextToken(); type != StreamTokenizer.TT_EOF; type = tok /**
.nextToken()) { * Read tokens from the input stream, and compile them into the rule tree
result++; * below this root.
final WordSequence newTuple = new WordSequence(); *
String token = tok.readBareToken(); * @param in the input stream from which I read.
* @param tupleLength the length of the tuples I read.
* @param root the ruleset to which I shall add.
* @return the number of tokens read.
* @throws IOException if can't read from file system.
*/
protected int digest(final InputStream in, final int tupleLength,
final RuleTreeNode root) throws IOException {
int result = 0;
final Queue<WordSequence> openTuples = new LinkedList<>();
final Tokeniser tok = new Tokeniser(in);
openTuples.add(newTuple); for (int type = tok.nextToken(); type != StreamTokenizer.TT_EOF; type = tok
for (WordSequence tuple : openTuples) { .nextToken()) {
tuple.add(token); result++;
} final WordSequence newTuple = new WordSequence();
String token = tok.readBareToken();
if (openTuples.size() > tupleLength) { openTuples.add(newTuple);
root.addSequence(openTuples.remove()); for (WordSequence tuple : openTuples) {
} tuple.add(token);
} }
return result; if (openTuples.size() > tupleLength) {
} root.addSequence(openTuples.remove());
}
}
return result;
}
} }

302
src/cc/journeyman/milkwood/Milkwood.java
View file

@ -1,3 +1,9 @@
/*
* Proprietary unpublished source code property of
* Simon Brooke <simon@journeyman.cc>.
*
* Copyright (c) 2013 Simon Brooke <simon@journeyman.cc>
*/
package cc.journeyman.milkwood; package cc.journeyman.milkwood;
import java.io.File; import java.io.File;
@ -8,182 +14,158 @@ import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
/*
* Proprietary unpublished source code property of
* Simon Brooke <simon@journeyman.cc>.
*
* Copyright (c) 2013 Simon Brooke <simon@journeyman.cc>
*/
/** /**
* Text mangler based on
* http://codekata.pragprog.com/2007/01/kata_fourteen_t.html
* *
* @author Simon Brooke <simon@journeyman.cc> * @author Simon Brooke <simon@journeyman.cc>
*/ */
public class Milkwood { public class Milkwood {
/**
* The magic token which is deemed to end sentences.
*/
public static final String PERIOD = ".";
/** /**
* Parse command line arguments and kick off the process. Expected arguments * The magic token which is deemed to end sentences.
* include: */
* <dl> public static final String PERIOD = ".";
* <dt>-d, -debug</dt>
* <dd>Print debugging output to standard error</dd>
* <dt>-i [FILE], -input [FILE]</dt>
* <dd>Input file, expected to be an English (or, frankly, other natural
* language) text. Defaults to standard in.</dd>
* <dt>-l [NN], -length [NN]</dt>
* <dd>The length in tuples of the desired output. Defaults to 100.
* <dt>-n [NN], -tuple-length [NN]</dt>
* <dd>The length of tuples into which the file will be analysed, default 2.
* </dd>
* <dt>-o [FILE], -output [FILE]</dt>
* <dd>Output file, to which generated text will be written. Defaults to
* standard out.</dd>
* </dl>
*
* @param args
* the command line arguments
* @exception FileNotFoundException
* if the user specifies a file which isn't available.
* @excpetion IOException if could not read from input or write to output.
*/
public static void main(String[] args) throws FileNotFoundException,
IOException {
/* defaults */
InputStream in = System.in;
OutputStream out = System.out;
int tupleLength = 2;
boolean debug = false;
int length = 100;
for (int cursor = 0; cursor < args.length; cursor++) { /**
String arg = args[cursor]; * Parse command line arguments and kick off the process. Expected arguments
* include:
* <dl>
* <dt>-d, -debug</dt>
* <dd>Print debugging output to standard error</dd>
* <dt>-i [FILE], -input [FILE]</dt>
* <dd>Input file, expected to be an English (or, frankly, other natural
* language) text. Defaults to standard in.</dd>
* <dt>-l [NN], -length [NN]</dt>
* <dd>The length in tuples of the desired output. Defaults to 100.
* <dt>-n [NN], -tuple-length [NN]</dt>
* <dd>The length of tuples into which the file will be analysed, default 2.
* </dd>
* <dt>-o [FILE], -output [FILE]</dt>
* <dd>Output file, to which generated text will be written. Defaults to
* standard out.</dd>
* </dl>
*
* @param args the command line arguments
* @exception FileNotFoundException if the user specifies a file which isn't
* available.
* @excpetion IOException if could not read from input or write to output.
*/
public static void main(String[] args) throws FileNotFoundException,
IOException {
/* defaults */
InputStream in = System.in;
OutputStream out = System.out;
int tupleLength = 2;
boolean debug = false;
int length = 100;
if (arg.startsWith("-") && arg.length() > 1) { for (int cursor = 0; cursor < args.length; cursor++) {
switch (arg.charAt(1)) { String arg = args[cursor];
case 'd':
debug = true;
break;
case 'i':
// input
in = new FileInputStream(new File(args[++cursor]));
break;
case 'o': // output
out = new FileOutputStream(new File(args[++cursor]));
break;
case 'l': // length
length = Integer.parseInt(args[++cursor]);
break;
case 'n':
case 't': // tuple length
tupleLength = Integer.parseInt(args[++cursor]);
break;
default:
throw new IllegalArgumentException(String.format(
"Unrecognised argument '%s'", arg));
}
}
}
try {
new Milkwood().readAndGenerate(in, out, tupleLength, length, debug);
} finally {
out.close();
}
}
/** if (arg.startsWith("-") && arg.length() > 1) {
* Read tokens from this input and use them to generate text on this output. switch (arg.charAt(1)) {
* case 'd':
* @param in debug = true;
* the input stream to read. break;
* @param out case 'i':
* the output stream to write to. // input
* @param tupleLength in = new FileInputStream(new File(args[++cursor]));
* the length of tuples to be used in generation. break;
* @param length case 'o': // output
* the length in tokens of the output to be generated. out = new FileOutputStream(new File(args[++cursor]));
* @param debug break;
* whether to print debugging output. case 'l': // length
* @throws IOException length = Integer.parseInt(args[++cursor]);
* if the file system buggers up, which is not, in the cosmic break;
* scheme of things, very likely. case 'n':
*/ case 't': // tuple length
void readAndGenerate(final InputStream in, final OutputStream out, tupleLength = Integer.parseInt(args[++cursor]);
final int tupleLength, int length, boolean debug) break;
throws IOException { default:
/* The root of the rule tree I shall build. */ throw new IllegalArgumentException(String.format(
RuleTreeNode root = new RuleTreeNode(); "Unrecognised argument '%s'", arg));
read(in, tupleLength, debug, root); }
}
}
try {
new Milkwood().readAndGenerate(in, out, tupleLength, length, debug);
} finally {
out.close();
}
}
WordSequence tokens = compose(tupleLength, debug, root, length); /**
* Read tokens from this input and use them to generate text on this output.
*
* @param in the input stream to read.
* @param out the output stream to write to.
* @param tupleLength the length of tuples to be used in generation.
* @param length the length in tokens of the output to be generated.
* @param debug whether to print debugging output.
* @throws IOException if the file system buggers up, which is not, in the
* cosmic scheme of things, very likely.
*/
void readAndGenerate(final InputStream in, final OutputStream out,
final int tupleLength, int length, boolean debug)
throws IOException {
/* The root of the rule tree I shall build. */
RuleTreeNode root = new RuleTreeNode();
read(in, tupleLength, debug, root);
write(out, debug, tokens); WordSequence tokens = compose(tupleLength, debug, root, length);
if ( debug) { write(out, debug, tokens);
System.err.println( "\n\nCompleted.");
}
}
/** if (debug) {
* Digest the input into a set of rules. System.err.println("\n\nCompleted.");
* }
* @param in }
* the input stream.
* @param tupleLength
* the length of tuples we shall consider.
* @param debug
* whether or not to print debugging output.
* @param root
* the root of the rule tree.
* @return the number of tokens read.
* @throws IOException
* if the file system buggers up, which is not, in the cosmic
* scheme of things, very likely.
*/
private int read(final InputStream in, final int tupleLength,
boolean debug, RuleTreeNode root) throws IOException {
int length = new Digester().read(in, tupleLength, root);
if (debug) { /**
System.err.println(root.toString()); * Digest the input into a set of rules.
} *
return length; * @param in the input stream.
} * @param tupleLength the length of tuples we shall consider.
* @param debug whether or not to print debugging output.
* @param root the root of the rule tree.
* @return the number of tokens read.
* @throws IOException if the file system buggers up, which is not, in the
* cosmic scheme of things, very likely.
*/
private int read(final InputStream in, final int tupleLength,
boolean debug, RuleTreeNode root) throws IOException {
int length = new Digester().digest(in, tupleLength, root);
private WordSequence compose(final int tupleLength, boolean debug, if (debug) {
RuleTreeNode root, int length) { System.err.println(root.toString());
WordSequence tokens = new Composer(debug).compose(root, length); }
return length;
}
if (tokens.contains(PERIOD)) { private WordSequence compose(final int tupleLength, boolean debug,
tokens = tokens.truncateAtLastInstance(PERIOD); RuleTreeNode root, int length) {
} WordSequence tokens = new Composer(debug).compose(root, length);
return tokens;
}
/** if (tokens.contains(PERIOD)) {
* Write this sequence of tokens to this output. tokens = tokens.truncateAtLastInstance(PERIOD);
* }
* @param out return tokens;
* the stream to which to write. }
* @param debug
* whether or not to print debugging output.
* @param tokens
* the sequence of tokens to write.
* @throws IOException
* if the file system buggers up, which is not, in the cosmic
* scheme of things, very likely.
*/
private void write(final OutputStream out, boolean debug,
WordSequence tokens) throws IOException {
Writer scrivenor = new Writer(out, debug);
try {
scrivenor.writeSequence(tokens);
} finally {
scrivenor.close();
}
}
/**
* Write this sequence of tokens to this output.
*
* @param out the stream to which to write.
* @param debug whether or not to print debugging output.
* @param tokens the sequence of tokens to write.
* @throws IOException if the file system buggers up, which is not, in the
* cosmic scheme of things, very likely.
*/
private void write(final OutputStream out, boolean debug,
WordSequence tokens) throws IOException {
try (Writer scrivenor = new Writer(out, debug)) {
scrivenor.writeSequence(tokens);
}
}
} }

17
src/cc/journeyman/milkwood/NoSuchPathException.java
View file

@ -1,17 +0,0 @@
/*
* Proprietary unpublished source code property of
* Simon Brooke <simon@journeyman.cc>.
*
* Copyright (c) 2013 Simon Brooke <simon@journeyman.cc>
*/
package cc.journeyman.milkwood;
/**
*
* @author Simon Brooke <simon@journeyman.cc>
*/
class NoSuchPathException extends Exception {
private static final long serialVersionUID = 1L;
}

127
src/cc/journeyman/milkwood/RuleTreeNode.java
View file

@ -16,17 +16,18 @@ import java.util.Random;
import java.util.Stack; import java.util.Stack;
/** /**
* Mapping a word to its successor words. This is probably highly * Mapping a word to its successor words. This is probably highly inefficient of
* inefficient of store, but for the present purposes my withers are unwrung. * store, but for the present purposes my withers are unwrung. Not thread safe
* Not thread safe in this form because of access to the random number generator. * in this form because of access to the random number generator.
* *
* @author Simon Brooke <simon@journeyman.cc> * @author Simon Brooke <simon@journeyman.cc>
*/ */
public class RuleTreeNode { public class RuleTreeNode {
/**
* The magic token which identifies the root node of a rule tree. /**
*/ * The magic token which identifies the root node of a rule tree.
public static final String ROOTMAGICTOKEN = "*ROOT*"; */
public static final String ROOTMAGICTOKEN = "*ROOT*";
/** /**
* The line separator on this platform. * The line separator on this platform.
*/ */
@ -35,63 +36,65 @@ public class RuleTreeNode {
* A random number generator. * A random number generator.
*/ */
private static Random RANDOM = new Random(); private static Random RANDOM = new Random();
/** /**
* The word at this node. * The word at this node.
*/ */
private final String word; private final String word;
/** /**
* Potential successors of this node * Potential successors of this node
*/ */
private Map<String,RuleTreeNode> rules = new HashMap<String,RuleTreeNode>(); private Map<String, RuleTreeNode> rules = new HashMap<>();
/** /**
* If no argument passed, generate a root node. * If no argument passed, generate a root node.
*/ */
public RuleTreeNode() { public RuleTreeNode() {
this( RuleTreeNode.ROOTMAGICTOKEN); this(RuleTreeNode.ROOTMAGICTOKEN);
} }
/** /**
* Create me wrapping this word. * Create me wrapping this word.
*
* @param word the word I represent. * @param word the word I represent.
*/ */
public RuleTreeNode(String word) { public RuleTreeNode(String word) {
this.word = word; this.word = word;
} }
/**
* Specialisation: neatly format the rule tree.
*
* @return a neatly formatted representation.
*/
@Override
public String toString() { public String toString() {
StringBuffer buffy = new StringBuffer(); StringBuffer buffy = new StringBuffer();
this.printToBuffer( buffy, 0); this.printToBuffer(buffy, 0);
return buffy.toString(); return buffy.toString();
} }
private void printToBuffer(StringBuffer buffy, int indent) { private void printToBuffer(StringBuffer buffy, int indent) {
for (int i = 0; i < indent; i++) { for (int i = 0; i < indent; i++) {
buffy.append( '\t'); buffy.append('\t');
} }
buffy.append( this.getWord()); buffy.append(this.getWord());
if ( this.rules.isEmpty()) { if (this.rules.isEmpty()) {
buffy.append(NEWLINE); buffy.append(NEWLINE);
} else { } else {
buffy.append( " ==>").append(NEWLINE); buffy.append(" ==>").append(NEWLINE);
for ( String successor : this.getSuccessors()) { for (String successor : this.getSuccessors()) {
rules.get(successor).printToBuffer(buffy, indent + 1); rules.get(successor).printToBuffer(buffy, indent + 1);
} }
buffy.append(NEWLINE); buffy.append(NEWLINE);
} }
} }
/**
/**
* *
* @return my word. * @return my word.
*/ */
@ -104,24 +107,24 @@ public class RuleTreeNode {
* @return a shuffled list of the words which could follow this one. * @return a shuffled list of the words which could follow this one.
*/ */
public Collection<String> getSuccessors() { public Collection<String> getSuccessors() {
ArrayList<String> result = new ArrayList<String>(); ArrayList<String> result = new ArrayList<>();
result.addAll(rules.keySet()); result.addAll(rules.keySet());
Collections.shuffle(result, RANDOM); Collections.shuffle(result, RANDOM);
return result; return result;
} }
/** /**
* Compile this sequence of tokens into rule nodes under me. * Compile this sequence of tokens into rule nodes under me.
*
* @param sequence the sequence of tokens to compile. * @param sequence the sequence of tokens to compile.
*/ */
public void addSequence(Queue<String> sequence) { public void addSequence(Queue<String> sequence) {
if (!sequence.isEmpty()) { if (!sequence.isEmpty()) {
String word = sequence.remove(); String token = sequence.remove();
RuleTreeNode successor = this.getRule(word); RuleTreeNode successor = this.getRule(token);
if (successor == null) { if (successor == null) {
successor = new RuleTreeNode(word); successor = new RuleTreeNode(token);
this.rules.put(word, successor); this.rules.put(token, successor);
} }
successor.addSequence(sequence); successor.addSequence(sequence);
@ -133,23 +136,23 @@ public class RuleTreeNode {
* *
* @return the successor chosen, or null if I have none. * @return the successor chosen, or null if I have none.
*/ */
protected RuleTreeNode getRule() { protected RuleTreeNode getRule() {
RuleTreeNode result = null; RuleTreeNode result = null;
if (!rules.isEmpty()) { if (!rules.isEmpty()) {
int target = RANDOM.nextInt(rules.keySet().size()); int target = RANDOM.nextInt(rules.keySet().size());
for (String key : rules.keySet()) { for (String key : rules.keySet()) {
/* /*
* NOTE: decrement after test. * NOTE: decrement after test.
*/ */
if (target-- == 0) { if (target-- == 0) {
result = rules.get(key); result = rules.get(key);
} }
} }
} }
return result; return result;
} }
/** /**
@ -161,10 +164,10 @@ public class RuleTreeNode {
return rules.get(token); return rules.get(token);
} }
protected String getWord(Stack<String> path) throws NoSuchPathException { protected String getWord(Stack<String> path) {
final String result; final String result;
if ( path.isEmpty()) { if (path.isEmpty()) {
result = this.getWord(); result = this.getWord();
} else { } else {
final RuleTreeNode successor = this.getRule(path.pop()); final RuleTreeNode successor = this.getRule(path.pop());
@ -180,25 +183,27 @@ public class RuleTreeNode {
} }
/** /**
* Find all the terminal strings in the current rule set which would match this path. * Find all the terminal strings in the current rule set which would match
* this path.
*
* @param path the path to match * @param path the path to match
* @return a collection (possibly empty) of potential successors. * @return a collection (possibly empty) of potential successors.
*/ */
public Collection<String> match(WordStack path) { public Collection<String> match(Window path) {
final Collection<String> result; final Collection<String> result;
if ( path.isEmpty()) { if (path.isEmpty()) {
result = this.getSuccessors(); result = this.getSuccessors();
} else { } else {
final RuleTreeNode successor = this.getRule(path.pop()); final RuleTreeNode successor = this.getRule(path.pop());
if (successor == null) { if (successor == null) {
result = new ArrayList<String>(); result = new ArrayList<>();
} else { } else {
result = successor.match(path); result = successor.match(path);
} }
} }
return result; return result;
} }
} }

107
src/cc/journeyman/milkwood/Tokeniser.java
View file

@ -22,60 +22,63 @@ import java.io.StreamTokenizer;
*/ */
public class Tokeniser extends StreamTokenizer { public class Tokeniser extends StreamTokenizer {
public Tokeniser(Reader r) { /**
super(r); * Initialise me appropriately wrapping this reader.
* @param r the reader to wrap.
*/
public Tokeniser(Reader r) {
super(r);
this.resetSyntax(); this.resetSyntax();
this.whitespaceChars(8, 15); this.whitespaceChars(8, 15);
this.whitespaceChars(28, 32); this.whitespaceChars(28, 32);
/* /*
* treat quotemarks as white space. Actually it would be better if quote * treat quotemarks as white space. Actually it would be better if quote
* marks were white space only if preceded or followed by whitespace, so * marks were white space only if preceded or followed by whitespace, so
* that, e.g., 'don't' and 'can't' appeared as single tokens. But that * that, e.g., 'don't' and 'can't' appeared as single tokens. But that
* means really reimplementing the parser and I don't have time. * means really reimplementing the parser and I don't have time.
*/ */
this.whitespaceChars((int) '\"', (int) '\"'); this.whitespaceChars((int) '\"', (int) '\"');
this.whitespaceChars((int) '\'', (int) '\''); this.whitespaceChars((int) '\'', (int) '\'');
/* /*
* treat underscore and hyphen as whitespace as well. Again, hyphen with * treat underscore and hyphen as whitespace as well. Again, hyphen with
* either leading or trailing non-whitespace probably ought to be * either leading or trailing non-whitespace probably ought to be
* treated specially, but... * treated specially, but...
*/ */
this.whitespaceChars((int) '_', (int) '_'); this.whitespaceChars((int) '_', (int) '_');
this.whitespaceChars((int) '-', (int) '-'); this.whitespaceChars((int) '-', (int) '-');
this.wordChars((int) '0', (int) '9'); this.wordChars((int) '0', (int) '9');
this.wordChars((int) 'A', (int) 'Z'); this.wordChars((int) 'A', (int) 'Z');
this.wordChars((int) 'a', (int) 'z'); this.wordChars((int) 'a', (int) 'z');
} }
public Tokeniser(InputStream in) { public Tokeniser(InputStream in) {
this(new BufferedReader(new InputStreamReader(in))); this(new BufferedReader(new InputStreamReader(in)));
} }
/** /**
* There surely must be a better way to get just the token out of a * There surely must be a better way to get just the token out of a
* StreamTokenizer...! * StreamTokenizer...!
*/ */
public String readBareToken() { public String readBareToken() {
final String token; final String token;
switch (this.ttype) {
case StreamTokenizer.TT_EOL:
token = "FIXME"; // TODO: fix this!
break;
case StreamTokenizer.TT_NUMBER:
token = new Double(this.nval).toString();
break;
case StreamTokenizer.TT_WORD:
token = this.sval.toLowerCase();
break;
default:
StringBuffer buffy = new StringBuffer();
buffy.append((char) this.ttype);
token = buffy.toString();
break;
}
return token;
}
switch (this.ttype) {
case StreamTokenizer.TT_EOL:
token = "FIXME"; // TODO: fix this!
break;
case StreamTokenizer.TT_NUMBER:
token = new Double(this.nval).toString();
break;
case StreamTokenizer.TT_WORD:
token = this.sval.toLowerCase();
break;
default:
StringBuilder bob = new StringBuilder();
bob.append((char) this.ttype);
token = bob.toString();
break;
}
return token;
}
} }

60
src/cc/journeyman/milkwood/TupleDictionary.java
View file

@ -1,60 +0,0 @@
/*
* Proprietary unpublished source code property of
* Simon Brooke <simon@journeyman.cc>.
*
* Copyright (c) 2013 Simon Brooke <simon@journeyman.cc>
*/
package cc.journeyman.milkwood;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
/**
*
* @author Simon Brooke <simon@journeyman.cc>
*/
public class TupleDictionary extends HashMap<String, Collection<WordSequence>> {
private static final long serialVersionUID = 1L;
/**
* Specialisation: if there isn't an existing entry, create one.
*
* @param token the token to look up
* @return the collection of possible tuples for that token.
*/
public Collection<WordSequence> get(String token) {
Collection<WordSequence> result = super.get(token);
if (result == null) {
result = new ArrayList<WordSequence>();
this.put(token, result);
}
return result;
}
/**
* Add a new, empty sequence to my entry for this token.
* @param token the token
* @return the new sequence which was added.
*/
protected WordSequence addSequence(String token) {
return this.addSequence(token, new WordSequence());
}
/**
* Add this sequence to my entry for this token.
* @param token the token.
* @param sequence the sequence to add. Must not be null!
* @return the sequence which was added.
*/
protected WordSequence addSequence(String token, WordSequence sequence) {
assert (sequence != null) : "invalid sequence argument";
this.get(token).add(sequence);
return sequence;
}
}

56
src/cc/journeyman/milkwood/Window.java Normal file
View file

@ -0,0 +1,56 @@
package cc.journeyman.milkwood;
import java.util.Stack;
/**
* Sliding window which rules may match.
*
* @author simon
*
*/
public class Window extends Stack<String> {
private static final long serialVersionUID = 1L;
/**
* Create a new, empty, wordstack.
*/
public Window() {
super();
}
/**
* create a new window from this window, having this new word as its
* terminal and ommitting the current first word. That is, the new window
* should be as long as the old, with each word shuffled up one place.
*
* @param prototype the window to copy from.
* @param terminal the new terminal word.
*/
public Window(Window prototype, String terminal) {
this();
Window copy = prototype.duplicate();
copy.pop();
this.populate(copy, terminal);
}
private void populate(Window copy, String terminal) {
if (copy.isEmpty()) {
this.push(terminal);
} else {
String token = copy.pop();
this.populate(copy, terminal);
this.push(token);
}
}
/**
* A wrapper round clone which hides all the ugly casting.
*
* @return a duplicate copy of myself.
*/
public Window duplicate() {
return (Window) this.clone();
}
}

88
src/cc/journeyman/milkwood/WordSequence.java
View file

@ -12,56 +12,58 @@ import java.util.Queue;
/** /**
* An ordered sequence of words. Of course it implements Queue since it is a * An ordered sequence of words. Of course it implements Queue since it is a
* LinkedList and LinkedList implements Queue, but I want to make it explicitly * LinkedList and LinkedList implements Queue, but I want to make it explicitly
* clear that this is a queue and can be used as such. * clear that this is a queue and can be used as such. Different from WordStack
* which is a Stack.
*
* @see WordStack
* *
* @author Simon Brooke <simon@journeyman.cc> * @author Simon Brooke <simon@journeyman.cc>
*/ */
class WordSequence extends LinkedList<String> implements Queue<String> { public class WordSequence extends LinkedList<String> implements Queue<String> {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
/** /**
* *
* @param tokens * @param tokens a sequence of tokens
* a sequence of tokens * @param marker a marker to terminate after the last occurrance of.
* @param marker * @return a copy of tokens, truncated at the last occurrance of the marker.
* a marker to terminate after the last occurrance of. */
* @return a copy of tokens, truncated at the last occurrance of the marker. public WordSequence truncateAtLastInstance(String marker) {
*/ final WordSequence result = new WordSequence();
public WordSequence truncateAtLastInstance(String marker) {
final WordSequence result = new WordSequence();
for (String token : this) { for (String token : this) {
if (token.endsWith(marker) && !this.contains(marker)) { result.add(token);
/* if (token.endsWith(marker) && !this.contains(marker)) {
* If the token we're looking at ends with the marker, and the /*
* remainder of the tokens does not include a token ending with * If the token we're looking at ends with the marker, and the
* the marker, we're done. Otherwise, we continue. OK? * remainder of the tokens does not include a token ending with
*/ * the marker, we're done. Otherwise, we continue. OK?
break; */
} break;
result.add(token); }
} }
return result; return result;
} }
/** /**
* Specialisation: Working around the bug that the tokeniser treats PERIOD as a word character. * Specialisation: Working around the bug that the tokeniser treats PERIOD
*/ * as a word character.
@Override */
public boolean contains(Object target) { @Override
boolean result = false; public boolean contains(Object target) {
if (target != null) { boolean result = false;
String marker = target.toString(); if (target != null) {
String marker = target.toString();
for (String token : this) { for (String token : this) {
if (token.endsWith(marker)) { if (token.endsWith(marker)) {
result = true; result = true;
break; break;
} }
} }
} }
return result; return result;
} }
} }

57
src/cc/journeyman/milkwood/WordStack.java
View file

@ -1,57 +0,0 @@
package cc.journeyman.milkwood;
import java.util.Stack;
/**
* Sliding window which rules may match.
*
* @author simon
*
*/
public class WordStack extends Stack<String> {
private static final long serialVersionUID = 1L;
/**
* Create a new, empty, wordstack.
*/
public WordStack() {
super();
}
/**
* create a new window from this window, having this new word as its
* terminal and ommitting the current first word. That is, the new window
* should be as long as the old, with each word shuffled up one place.
*
* @param prototype the window to copy from.
* @param terminal the new terminal word.
*/
public WordStack(WordStack prototype, String terminal) {
this();
WordStack copy = prototype.duplicate();
copy.pop();
this.populate( copy, terminal);
}
private void populate(WordStack copy, String terminal) {
if ( copy.isEmpty()) {
this.push(terminal);
} else {
String token = copy.pop();
this.populate(copy, terminal);
this.push( token);
}
}
/**
* A wrapper round clone which hides all the ugly casting.
*
* @return a duplicate copy of myself.
*/
public WordStack duplicate() {
return (WordStack) this.clone();
}
}

259
src/cc/journeyman/milkwood/Writer.java
View file

@ -20,152 +20,137 @@ import java.util.Random;
* @author Simon Brooke <simon@journeyman.cc> * @author Simon Brooke <simon@journeyman.cc>
*/ */
class Writer extends BufferedWriter { class Writer extends BufferedWriter {
/**
* The average number of sentences in a paragraph.
*/
public static final int AVSENTENCESPERPARA = 5;
/**
* A random number generator.
*/
private static Random RANDOM = new Random();
/**
* Dictionary of first-words we know about; each first-word maps onto a
* tuple of tuples of word sequences beginning with that word, so 'I' might
* map onto [[I, CAME, COMMA],[I, SAW, COMMA],[I CONQUERED COMMA]].
*/
TupleDictionary dictionary = new TupleDictionary();
/** /**
* Whether or not I am in debugging mode. * Line separator on this platform.
*/ */
@SuppressWarnings("unused") public static final String NEWLINE = System.getProperty("line.separator");
private final boolean debug; /**
* The average number of sentences in a paragraph.
*/
public static final int AVSENTENCESPERPARA = 5;
/**
* A random number generator.
*/
private static Random RANDOM = new Random();
/**
* Whether or not I am in debugging mode.
*/
@SuppressWarnings("unused")
private final boolean debug;
/** /**
* @param out * @param out the output stream to which I shall write.
* the output stream to which I shall write. * @param debug Whether or not I am in debugging mode.
* @param debug */
* Whether or not I am in debugging mode. public Writer(OutputStream out, final boolean debug) {
*/ super(new OutputStreamWriter(out));
public Writer(OutputStream out, final boolean debug) { this.debug = debug;
super(new OutputStreamWriter(out)); }
this.debug = debug;
}
/** /**
* Write this sequence of tokens on this stream, sorting out minor issues of * Write this sequence of tokens on this stream, sorting out minor issues of
* orthography. * orthography.
* *
* @param tokens * @param tokens the tokens.
* the tokens. * @throws IOException if it is impossible to write (e.g. file system full).
* @throws IOException */
* if it is impossible to write (e.g. file system full). public void writeSequence(WordSequence tokens) throws IOException {
*/ boolean capitaliseNext = true;
public void writeSequence(WordSequence tokens) throws IOException {
boolean capitaliseNext = true;
try { for (String token : tokens) {
for (String token : tokens) { capitaliseNext = writeToken(capitaliseNext, token);
capitaliseNext = writeToken(capitaliseNext, token); }
} this.write(NEWLINE);
} finally { }
this.flush();
this.close();
}
}
/** /**
* Deal with end of paragraph, capital after full stop, and other minor * Deal with end of paragraph, capital after full stop, and other minor
* orthographic conventions. * orthographic conventions.
* *
* @param capitalise * @param capitalise whether or not the token should be capitalised
* whether or not the token should be capitalised * @param token the token to write;
* @param token * @returnvtrue if the next token to be written should be capitalised.
* the token to write; * @throws IOException
* @returnvtrue if the next token to be written should be capitalised. */
* @throws IOException private boolean writeToken(boolean capitalise, String token)
*/ throws IOException {
private boolean writeToken(boolean capitalise, String token) if (this.spaceBefore(token)) {
throws IOException { this.write(" ");
if (this.spaceBefore(token)) { }
this.write(" "); if (capitalise) {
} this.write(token.substring(0, 1).toUpperCase(Locale.getDefault()));
if (capitalise) { this.write(token.substring(1));
this.write(token.substring(0, 1).toUpperCase(Locale.getDefault())); } else {
this.write(token.substring(1)); this.write(token);
} else { }
this.write(token);
}
this.maybeParagraph(token); this.maybeParagraph(token);
return (token.endsWith(Milkwood.PERIOD)); return (token.endsWith(Milkwood.PERIOD));
} }
/** /**
* Return false if token is punctuation, else true. Wouldn't it be nice if * Return false if token is punctuation, else true. Wouldn't it be nice if
* Java provided Character.isPunctuation(char)? However, since it doesn't, I * Java provided Character.isPunctuation(char)? However, since it doesn't, I
* can give this slightly special semantics: return true only if this is * can give this slightly special semantics: return true only if this is
* punctuation which would not normally be preceded with a space. * punctuation which would not normally be preceded with a space.
* *
* @param ch * @param ch a character.
* a character. * @return true if the should be preceded by a space, else false.
* @return true if the should be preceded by a space, else false. */
*/ private boolean spaceBefore(String token) {
private boolean spaceBefore(String token) { final boolean result;
final boolean result;
switch (token.length()) { switch (token.length()) {
case 0: case 0:
result = false; result = false;
break; break;
case 1: case 1:
switch (token.charAt(0)) { switch (token.charAt(0)) {
case '.': case '.':
case ',': case ',':
case ':': case ':':
case ';': case ';':
case 's': case 's':
/* /*
* an 's' on its own is probably evidence of a possessive with * an 's' on its own is probably evidence of a possessive with
* the apostrophe lost * the apostrophe lost
*/ */
case 't': case 't':
/* /*
* similar; probably 'doesn't' or 'shouldn't' or other cases of * similar; probably 'doesn't' or 'shouldn't' or other cases of
* 'not' with an elided 'o'. * 'not' with an elided 'o'.
*/ */
result = false; result = false;
break; break;
default: default:
result = true; result = true;
break; break;
} }
break; break;
default: default:
result = true; result = true;
} }
return result; return result;
} }
/**
* If this token is an end-of-sentence token, then, on one chance in some,
* have the writer write two new lines. NOTE: The tokeniser is treating
* PERIOD ('.') as a word character, even though it has not been told to.
* Token.endsWith( PERIOD) is a hack to get round this problem. TODO:
* investigate and fix.
*
* @param token
* a token
* @throws IOException
* if Mr this has run out of ink
*/
private void maybeParagraph(String token) throws IOException {
if (token.endsWith(Milkwood.PERIOD)
&& RANDOM.nextInt(AVSENTENCESPERPARA) == 0) {
this.write("\n\n");
}
}
/**
* If this token is an end-of-sentence token, then, on one chance in some,
* have the writer write two new lines. NOTE: The tokeniser is treating
* PERIOD ('.') as a word character, even though it has not been told to.
* Token.endsWith( PERIOD) is a hack to get round this problem. TODO:
* investigate and fix.
*
* @param token a token
* @throws IOException if Mr this has run out of ink
*/
private void maybeParagraph(String token) throws IOException {
if (token.endsWith(Milkwood.PERIOD)
&& RANDOM.nextInt(AVSENTENCESPERPARA) == 0) {
this.write(NEWLINE);
this.write(NEWLINE);
}
}
} }