Rebased with JOSHUA-252 and resolved merge conflicts

Author: thammegowda

.gitmodules

@@ -1,7 +0,0 @@
-[submodule "berkeleylm"]
-	path = ext/berkeleylm
-	url = https://github.com/joshua-decoder/berkeleylm.git
-[submodule "ext/kenlm"]
-	path = ext/kenlm
-	url = https://github.com/kpu/kenlm.git
-	branch = 56fdb5c44fca34d5a2e07d96139c28fb163983c5

bin/bleu

@@ -1,5 +1,20 @@
 #!/usr/bin/env bash
 
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 if [[ -z $2 ]]; then
   echo "Usage: bleu output reference"
   exit 1

bin/extract-1best

@@ -1,3 +1,18 @@
 #!/bin/bash
 
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 java -Xmx500m -cp $JOSHUA/class -Dfile.encoding=utf8 joshua.util.ExtractTopCand $1 - $2

bin/joshua-decoder

@@ -1,4 +1,20 @@
 #!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 #
 # Joshua decoder invocation script.
 # 

bin/meteor

@@ -1,5 +1,20 @@
 #!/usr/bin/env bash
 
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 if [[ -z $3 ]]; then
   echo "Usage: meteor output reference lang"
   exit 1

Dockerfile renamed to docker/Dockerfile

@@ -1,4 +1,4 @@
-# Examples
+# Joshua Examples
 
 The examples in this directory demonstrate how to exercise different
 Joshua features. If you have any comments or questions please submit 
@@ -10,4 +10,39 @@ Bugs or source code issues should be logged in our
 The decoding examples and model training examples in the subdirectories of this
 directory assume you have downloaded the Fisher Spanish--English dataset, which
 contains speech-recognizer output paired with English translations. This data
-can be downloaded by running the [download.sh](https://github.com/apache/incubator-joshua/blob/master/examples/download.sh) script.
+can be downloaded by running the [download.sh](https://github.com/apache/incubator-joshua/blob/master/src/examples/resources/download.sh) script.
+
+# Building a Spanish --> English Translation Model using the Fisher Spanish CALLHOME corpus
+
+An example of how to build a model using the Fisher Spanish CALLHOME corpus
+
+A) Download the corpus:
+    1) mkdir $HOME/git
+    2) cd $HOME/git
+    3) curl -o fisher-callhome-corpus.zip https://codeload.github.com/joshua-decoder/fisher-callhome-corpus/legacy.zip/master
+    4) unzip fisher-callhome-corpus.zip
+    5) # Set environment variable SPANISH=$HOME/git/fisher-callhome-corpus
+    5) mv joshua-decoder-*/ fisher-callhome-corpus
+
+B) Download and install Joshua:
+    1) cd /directory/to/install/
+    2) git clone https://github.com/apache/incubator-joshua.git
+    3) cd incubator-joshua
+    4) # Set environment variable JAVA_HOME=/path/to/java    # Try $(readlink -f /usr/bin/javac | sed "s:/bin/javac::")
+    5) # Set environment variable JOSHUA=/directory/to/install/joshua
+    6) mvn install
+
+C) Train the model:
+    1) mkdir -p $HOME/expts/joshua && cd $HOME/expts/joshua
+    2) $JOSHUA/bin/pipeline.pl \
+        --rundir 1 \
+        --readme "Baseline Hiero run" \
+        --source es \
+        --target en \
+        --lm-gen srilm \
+        --witten-bell \
+        --corpus $SPANISH/corpus/asr/callhome_train \
+        --corpus $SPANISH/corpus/asr/fisher_train \
+        --tune  $SPANISH/corpus/asr/fisher_dev \
+        --test  $SPANISH/corpus/asr/callhome_devtest \
+        --lm-order 3