Merge branch 'print_counts'

Author: mjpost

example/counts/corpus.a

@@ -0,0 +1,77 @@
+# this is an example Thrax configuration file
+# <- this symbol indicates a comment
+# each line should be a key-value pair separated by whitespace
+
+###
+### GRAMMAR OPTIONS
+###
+
+grammar     hiero   # or samt
+reverse     false
+source-is-parsed    false
+target-is-parsed    false
+# default-nt    X   # X is the default anyway
+
+min-rule-count 1
+
+# the number of reducers
+reducers 16
+
+# Maximum length of initial phrase pairs. These are set to be shorter than
+# used by Hiero.
+initial-phrase-length   5
+lex-source-words        5
+lex-target-words        5
+
+# maximum number of NTs in a rule
+arity                   0
+
+# minimum number of aligned terminals in a rule
+lexicality              1   
+
+# allow adjacent nonterminals on source side
+adjacent-nts    false   
+
+# allow unaligned words at boundaries of phrases
+loose           true
+
+allow-abstract-rules    false
+allow-nonlexical-x      false
+allow-full-sentence-rules   false
+
+nonlex-source-length    5
+nonlex-target-length    5
+nonlex-source-words     5
+nonlex-target-words     5
+
+allow-double-plus    false
+
+rule-span-limit         12
+
+phrase-penalty  2.718
+
+# a whitespace seperated list of features
+# in this example, the features are phrase translation probability,
+# lexical probability, and phrase penalty
+# features        phrase-penalty e2fphrase f2ephrase lexprob lexical abstract adjacent x-rule source-terminals-without-target target-terminals-without-source monotonic glue-rule rarity target-word-count unaligned-count
+features        e_given_f_phrase f_given_e_phrase e_given_f_lex f_given_e_lex rarity phrase-penalty alignment count
+
+# the only option and default later we will want to add formats for other decoders such as moses and
+# cdec, if they use other formats
+output-format   joshua  
+
+# label feature scores? each score will be output as name=score
+label-feature-scores false
+
+amazon-work s3://edu.jhu.cs.jonny/wmt11/fr-en/hiero
+amazon-jar  s3://edu.jhu.cs.jonny/thrax.jar
+amazon-num-instances    15
+
+max-split-size  8388608
+
+# the format should be:
+# foreign sentence ||| english sentence ||| alignment
+# where the english is either parsed or not depending on whether you want
+# SAMT or you want Hiero.
+#input-file  s3://edu.jhu.cs.jonny/wmt11/corpus.fr-en
+input-file pipeline-es-en-phrase-_export_projects_mpost_language-packs_es-en_1.3/input-file

example/counts/corpus.en

@@ -100,6 +100,7 @@ private synchronized void scheduleJobs() throws SchedulerException {
     String features = BackwardsCompatibility.equivalent(conf.get("thrax.features", ""));
 
     System.err.println("Running in mode: " + type);
+    System.err.println("Features: " + features);
 
     scheduler.schedule(VocabularyJob.class);
 
@@ -174,7 +175,7 @@ public static void main(String[] argv) throws Exception {
     int returnCode = ToolRunner.run(null, new Thrax(), argv);
     System.exit(returnCode);
   }
-
+  
   protected synchronized void workerDone(Class<? extends ThraxJob> theClass, boolean success) {
     try {
       scheduler.setState(theClass, success ? JobState.SUCCESS : JobState.FAILED);

example/counts/corpus.es

@@ -22,7 +22,8 @@ else if (name.equals(SourceGivenTargetLexicalProbabilityFeature.NAME))
       return new SourceGivenTargetLexicalProbabilityFeature();
     else if (name.equals(TargetGivenSourceLexicalProbabilityFeature.NAME))
       return new TargetGivenSourceLexicalProbabilityFeature();
-    else if (name.equals(AlignmentFeature.NAME)) return new AlignmentFeature();
+    else if (name.equals(AlignmentFeature.NAME)) 
+      return new AlignmentFeature();
 
     return null;
   }

example/counts/thrax-phrase.conf

@@ -3,6 +3,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import edu.jhu.thrax.hadoop.features.annotation.CountFeature;
 import edu.jhu.thrax.util.FormatUtils;
 
 public class MapReduceFeatureFactory {

src/edu/jhu/thrax/Thrax.java

@@ -99,7 +99,10 @@ public static int[] applyIndices(int[] input, boolean monotonic) {
   public static Text ruleToText(RuleWritable r, Map<String, Writable> fs, boolean label,
       boolean sparse) {
     if (r == null) throw new IllegalArgumentException("Cannot convert a null rule to text.");
+    
+    // Both alignment and count are features that are handled specially
     String alignment = null;    
+    String ruleCount = "";
     StringBuilder sb = new StringBuilder();
     sb.append(Vocabulary.word(r.lhs));
     sb.append(DELIM);
@@ -125,7 +128,10 @@ public static Text ruleToText(RuleWritable r, Map<String, Writable> fs, boolean
     for (String t : fs.keySet()) {
       String score;
       Writable val = fs.get(t);
-      if (val instanceof FloatWritable) {
+      if (t.equals("Count")) {
+        ruleCount = String.format("%d", ((IntWritable) fs.get(t)).get());
+        continue;
+      } else if (val instanceof FloatWritable) {
         float value = ((FloatWritable) fs.get(t)).get();
         if (value == -0.0 || Math.abs(value) < 0.000005)
           score = "0";
@@ -150,6 +156,14 @@ public static Text ruleToText(RuleWritable r, Map<String, Writable> fs, boolean
     }
     if (alignment != null)
       sb.append(DELIMITER + " ").append(alignment + " ");
+
+    if (! ruleCount.equals("")) {
+      // If there was no alignment, output a blank
+      if (alignment == null)
+        sb.append(DELIMITER + "  ");
+
+      sb.append(DELIMITER + " ").append(ruleCount).append(" ");
+    }
     return new Text(sb.substring(0, sb.length() - 1));
   }