Collect more stats.

Author: Mark Hale

tools/src/main/java/com/msd/gin/halyard/tools/HalyardStats.java

@@ -121,6 +121,7 @@ public final class HalyardStats extends AbstractHalyardTool {
 
     private static final long DEFAULT_GRAPH_THRESHOLD = 1000;
     private static final long DEFAULT_PARTITION_THRESHOLD = 5000;
+    private static final int DEFAULT_DISTINCT2_THRESHOLD = 1000;
 
 	enum Counters {
 		REMOVED_STATEMENTS,
@@ -185,9 +186,11 @@ static final class StatsMapper extends RdfTableMapper<ImmutableBytesWritable, Lo
 		Value subsetId;
 		Set<Value> lastSubsetIds;
 		HashTracker hashTracker;
-        long subsetDistincts;
         Set<Value> lastSubsetDistincts;
         IRI subsetDistinctType;
+        Set<Value> lastSubsetDistincts2;
+        int distinct2Threshold = DEFAULT_DISTINCT2_THRESHOLD;
+        IRI subsetDistinct2Type;
 		HashTracker subhashTracker;
         long setThreshold, setCounter, subsetThreshold, subsetCounter;
         long instanceOfCounter, classPartitionThreshold;
@@ -246,20 +249,23 @@ protected void map(ImmutableBytesWritable rowKey, Result value, Context output)
                         subhashLen = index.getRole(TermRole.PREDICATE).keyHashSize();
                         subsetType = VOID_EXT.SUBJECT;
                         subsetDistinctType = VOID.PROPERTIES;
+                        subsetDistinct2Type = VOID.DISTINCT_OBJECTS;
                         break;
                     case POS:
                     case CPOS:
                         hashLen = index.getRole(TermRole.PREDICATE).keyHashSize();
                         subhashLen = index.getRole(TermRole.OBJECT).keyHashSize();
                         subsetType = VOID.PROPERTY;
                         subsetDistinctType = VOID.DISTINCT_OBJECTS;
+                        subsetDistinct2Type = VOID.DISTINCT_SUBJECTS;
                         break;
                     case OSP:
                     case COSP:
                         hashLen = index.getRole(TermRole.OBJECT).keyHashSize();
                         subhashLen = index.getRole(TermRole.SUBJECT).keyHashSize();
                         subsetType = VOID_EXT.OBJECT;
                         subsetDistinctType = VOID.DISTINCT_SUBJECTS;
+                        subsetDistinct2Type = VOID.PROPERTIES;
                         break;
                     default:
                         throw new IOException("Unknown index #" + index);
@@ -273,10 +279,12 @@ protected void map(ImmutableBytesWritable rowKey, Result value, Context output)
             if (!hashTracker.equals(key)) {
             	lastSubsetIds = new HashSet<>();
             	lastSubsetDistincts = new HashSet<>();
+            	lastSubsetDistincts2 = new HashSet<>(distinct2Threshold);
             }
 
             if (!subhashTracker.equals(key)) {
             	lastSubsetDistincts = new HashSet<>();
+            	lastSubsetDistincts2 = new HashSet<>(distinct2Threshold);
             }
 
             Statement[] stmts = stmtIndices.parseStatements(null, null, null, null, value, vf);
@@ -320,8 +328,13 @@ protected void map(ImmutableBytesWritable rowKey, Result value, Context output)
 			                        subsetId = subj;
 		                        }
 	                			IRI pred = stmt.getPredicate();
-	                			if (lastSubsetDistincts.add(pred)) {
-	                				subsetDistincts++;
+	                			lastSubsetDistincts.add(pred);
+	                			if (lastSubsetDistincts2 != null) {
+		                			Value obj = stmt.getObject();
+		                			lastSubsetDistincts2.add(obj);
+		                			if (lastSubsetDistincts2.size() > distinct2Threshold) {
+		                				lastSubsetDistincts2 = null;
+		                			}
 	                			}
 		                		triples++;
 		                	}
@@ -336,8 +349,13 @@ protected void map(ImmutableBytesWritable rowKey, Result value, Context output)
 			                        subsetId = pred;
 	                			}
                     			Value obj = stmt.getObject();
-	                			if (lastSubsetDistincts.add(obj)) {
-	                				subsetDistincts++;
+	                			lastSubsetDistincts.add(obj);
+	                			if (lastSubsetDistincts2 != null) {
+		                			Value subj = stmt.getSubject();
+		                			lastSubsetDistincts2.add(subj);
+		                			if (lastSubsetDistincts2.size() > distinct2Threshold) {
+		                				lastSubsetDistincts2 = null;
+		                			}
 	                			}
 	                    		if (RDF.TYPE.equals(pred)) {
 	                    			if (!obj.equals(rdfClass)) {
@@ -368,8 +386,13 @@ protected void map(ImmutableBytesWritable rowKey, Result value, Context output)
 			                        subsetId = obj;
 		                        }
                     			Value subj = stmt.getSubject();
-	                			if (lastSubsetDistincts.add(subj)) {
-	                				subsetDistincts++;
+	                			lastSubsetDistincts.add(subj);
+	                			if (lastSubsetDistincts2 != null) {
+		                			Value pred = stmt.getPredicate();
+		                			lastSubsetDistincts2.add(pred);
+		                			if (lastSubsetDistincts2.size() > distinct2Threshold) {
+		                				lastSubsetDistincts2 = null;
+		                			}
 	                			}
 		                	}
 		                    break;
@@ -482,14 +505,15 @@ private void reset(Context output) throws IOException, InterruptedException {
 		}
 
 		private void resetSubset(Context output) throws IOException, InterruptedException {
-			assert subsetDistincts <= subsetCounter;
+			assert lastSubsetDistincts.size() <= subsetCounter;
             if (subsetCounter >= subsetThreshold) {
                 report(output, subsetType, subsetId, VOID.TRIPLES, subsetCounter);
-                report(output, subsetType, subsetId, subsetDistinctType, subsetDistincts);
+                report(output, subsetType, subsetId, subsetDistinctType, lastSubsetDistincts.size());
+                report(output, subsetType, subsetId, subsetDistinct2Type, lastSubsetDistincts2.size());
             }
             subsetCounter = 0;
-            subsetDistincts = 0;
             lastSubsetDistincts = new HashSet<>();
+        	lastSubsetDistincts2 = new HashSet<>(distinct2Threshold);
             resetClass(output);
         }