Run two TRC containers locally for MDA and regular traceroutes (#139)

* Run two TRC containers locally for MDA and regular traceroutes

This commit contains changes for running two instances of the
traceroute-caller container locally via docker-compose. One
instance runs MDA traceroutes (scamper1 datatype) and the other
instance runs regular traceroutes (scamper2 datatype).

Changes tested locally with "go test -race" and also with
"docker-compose".

Author: Saied Kazemi

Dockerfile

@@ -43,6 +43,12 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Create /var/empty to avoid a race condition in scamper that results
+# in the following failure:
+#   scamper_privsep_init: could not mkdir /var/empty: File exists
+RUN mkdir -p /var/empty && \
+    chmod 555 /var/empty
+
 # Bring the statically-linked traceroute-caller binary from the go build image.
 COPY --from=build_caller /go/bin/traceroute-caller /
 

docker-compose.yaml

@@ -19,6 +19,10 @@ services:
         published: 9992
         protocol: tcp
         mode: bridge
+      - target: 9993
+        published: 9993
+        protocol: tcp
+        mode: bridge
     command:
       - -prometheusx.listen-address=:9990
       - -output=/local/tcpinfo
@@ -44,7 +48,7 @@ services:
       - -siteinfo.url=file:///testdata/annotations-incomplete.json
       - -hostname=mlab1-lga0t.mlab-sandbox.measurement-lab.org
 
-  traceroute-caller:
+  trc-scamper1:
     build:
       context: .
       dockerfile: Dockerfile
@@ -65,11 +69,33 @@ services:
       - -IPCacheTimeout=1m
       - -IPCacheUpdatePeriod=10s
       - -hopannotation-output=/local/hopannotation1
+      - -scamper.trace-type=mda
+      - -traceroute-output=/local/scamper1
+      - -scamper.timeout=30m
+      - -scamper.tracelb-W=15
+      - -scamper.tracelb-ptr=true
+
+  trc-scamper2:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - ./local:/local
+    network_mode: "service:tcpinfo"
+    deploy:
+      # NOTE: traceroute-caller may fail on startup if it tries to read a
+      # socket that the above services are not yet listening on.
+      # So, allow restart.
+      restart_policy:
+        condition: any
+        delay: 5s
+    command:
+      - -prometheusx.listen-address=:9993
+      - -tcpinfo.eventsocket=/local/tcpevents.sock
+      - -ipservice.sock=/local/uuid-annotator.sock
+      - -IPCacheTimeout=1m
+      - -IPCacheUpdatePeriod=10s
+      - -hopannotation-output=/local/hopannotation1
       - -scamper.trace-type=regular
       - -traceroute-output=/local/scamper2
       - -scamper.timeout=10m
-      # -scamper.trace-type=mda
-      #- -traceroute-output=/local/scamper1
-      #- -scamper.timeout=30m
-      #- -scamper.tracelb-W=15
-      #- -scamper.tracelb-ptr=false

hopannotation/hopannotation.go

@@ -64,8 +64,9 @@ var (
 	hostname string
 
 	// Package testing aid.
-	tickerDuration = int64(60 * 1000 * time.Millisecond) // ticker duration for cache resetter
-	writeFile      = ioutil.WriteFile
+	tickerDuration   = int64(60 * 1000 * time.Millisecond) // ticker duration for cache resetter
+	writeFile        = ioutil.WriteFile
+	errInvalidConfig = errors.New("invalid context or hop annotation configuration")
 )
 
 // HopAnnotation1 is the datatype that is written to the hop annotation file.
@@ -107,8 +108,8 @@ func init() {
 // passage of the midnight every minute to reset the cache.  The goroutine
 // will terminate when the ctx is cancelled.
 func New(ctx context.Context, haCfg Config) (*HopCache, error) {
-	if haCfg.AnnotatorClient == nil || haCfg.OutputPath == "" {
-		return nil, fmt.Errorf("invalid hop annotation configuration: %+v", haCfg)
+	if ctx == nil || haCfg.AnnotatorClient == nil || haCfg.OutputPath == "" {
+		return nil, fmt.Errorf("%v: %+v", errInvalidConfig, haCfg)
 	}
 	hc := &HopCache{
 		hops:       make(map[string]bool, 10000), // based on observation

hopannotation/hopannotation_test.go

@@ -76,6 +76,15 @@ func newHopCache(ctx context.Context, t *testing.T, path string) (*HopCache, *fa
 }
 
 func TestNew(t *testing.T) {
+	// First check with invalid parameters.
+	haCfg := Config{
+		AnnotatorClient: nil,
+		OutputPath:      "",
+	}
+	if _, err := New(context.TODO(), haCfg); !strings.Contains(err.Error(), errInvalidConfig.Error()) {
+		t.Fatalf("New() = %v, want %v", err, errInvalidConfig)
+	}
+
 	// Change ticker duration to 100ms to avoid waiting a long time for
 	// the resetter goroutine to notice passage of midnight or cancelled
 	// context.

internal/triggertrace/triggertrace.go

@@ -157,7 +157,13 @@ func (h *Handler) traceAnnotateAndArchive(ctx context.Context, dest Destination)
 		log.Printf("failed to annotate some or all hops (errors: %+v)\n", allErrs)
 	}
 	if len(annotations) > 0 {
-		h.HopAnnotator.WriteAnnotations(annotations, traceStartTime)
+		allErrs := h.HopAnnotator.WriteAnnotations(annotations, traceStartTime)
+		if allErrs != nil {
+			log.Printf("failed to write some or all annotations due to the following error(s):\n")
+			for _, err := range allErrs {
+				log.Printf("error: %v\n", err)
+			}
+		}
 	}
 }
 

tracer/scamper.go

@@ -106,7 +106,8 @@ func (s *Scamper) CachedTrace(cookie, uuid string, t time.Time, cachedTrace []by
 
 	// Create and add the first line to the cached traceroute.
 	newTrace := append(createMetaline(uuid, true, extractUUID(cachedTrace[:split])), cachedTrace[split+1:]...)
-	return ioutil.WriteFile(filename, []byte(newTrace), 0666)
+	// Make the file readable so it won't be overwritten.
+	return ioutil.WriteFile(filename, []byte(newTrace), 0444)
 }
 
 // DontTrace is called when a previous traceroute that we were waiting for
@@ -159,7 +160,7 @@ func runCmd(ctx context.Context, label string, cmd []string) ([]byte, error) {
 	var outb, errb bytes.Buffer
 	c.Stdout = &outb
 	c.Stderr = &errb
-	log.Printf("context %p: command %s started\n", ctx, strings.Join(cmd, " "))
+	log.Printf("context %p: command started: %s\n", ctx, strings.Join(cmd, " "))
 	start := time.Now()
 	err := c.Run()
 	latency := time.Since(start).Seconds()
@@ -179,7 +180,7 @@ func runCmd(ctx context.Context, label string, cmd []string) ([]byte, error) {
 		return outb.Bytes(), err
 	}
 
-	log.Printf("Command succeeded in context %p\n", ctx)
+	log.Printf("context %p: command succeeded\n", ctx)
 	traceTimeHistogram.WithLabelValues("success").Observe(latency)
 	return outb.Bytes(), nil
 }