better insert error log (#843)

* better insert error log

* remove obsolete TODO

Author: gfr10598

bq/insert.go

@@ -233,8 +233,10 @@ func (in *BQInserter) updateMetrics(err error) error {
 		}
 		// If ALL rows failed...
 		if len(typedErr) == in.pending {
+			log.Printf("InsertErr %v %v\n", err, typedErr.Error()) // Log the first RowInsertionError detail
+			// Backend failure counts failed RPCs.
 			metrics.BackendFailureCount.WithLabelValues(
-				in.TableBase(), "failed insert").Inc()
+				in.TableBase(), "putmulti failed insert").Inc()
 			in.failures++
 		}
 
@@ -243,43 +245,46 @@ func (in *BQInserter) updateMetrics(err error) error {
 			// Handle each error individually.
 			for i, rowError := range typedErr {
 				// These are rowInsertionErrors
-				log.Printf("Insert error: %d %s on %s\n", i, rowError.Error(), in.FullTableName())
+				log.Printf("InsertErr %d %s on %s\n", i, rowError.Error(), in.FullTableName())
 			}
 		} else if len(typedErr) > 0 {
 			// Otherwise, just log the first RowInsertionError detail
-			log.Printf("%d insert errors: %v %s on %s\n", len(typedErr), err, typedErr[0].Error(), in.FullTableName())
+			log.Printf("InsertErr (%d) %v %s on %s\n", len(typedErr), err, typedErr[0].Error(), in.FullTableName())
 		}
 
+		// ErrorCount counts failed rows.
 		metrics.ErrorCount.WithLabelValues(
-			in.TableBase(), "PutMultiError", "insert row error").
+			in.TableBase(), "PutMultiError", "putmulti error").
 			Add(float64(len(typedErr)))
 		in.inserted -= len(typedErr)
 		in.badRows += len(typedErr)
 		err = nil
 	case *url.Error:
-		log.Printf("Insert url.Error: %v on %s", typedErr, in.FullTableName())
+		log.Printf("InsertErr url.Error: %v on %s", typedErr, in.FullTableName())
 		metrics.BackendFailureCount.WithLabelValues(
-			in.TableBase(), "failed insert").Inc()
+			in.TableBase(), "url failed insert").Inc()
 		metrics.ErrorCount.WithLabelValues(
-			in.TableBase(), "url.Error", "UNHANDLED insert error").Inc()
+			in.TableBase(), "url.Error", "UNHANDLED url insert error").Inc()
 		// TODO - Conservative, but possibly not correct.
 		// This at least preserves the count invariance.
 		in.inserted -= in.pending
 		in.badRows += in.pending
 		err = nil
 	case *googleapi.Error:
 		// TODO add special handling for Quota Exceeded
-		log.Printf("Insert error: %v on %s", typedErr, in.FullTableName())
+		log.Printf("InsertErr %v on %s", typedErr, in.FullTableName())
 		if strings.Contains(err.Error(), "Quota exceeded:") {
 			metrics.BackendFailureCount.WithLabelValues(
 				in.TableBase(), "quota exceeded").Inc()
 			metrics.ErrorCount.WithLabelValues(
-				in.TableBase(), "googleapi.Error", "Insert: Quota Exceeded").Inc()
+				in.TableBase(), "googleapi.Error", "Insert: Quota Exceeded").
+				Add(float64(in.pending))
 		} else {
 			metrics.BackendFailureCount.WithLabelValues(
-				in.TableBase(), "failed insert").Inc()
+				in.TableBase(), "googleapi failed insert").Inc()
 			metrics.ErrorCount.WithLabelValues(
-				in.TableBase(), "googleapi.Error", "UNHANDLED insert error").Inc()
+				in.TableBase(), "googleapi.Error", "UNHANDLED googleapi error").
+				Add(float64(in.pending))
 		}
 		// TODO - Conservative, but possibly not correct.
 		// This at least preserves the count invariance.
@@ -289,14 +294,15 @@ func (in *BQInserter) updateMetrics(err error) error {
 
 	default:
 		// With Elem(), this was causing panics.
-		log.Printf("Unhandled %v: %v on %s\n", reflect.TypeOf(typedErr),
+		log.Printf("InsertErr (Unhandled) %v: %v on %s\n", reflect.TypeOf(typedErr),
 			typedErr, in.FullTableName())
 		metrics.BackendFailureCount.WithLabelValues(
 			in.TableBase(), "failed insert").Inc()
-		metrics.ErrorCount.WithLabelValues(
-			in.TableBase(), "unknown", "UNHANDLED insert error").Inc()
-		// TODO - Conservative, but possibly not correct.
+		// TODO - This accounting is conservative, but possibly overestimates failed rows.
 		// This at least preserves the count invariance.
+		metrics.ErrorCount.WithLabelValues(
+			in.TableBase(), "unknown", "UNHANDLED insert error").
+			Add(float64(in.pending))
 		in.inserted -= in.pending
 		in.badRows += in.pending
 		err = nil
@@ -357,6 +363,7 @@ func (in *BQInserter) Flush() error {
 // Commit implements row.Sink.
 // NOTE: the label is ignored, and the TableBase is used instead.
 func (in *BQInserter) Commit(rows []interface{}, label string) error {
+	// TODO - this causes large memory and large number of goroutines.
 	in.acquire()
 	defer in.release()
 	return in.flushSlice(rows)