Merge branch 'decimation'

Author: bemasher

README.md

@@ -25,6 +25,7 @@ Available command-line flags are as follows:
 ```
 Usage of rtlamr:
   -cpuprofile=: write cpu profile to this file
+  -decimation=1: integer decimation factor, keep every nth sample
   -duration=0: time to run for, 0 for infinite, ex. 1h5m10s
   -fastmag=false: use faster alpha max + beta min magnitude approximation
   -filterid=: display only messages matching an id in a comma-separated list of ids.
@@ -36,7 +37,7 @@ Usage of rtlamr:
   -quiet=false: suppress printing state information at startup
   -samplefile=/dev/null: raw signal dump file
   -single=false: one shot execution
-  -symbollength=73: symbol length in samples, see -help for valid lengths
+  -symbollength=72: symbol length in samples, see -help for valid lengths
 
 rtltcp specific:
   -agcmode=false: enable/disable rtl agc
@@ -67,7 +68,9 @@ $ rtlamr
 If you want to run the spectrum server on a different machine than the receiver you'll want to specify an address to listen on that is accessible from the machine `rtlamr` will run on with the `-a` option for `rtl_tcp` with an address accessible by the system running the receiver.
 
 ### Messages
-Currently both SCM (Standard Consumption Message) and IDM (Interval Data Message) packets can be decoded but are mutually exclusive, you cannot receive both simultaneously. See [Wikipedia: Encoder Receiver Transmitter](http://en.wikipedia.org/wiki/Encoder_receiver_transmitter) for more details on packet structure.
+Currently both SCM (Standard Consumption Message) and IDM (Interval Data Message) packets can be decoded but are mutually exclusive, you cannot receive both simultaneously. See [RTLAMR: Protocol](http://bemasher.github.io/rtlamr/protocol.html) for more details on packet structure.
+
+There's now experimental support for meters with R900 transmitters!
 
 ### Sensitivity
 Using a NooElec NESDR Nano R820T with the provided antenna, I can reliably receive standard consumption messages from ~300 different meters and intermittently from another ~600 meters. These figures are calculated from the number of messages received during a 25 minute window. Reliably in this case means receiving at least 10 of the expected 12 messages and intermittently means 3-9 messages.
@@ -79,8 +82,6 @@ Check out the table of meters I've been compiling from various internet sources:
 
 If you've got a meter not on the list that you've successfully received messages from, you can submit this info via a form available at the link above.
 
-There's now experimental support for meters with R900 transmitters!
-
 ### Ethics
 _Do not use this for nefarious purposes._ If you do, I don't want to know about it, I am not and will not be responsible for your lack of common decency and/or foresight. However, if you find a clever non-evil use for this, by all means, share.
 

decode/decode.go

@@ -25,36 +25,79 @@ import (
 
 // PacketConfig specifies packet-specific radio configuration.
 type PacketConfig struct {
-	DataRate                    int
+	DataRate int
+
 	BlockSize, BlockSize2       int
 	SymbolLength, SymbolLength2 int
 	SampleRate                  int
 
 	PreambleSymbols, PacketSymbols int
 	PreambleLength, PacketLength   int
-	BufferLength                   int
 	Preamble                       string
 
+	BufferLength int
+
 	CenterFreq uint32
 }
 
-func (cfg PacketConfig) Log() {
-	log.Println("BlockSize:", cfg.BlockSize)
-	log.Println("CenterFreq:", cfg.CenterFreq)
-	log.Println("SampleRate:", cfg.SampleRate)
-	log.Println("DataRate:", cfg.DataRate)
-	log.Println("SymbolLength:", cfg.SymbolLength)
-	log.Println("PreambleSymbols:", cfg.PreambleSymbols)
-	log.Println("PreambleLength:", cfg.PreambleLength)
-	log.Println("PacketSymbols:", cfg.PacketSymbols)
-	log.Println("PacketLength:", cfg.PacketLength)
-	log.Println("Preamble:", cfg.Preamble)
+func (cfg PacketConfig) Decimate(decimation int) PacketConfig {
+	cfg.BlockSize /= decimation
+	cfg.BlockSize2 /= decimation
+	cfg.SymbolLength /= decimation
+	cfg.SymbolLength2 /= decimation
+	cfg.SampleRate /= decimation
+	cfg.DataRate /= decimation
+
+	cfg.PreambleLength /= decimation
+	cfg.PacketLength /= decimation
+
+	cfg.BufferLength /= decimation
+
+	return cfg
+}
+
+func (d Decoder) Log() {
+	if d.Decimation != 1 {
+		log.Printf("BlockSize: %d|%d\n", d.Cfg.BlockSize, d.DecCfg.BlockSize)
+		log.Println("CenterFreq:", d.Cfg.CenterFreq)
+		log.Printf("SampleRate: %d|%d\n", d.Cfg.SampleRate, d.DecCfg.SampleRate)
+		log.Printf("DataRate: %d|%d\n", d.Cfg.DataRate, d.DecCfg.DataRate)
+		log.Printf("SymbolLength: %d|%d\n", d.Cfg.SymbolLength, d.DecCfg.SymbolLength)
+		log.Println("PreambleSymbols:", d.Cfg.PreambleSymbols)
+		log.Printf("PreambleLength: %d|%d\n", d.Cfg.PreambleLength, d.DecCfg.PreambleLength)
+		log.Println("PacketSymbols:", d.Cfg.PacketSymbols)
+		log.Printf("PacketLength: %d|%d\n", d.Cfg.PacketLength, d.DecCfg.PacketLength)
+		log.Println("Preamble:", d.Cfg.Preamble)
+
+		if d.Cfg.SymbolLength%d.Decimation != 0 {
+			log.Println("Warning: decimated symbol length is non-integral, sensitivity may be poor")
+		}
+
+		if d.DecCfg.SymbolLength < 3 {
+			log.Fatal("Error: illegal decimation factor, choose a smaller factor")
+		}
+
+		return
+	}
+
+	log.Println("CenterFreq:", d.Cfg.CenterFreq)
+	log.Println("SampleRate:", d.Cfg.SampleRate)
+	log.Println("DataRate:", d.Cfg.DataRate)
+	log.Println("SymbolLength:", d.Cfg.SymbolLength)
+	log.Println("PreambleSymbols:", d.Cfg.PreambleSymbols)
+	log.Println("PreambleLength:", d.Cfg.PreambleLength)
+	log.Println("PacketSymbols:", d.Cfg.PacketSymbols)
+	log.Println("PacketLength:", d.Cfg.PacketLength)
+	log.Println("Preamble:", d.Cfg.Preamble)
 }
 
 // Decoder contains buffers and radio configuration.
 type Decoder struct {
 	Cfg PacketConfig
 
+	Decimation int
+	DecCfg     PacketConfig
+
 	IQ        []byte
 	Signal    []float64
 	Filtered  []float64
@@ -70,16 +113,30 @@ type Decoder struct {
 }
 
 // Create a new decoder with the given packet configuration.
-func NewDecoder(cfg PacketConfig, fastMag bool) (d Decoder) {
+func NewDecoder(cfg PacketConfig, decimation int, fastMag bool) (d Decoder) {
 	d.Cfg = cfg
 
+	d.Cfg.SymbolLength2 = d.Cfg.SymbolLength << 1
+	d.Cfg.SampleRate = d.Cfg.DataRate * d.Cfg.SymbolLength
+
+	d.Cfg.PreambleLength = d.Cfg.PreambleSymbols * d.Cfg.SymbolLength2
+	d.Cfg.PacketLength = d.Cfg.PacketSymbols * d.Cfg.SymbolLength2
+
+	d.Cfg.BlockSize = NextPowerOf2(d.Cfg.PreambleLength)
+	d.Cfg.BlockSize2 = d.Cfg.BlockSize << 1
+
+	d.Cfg.BufferLength = d.Cfg.PacketLength + d.Cfg.BlockSize
+
+	d.Decimation = decimation
+	d.DecCfg = d.Cfg.Decimate(d.Decimation)
+
 	// Allocate necessary buffers.
 	d.IQ = make([]byte, d.Cfg.BufferLength<<1)
-	d.Signal = make([]float64, d.Cfg.BufferLength)
-	d.Filtered = make([]float64, d.Cfg.BufferLength)
-	d.Quantized = make([]byte, d.Cfg.BufferLength)
+	d.Signal = make([]float64, d.DecCfg.BufferLength)
+	d.Filtered = make([]float64, d.DecCfg.BufferLength)
+	d.Quantized = make([]byte, d.DecCfg.BufferLength)
 
-	d.csum = make([]float64, d.Cfg.BlockSize+d.Cfg.SymbolLength2+1)
+	d.csum = make([]float64, (d.DecCfg.PacketLength - d.DecCfg.SymbolLength2 + 1))
 
 	// Calculate magnitude lookup table specified by -fastmag flag.
 	if fastMag {
@@ -99,10 +156,10 @@ func NewDecoder(cfg PacketConfig, fastMag bool) (d Decoder) {
 	// Slice quantized sample buffer to make searching for the preamble more
 	// memory local. Pre-allocate a flat buffer so memory is contiguous and
 	// assign slices to the buffer.
-	d.slices = make([][]byte, d.Cfg.SymbolLength2)
-	flat := make([]byte, d.Cfg.BlockSize2-(d.Cfg.BlockSize2%d.Cfg.SymbolLength2))
+	d.slices = make([][]byte, d.DecCfg.SymbolLength2)
+	flat := make([]byte, d.DecCfg.BlockSize2-(d.DecCfg.BlockSize2%d.DecCfg.SymbolLength2))
 
-	symbolsPerBlock := d.Cfg.BlockSize2 / d.Cfg.SymbolLength2
+	symbolsPerBlock := d.DecCfg.BlockSize2 / d.DecCfg.SymbolLength2
 	for symbolOffset := range d.slices {
 		lower := symbolOffset * symbolsPerBlock
 		upper := (symbolOffset + 1) * symbolsPerBlock
@@ -111,7 +168,7 @@ func NewDecoder(cfg PacketConfig, fastMag bool) (d Decoder) {
 
 	// Signal up to the final stage is 1-bit per byte. Allocate a buffer to
 	// store packed version 8-bits per byte.
-	d.pkt = make([]byte, (d.Cfg.PacketSymbols+7)>>3)
+	d.pkt = make([]byte, (d.DecCfg.PacketSymbols+7)>>3)
 
 	return
 }
@@ -120,28 +177,28 @@ func NewDecoder(cfg PacketConfig, fastMag bool) (d Decoder) {
 func (d Decoder) Decode(input []byte) []int {
 	// Shift buffers to append new block.
 	copy(d.IQ, d.IQ[d.Cfg.BlockSize<<1:])
-	copy(d.Signal, d.Signal[d.Cfg.BlockSize:])
-	copy(d.Filtered, d.Filtered[d.Cfg.BlockSize:])
-	copy(d.Quantized, d.Quantized[d.Cfg.BlockSize:])
+	copy(d.Signal, d.Signal[d.DecCfg.BlockSize:])
+	copy(d.Filtered, d.Filtered[d.DecCfg.BlockSize:])
+	copy(d.Quantized, d.Quantized[d.DecCfg.BlockSize:])
 	copy(d.IQ[d.Cfg.PacketLength<<1:], input[:])
 
 	iqBlock := d.IQ[d.Cfg.PacketLength<<1:]
-	signalBlock := d.Signal[d.Cfg.PacketLength:]
+	signalBlock := d.Signal[d.DecCfg.PacketLength:]
 
 	// Compute the magnitude of the new block.
 	d.demod.Execute(iqBlock, signalBlock)
 
-	signalBlock = d.Signal[d.Cfg.PacketLength-d.Cfg.SymbolLength2:]
-	filterBlock := d.Filtered[d.Cfg.PacketLength-d.Cfg.SymbolLength2:]
+	signalBlock = d.Signal[d.DecCfg.PacketLength-d.DecCfg.SymbolLength2:]
+	filterBlock := d.Filtered[d.DecCfg.PacketLength-d.DecCfg.SymbolLength2:]
 
 	// Perform matched filter on new block.
 	d.Filter(signalBlock, filterBlock)
 
 	// Perform bit-decision on new block.
-	Quantize(filterBlock, d.Quantized[d.Cfg.PacketLength-d.Cfg.SymbolLength2:])
+	Quantize(filterBlock, d.Quantized[d.DecCfg.PacketLength-d.DecCfg.SymbolLength2:])
 
 	// Pack the quantized signal into slices for searching.
-	d.Pack(d.Quantized[:d.Cfg.BlockSize2], d.slices)
+	d.Pack(d.Quantized[:d.DecCfg.BlockSize2], d.slices)
 
 	// Return a list of indexes the preamble exists at.
 	return d.Search(d.slices, d.preamble)
@@ -168,8 +225,12 @@ func NewSqrtMagLUT() (lut MagLUT) {
 
 // Calculates complex magnitude on given IQ stream writing result to output.
 func (lut MagLUT) Execute(input []byte, output []float64) {
-	for idx := 0; idx < len(input); idx += 2 {
-		output[idx>>1] = math.Sqrt(lut[input[idx]] + lut[input[idx+1]])
+	decIdx := 0
+	dec := (len(input) / len(output))
+
+	for idx := 0; decIdx < len(output); idx += dec {
+		output[decIdx] = math.Sqrt(lut[input[idx]] + lut[input[idx+1]])
+		decIdx++
 	}
 }
 
@@ -192,14 +253,18 @@ func (lut AlphaMaxBetaMinLUT) Execute(input []byte, output []float64) {
 		ß = 0.392699081699
 	)
 
-	for idx := 0; idx < len(input); idx += 2 {
+	decIdx := 0
+	dec := (len(input) / len(output))
+
+	for idx := 0; decIdx < len(output); idx += dec {
 		i := lut[input[idx]]
 		q := lut[input[idx+1]]
 		if i > q {
-			output[idx>>1] = α*i + ß*q
+			output[decIdx] = α*i + ß*q
 		} else {
-			output[idx>>1] = α*q + ß*i
+			output[decIdx] = α*q + ß*i
 		}
+		decIdx++
 	}
 }
 
@@ -215,9 +280,10 @@ func (d Decoder) Filter(input, output []float64) {
 	}
 
 	// Filter result is difference of summation of lower and upper symbols.
-	lower := d.csum[d.Cfg.SymbolLength:]
-	upper := d.csum[d.Cfg.SymbolLength2:]
-	for idx := range input[:len(input)-d.Cfg.SymbolLength2] {
+	lower := d.csum[d.DecCfg.SymbolLength:]
+	upper := d.csum[d.DecCfg.SymbolLength2:]
+	n := len(input) - d.DecCfg.SymbolLength2
+	for idx := 0; idx < n; idx++ {
 		output[idx] = (lower[idx] - d.csum[idx]) - (upper[idx] - lower[idx])
 	}
 
@@ -239,7 +305,7 @@ func Quantize(input []float64, output []byte) {
 func (d Decoder) Pack(input []byte, slices [][]byte) {
 	for symbolOffset, slice := range slices {
 		for symbolIdx := range slice {
-			slice[symbolIdx] = input[symbolIdx*d.Cfg.SymbolLength2+symbolOffset]
+			slice[symbolIdx] = input[symbolIdx*d.DecCfg.SymbolLength2+symbolOffset]
 		}
 	}
 
@@ -254,7 +320,7 @@ func (d Decoder) Search(slices [][]byte, preamble []byte) (indexes []int) {
 	for symbolOffset, slice := range slices {
 		for symbolIdx := range slice[:len(slice)-preambleLength] {
 			if bytes.Equal(preamble, slice[symbolIdx:][:preambleLength]) {
-				indexes = append(indexes, symbolIdx*d.Cfg.SymbolLength2+symbolOffset)
+				indexes = append(indexes, symbolIdx*d.DecCfg.SymbolLength2+symbolOffset)
 			}
 		}
 	}
@@ -274,14 +340,14 @@ func (d Decoder) Slice(indices []int) (pkts [][]byte) {
 	for _, qIdx := range indices {
 		// Check that we're still within the first sample block. We'll catch
 		// the message on the next sample block otherwise.
-		if qIdx > d.Cfg.BlockSize {
+		if qIdx > d.DecCfg.BlockSize {
 			continue
 		}
 
 		// Packet is 1 bit per byte, pack to 8-bits per byte.
-		for pIdx := 0; pIdx < d.Cfg.PacketSymbols; pIdx++ {
+		for pIdx := 0; pIdx < d.DecCfg.PacketSymbols; pIdx++ {
 			d.pkt[pIdx>>3] <<= 1
-			d.pkt[pIdx>>3] |= d.Quantized[qIdx+(pIdx*d.Cfg.SymbolLength2)]
+			d.pkt[pIdx>>3] |= d.Quantized[qIdx+(pIdx*d.DecCfg.SymbolLength2)]
 		}
 
 		// Store the packet in the seen map and append to the packet list.

flags.go

@@ -39,7 +39,9 @@ var sampleFile *os.File
 var msgType = flag.String("msgtype", "scm", "message type to receive: scm, idm or r900")
 var fastMag = flag.Bool("fastmag", false, "use faster alpha max + beta min magnitude approximation")
 
-var symbolLength = flag.Int("symbollength", 73, "symbol length in samples, see -help for valid lengths")
+var symbolLength = flag.Int("symbollength", 72, "symbol length in samples, see -help for valid lengths")
+
+var decimation = flag.Int("decimation", 1, "integer decimation factor, keep every nth sample")
 
 var timeLimit = flag.Duration("duration", 0, "time to run for, 0 for infinite, ex. 1h5m10s")
 var meterID UintMap
@@ -64,6 +66,7 @@ func RegisterFlags() {
 		"samplefile":   true,
 		"msgtype":      true,
 		"symbollength": true,
+		"decimation":   true,
 		"duration":     true,
 		"filterid":     true,
 		"filtertype":   true,

idm/idm.go

@@ -28,27 +28,13 @@ import (
 )
 
 func NewPacketConfig(symbolLength int) (cfg decode.PacketConfig) {
-	cfg.DataRate = 32768
-
 	cfg.CenterFreq = 912600155
-
+	cfg.DataRate = 32768
 	cfg.SymbolLength = symbolLength
-	cfg.SymbolLength2 = cfg.SymbolLength << 1
-
-	cfg.SampleRate = cfg.DataRate * cfg.SymbolLength
-
 	cfg.PreambleSymbols = 32
 	cfg.PacketSymbols = 92 * 8
-
-	cfg.PreambleLength = cfg.PreambleSymbols * cfg.SymbolLength2
-	cfg.PacketLength = cfg.PacketSymbols * cfg.SymbolLength2
-
-	cfg.BlockSize = decode.NextPowerOf2(cfg.PreambleLength)
-	cfg.BlockSize2 = cfg.BlockSize << 1
-
-	cfg.BufferLength = cfg.PacketLength + cfg.BlockSize
-
 	cfg.Preamble = "01010101010101010001011010100011"
+
 	return
 }
 
@@ -65,8 +51,8 @@ func (p Parser) Cfg() decode.PacketConfig {
 	return p.Decoder.Cfg
 }
 
-func NewParser(symbolLength int, fastMag bool) (p Parser) {
-	p.Decoder = decode.NewDecoder(NewPacketConfig(symbolLength), fastMag)
+func NewParser(symbolLength, decimation int, fastMag bool) (p Parser) {
+	p.Decoder = decode.NewDecoder(NewPacketConfig(symbolLength), decimation, fastMag)
 	p.CRC = crc.NewCRC("CCITT", 0xFFFF, 0x1021, 0x1D0F)
 	return
 }
@@ -158,10 +144,6 @@ func NewIDM(data parse.Data) (idm IDM) {
 
 type Interval [47]uint16
 
-// func (interval Interval) MarshalText() (text []byte, err error) {
-// 	return []byte(fmt.Sprintf("%+v", interval)), nil
-// }
-
 func (interval Interval) Record() (r []string) {
 	for _, val := range interval {
 		r = append(r, strconv.FormatUint(uint64(val), 10))