[WIP] bugfixes

cmd/autostart/autostart.go

@@ -18,11 +18,11 @@ func main() {
 	service := core.TrackingService(repo, disp, conf)
 
 	go func() {
-		service.NewTracking(core.TCP, core.SERIAL)
+		service.NewTracking(core.TCP)
 		time.Sleep(5 * time.Second)
 		service.StartRecord()
 		time.Sleep(15 * time.Second)
-		service.StopRecord()
+		//service.StopRecord()
 	}()
 
 	web.CreateServer(service, disp, conf)

core/collectors.go

@@ -2,13 +2,11 @@ package core
 
 import (
 	"fmt"
+	"git.timovolkmann.de/gyrogpsc/ublox"
+	"go.bug.st/serial"
 	"log"
 	"net"
 	"os"
-	"time"
-
-	"git.timovolkmann.de/gyrogpsc/ublox"
-	"go.bug.st/serial"
 )
 
 type Collector interface {
@@ -74,11 +72,14 @@ func (s *serialCollector) Collect() {
 				break
 			}
 			sd, err := ConvertUbxToSensorData(meas)
-			if err != nil || sd == nil {
+			if err != nil {
 				log.Println("convert err:", err, meas, sd)
 				continue
 			}
-
+			// skip irrelevant messages
+			if sd == nil {
+				continue
+			}
 
 			err = s.proc.Push(sd)
 			if err != nil {
@@ -172,7 +173,7 @@ func (c *tcpCollector) jsonHandler(conn net.Conn) {
 			continue
 		}
 		if !c.active {
-			time.Sleep(50 * time.Millisecond)
+			//time.Sleep(50 * time.Millisecond)
 			continue
 		}
 		err = c.processor.Push(sd)

core/pipeline.go

@@ -1,180 +1,194 @@
 package core
 
 import (
-	"encoding/json"
+    "encoding/json"
-	"errors"
+    "errors"
-	"fmt"
+    "fmt"
-	"log"
+    "github.com/google/go-cmp/cmp"
-	"sync"
+    "github.com/google/go-cmp/cmp/cmpopts"
-	"time"
+    "log"
+    "sync"
+    "time"
 )
 
 type pipeline struct {
-	active        bool
+    active        bool
-	record   	  bool
+    record        bool
-	syn           synchronizer
+    synchroniz    synchronizer
-	agr           aggregator
+    buffer        pipeBuffer
-	pub           Publisher
+    publisher     Publisher
-	stor          Storer
+    storer        Storer
-	publishTicker *time.Ticker
+    publishTicker *time.Ticker
 }
 
 // pipe implements Runner & Pusher
 func NewPipeline(d Publisher, s Storer, conf *Configuration) *pipeline {
-	return &pipeline{
+    return &pipeline{
-		false,
+        false,
-		false,
+        false,
-		synchronizer{
+        synchronizer{
-			//bufferSize:   100,
+            //bufferSize:   100,
-			mutex:        &sync.Mutex{},
+            mutex:        &sync.Mutex{},
-			updateTicker: time.NewTicker(time.Duration(conf.Pipeline.SyncUpdateIntervalMs) * time.Millisecond),
+            updateTicker: time.NewTicker(time.Duration(conf.Pipeline.SyncUpdateIntervalMs) * time.Millisecond),
-		},
+        },
-		aggregator{
+        pipeBuffer{
-			tcpMutex:    &sync.Mutex{},
+            tcpMutex:    &sync.Mutex{},
-			serialMutex: &sync.Mutex{},
+            serialMutex: &sync.Mutex{},
-		},
+        },
-		d,
+        d,
-		s,
+        s,
-		time.NewTicker(time.Duration(conf.Pipeline.PublishIntervalMs) * time.Millisecond),
+        time.NewTicker(time.Duration(conf.Pipeline.PublishIntervalMs) * time.Millisecond),
-	}
+    }
 }
 
 func (p *pipeline) Run() {
-	p.active = true
+    p.active = true
-	log.Println("pipe: processing service started")
+    log.Println("pipe: processing service started")
-	go func() {
+    go func() {
-		for p.active {
+        for p.active {
-			<-p.syn.updateTicker.C
+            <-p.synchroniz.updateTicker.C
-			err := p.refreshDelay()
+            err := p.refreshDelay()
-			if err != nil {
+            if err != nil {
-				log.Println(err)
+                log.Println(err)
-			}
+            }
-		}
+        }
-		log.Println("pipe: updater stopped")
+        log.Println("pipe: updater stopped")
-	}()
+    }()
-	go func() {
+    go func() {
-		for p.active {
+        for p.active {
-			<-p.publishTicker.C
+            <-p.publishTicker.C
-			err := p.Publish()
+            err := p.publish()
-			if err != nil && err.Error() != "no data available" {
+            if err != nil && err.Error() != "no data available" {
-				log.Println(err)
+                log.Println(err)
-			}
+            }
-		}
+        }
-		log.Println("pipe: publisher stopped")
+        log.Println("pipe: publisher stopped")
-	}()
+    }()
 }
 
 func (p *pipeline) Record() {
-	p.record = true
+    p.record = true
 }
 func (p *pipeline) Stop() {
-	p.record = false
+    p.record = false
 }
 
-func (p *pipeline) Publish() error {
+func (p *pipeline) publish() error {
-	p.agr.tcpMutex.Lock()
+    p.buffer.tcpMutex.Lock()
-	p.agr.serialMutex.Lock()
+    p.buffer.serialMutex.Lock()
 
-	if (p.agr.tcpSensorData == sensorData{} && p.agr.serialSensorData == sensorData{}) {
+    if (p.buffer.MeasTcp == sensorData{} && p.buffer.MeasSerial == sensorData{}) {
-		p.agr.tcpMutex.Unlock()
+        p.buffer.tcpMutex.Unlock()
-		p.agr.serialMutex.Unlock()
+        p.buffer.serialMutex.Unlock()
-		return errors.New("no data available")
+        return errors.New("no data available")
-	}
+    }
+    if cmp.Equal(p.buffer.MeasTcp, p.buffer.LastMeasTcp, cmpopts.IgnoreUnexported(sensorData{})) &&
+        cmp.Equal(p.buffer.MeasSerial, p.buffer.LastMeasSerial, cmpopts.IgnoreUnexported(sensorData{})) {
+        p.buffer.tcpMutex.Unlock()
+        p.buffer.serialMutex.Unlock()
+        return errors.New("same data")
+    }
+    log.Println("––––––––––––––––––––––––––––––––––––")
+    log.Printf("MEAS old: %v", p.buffer.LastMeasTcp)
+    log.Printf("MEAS new: %v", p.buffer.MeasTcp)
+    log.Println("––––––––––––––––––––––––––––––––––––")
+    p.buffer.LastMeasTcp = p.buffer.MeasTcp
+    p.buffer.LastMeasSerial = p.buffer.MeasSerial
+    p.storer.EnqueuePair(p.buffer.MeasTcp, p.buffer.MeasSerial)
 
-	p.stor.EnqueuePair(p.agr.tcpSensorData, p.agr.serialSensorData)
+    data := map[string]sensorData{
+        string(SOURCE_TCP):    p.buffer.MeasTcp,
+        string(SOURCE_SERIAL): p.buffer.MeasSerial,
+    }
 
-	data := map[string]sensorData{
+    p.buffer.tcpMutex.Unlock()
-		string(SOURCE_TCP):    p.agr.tcpSensorData,
+    p.buffer.serialMutex.Unlock()
-		string(SOURCE_SERIAL): p.agr.serialSensorData,
-	}
 
-	p.agr.tcpMutex.Unlock()
+    jdata, err := json.Marshal(data)
-	p.agr.serialMutex.Unlock()
+    //log.Println(string(pretty.Pretty(jdata)))
-
+    if err != nil {
-	jdata, err := json.Marshal(data)
+        return err
-	//log.Println(string(pretty.Pretty(jdata)))
+    }
-	if err != nil {
+    p.publisher.Publish(string(jdata))
-		return err
+    return nil
-	}
-	p.pub.Publish(string(jdata))
-	return nil
 }
 
-type aggregator struct {
+type pipeBuffer struct {
-	tcpSensorData    sensorData
+    MeasTcp        sensorData
-	serialSensorData sensorData
+    MeasSerial     sensorData
-	tcpMutex         *sync.Mutex
+    LastMeasTcp    sensorData
-	serialMutex      *sync.Mutex
+    LastMeasSerial sensorData
+    tcpMutex       *sync.Mutex
+    serialMutex    *sync.Mutex
 }
 
 type UnixNanoTime int64
 
 type synchronizer struct {
-	tcpSerialDelayMs int64
+    tcpSerialDelayMs int64
-	mutex        *sync.Mutex
+    mutex            *sync.Mutex
-	updateTicker *time.Ticker
+    updateTicker     *time.Ticker
 }
 
 func (p *pipeline) refreshDelay() error {
-	log.Println("refreshing delay....")
+    log.Println("refreshing delay....")
-	fmt.Println("Delay TCP/SERIAL", p.syn.tcpSerialDelayMs)
+    fmt.Println("Delay TCP/SERIAL", p.synchroniz.tcpSerialDelayMs)
-	p.agr.serialMutex.Lock()
+    p.buffer.serialMutex.Lock()
-	p.agr.tcpMutex.Lock()
+    p.buffer.tcpMutex.Lock()
-	tcpTime := time.Unix(0, p.agr.tcpSensorData.Timestamp)
+    tcpTime := time.Unix(0, p.buffer.MeasTcp.Timestamp)
-	serTime := time.Unix(0, p.agr.serialSensorData.Timestamp)
+    serTime := time.Unix(0, p.buffer.MeasSerial.Timestamp)
-	p.agr.serialMutex.Unlock()
+    p.buffer.serialMutex.Unlock()
-	p.agr.tcpMutex.Unlock()
+    p.buffer.tcpMutex.Unlock()
-	if tcpTime.UnixNano() == 0 || serTime.UnixNano() == 0 {
+    if tcpTime.UnixNano() == 0 || serTime.UnixNano() == 0 {
-		return errors.New("no sync possible. check if both collectors running. otherwise check GPS fix")
+        return errors.New("no sync possible. check if both collectors running. otherwise check GPS fix")
-	}
+    }
-	currentDelay := tcpTime.Sub(serTime).Milliseconds()
+    currentDelay := tcpTime.Sub(serTime).Milliseconds()
-	if currentDelay > 5000 || currentDelay < -5000 {
+    if currentDelay > 5000 || currentDelay < -5000 {
-		p.syn.tcpSerialDelayMs = 0
+        p.synchroniz.tcpSerialDelayMs = 0
-		return errors.New("skipping synchronisation! time not properly configured or facing network problems.")
+        return errors.New("skipping synchronisation! time not properly configured or facing network problems.")
-	}
+    }
-	log.Println("TCP", tcpTime.String())
+    log.Println("TCP", tcpTime.String())
-	log.Println("SER", serTime.String())
+    log.Println("SER", serTime.String())
-	log.Println("Difference", tcpTime.Sub(serTime).Milliseconds(), "ms")
+    log.Println("Difference", tcpTime.Sub(serTime).Milliseconds(), "ms")
-	delay := tcpTime.Sub(serTime).Milliseconds()
+    delay := tcpTime.Sub(serTime).Milliseconds()
-	p.syn.tcpSerialDelayMs += delay
+    p.synchroniz.tcpSerialDelayMs += delay
-	return nil
+    return nil
 }
 
 func (p *pipeline) Push(data *sensorData) error {
-	if data == nil {
+    if data == nil {
-		return errors.New("nil processing not allowed")
+        return errors.New("nil processing not allowed")
-	}
+    }
-	//log.Println(string(data.source))
+    //log.Println("push data to pipe:", string(data.source))
-	// TODO: persist data here with current timestamp
+    p.storer.EnqueueRaw(*data)
-	p.stor.EnqueueRaw(*data)
+    switch data.source {
-	switch data.source {
+    case SOURCE_TCP:
-	case SOURCE_TCP:
+        go p.pushTcpDataToBuffer(*data)
-		go p.pushTcpDataToBuffer(*data)
+    case SOURCE_SERIAL:
-	case SOURCE_SERIAL:
+        go p.pushSerialDataToBuffer(*data)
-		go p.pushSerialDataToBuffer(*data)
+    default:
-	default:
+        panic("pipe: invalid data source")
-		panic("pipe: invalid data source")
+    }
-	}
+    return nil
-	return nil
 }
 
 func (p *pipeline) pushTcpDataToBuffer(data sensorData) {
-	if p.syn.tcpSerialDelayMs > 0 {
+    if p.synchroniz.tcpSerialDelayMs > 0 {
-		time.Sleep(time.Duration(p.syn.tcpSerialDelayMs) * time.Millisecond)
+        time.Sleep(time.Duration(p.synchroniz.tcpSerialDelayMs) * time.Millisecond)
-	}
+    }
-	p.agr.tcpMutex.Lock()
+    p.buffer.tcpMutex.Lock()
-	p.agr.tcpSensorData = p.agr.tcpSensorData.ConsolidateExTime(data)
+    p.buffer.MeasTcp = p.buffer.MeasTcp.ConsolidateExTime(data)
-	p.agr.tcpMutex.Unlock()
+    p.buffer.tcpMutex.Unlock()
 }
 func (p *pipeline) pushSerialDataToBuffer(data sensorData) {
-	if p.syn.tcpSerialDelayMs < 0 {
+    if p.synchroniz.tcpSerialDelayMs < 0 {
-		time.Sleep(time.Duration(-p.syn.tcpSerialDelayMs) * time.Millisecond)
+        time.Sleep(time.Duration(-p.synchroniz.tcpSerialDelayMs) * time.Millisecond)
-	}
+    }
-	p.agr.serialMutex.Lock()
+    p.buffer.serialMutex.Lock()
-	p.agr.serialSensorData = p.agr.serialSensorData.ConsolidateEpochsOnly(data)
+    p.buffer.MeasSerial = p.buffer.MeasSerial.ConsolidateEpochsOnly(data)
-	p.agr.serialMutex.Unlock()
+    p.buffer.serialMutex.Unlock()
 }
 
 func (p *pipeline) Close() {
-	p.active = false
+    p.active = false
 }

go.mod

@@ -5,6 +5,7 @@ go 1.15
 require (
 	github.com/gofiber/fiber/v2 v2.2.4
 	github.com/gofiber/websocket/v2 v2.0.2
+	github.com/google/go-cmp v0.3.0
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/websocket v1.4.2
 	github.com/spf13/viper v1.7.1

go.sum

@@ -74,6 +74,7 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=