some improvements in logging and naming/structuring

core/dispatcher.go

@@ -18,7 +18,7 @@ func NewDispatcher() *dispatcher {
 }
 
 func (d *dispatcher) Publish(message string) {
-    logrus.Debugf("publish to %v listeners:\n%v\n", len(d.listeners))
+	logrus.Debugf("publish to %v listeners\n", len(d.listeners))
 	logrus.Debug(message)
 	for _, ch := range d.listeners {
 		ch <- message

core/sensordata.go

@@ -25,16 +25,6 @@ type sensorData struct {
 	Orientation [3]float64
 }
 
-type recordPair struct {
-    RecordTime time.Time
-    data       map[sourceId]sensorData
-}
-
-type rawRecord struct {
-    RecordTime time.Time
-    sensorData
-}
-
 func (s sensorData) isSameEpoch(n sensorData) bool {
 	if n.itow == 0 {
 		return false

core/service.go

@@ -92,11 +92,11 @@ func (t *trackingService) StopRecord() {
 	t.opMode = LIVE
 	t.pipe.StopRecord()
 
-	m1.Lock()
+	mRec.Lock()
-	m2.Lock()
+	mRaw.Lock()
 	err := t.repo.Save(*t.current)
-	m2.Unlock()
+	mRaw.Unlock()
-	m1.Unlock()
+	mRec.Unlock()
 
 	if err != nil {
 		logrus.Println(err)
@@ -140,9 +140,9 @@ func (t *trackingService) LoadTracking(trackingId uuid.UUID) {
 }
 
 func (t *trackingService) safelyReplaceTracking(tr Tracking) {
-	m1.Lock()
+	mRec.Lock()
-	m2.Lock()
+	mRaw.Lock()
 	*t.current = tr
-	m2.Unlock()
+	mRaw.Unlock()
-	m1.Unlock()
+	mRec.Unlock()
 }

core/trackings.go

@@ -7,33 +7,46 @@ import (
 	"time"
 )
 
-var m1 sync.RWMutex
-var m2 sync.RWMutex
-
 type Tracking struct {
 	TrackingMetadata
 	Records []recordPair
 	Rawdata []rawRecord
 }
 
+var mRec sync.RWMutex
+var mRaw sync.RWMutex
+
 type TrackingMetadata struct {
 	UUID        uuid.UUID
 	TimeCreated time.Time
 	Collectors  []CollectorType
 }
 
+// persistence wrapper for sensordata
+type recordPair struct {
+	RecordTimeKey time.Time // uniqueness ensured through mutex
+	DataPair      map[sourceId]sensorData
+}
+
+type rawRecord struct {
+	RecordTimeKey time.Time // uniqueness ensured through mutex
+	Data          sensorData
+}
+
+// END persistence wrapper for sensordata
+
 func (s *Tracking) EnqueuePair(tcp sensorData, ser sensorData) {
 	rp := recordPair{
-        RecordTime: time.Now(),
+		RecordTimeKey: time.Now(),
-        data: map[sourceId]sensorData{
+		DataPair: map[sourceId]sensorData{
 			tcp.source: tcp,
 			ser.source: ser,
 		},
 	}
-    m1.Lock()
+	mRec.Lock()
 	s.Records = append(s.Records, rp)
 	logrus.Debugln("tracking Records: len->", len(s.Records))
-    m1.Unlock()
+	mRec.Unlock()
 }
 
 func (s *Tracking) EnqueueRaw(data sensorData) {
@@ -41,10 +54,10 @@ func (s *Tracking) EnqueueRaw(data sensorData) {
 		time.Now(),
 		data,
 	}
-    m1.Lock()
+	mRec.Lock()
 	s.Rawdata = append(s.Rawdata, sr)
 	logrus.Debugln("raw data points: len->", len(s.Rawdata))
-    m1.Unlock()
+	mRec.Unlock()
 }
 
 func emptyTracking() Tracking {

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/spf13/viper v1.7.1
 	github.com/stretchr/testify v1.6.1 // indirect
 	github.com/tidwall/gjson v1.6.0
-	github.com/tidwall/pretty v1.0.2 // indirect
+	github.com/tidwall/pretty v1.0.2
 	go.bug.st/serial v1.1.1
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 )

storage/kvstore.go

@@ -7,15 +7,17 @@ import (
 	"github.com/dgraph-io/badger/v2"
 	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
+	"github.com/tidwall/pretty"
 	"os"
 	"path/filepath"
+	"strconv"
 )
 
 // Must implement Repo
 type badgerStore struct {
-    trackings *badger.DB
+	trackingsDb *badger.DB
-    records   *badger.DB
+	recordsDb   *badger.DB
-    rawdata   *badger.DB
+	rawdataDb   *badger.DB
 }
 
 func NewRepository(c *core.Configuration) *badgerStore {
@@ -32,11 +34,11 @@ func NewRepository(c *core.Configuration) *badgerStore {
 	if err != nil {
 		logrus.Error(err)
 	}
-    return &badgerStore{trackings: tr, records: dp, rawdata: rd}
+	return &badgerStore{trackingsDb: tr, recordsDb: dp, rawdataDb: rd}
 }
 
 func (r *badgerStore) isDbAvailable() bool {
-    return r.trackings.IsClosed() || r.records.IsClosed() || r.rawdata.IsClosed()
+	return r.trackingsDb.IsClosed() || r.recordsDb.IsClosed() || r.rawdataDb.IsClosed()
 }
 
 func (r *badgerStore) Save(tr core.Tracking) error {
@@ -54,10 +56,12 @@ func (r *badgerStore) Save(tr core.Tracking) error {
 		logrus.Error(err, tr)
 		return err
 	}
-    err = r.records.Update(func(txn *badger.Txn) error {
+	err = r.recordsDb.Update(func(txn *badger.Txn) error {
 		for _, v := range tr.Records {
-            k := createDataKey(tr.UUID, v.RecordTime.UnixNano())
+			k := createRecordKey(tr.UUID, v.RecordTimeKey.UnixNano())
-            j, err := json.Marshal(v)
+			j, err := json.Marshal(v.DataPair)
+			logrus.Debugln("save record k/v:\n", tr.UUID.String(), strconv.FormatInt(v.RecordTimeKey.UnixNano(), 10))
+			logrus.Debugln(string(pretty.Pretty(j)))
 			if err != nil {
 				return err
 			}
@@ -69,10 +73,12 @@ func (r *badgerStore) Save(tr core.Tracking) error {
 		logrus.Error(err, tr)
 		return err
 	}
-    err = r.records.Update(func(txn *badger.Txn) error {
+	err = r.rawdataDb.Update(func(txn *badger.Txn) error {
 		for _, v := range tr.Rawdata {
-            k := createDataKey(tr.UUID, v.Timestamp)
+			k := createRecordKey(tr.UUID, v.RecordTimeKey.UnixNano())
 			j, err := json.Marshal(v)
+			logrus.Debugln("save raw k/v:\n", tr.UUID.String(), strconv.FormatInt(v.RecordTimeKey.UnixNano(), 10))
+			logrus.Debugln(string(pretty.Pretty(j)))
 			if err != nil {
 				return err
 			}
@@ -84,7 +90,8 @@ func (r *badgerStore) Save(tr core.Tracking) error {
 		logrus.Error(err, tr)
 		return err
 	}
-    err = r.trackings.Update(func(txn *badger.Txn) error {
+	err = r.trackingsDb.Update(func(txn *badger.Txn) error {
+		logrus.Debug("save tracking meta k/v:\n", string(ts), string(meta))
 		err := txn.Set(ts, meta)
 		return err
 	})
@@ -92,6 +99,16 @@ func (r *badgerStore) Save(tr core.Tracking) error {
 		logrus.Error(err, tr)
 		return err
 	}
+
+	r.trackingsDb.PrintHistogram(nil)
+	dr := 0.5
+	err = r.trackingsDb.RunValueLogGC(dr)
+	logrus.Debug("DB GC:", err)
+	err = r.recordsDb.RunValueLogGC(dr)
+	logrus.Debug("DB GC:", err)
+	err = r.rawdataDb.RunValueLogGC(dr)
+	logrus.Debug("DB GC:", err)
+	r.trackingsDb.PrintHistogram(nil)
 	logrus.Info("sucessfully saved tracking")
 	return nil
 }
@@ -112,9 +129,9 @@ func (r *badgerStore) Save(tr core.Tracking) error {
 //    ch := make(chan error, 3)
 //    go func() {
 //        defer wg.Done()
-//        err = r.records.Update(func(txn *badger.Txn) error {
+//        err = r.recordsDb.Update(func(txn *badger.Txn) error {
 //            for _, v := range tracking.Records {
-//                k := createDataKey(tracking.UUID, v.RecordTime.UnixNano())
+//                k := createRecordKey(tracking.UUID, v.RecordTime.UnixNano())
 //                j, err := json.Marshal(v)
 //                if err != nil {
 //                    return err
@@ -127,9 +144,9 @@ func (r *badgerStore) Save(tr core.Tracking) error {
 //    }()
 //    go func() {
 //        defer wg.Done()
-//        err = r.records.Update(func(txn *badger.Txn) error {
+//        err = r.recordsDb.Update(func(txn *badger.Txn) error {
 //            for _, v := range tracking.Rawdata {
-//                k := createDataKey(tracking.UUID, v.Timestamp)
+//                k := createRecordKey(tracking.UUID, v.Timestamp)
 //                j, err := json.Marshal(v)
 //                if err != nil {
 //                    return err
@@ -142,7 +159,7 @@ func (r *badgerStore) Save(tr core.Tracking) error {
 //    }()
 //    go func() {
 //        defer wg.Done()
-//        err = r.trackings.Update(func(txn *badger.Txn) error {
+//        err = r.trackingsDb.Update(func(txn *badger.Txn) error {
 //            err := txn.Set(ts, meta)
 //            return err
 //        })
@@ -172,13 +189,14 @@ func (r *badgerStore) Load(id uuid.UUID) (core.Tracking, error) {
 	panic("implement me")
 }
 
-func createDataKey(uid uuid.UUID, timestamp int64) []byte {
+func createRecordKey(uid uuid.UUID, timestamp int64) []byte {
 	prefix, err := uid.MarshalText()
 	if err != nil || timestamp < 0 {
 		logrus.Error("unable to create key", err)
 	}
 	suffix := make([]byte, 8)
-    binary.LittleEndian.PutUint64(suffix, uint64(timestamp))
+	binary.BigEndian.PutUint64(suffix, uint64(timestamp))
+
 	return append(prefix, suffix...)
 }
 
@@ -192,6 +210,6 @@ func unmarshalDataKey(key []byte) (uuid.UUID, int64) {
 	if err != nil {
 		panic("corrupted key")
 	}
-    timestamp := int64(binary.LittleEndian.Uint64(suffix))
+	timestamp := int64(binary.BigEndian.Uint64(suffix))
 	return uid, timestamp
 }