tvolkmann/gyrogpsc
1
0
Fork 0
Code Issues Pull Requests Projects 1 Releases 1 Wiki Activity

some improvements in logging and naming/structuring

Browse Source
This commit is contained in:
Timo Volkmann 2020-12-12 16:34:35 +01:00
parent 1ea6822202
commit 391020ec47
6 changed files with 329 additions and 308 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/dispatcher.go
View File

@ -18,7 +18,7 @@ func NewDispatcher() *dispatcher {
}
func (d *dispatcher) Publish(message string) {
logrus.Debugf("publish to %v listeners:\n%v\n", len(d.listeners))
logrus.Debugf("publish to %v listeners\n", len(d.listeners))
logrus.Debug(message)
for _, ch := range d.listeners {
ch <- message

10
core/sensordata.go
View File

@ -25,16 +25,6 @@ type sensorData struct {
Orientation [3]float64
}
type recordPair struct {
RecordTime time.Time
data map[sourceId]sensorData
}
type rawRecord struct {
RecordTime time.Time
sensorData
}
func (s sensorData) isSameEpoch(n sensorData) bool {
if n.itow == 0 {
return false

16
core/service.go
View File

@ -92,11 +92,11 @@ func (t *trackingService) StopRecord() {
t.opMode = LIVE
t.pipe.StopRecord()
m1.Lock()
m2.Lock()
mRec.Lock()
mRaw.Lock()
err := t.repo.Save(*t.current)
m2.Unlock()
m1.Unlock()
mRaw.Unlock()
mRec.Unlock()
if err != nil {
logrus.Println(err)
@ -140,9 +140,9 @@ func (t *trackingService) LoadTracking(trackingId uuid.UUID) {
}
func (t *trackingService) safelyReplaceTracking(tr Tracking) {
m1.Lock()
m2.Lock()
mRec.Lock()
mRaw.Lock()
*t.current = tr
m2.Unlock()
m1.Unlock()
mRaw.Unlock()
mRec.Unlock()
}

31
core/trackings.go
View File

@ -7,33 +7,46 @@ import (
"time"
)
var m1 sync.RWMutex
var m2 sync.RWMutex
type Tracking struct {
TrackingMetadata
Records []recordPair
Rawdata []rawRecord
}
var mRec sync.RWMutex
var mRaw sync.RWMutex
type TrackingMetadata struct {
UUID uuid.UUID
TimeCreated time.Time
Collectors []CollectorType
}
// persistence wrapper for sensordata
type recordPair struct {
RecordTimeKey time.Time // uniqueness ensured through mutex
DataPair map[sourceId]sensorData
}
type rawRecord struct {
RecordTimeKey time.Time // uniqueness ensured through mutex
Data sensorData
}
// END persistence wrapper for sensordata
func (s *Tracking) EnqueuePair(tcp sensorData, ser sensorData) {
rp := recordPair{
RecordTime: time.Now(),
data: map[sourceId]sensorData{
RecordTimeKey: time.Now(),
DataPair: map[sourceId]sensorData{
tcp.source: tcp,
ser.source: ser,
},
}
m1.Lock()
mRec.Lock()
s.Records = append(s.Records, rp)
logrus.Debugln("tracking Records: len->", len(s.Records))
m1.Unlock()
mRec.Unlock()
}
func (s *Tracking) EnqueueRaw(data sensorData) {
@ -41,10 +54,10 @@ func (s *Tracking) EnqueueRaw(data sensorData) {
time.Now(),
data,
}
m1.Lock()
mRec.Lock()
s.Rawdata = append(s.Rawdata, sr)
logrus.Debugln("raw data points: len->", len(s.Rawdata))
m1.Unlock()
mRec.Unlock()
}
func emptyTracking() Tracking {

2
go.mod
View File

@ -12,7 +12,7 @@ require (
github.com/spf13/viper v1.7.1
github.com/stretchr/testify v1.6.1 // indirect
github.com/tidwall/gjson v1.6.0
github.com/tidwall/pretty v1.0.2 // indirect
github.com/tidwall/pretty v1.0.2
go.bug.st/serial v1.1.1
golang.org/x/sync v0.0.0-20190423024810-112230192c58
)

60
storage/kvstore.go
View File

@ -7,22 +7,24 @@ import (
"github.com/dgraph-io/badger/v2"
"github.com/google/uuid"
"github.com/sirupsen/logrus"
"github.com/tidwall/pretty"
"os"
"path/filepath"
"strconv"
)
// Must implement Repo
type badgerStore struct {
trackings *badger.DB
records *badger.DB
rawdata *badger.DB
trackingsDb *badger.DB
recordsDb *badger.DB
rawdataDb *badger.DB
}
func NewRepository(c *core.Configuration) *badgerStore {
dir, _ := os.Getwd()
logrus.Debug(dir)
if _, err := os.Stat(filepath.Join(dir,"_db")); os.IsNotExist(err) {
os.Mkdir(filepath.Join(dir,"_db"), os.ModePerm)
if _, err := os.Stat(filepath.Join(dir, "_db")); os.IsNotExist(err) {
os.Mkdir(filepath.Join(dir, "_db"), os.ModePerm)
}
tr, err := badger.Open(badger.DefaultOptions("_db/trackings"))
@ -32,11 +34,11 @@ func NewRepository(c *core.Configuration) *badgerStore {
if err != nil {
logrus.Error(err)
}
return &badgerStore{trackings: tr, records: dp, rawdata: rd}
return &badgerStore{trackingsDb: tr, recordsDb: dp, rawdataDb: rd}
}
func (r *badgerStore) isDbAvailable() bool {
return r.trackings.IsClosed() || r.records.IsClosed() || r.rawdata.IsClosed()
return r.trackingsDb.IsClosed() || r.recordsDb.IsClosed() || r.rawdataDb.IsClosed()
}
func (r *badgerStore) Save(tr core.Tracking) error {
@ -54,10 +56,12 @@ func (r *badgerStore) Save(tr core.Tracking) error {
logrus.Error(err, tr)
return err
}
err = r.records.Update(func(txn *badger.Txn) error {
err = r.recordsDb.Update(func(txn *badger.Txn) error {
for _, v := range tr.Records {
k := createDataKey(tr.UUID, v.RecordTime.UnixNano())
j, err := json.Marshal(v)
k := createRecordKey(tr.UUID, v.RecordTimeKey.UnixNano())
j, err := json.Marshal(v.DataPair)
logrus.Debugln("save record k/v:\n", tr.UUID.String(), strconv.FormatInt(v.RecordTimeKey.UnixNano(), 10))
logrus.Debugln(string(pretty.Pretty(j)))
if err != nil {
return err
}
@ -69,10 +73,12 @@ func (r *badgerStore) Save(tr core.Tracking) error {
logrus.Error(err, tr)
return err
}
err = r.records.Update(func(txn *badger.Txn) error {
err = r.rawdataDb.Update(func(txn *badger.Txn) error {
for _, v := range tr.Rawdata {
k := createDataKey(tr.UUID, v.Timestamp)
k := createRecordKey(tr.UUID, v.RecordTimeKey.UnixNano())
j, err := json.Marshal(v)
logrus.Debugln("save raw k/v:\n", tr.UUID.String(), strconv.FormatInt(v.RecordTimeKey.UnixNano(), 10))
logrus.Debugln(string(pretty.Pretty(j)))
if err != nil {
return err
}
@ -84,7 +90,8 @@ func (r *badgerStore) Save(tr core.Tracking) error {
logrus.Error(err, tr)
return err
}
err = r.trackings.Update(func(txn *badger.Txn) error {
err = r.trackingsDb.Update(func(txn *badger.Txn) error {
logrus.Debug("save tracking meta k/v:\n", string(ts), string(meta))
err := txn.Set(ts, meta)
return err
})
@ -92,6 +99,16 @@ func (r *badgerStore) Save(tr core.Tracking) error {
logrus.Error(err, tr)
return err
}
r.trackingsDb.PrintHistogram(nil)
dr := 0.5
err = r.trackingsDb.RunValueLogGC(dr)
logrus.Debug("DB GC:", err)
err = r.recordsDb.RunValueLogGC(dr)
logrus.Debug("DB GC:", err)
err = r.rawdataDb.RunValueLogGC(dr)
logrus.Debug("DB GC:", err)
r.trackingsDb.PrintHistogram(nil)
logrus.Info("sucessfully saved tracking")
return nil
}
@ -112,9 +129,9 @@ func (r *badgerStore) Save(tr core.Tracking) error {
// ch := make(chan error, 3)
// go func() {
// defer wg.Done()
// err = r.records.Update(func(txn *badger.Txn) error {
// err = r.recordsDb.Update(func(txn *badger.Txn) error {
// for _, v := range tracking.Records {
// k := createDataKey(tracking.UUID, v.RecordTime.UnixNano())
// k := createRecordKey(tracking.UUID, v.RecordTime.UnixNano())
// j, err := json.Marshal(v)
// if err != nil {
// return err
@ -127,9 +144,9 @@ func (r *badgerStore) Save(tr core.Tracking) error {
// }()
// go func() {
// defer wg.Done()
// err = r.records.Update(func(txn *badger.Txn) error {
// err = r.recordsDb.Update(func(txn *badger.Txn) error {
// for _, v := range tracking.Rawdata {
// k := createDataKey(tracking.UUID, v.Timestamp)
// k := createRecordKey(tracking.UUID, v.Timestamp)
// j, err := json.Marshal(v)
// if err != nil {
// return err
@ -142,7 +159,7 @@ func (r *badgerStore) Save(tr core.Tracking) error {
// }()
// go func() {
// defer wg.Done()
// err = r.trackings.Update(func(txn *badger.Txn) error {
// err = r.trackingsDb.Update(func(txn *badger.Txn) error {
// err := txn.Set(ts, meta)
// return err
// })
@ -172,13 +189,14 @@ func (r *badgerStore) Load(id uuid.UUID) (core.Tracking, error) {
panic("implement me")
}
func createDataKey(uid uuid.UUID, timestamp int64) []byte {
func createRecordKey(uid uuid.UUID, timestamp int64) []byte {
prefix, err := uid.MarshalText()
if err != nil || timestamp < 0 {
logrus.Error("unable to create key", err)
}
suffix := make([]byte, 8)
binary.LittleEndian.PutUint64(suffix, uint64(timestamp))
binary.BigEndian.PutUint64(suffix, uint64(timestamp))
return append(prefix, suffix...)
}
@ -192,6 +210,6 @@ func unmarshalDataKey(key []byte) (uuid.UUID, int64) {
if err != nil {
panic("corrupted key")
}
timestamp := int64(binary.LittleEndian.Uint64(suffix))
timestamp := int64(binary.BigEndian.Uint64(suffix))
return uid, timestamp
}