From 05946c32304d55a166cc74111a172593282ce5c0 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Wed, 25 Nov 2020 21:20:21 +0100 Subject: [PATCH 01/20] moved dispatcher, preparing for sensor data consolidation --- .idea/workspace.xml | 154 +++++++++++++---------------- cmd/server/server.go | 17 +++- cmd/server_only/server_only.go | 25 +++++ {dispatcher => gnet}/dispatcher.go | 6 +- gnet/net.go | 131 ++++++++++++++++++++++++ hyperimu.json | 2 +- serial_ubx/serial.go | 6 +- 7 files changed, 246 insertions(+), 95 deletions(-) create mode 100644 cmd/server_only/server_only.go rename {dispatcher => gnet}/dispatcher.go (87%) create mode 100644 gnet/net.go diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 230effa..4881256 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -16,85 +16,19 @@ - - + - - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - + + + + + + + + + + @@ -165,6 +108,14 @@ + + + + + + + + @@ -173,13 +124,23 @@ + + + + + + + + + + + - @@ -227,7 +188,14 @@ @@ -239,12 +207,17 @@ + true @@ -301,7 +275,7 @@ - + @@ -309,9 +283,9 @@ - + - + @@ -319,9 +293,9 @@ - + - + @@ -329,9 +303,9 @@ - + - + @@ -339,7 +313,7 @@ - + @@ -362,10 +336,18 @@ + + + + + + + + diff --git a/cmd/server/server.go b/cmd/server/server.go index 5e43977..dc22536 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -1,6 +1,7 @@ package main import ( +<<<<<<< Updated upstream "encoding/json" "fmt" "git.timovolkmann.de/gyrogpsc/dispatcher" @@ -17,6 +18,16 @@ import ( const ( TCP_PORT = ":3010" SERIAL_PORT = "/dev/tty.usbmodem14201" +======= + "git.timovolkmann.de/gyrogpsc/gnet" +) + +const ( + TCP_PORT = ":3010" + HTTP_PORT = ":3011" + SERIAL_PORT = "/dev/tty.usbmodem14201" + MAX_WS_DELAY = 20 +>>>>>>> Stashed changes ) var upgrader = websocket.Upgrader{} // use default options @@ -72,7 +83,7 @@ func home(w http.ResponseWriter, r *http.Request) { } func main() { - d := dispatcher.New() + d := gnet.NewDispatcher() collectRoutines(d) http.HandleFunc("/echo", echo(d)) http.HandleFunc("/", home) @@ -81,7 +92,9 @@ func main() { log.Fatal(http.ListenAndServe(":3011", nil)) } -func collectRoutines(d *dispatcher.Dispatcher) { +func collectRoutines(d *gnet.Dispatcher) { + // TODO: Hier die Sensordaten zwischenspeichern und per + // collectRoutines Serial UBX Sensor Data go serialUbxCollector(d) // collectRoutines TCP JSON Sensor Data diff --git a/cmd/server_only/server_only.go b/cmd/server_only/server_only.go new file mode 100644 index 0000000..bd77897 --- /dev/null +++ b/cmd/server_only/server_only.go @@ -0,0 +1,25 @@ +package main + +import ( + "git.timovolkmann.de/gyrogpsc/gnet" +) + +const ( + TCP_PORT = ":3010" + HTTP_PORT = ":3011" + SERIAL_PORT = "/dev/tty.usbmodem14201" + MAX_WS_DELAY = 20 +) + +func main() { + d := gnet.NewDispatcher() + collectRoutines(d) + gnet.NewHttpServer(d, HTTP_PORT) +} + +func collectRoutines(d *gnet.Dispatcher) { + // collectRoutines Serial UBX Sensor Data + // go gnet.SerialUbxCollector(d, SERIAL_PORT) + // collectRoutines TCP JSON Sensor Data + go gnet.TcpJsonCollector(d, TCP_PORT) +} diff --git a/dispatcher/dispatcher.go b/gnet/dispatcher.go similarity index 87% rename from dispatcher/dispatcher.go rename to gnet/dispatcher.go index 05bcce4..0dd7144 100644 --- a/dispatcher/dispatcher.go +++ b/gnet/dispatcher.go @@ -1,4 +1,4 @@ -package dispatcher +package gnet import ( "errors" @@ -10,7 +10,7 @@ type Dispatcher struct { counter int16 } -func New() *Dispatcher { +func NewDispatcher() *Dispatcher { fmt.Println("new dispatcher") return &Dispatcher{ listeners: make(map[int16]chan string), @@ -19,7 +19,7 @@ func New() *Dispatcher { } func (d *Dispatcher) Publish(message string) { - fmt.Println("publish to listeners", len(d.listeners)) + //fmt.Println("publish to listeners", len(d.listeners)) for _, ch := range d.listeners { ch <- message } diff --git a/gnet/net.go b/gnet/net.go new file mode 100644 index 0000000..c8981e9 --- /dev/null +++ b/gnet/net.go @@ -0,0 +1,131 @@ +package gnet + +import ( + "encoding/json" + "fmt" + "git.timovolkmann.de/gyrogpsc/serial_ubx" + "github.com/gorilla/websocket" + "github.com/tidwall/pretty" + "html/template" + "log" + "net" + "net/http" + "os" +) + +func echo(d *Dispatcher) func(w http.ResponseWriter, r *http.Request) { + var upgrader = websocket.Upgrader{} // use default options + return func(w http.ResponseWriter, r *http.Request) { + fmt.Println("upgrading to ws") + c, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Print("upgrade:", err) + return + } + //defer c.Close() + go func() { + for { + if _, _, err := c.NextReader(); err != nil { + c.Close() + break + } + } + }() + + dispatcherId, channel := d.Subscribe() + defer d.Unsubscribe(dispatcherId) + for { + log.Println("") + //if err != nil { + // log.Println("read:", err) + // break + //} + cmsg := <-channel + err = c.WriteMessage(websocket.TextMessage, []byte(cmsg)) + if err != nil { + log.Println("write:", err) + + break + } + } + } +} + +func home(w http.ResponseWriter, r *http.Request) { + //var homeTemplate = template.Must(template.NewDispatcher("").ParseFiles("index.html")) + tpl, err := template.ParseFiles("index.html") + if err != nil { + log.Fatalln(err) + } + err = tpl.Execute(w, "ws://"+r.Host+"/echo") + if err != nil { + log.Fatalln(err) + } +} + +func NewHttpServer(d *Dispatcher, httpPort string) { + http.HandleFunc("/echo", echo(d)) + http.HandleFunc("/", home) + http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(".")))) + + log.Fatal(http.ListenAndServe(httpPort, nil)) +} + +func SerialUbxCollector(d *Dispatcher, serialPort string) { + r, err := serial_ubx.Setup(serialPort) + if err != nil { + log.Fatalln(err) + } + + for { + meas, err := r.NextMeasurement() + if err != nil { + continue + } + measjson, err := json.Marshal(meas) + fmt.Println(string(pretty.Pretty(measjson))) + d.Publish(string(measjson)) + } +} + +func TcpJsonCollector(d *Dispatcher, tcpPort string) { + listener, err := net.Listen("tcp", tcpPort) + if err != nil { + fmt.Println("Error listening:", err.Error()) + os.Exit(1) + } + // Close the listener when the application closes. + defer listener.Close() + + for { + // Listen for an incoming connection. + conn, err := listener.Accept() + if err != nil { + fmt.Println("Error accepting: ", err.Error()) + os.Exit(1) + } + // Handle connections in a new goroutine. + go handleTcpJsonSensorData(conn, d) + } +} + +// Handles incoming requests. +func handleTcpJsonSensorData(conn net.Conn, d *Dispatcher) { + defer conn.Close() + // Make a buffer to hold incoming data. + for { + buf := make([]byte, 2048) + // Read the incoming connection into the buffer. + _, err := conn.Read(buf) + if err != nil { + fmt.Println("Error reading:", err.Error()) + break + } + json := pretty.Pretty(buf) + fmt.Println(string(json)) + d.Publish(string(buf)) + // Send a response back to person contacting us. + //conn.Write([]byte("success")) + // Close the connection when you're done with it. + } +} diff --git a/hyperimu.json b/hyperimu.json index 1567da8..c5b6bfa 100644 --- a/hyperimu.json +++ b/hyperimu.json @@ -63,4 +63,4 @@ -0.006618400104343891 ], "tmd3702_proximity proximity sensor": [5, 0, 0] -} +} \ No newline at end of file diff --git a/serial_ubx/serial.go b/serial_ubx/serial.go index 6201a0f..30ae70c 100644 --- a/serial_ubx/serial.go +++ b/serial_ubx/serial.go @@ -61,7 +61,7 @@ func (u *ubxReceiver) NextMeasurement() (*Measurement, error) { u.currentMeas.Position[0] = v.Lat_dege7 u.currentMeas.Position[1] = v.Lon_dege7 u.currentMeas.Position[2] = v.Height_mm - fmt.Printf("%T %v\n", *v, *v) + //fmt.Printf("%T %v\n", *v, *v) case *ublox.HnrPvt: t, err := time.Parse(time.RFC3339Nano, formatTime(v.Year_y, v.Month_month, v.Day_d, v.Hour_h, v.Min_min, v.Sec_s, v.Nano_ns)) if err != nil { @@ -71,12 +71,12 @@ func (u *ubxReceiver) NextMeasurement() (*Measurement, error) { u.currentMeas.Position[0] = v.Lat_dege7 u.currentMeas.Position[1] = v.Lon_dege7 u.currentMeas.Position[2] = v.Height_mm - fmt.Printf("%T %v\n", *v, *v) + //fmt.Printf("%T %v\n", *v, *v) case *ublox.NavAtt: u.currentMeas.Orientation[0] = v.Pitch_deg u.currentMeas.Orientation[1] = v.Roll_deg u.currentMeas.Orientation[2] = v.Heading_deg - fmt.Printf("%T %v\n", *v, *v) + //fmt.Printf("%T %v\n", *v, *v) //case *ublox.RawMessage: // //fmt.Printf("%T %v\n\n", *v, *v) From cefb0153b2b5de96b87f0c1ac71db3d1ad0461f5 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Wed, 2 Dec 2020 19:41:41 +0100 Subject: [PATCH 02/20] refactor and implemented data pipeline --- .idea/.gitignore | 8 ++ .idea/misc.xml | 6 ++ .idea/vcs.xml | 1 - .idea/workspace.xml | 103 ++++++++++++--------- cmd/serial_only/serial_only.go | 21 +++-- cmd/server/server.go | 158 +++------------------------------ cmd/server_only/server_only.go | 25 ------ cmd/tcp_only/tcp_only.go | 22 +++++ core/collectors.go | 93 +++++++++++++++++++ {gnet => core}/dispatcher.go | 23 +++-- core/format.go | 108 ++++++++++++++++++++++ core/http.go | 67 ++++++++++++++ core/pipeline.go | 127 ++++++++++++++++++++++++++ gnet/net.go | 131 --------------------------- go.mod | 4 +- go.sum | 39 +++----- serial_ubx/serial.go | 124 -------------------------- ublox/decode.go | 18 ++-- ublox/messages.go | 18 ++-- 19 files changed, 564 insertions(+), 532 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/misc.xml delete mode 100644 cmd/server_only/server_only.go create mode 100644 cmd/tcp_only/tcp_only.go create mode 100644 core/collectors.go rename {gnet => core}/dispatcher.go (59%) create mode 100644 core/format.go create mode 100644 core/http.go create mode 100644 core/pipeline.go delete mode 100644 gnet/net.go delete mode 100644 serial_ubx/serial.go diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..73f69e0 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..0e93714 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml index ac4ad06..94a25f7 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -2,6 +2,5 @@ - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 4881256..7ca8493 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -22,13 +22,25 @@ + + + + + + + + + - - - - - + + + + + + + + + - - - - - - - - - - - + @@ -99,6 +102,15 @@ + + + + + + + + + @@ -124,23 +136,9 @@ - - - - - - - - - - - - - - - + @@ -231,6 +229,9 @@ + + + @@ -247,6 +248,10 @@ + + + + @@ -275,8 +280,8 @@ - - + + @@ -285,8 +290,9 @@ - - + + + @@ -295,8 +301,9 @@ - - + + + @@ -305,8 +312,9 @@ - - + + + @@ -315,6 +323,7 @@ + @@ -331,11 +340,12 @@ - - + + + @@ -344,14 +354,19 @@ + + + + - - + + + diff --git a/cmd/serial_only/serial_only.go b/cmd/serial_only/serial_only.go index 3cd3139..352585e 100644 --- a/cmd/serial_only/serial_only.go +++ b/cmd/serial_only/serial_only.go @@ -1,14 +1,21 @@ package main import ( - "git.timovolkmann.de/gyrogpsc/serial_ubx" - "log" + "fmt" + "git.timovolkmann.de/gyrogpsc/core" +) + +const ( + SERIAL_PORT = "/dev/tty.usbmodem14201" ) func main() { - r, err := serial_ubx.Setup("/dev/tty.usbmodem14201") - if err != nil { - log.Fatal(err) - } - r.Printloop() + core.SerialCollector(&printer{}, SERIAL_PORT) +} + +type printer struct{} + +func (p *printer) Process(data *core.Sensordata) error { + fmt.Println(data) + return nil } diff --git a/cmd/server/server.go b/cmd/server/server.go index dc22536..536320a 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -1,161 +1,25 @@ package main import ( -<<<<<<< Updated upstream - "encoding/json" - "fmt" - "git.timovolkmann.de/gyrogpsc/dispatcher" - "git.timovolkmann.de/gyrogpsc/serial_ubx" - "github.com/gorilla/websocket" - "github.com/tidwall/pretty" - "html/template" - "log" - "net" - "net/http" - "os" + "git.timovolkmann.de/gyrogpsc/core" ) const ( TCP_PORT = ":3010" + HTTP_PORT = ":3011" SERIAL_PORT = "/dev/tty.usbmodem14201" -======= - "git.timovolkmann.de/gyrogpsc/gnet" ) -const ( - TCP_PORT = ":3010" - HTTP_PORT = ":3011" - SERIAL_PORT = "/dev/tty.usbmodem14201" - MAX_WS_DELAY = 20 ->>>>>>> Stashed changes -) - -var upgrader = websocket.Upgrader{} // use default options - -func echo(d *dispatcher.Dispatcher) func(w http.ResponseWriter, r *http.Request) { - fmt.Println("echo") - return func(w http.ResponseWriter, r *http.Request) { - fmt.Println("upgrading to ws") - c, err := upgrader.Upgrade(w, r, nil) - if err != nil { - log.Print("upgrade:", err) - return - } - //defer c.Close() - go func() { - for { - if _, _, err := c.NextReader(); err != nil { - c.Close() - break - } - } - }() - - dispatcherId, channel := d.Subscribe() - defer d.Unsubscribe(dispatcherId) - for { - log.Println("") - //if err != nil { - // log.Println("read:", err) - // break - //} - cmsg := <-channel - err = c.WriteMessage(websocket.TextMessage, []byte(cmsg)) - if err != nil { - log.Println("write:", err) - - break - } - } - } -} - -func home(w http.ResponseWriter, r *http.Request) { - //var homeTemplate = template.Must(template.New("").ParseFiles("index.html")) - tpl, err := template.ParseFiles("index.html") - if err != nil { - log.Fatalln(err) - } - err = tpl.Execute(w, "ws://"+r.Host+"/echo") - if err != nil { - log.Fatalln(err) - } -} - func main() { - d := gnet.NewDispatcher() - collectRoutines(d) - http.HandleFunc("/echo", echo(d)) - http.HandleFunc("/", home) - http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(".")))) - - log.Fatal(http.ListenAndServe(":3011", nil)) + dispatcher := core.NewDispatcher() + processor := core.NewPipeline(dispatcher, 20, 10000) + collectRoutines(processor) + core.NewHttpServer(dispatcher, HTTP_PORT) } -func collectRoutines(d *gnet.Dispatcher) { - // TODO: Hier die Sensordaten zwischenspeichern und per - - // collectRoutines Serial UBX Sensor Data - go serialUbxCollector(d) - // collectRoutines TCP JSON Sensor Data - go tcpJsonCollector(d) -} - -func serialUbxCollector(d *dispatcher.Dispatcher) { - r, err := serial_ubx.Setup(SERIAL_PORT) - if err != nil { - log.Fatalln(err) - } - - for { - meas, err := r.NextMeasurement() - if err != nil { - continue - } - fmt.Println(meas) - measjson, err := json.Marshal(meas) - d.Publish(string(measjson)) - } -} - -func tcpJsonCollector(d *dispatcher.Dispatcher) { - listener, err := net.Listen("tcp", TCP_PORT) - if err != nil { - fmt.Println("Error listening:", err.Error()) - os.Exit(1) - } - // Close the listener when the application closes. - defer listener.Close() - - for { - // Listen for an incoming connection. - conn, err := listener.Accept() - if err != nil { - fmt.Println("Error accepting: ", err.Error()) - os.Exit(1) - } - // Handle connections in a new goroutine. - go handleTcpJsonSensorData(conn, d) - } -} - -// Handles incoming requests. -func handleTcpJsonSensorData(conn net.Conn, d *dispatcher.Dispatcher) { - defer conn.Close() - // Make a buffer to hold incoming data. - for { - buf := make([]byte, 2048) - // Read the incoming connection into the buffer. - _, err := conn.Read(buf) - if err != nil { - fmt.Println("Error reading:", err.Error()) - break - } - json := pretty.Pretty(buf) - fmt.Println(string(json)) - d.Publish(string(json)) - // Send a response back to person contacting us. - //conn.Write([]byte("success")) - // Close the connection when you're done with it. - } +func collectRoutines(proc core.Processor) { + // collect Sensor data from Serial UBX in Goroutine + go core.SerialCollector(proc, SERIAL_PORT) + // collect Sensor data from JSON over TCP in Goroutine + go core.TcpCollector(proc, TCP_PORT) } diff --git a/cmd/server_only/server_only.go b/cmd/server_only/server_only.go deleted file mode 100644 index bd77897..0000000 --- a/cmd/server_only/server_only.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "git.timovolkmann.de/gyrogpsc/gnet" -) - -const ( - TCP_PORT = ":3010" - HTTP_PORT = ":3011" - SERIAL_PORT = "/dev/tty.usbmodem14201" - MAX_WS_DELAY = 20 -) - -func main() { - d := gnet.NewDispatcher() - collectRoutines(d) - gnet.NewHttpServer(d, HTTP_PORT) -} - -func collectRoutines(d *gnet.Dispatcher) { - // collectRoutines Serial UBX Sensor Data - // go gnet.SerialUbxCollector(d, SERIAL_PORT) - // collectRoutines TCP JSON Sensor Data - go gnet.TcpJsonCollector(d, TCP_PORT) -} diff --git a/cmd/tcp_only/tcp_only.go b/cmd/tcp_only/tcp_only.go new file mode 100644 index 0000000..b9383f1 --- /dev/null +++ b/cmd/tcp_only/tcp_only.go @@ -0,0 +1,22 @@ +package main + +import ( + "git.timovolkmann.de/gyrogpsc/core" +) + +const ( + TCP_PORT = ":3010" + HTTP_PORT = ":3011" +) + +func main() { + dispatcher := core.NewDispatcher() + processor := core.NewPipeline(dispatcher, 20, 10000) + collectRoutines(processor) + core.NewHttpServer(dispatcher, HTTP_PORT) +} + +func collectRoutines(proc core.Processor) { + // collect Sensor data from JSON over TCP in Goroutine + go core.TcpCollector(proc, TCP_PORT) +} diff --git a/core/collectors.go b/core/collectors.go new file mode 100644 index 0000000..b5a5b21 --- /dev/null +++ b/core/collectors.go @@ -0,0 +1,93 @@ +package core + +import ( + "fmt" + "git.timovolkmann.de/gyrogpsc/ublox" + "go.bug.st/serial" + "log" + "net" + "os" +) + +func TcpCollector(proc Processor, tcpPort string) { + listener, err := net.Listen("tcp", tcpPort) + if err != nil { + fmt.Println("Error listening:", err.Error()) + os.Exit(1) + } + // Close the listener when the application closes. + defer listener.Close() + + for { + // Listen for an incoming connection. + conn, err := listener.Accept() + if err != nil { + fmt.Println("Error accepting: ", err.Error()) + os.Exit(1) + } + // Handle connections in a new goroutine. + go jsonHandler(conn, proc) + } +} + +// handles incoming tcp connections with json payload. +func jsonHandler(conn net.Conn, proc Processor) { + defer conn.Close() + + // TRY reader := bufio.NewReader(conn) OR NewScanner(conn) + buf := make([]byte, 2048) + for { + // Read the incoming connection into the buffer. + n, err := conn.Read(buf) + if err != nil { + fmt.Println("TCP error - reading from connection:", n, err.Error()) + break + } + //json := pretty.Pretty(buf[:n]) + //fmt.Println(string(json)) + fmt.Println(string(buf[:n])) + sd, err := ConvertSensorDataPhone(buf[:n]) + if err != nil { + log.Println(err) + continue + } + + err = proc.Process(sd) + if err != nil { + log.Println(err) + continue + } + } +} + +func SerialCollector(proc Processor, serialPort string) { + mode := &serial.Mode{ + BaudRate: 115200, + } + port, err := serial.Open(serialPort, mode) + if err != nil { + log.Fatalln(err) + } + + decoder := ublox.NewDecoder(port) + + for { + meas, err := decoder.Decode() + if err != nil { + log.Println(err) + continue + } + sd, err := ConvertUbxToSensorData(meas) + if err != nil { + log.Println(err) + continue + } + + err = proc.Process(sd) + if err != nil { + log.Println(err) + continue + } + + } +} diff --git a/gnet/dispatcher.go b/core/dispatcher.go similarity index 59% rename from gnet/dispatcher.go rename to core/dispatcher.go index 0dd7144..fb5455c 100644 --- a/gnet/dispatcher.go +++ b/core/dispatcher.go @@ -1,31 +1,40 @@ -package gnet +package core import ( "errors" "fmt" ) -type Dispatcher struct { +type Subscriber interface { + Subscribe() (int16, <-chan string) + Unsubscribe(id int16) error +} + +type Publisher interface { + Publish(message string) +} + +type dispatcher struct { listeners map[int16]chan string counter int16 } -func NewDispatcher() *Dispatcher { +func NewDispatcher() *dispatcher { fmt.Println("new dispatcher") - return &Dispatcher{ + return &dispatcher{ listeners: make(map[int16]chan string), counter: 0, } } -func (d *Dispatcher) Publish(message string) { +func (d *dispatcher) Publish(message string) { //fmt.Println("publish to listeners", len(d.listeners)) for _, ch := range d.listeners { ch <- message } } -func (d *Dispatcher) Subscribe() (id int16, receiver <-chan string) { +func (d *dispatcher) Subscribe() (id int16, receiver <-chan string) { fmt.Println("subscribe") key := d.counter d.counter++ @@ -34,7 +43,7 @@ func (d *Dispatcher) Subscribe() (id int16, receiver <-chan string) { return key, rec } -func (d *Dispatcher) Unsubscribe(id int16) error { +func (d *dispatcher) Unsubscribe(id int16) error { fmt.Println("unsubscribe") receiver, ok := d.listeners[id] if !ok { diff --git a/core/format.go b/core/format.go new file mode 100644 index 0000000..0ef83d1 --- /dev/null +++ b/core/format.go @@ -0,0 +1,108 @@ +package core + +import ( + "errors" + "git.timovolkmann.de/gyrogpsc/ublox" + "log" + "time" + + "github.com/m7shapan/njson" +) + +/*{ +"smartphone": { +// hier daten von hyperimu +}, +"serial": { +// hier Daten von M8U: +"timestamp": 37539672354 +"position": [0, 0, 0], +"orientation": [0, 0, 0] +} +} +*/ + +type sourceId string + +const ( + SOURCE_TCP sourceId = "SOURCE_TCP" + SOURCE_SERIAL sourceId = "SOURCE_SERIAL" +) + +type Sensordata struct { + itow uint32 + SourceId sourceId + Timestamp int64 + Position [3]float64 + Orientation [3]float64 +} + +func (s *Sensordata) isSameEpoch(n *Sensordata) bool { + if n.itow == 0 { + return false + } + return s.itow == n.itow +} + +func (s *Sensordata) Consolidate(n *Sensordata) *Sensordata { + if s.SourceId != n.SourceId { + log.Fatalln("Do not consolidate Sensordata from different Sources") + } + if s.isSameEpoch(n) { + // TODO: implement epoch consolidation logic + } + return n +} + +var ( + errNotImplemented = errors.New("message not implemented") +) + +func ConvertUbxToSensorData(msg interface{}) (*Sensordata, error) { + sd := &Sensordata{} + switch v := msg.(type) { + case *ublox.NavPvt: + sd.itow = v.ITOW_ms + sd.Timestamp = time.Date(int(v.Year_y), time.Month(v.Month_month), int(v.Day_d), int(v.Hour_h), int(v.Min_min), int(v.Sec_s), int(v.Nano_ns), time.UTC).UnixNano() + sd.Position[0] = float64(v.Lat_dege7) + sd.Position[1] = float64(v.Lon_dege7) + sd.Position[2] = float64(v.Height_mm) + case *ublox.HnrPvt: + sd.itow = v.ITOW_ms + sd.Timestamp = time.Date(int(v.Year_y), time.Month(v.Month_month), int(v.Day_d), int(v.Hour_h), int(v.Min_min), int(v.Sec_s), int(v.Nano_ns), time.UTC).UnixNano() + sd.Position[0] = float64(v.Lat_dege7) + sd.Position[1] = float64(v.Lon_dege7) + sd.Position[2] = float64(v.Height_mm) + case *ublox.NavAtt: + sd.itow = v.ITOW_ms + sd.Orientation[0] = float64(v.Pitch_deg) + sd.Orientation[1] = float64(v.Roll_deg) + sd.Orientation[2] = float64(v.Heading_deg) + default: + return nil, errNotImplemented + } + return sd, nil +} + +func ConvertSensorDataPhone(jsonData []byte) (*Sensordata, error) { + return convertAndroidHyperImu(jsonData) +} + +func convertAndroidHyperImu(jsonData []byte) (*Sensordata, error) { + prep := struct { + Timestamp int64 `njson:"Timestamp"` + Position [3]float64 `njson:"GPS"` + Orientation [3]float64 `njson:"orientation"` + }{} + err := njson.Unmarshal(jsonData, &prep) + if err != nil { + return nil, err + } + sd := &Sensordata{ + Timestamp: prep.Timestamp * int64(time.Millisecond), + //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), + Position: prep.Position, + Orientation: prep.Orientation, + } + return sd, nil +} diff --git a/core/http.go b/core/http.go new file mode 100644 index 0000000..0328766 --- /dev/null +++ b/core/http.go @@ -0,0 +1,67 @@ +package core + +import ( + "fmt" + "github.com/gorilla/websocket" + "html/template" + "log" + "net/http" +) + +func echo(sub Subscriber) func(w http.ResponseWriter, r *http.Request) { + var upgrader = websocket.Upgrader{} // use default options + return func(w http.ResponseWriter, r *http.Request) { + fmt.Println("upgrading to ws") + c, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Print("upgrade:", err) + return + } + //defer c.Close() + go func() { + for { + if _, _, err := c.NextReader(); err != nil { + c.Close() + break + } + } + }() + + dispatcherId, channel := sub.Subscribe() + defer sub.Unsubscribe(dispatcherId) + for { + log.Println("") + //if err != nil { + // log.Println("read:", err) + // break + //} + cmsg := <-channel + err = c.WriteMessage(websocket.TextMessage, []byte(cmsg)) + if err != nil { + log.Println("write:", err) + + break + } + } + } +} + +func home(w http.ResponseWriter, r *http.Request) { + //var homeTemplate = template.Must(template.NewDispatcher("").ParseFiles("index.html")) + tpl, err := template.ParseFiles("index.html") + if err != nil { + log.Fatalln(err) + } + err = tpl.Execute(w, "ws://"+r.Host+"/echo") + if err != nil { + log.Fatalln(err) + } +} + +func NewHttpServer(sub Subscriber, httpPort string) { + http.HandleFunc("/echo", echo(sub)) + http.HandleFunc("/", home) + http.Handle("/static/", http.FileServer(http.Dir("."))) + + log.Fatal(http.ListenAndServe(httpPort, nil)) +} diff --git a/core/pipeline.go b/core/pipeline.go new file mode 100644 index 0000000..8d98e81 --- /dev/null +++ b/core/pipeline.go @@ -0,0 +1,127 @@ +package core + +import ( + "encoding/json" + "errors" + "log" + "sync" + "time" +) + +type Processor interface { + Process(data *Sensordata) error +} + +type pipeline struct { + synchronizer + aggregator + Publisher + publishTicker *time.Ticker +} + +func NewPipeline(d Publisher, publishIntervalMs int, delayUpdateIntervalMs int) *pipeline { + return &pipeline{ + synchronizer{ + bufferSize: 100, + updateTicker: time.NewTicker(time.Duration(delayUpdateIntervalMs) * time.Second), + }, + aggregator{}, + d, + time.NewTicker(time.Duration(publishIntervalMs) * time.Second), + } +} + +func (p *pipeline) Run() { + go p.schedule() + go func() { + for { + <-p.publishTicker.C + err := p.Publish() + if err != nil { + log.Println(err) + } + } + }() +} + +func (p *pipeline) Publish() error { + if (p.tcpSensorData == Sensordata{} && p.serialSensorData == Sensordata{}) { + return errors.New("No Data available") + } + data := map[string]Sensordata{ + string(p.tcpSensorData.SourceId): p.tcpSensorData, + string(p.serialSensorData.SourceId): p.serialSensorData, + } + jdata, err := json.Marshal(data) + if err != nil { + return err + } + p.Publisher.Publish(string(jdata)) + return nil +} + +type aggregator struct { + tcpSensorData Sensordata + serialSensorData Sensordata + tcpMutex *sync.Mutex + serialMutex *sync.Mutex +} + +type UnixNanoTime int64 + +type synchronizer struct { + tcpDelayMs int + serialDelayMs int + tcpBuffer map[UnixNanoTime]Sensordata + serialBuffer map[UnixNanoTime]Sensordata + bufferSize int + mutex *sync.Mutex + updateTicker *time.Ticker + // should run concurrently + // + // Methods: + // pushSensordata(Sensordata), remove oldest if larger than bufferSize + // refreshDelay() + // Schedule() +} + +func (s *synchronizer) schedule() { + for { + <-s.updateTicker.C + err := s.refreshDelay() + if err != nil { + log.Println(err) + } + } +} + +func (s *synchronizer) refreshDelay() error { + // TODO: implement + return nil +} + +func (p *pipeline) Process(data *Sensordata) error { + switch data.SourceId { + case SOURCE_TCP: + go p.pushTcpDataToBuffer(*data) + return nil + case SOURCE_SERIAL: + go p.pushSerialDataToBuffer(*data) + return nil + default: + return errors.New("invalid data source") + } +} + +func (p *pipeline) pushTcpDataToBuffer(data Sensordata) { + time.Sleep(time.Duration(p.tcpDelayMs)) + p.tcpMutex.Lock() + p.tcpSensorData = data + p.tcpMutex.Unlock() +} +func (p *pipeline) pushSerialDataToBuffer(data Sensordata) { + time.Sleep(time.Duration(p.serialDelayMs)) + p.serialMutex.Lock() + p.serialSensorData = data + p.serialMutex.Unlock() +} diff --git a/gnet/net.go b/gnet/net.go deleted file mode 100644 index c8981e9..0000000 --- a/gnet/net.go +++ /dev/null @@ -1,131 +0,0 @@ -package gnet - -import ( - "encoding/json" - "fmt" - "git.timovolkmann.de/gyrogpsc/serial_ubx" - "github.com/gorilla/websocket" - "github.com/tidwall/pretty" - "html/template" - "log" - "net" - "net/http" - "os" -) - -func echo(d *Dispatcher) func(w http.ResponseWriter, r *http.Request) { - var upgrader = websocket.Upgrader{} // use default options - return func(w http.ResponseWriter, r *http.Request) { - fmt.Println("upgrading to ws") - c, err := upgrader.Upgrade(w, r, nil) - if err != nil { - log.Print("upgrade:", err) - return - } - //defer c.Close() - go func() { - for { - if _, _, err := c.NextReader(); err != nil { - c.Close() - break - } - } - }() - - dispatcherId, channel := d.Subscribe() - defer d.Unsubscribe(dispatcherId) - for { - log.Println("") - //if err != nil { - // log.Println("read:", err) - // break - //} - cmsg := <-channel - err = c.WriteMessage(websocket.TextMessage, []byte(cmsg)) - if err != nil { - log.Println("write:", err) - - break - } - } - } -} - -func home(w http.ResponseWriter, r *http.Request) { - //var homeTemplate = template.Must(template.NewDispatcher("").ParseFiles("index.html")) - tpl, err := template.ParseFiles("index.html") - if err != nil { - log.Fatalln(err) - } - err = tpl.Execute(w, "ws://"+r.Host+"/echo") - if err != nil { - log.Fatalln(err) - } -} - -func NewHttpServer(d *Dispatcher, httpPort string) { - http.HandleFunc("/echo", echo(d)) - http.HandleFunc("/", home) - http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(".")))) - - log.Fatal(http.ListenAndServe(httpPort, nil)) -} - -func SerialUbxCollector(d *Dispatcher, serialPort string) { - r, err := serial_ubx.Setup(serialPort) - if err != nil { - log.Fatalln(err) - } - - for { - meas, err := r.NextMeasurement() - if err != nil { - continue - } - measjson, err := json.Marshal(meas) - fmt.Println(string(pretty.Pretty(measjson))) - d.Publish(string(measjson)) - } -} - -func TcpJsonCollector(d *Dispatcher, tcpPort string) { - listener, err := net.Listen("tcp", tcpPort) - if err != nil { - fmt.Println("Error listening:", err.Error()) - os.Exit(1) - } - // Close the listener when the application closes. - defer listener.Close() - - for { - // Listen for an incoming connection. - conn, err := listener.Accept() - if err != nil { - fmt.Println("Error accepting: ", err.Error()) - os.Exit(1) - } - // Handle connections in a new goroutine. - go handleTcpJsonSensorData(conn, d) - } -} - -// Handles incoming requests. -func handleTcpJsonSensorData(conn net.Conn, d *Dispatcher) { - defer conn.Close() - // Make a buffer to hold incoming data. - for { - buf := make([]byte, 2048) - // Read the incoming connection into the buffer. - _, err := conn.Read(buf) - if err != nil { - fmt.Println("Error reading:", err.Error()) - break - } - json := pretty.Pretty(buf) - fmt.Println(string(json)) - d.Publish(string(buf)) - // Send a response back to person contacting us. - //conn.Write([]byte("success")) - // Close the connection when you're done with it. - } -} diff --git a/go.mod b/go.mod index 2773932..77abb86 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,9 @@ module git.timovolkmann.de/gyrogpsc go 1.15 require ( - github.com/daedaleanai/ublox v0.0.0-20201103121443-9befa131d32d github.com/gorilla/websocket v1.4.2 - github.com/sparkfun/SparkFun_Ublox_Arduino_Library v1.8.7 // indirect + github.com/m7shapan/njson v1.0.1 github.com/tidwall/pretty v1.0.2 go.bug.st/serial v1.1.1 golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf // indirect - golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb // indirect ) diff --git a/go.sum b/go.sum index 8877819..6d0f678 100644 --- a/go.sum +++ b/go.sum @@ -1,50 +1,33 @@ github.com/creack/goselect v0.1.1 h1:tiSSgKE1eJtxs1h/VgGQWuXUP0YS4CDIFMp6vaI1ls0= github.com/creack/goselect v0.1.1/go.mod h1:a/NhLweNvqIYMuxcMOuWY516Cimucms3DglDzQP3hKY= -github.com/daedaleanai/ublox v0.0.0-20201103121443-9befa131d32d h1:WbFmX8L79E02PgDJYWINhWvceaMGUzgmrwdE5CuUBBk= -github.com/daedaleanai/ublox v0.0.0-20201103121443-9befa131d32d/go.mod h1:pfcwlN8XUYXVYAkPU2LrFZnXIS4EvpZaXh+qRKCN9Sg= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/m7shapan/njson v1.0.1 h1:s+odQrPkzcCCGRTp46cD0XCVYN3pvdoaVwbFVmjAvys= +github.com/m7shapan/njson v1.0.1/go.mod h1:4sidL3oRZO1KV5FkclRBPI7nqFzlIq3BwdxHRMlOa9U= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sparkfun/SparkFun_Ublox_Arduino_Library v1.8.7 h1:T112CHmp+v1bh0W7sp49tZe8SUbw3viSGRTVLziZxfc= -github.com/sparkfun/SparkFun_Ublox_Arduino_Library v1.8.7/go.mod h1:FVoZAzJFrR5D6P8qd2rgpJAV/qF5oODtIT9YjVV+xzY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= +github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= +github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= +github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.bug.st/serial v1.1.1 h1:5J1DpaIaSIruBi7jVnKXnhRS+YQ9+2PLJMtIZKoIgnc= go.bug.st/serial v1.1.1/go.mod h1:VmYBeyJWp5BnJ0tw2NUJHZdJTGl2ecBGABHlzRK1knY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf h1:kt3wY1Lu5MJAnKTfoMR52Cu4gwvna4VTzNOiT8tY73s= golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb h1:z5+u0pkAUPUWd3taoTialQ2JAMo4Wo1Z3L25U4ZV9r0= -golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/serial_ubx/serial.go b/serial_ubx/serial.go deleted file mode 100644 index 30ae70c..0000000 --- a/serial_ubx/serial.go +++ /dev/null @@ -1,124 +0,0 @@ -package serial_ubx - -import ( - "errors" - "fmt" - "git.timovolkmann.de/gyrogpsc/ublox" - "go.bug.st/serial" - "log" - "time" -) - -type Measurement struct { - //Timestamp int64 `json:"timestamp"` - Timestamp time.Time `json:"timestamp"` - Position [3]int32 `json:"position"` // Latitude, Longitude, Height - Orientation [3]int32 `json:"orientation"` // Pitch, Roll, Heading -} - -type ubxReceiver struct { - decoder *ublox.Decoder - currentMeas Measurement -} - -func Setup(portname string) (*ubxReceiver, error) { - mode := &serial.Mode{ - BaudRate: 115200, - } - port, err := serial.Open(portname, mode) - if err != nil { - return nil, err - } - return &ubxReceiver{ - decoder: ublox.NewDecoder(port), - }, nil -} - -func (u *ubxReceiver) Next() (ublox.Message, error) { - return u.decoder.Decode() -} - -var ( - errNotImplemented = errors.New("message not implemented") -) - -// TODO: additional callback with adjustable timing -func (u *ubxReceiver) NextMeasurement() (*Measurement, error) { - - msg, err := u.decoder.Decode() - if err != nil { - return nil, err - } - //t := time.Time{} - switch v := msg.(type) { - case *ublox.NavPvt: - t, err := time.Parse(time.RFC3339Nano, formatTime(v.Year_y, v.Month_month, v.Day_d, v.Hour_h, v.Min_min, v.Sec_s, v.Nano_ns)) - if err != nil { - log.Println(err) - } - //u.currentMeas.Timestamp = t.UnixNano() - u.currentMeas.Timestamp = t - u.currentMeas.Position[0] = v.Lat_dege7 - u.currentMeas.Position[1] = v.Lon_dege7 - u.currentMeas.Position[2] = v.Height_mm - //fmt.Printf("%T %v\n", *v, *v) - case *ublox.HnrPvt: - t, err := time.Parse(time.RFC3339Nano, formatTime(v.Year_y, v.Month_month, v.Day_d, v.Hour_h, v.Min_min, v.Sec_s, v.Nano_ns)) - if err != nil { - log.Println(err) - } - u.currentMeas.Timestamp = t - u.currentMeas.Position[0] = v.Lat_dege7 - u.currentMeas.Position[1] = v.Lon_dege7 - u.currentMeas.Position[2] = v.Height_mm - //fmt.Printf("%T %v\n", *v, *v) - case *ublox.NavAtt: - u.currentMeas.Orientation[0] = v.Pitch_deg - u.currentMeas.Orientation[1] = v.Roll_deg - u.currentMeas.Orientation[2] = v.Heading_deg - //fmt.Printf("%T %v\n", *v, *v) - - //case *ublox.RawMessage: - // //fmt.Printf("%T %v\n\n", *v, *v) - default: - return nil, errNotImplemented - } - m := u.currentMeas - return &m, nil -} - -func formatTime(Year_y uint16, Month_month byte, Day_d byte, Hour_h byte, Min_min byte, Sec_s byte, Nano_ns int32) string { - //Nano_ns *= 1e+3 - if Nano_ns < 0 { - Nano_ns += int32(time.Second) - if Sec_s > 0 { - Sec_s-- - } else if Min_min > 0 { - Sec_s = 59 - Min_min-- - } else if Hour_h > 0 { - Sec_s = 59 - Min_min = 59 - Hour_h-- - } else if Day_d > 1 { - Sec_s = 59 - Min_min = 59 - Hour_h = 23 - Day_d-- - } // TODO: more cases for exact behavior! good for now... - } - //fmt.Printf("%04d-%02d-%02dT%02d:%02d:%02d.%09dZ00:00\n", Year_y, Month_month, Day_d, Hour_h, Min_min, Sec_s, Nano_ns ) - return fmt.Sprintf("%04d-%02d-%02dT%02d:%02d:%02d.%09dZ", Year_y, Month_month, Day_d, Hour_h, Min_min, Sec_s, Nano_ns) -} - -func (u *ubxReceiver) Printloop() { - - for { - meas, err := u.NextMeasurement() - if err != nil { - continue - } - fmt.Println(meas) - } - -} diff --git a/ublox/decode.go b/ublox/decode.go index ba019af..a940e08 100644 --- a/ublox/decode.go +++ b/ublox/decode.go @@ -14,17 +14,17 @@ import ( "io" ) -// A Decoder scans an io stream into UBX (0xB5-0x62 separated) or NMEA ("$xxx,,,,*FF\r\n") frames. +// A decoder scans an io stream into UBX (0xB5-0x62 separated) or NMEA ("$xxx,,,,*FF\r\n") frames. // If you have an unmixed stream of NMEA-only data you can use nmea.Decode() on bufio.Scanner.Bytes() directly. -type Decoder struct { +type decoder struct { s *bufio.Scanner } // NewDecoder creates a new bufio Scanner with a splitfunc that can handle both UBX and NMEA frames. -func NewDecoder(r io.Reader) *Decoder { +func NewDecoder(r io.Reader) *decoder { d := bufio.NewScanner(r) d.Split(splitFunc) - return &Decoder{s: d} + return &decoder{s: d} } // Assume we're either at the start of an NMEA sentence or at the start of a UBX message @@ -74,8 +74,8 @@ func splitFunc(data []byte, atEOF bool) (advance int, token []byte, err error) { return 1 + i1, nil, nil } -// Decode reads on NMEA or UBX frame and calls DecodeUbx accordingly to parse the message. Skips NMEA. -func (d *Decoder) Decode() (msg Message, err error) { +// Decode reads on NMEA or UBX frame and calls decodeUbx accordingly to parse the message, while skipping NMEA. +func (d *decoder) Decode() (msg interface{}, err error) { if !d.s.Scan() { if err = d.s.Err(); err == nil { err = io.EOF @@ -89,7 +89,7 @@ func (d *Decoder) Decode() (msg Message, err error) { return nil, err //return nmea.Decode(d.s.Bytes()) case 0xB5: - return DecodeUbx(d.s.Bytes()) + return decodeUbx(d.s.Bytes()) } panic("impossible frame") } @@ -99,7 +99,7 @@ var ( errInvalidChkSum = errors.New("invalid UBX checksum") ) -func DecodeUbx(frame []byte) (msg Message, err error) { +func decodeUbx(frame []byte) (msg Message, err error) { buf := bytes.NewReader(frame) @@ -149,7 +149,7 @@ func DecodeUbx(frame []byte) (msg Message, err error) { if msg != nil { err = binary.Read(buf, binary.LittleEndian, msg) } else { - msg = &RawMessage{ClassID: header.ClassID, Data: append([]byte(nil), frame[6:len(frame)-2]...)} + msg = &RawMessage{classID: header.ClassID, Data: append([]byte(nil), frame[6:len(frame)-2]...)} } //fmt.Println(msg) diff --git a/ublox/messages.go b/ublox/messages.go index ad3a078..610f7b2 100644 --- a/ublox/messages.go +++ b/ublox/messages.go @@ -1,15 +1,21 @@ package ublox type Message interface { - classID() uint16 + ClassID() uint16 } +//type UbxMessage interface { +// Timestamp() (time.Time, error) +// Position() ([3]float64, error) +// Orientation() ([3]float64, error) +//} + type RawMessage struct { - ClassID uint16 + classID uint16 Data []byte } -func (msg *RawMessage) classID() uint16 { return msg.ClassID } +func (msg *RawMessage) ClassID() uint16 { return msg.classID } type NavPvt struct { ITOW_ms uint32 // - GPS time of week of the navigation epoch. See the description of iTOW for details. @@ -47,7 +53,7 @@ type NavPvt struct { MagAcc_deg2e uint16 // 1e-2 Magnetic declination accuracy. Only supported in ADR 4.10 and later. } -func (NavPvt) classID() uint16 { return 0x0701 } +func (NavPvt) ClassID() uint16 { return 0x0701 } type HnrPvt struct { ITOW_ms uint32 // - GPS time of week of the navigation epoch. See the description of iTOW for details. @@ -77,7 +83,7 @@ type HnrPvt struct { Reserved1 [4]byte // - Reserved } -func (HnrPvt) classID() uint16 { return 0x0028 } +func (HnrPvt) ClassID() uint16 { return 0x0028 } type NavAtt struct { ITOW_ms uint32 // - GPS time of week of the navigation epoch. See the description of iTOW for details. @@ -91,7 +97,7 @@ type NavAtt struct { AccHeading_deg uint32 } -func (NavAtt) classID() uint16 { return 0x0501 } +func (NavAtt) ClassID() uint16 { return 0x0501 } //go:generate stringer -output=strings_navpvt.go -trimprefix NavPVT -type=NavPVTFixType,NavPVTValid,NavPVTFlags,NavPVTFlags2,NavPVTFlags3 From e33761bc46daf5aa30fc742f50a44cab0d48548d Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Thu, 3 Dec 2020 13:07:53 +0100 Subject: [PATCH 03/20] fixed errors --- .idea/workspace.xml | 82 +++++++++++-------- cmd/server/server.go | 10 ++- cmd/tcp_only/tcp_only.go | 2 +- core/collectors.go | 13 ++- core/dispatcher.go | 7 +- core/format.go | 37 +++++++-- core/http.go | 8 +- core/pipeline.go | 43 +++++++--- .../ex_websocketMessage.json | 0 hyperimu.json => static/hyperimu.json | 0 index.html => static/index.html | 0 ublox/decode.go | 3 +- 12 files changed, 141 insertions(+), 64 deletions(-) rename ex_websocketMessage.json => static/ex_websocketMessage.json (100%) rename hyperimu.json => static/hyperimu.json (100%) rename index.html => static/index.html (100%) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 7ca8493..202b72a 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -22,25 +22,18 @@ - - - - - - - - - - - - - - - + + + + + + + + + - - + + - - + @@ -193,7 +186,14 @@ @@ -238,12 +238,17 @@ - true + + + + @@ -256,6 +261,10 @@ + + + + @@ -280,7 +289,7 @@ - + @@ -290,8 +299,8 @@ - - + + @@ -301,8 +310,8 @@ - - + + @@ -312,8 +321,8 @@ - - + + @@ -323,7 +332,7 @@ - + @@ -354,22 +363,29 @@ - + - + - + - + + + + + + + + \ No newline at end of file diff --git a/cmd/server/server.go b/cmd/server/server.go index 536320a..3b6bb8f 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -2,6 +2,7 @@ package main import ( "git.timovolkmann.de/gyrogpsc/core" + "log" ) const ( @@ -11,10 +12,15 @@ const ( ) func main() { + log.Println("setup dispatcher") dispatcher := core.NewDispatcher() - processor := core.NewPipeline(dispatcher, 20, 10000) + log.Println("initialize processing pipeline") + processor := core.NewPipeline(dispatcher, 50, 1000) + processor.Run() + log.Println("start data collectors") collectRoutines(processor) - core.NewHttpServer(dispatcher, HTTP_PORT) + log.Println("start http server") + core.HttpListenAndServe(dispatcher, HTTP_PORT) } func collectRoutines(proc core.Processor) { diff --git a/cmd/tcp_only/tcp_only.go b/cmd/tcp_only/tcp_only.go index b9383f1..5e21bac 100644 --- a/cmd/tcp_only/tcp_only.go +++ b/cmd/tcp_only/tcp_only.go @@ -13,7 +13,7 @@ func main() { dispatcher := core.NewDispatcher() processor := core.NewPipeline(dispatcher, 20, 10000) collectRoutines(processor) - core.NewHttpServer(dispatcher, HTTP_PORT) + core.HttpListenAndServe(dispatcher, HTTP_PORT) } func collectRoutines(proc core.Processor) { diff --git a/core/collectors.go b/core/collectors.go index b5a5b21..47146c4 100644 --- a/core/collectors.go +++ b/core/collectors.go @@ -66,7 +66,7 @@ func SerialCollector(proc Processor, serialPort string) { } port, err := serial.Open(serialPort, mode) if err != nil { - log.Fatalln(err) + log.Fatalln(err.Error()) } decoder := ublox.NewDecoder(port) @@ -74,20 +74,25 @@ func SerialCollector(proc Processor, serialPort string) { for { meas, err := decoder.Decode() if err != nil { + if err.Error() == "NMEA not implemented" { + continue + } log.Println(err) continue } sd, err := ConvertUbxToSensorData(meas) if err != nil { - log.Println(err) + log.Println("convert err:", err, meas) + continue + } + if sd == nil { continue } err = proc.Process(sd) if err != nil { - log.Println(err) + log.Println("process err:", err, *sd) continue } - } } diff --git a/core/dispatcher.go b/core/dispatcher.go index fb5455c..e6a97b7 100644 --- a/core/dispatcher.go +++ b/core/dispatcher.go @@ -2,7 +2,7 @@ package core import ( "errors" - "fmt" + "log" ) type Subscriber interface { @@ -20,7 +20,6 @@ type dispatcher struct { } func NewDispatcher() *dispatcher { - fmt.Println("new dispatcher") return &dispatcher{ listeners: make(map[int16]chan string), counter: 0, @@ -28,14 +27,13 @@ func NewDispatcher() *dispatcher { } func (d *dispatcher) Publish(message string) { - //fmt.Println("publish to listeners", len(d.listeners)) + log.Printf("publish to %v listeners:\n%v\n", len(d.listeners), message) for _, ch := range d.listeners { ch <- message } } func (d *dispatcher) Subscribe() (id int16, receiver <-chan string) { - fmt.Println("subscribe") key := d.counter d.counter++ rec := make(chan string) @@ -44,7 +42,6 @@ func (d *dispatcher) Subscribe() (id int16, receiver <-chan string) { } func (d *dispatcher) Unsubscribe(id int16) error { - fmt.Println("unsubscribe") receiver, ok := d.listeners[id] if !ok { return errors.New("no subscription with id") diff --git a/core/format.go b/core/format.go index 0ef83d1..5da9ef1 100644 --- a/core/format.go +++ b/core/format.go @@ -37,47 +37,74 @@ type Sensordata struct { Orientation [3]float64 } -func (s *Sensordata) isSameEpoch(n *Sensordata) bool { +func (s Sensordata) isSameEpoch(n Sensordata) bool { if n.itow == 0 { return false } return s.itow == n.itow } -func (s *Sensordata) Consolidate(n *Sensordata) *Sensordata { - if s.SourceId != n.SourceId { +func (s Sensordata) Consolidate(n Sensordata) Sensordata { + if (s.SourceId != n.SourceId && s != Sensordata{}) { + log.Println(s) + log.Println(n) log.Fatalln("Do not consolidate Sensordata from different Sources") } if s.isSameEpoch(n) { - // TODO: implement epoch consolidation logic + null := Sensordata{} + //if s.Timestamp == null.Timestamp { s.Timestamp = n.Timestamp } + //if s.Position == null.Position { s.Position = n.Position } + //if s.Orientation == null.Orientation { s.Orientation = n.Orientation } + if n.Timestamp != null.Timestamp && s.Timestamp != n.Timestamp { + s.Timestamp = n.Timestamp + } + if n.Position != null.Position && s.Position != n.Position { + s.Position = n.Position + } + if n.Orientation != null.Orientation && s.Orientation != n.Orientation { + s.Orientation = n.Orientation + } + + return s } return n } var ( errNotImplemented = errors.New("message not implemented") + errRawMessage = errors.New("raw message") ) func ConvertUbxToSensorData(msg interface{}) (*Sensordata, error) { - sd := &Sensordata{} + sd := &Sensordata{ + SourceId: SOURCE_SERIAL, + } switch v := msg.(type) { case *ublox.NavPvt: + //log.Println("NAV-PVT") sd.itow = v.ITOW_ms sd.Timestamp = time.Date(int(v.Year_y), time.Month(v.Month_month), int(v.Day_d), int(v.Hour_h), int(v.Min_min), int(v.Sec_s), int(v.Nano_ns), time.UTC).UnixNano() sd.Position[0] = float64(v.Lat_dege7) sd.Position[1] = float64(v.Lon_dege7) sd.Position[2] = float64(v.Height_mm) case *ublox.HnrPvt: + //log.Println("HNR-PVT") sd.itow = v.ITOW_ms sd.Timestamp = time.Date(int(v.Year_y), time.Month(v.Month_month), int(v.Day_d), int(v.Hour_h), int(v.Min_min), int(v.Sec_s), int(v.Nano_ns), time.UTC).UnixNano() sd.Position[0] = float64(v.Lat_dege7) sd.Position[1] = float64(v.Lon_dege7) sd.Position[2] = float64(v.Height_mm) case *ublox.NavAtt: + //log.Println("NAV-ATT") sd.itow = v.ITOW_ms sd.Orientation[0] = float64(v.Pitch_deg) sd.Orientation[1] = float64(v.Roll_deg) sd.Orientation[2] = float64(v.Heading_deg) + case *ublox.RawMessage: + //class := make([]byte, 2) + //binary.LittleEndian.PutUint16(class, v.ClassID()) + //log.Printf("%#v, %#v", class[0],class[1]) + return nil, nil default: return nil, errNotImplemented } diff --git a/core/http.go b/core/http.go index 0328766..b957ac6 100644 --- a/core/http.go +++ b/core/http.go @@ -48,7 +48,7 @@ func echo(sub Subscriber) func(w http.ResponseWriter, r *http.Request) { func home(w http.ResponseWriter, r *http.Request) { //var homeTemplate = template.Must(template.NewDispatcher("").ParseFiles("index.html")) - tpl, err := template.ParseFiles("index.html") + tpl, err := template.ParseFiles("static/index.html") if err != nil { log.Fatalln(err) } @@ -58,10 +58,14 @@ func home(w http.ResponseWriter, r *http.Request) { } } -func NewHttpServer(sub Subscriber, httpPort string) { +func HttpListenAndServe(sub Subscriber, httpPort string) { + log.Println("register websocket handler") http.HandleFunc("/echo", echo(sub)) + log.Println("register index handler") http.HandleFunc("/", home) + log.Println("register static file handler") http.Handle("/static/", http.FileServer(http.Dir("."))) + log.Println("start server") log.Fatal(http.ListenAndServe(httpPort, nil)) } diff --git a/core/pipeline.go b/core/pipeline.go index 8d98e81..9f031ee 100644 --- a/core/pipeline.go +++ b/core/pipeline.go @@ -8,6 +8,8 @@ import ( "time" ) +// TODO: adapt HNR-INS data to continue orientation stream + type Processor interface { Process(data *Sensordata) error } @@ -23,11 +25,15 @@ func NewPipeline(d Publisher, publishIntervalMs int, delayUpdateIntervalMs int) return &pipeline{ synchronizer{ bufferSize: 100, - updateTicker: time.NewTicker(time.Duration(delayUpdateIntervalMs) * time.Second), + mutex: &sync.Mutex{}, + updateTicker: time.NewTicker(time.Duration(delayUpdateIntervalMs) * time.Millisecond), + }, + aggregator{ + tcpMutex: &sync.Mutex{}, + serialMutex: &sync.Mutex{}, }, - aggregator{}, d, - time.NewTicker(time.Duration(publishIntervalMs) * time.Second), + time.NewTicker(time.Duration(publishIntervalMs) * time.Millisecond), } } @@ -37,22 +43,35 @@ func (p *pipeline) Run() { for { <-p.publishTicker.C err := p.Publish() - if err != nil { + if err != nil /*&& err.Error() != "no data available"*/ { log.Println(err) } } }() + log.Println("pipeline: processing service started") } func (p *pipeline) Publish() error { + p.tcpMutex.Lock() + p.serialMutex.Lock() + //log.Println(p.tcpSensorData) + //log.Println(p.serialSensorData) if (p.tcpSensorData == Sensordata{} && p.serialSensorData == Sensordata{}) { - return errors.New("No Data available") + p.tcpMutex.Unlock() + p.serialMutex.Unlock() + return errors.New("no data available") } data := map[string]Sensordata{ - string(p.tcpSensorData.SourceId): p.tcpSensorData, - string(p.serialSensorData.SourceId): p.serialSensorData, + string(SOURCE_TCP): p.tcpSensorData, + string(SOURCE_SERIAL): p.serialSensorData, } + p.tcpSensorData = Sensordata{} + p.serialSensorData = Sensordata{} + p.tcpMutex.Unlock() + p.serialMutex.Unlock() + jdata, err := json.Marshal(data) + log.Println(string(jdata)) if err != nil { return err } @@ -86,6 +105,7 @@ type synchronizer struct { } func (s *synchronizer) schedule() { + log.Println("synchronizer: started") for { <-s.updateTicker.C err := s.refreshDelay() @@ -101,16 +121,19 @@ func (s *synchronizer) refreshDelay() error { } func (p *pipeline) Process(data *Sensordata) error { + if data == nil { + return errors.New("nil processing not allowed") + } + //log.Println(string(data.SourceId)) switch data.SourceId { case SOURCE_TCP: go p.pushTcpDataToBuffer(*data) - return nil case SOURCE_SERIAL: go p.pushSerialDataToBuffer(*data) - return nil default: return errors.New("invalid data source") } + return nil } func (p *pipeline) pushTcpDataToBuffer(data Sensordata) { @@ -122,6 +145,6 @@ func (p *pipeline) pushTcpDataToBuffer(data Sensordata) { func (p *pipeline) pushSerialDataToBuffer(data Sensordata) { time.Sleep(time.Duration(p.serialDelayMs)) p.serialMutex.Lock() - p.serialSensorData = data + p.serialSensorData = p.serialSensorData.Consolidate(data) p.serialMutex.Unlock() } diff --git a/ex_websocketMessage.json b/static/ex_websocketMessage.json similarity index 100% rename from ex_websocketMessage.json rename to static/ex_websocketMessage.json diff --git a/hyperimu.json b/static/hyperimu.json similarity index 100% rename from hyperimu.json rename to static/hyperimu.json diff --git a/index.html b/static/index.html similarity index 100% rename from index.html rename to static/index.html diff --git a/ublox/decode.go b/ublox/decode.go index a940e08..3e0ccab 100644 --- a/ublox/decode.go +++ b/ublox/decode.go @@ -85,8 +85,7 @@ func (d *decoder) Decode() (msg interface{}, err error) { switch d.s.Bytes()[0] { case '$': - //fmt.Println("NMEA message: skipping!") - return nil, err + return nil, errors.New("NMEA not implemented") //return nmea.Decode(d.s.Bytes()) case 0xB5: return decodeUbx(d.s.Bytes()) From 21a20727f605b37f1961d603a96a272fac144db8 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Wed, 9 Dec 2020 11:27:39 +0100 Subject: [PATCH 04/20] added gitignore --- .gitignore | 158 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d96ad9b --- /dev/null +++ b/.gitignore @@ -0,0 +1,158 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/macos,go,intellij +# Edit at https://www.toptal.com/developers/gitignore?templates=macos,go,intellij + +### Go ### +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +### Go Patch ### +/vendor/ +/Godeps/ + +### Intellij ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Intellij Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# End of https://www.toptal.com/developers/gitignore/api/macos,go,intellij \ No newline at end of file From 79325d92094a3d63749ee7ebb5b98025372e655f Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Wed, 9 Dec 2020 12:44:21 +0100 Subject: [PATCH 05/20] removed workspace.xml --- .idea/workspace.xml | 234 -------------------------------------------- 1 file changed, 234 deletions(-) delete mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index 81e9551..0000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,234 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1604516509922 - - - 1604747925521 - - - 1604747940729 - - - 1604768327573 - - - 1604777752387 - - - 1605444355479 - - - - - - - - - - - - - - - - true - - \ No newline at end of file From f5f98ced5b968ad70e7c75b178daf165ae66f59b Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Thu, 10 Dec 2020 09:27:17 +0100 Subject: [PATCH 06/20] renamed format.go to sensordata.go --- core/{format.go => sensordata.go} | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) rename core/{format.go => sensordata.go} (93%) diff --git a/core/format.go b/core/sensordata.go similarity index 93% rename from core/format.go rename to core/sensordata.go index 4124f64..a2cb8cc 100644 --- a/core/format.go +++ b/core/sensordata.go @@ -9,19 +9,6 @@ import ( "time" ) -/*{ -"smartphone": { -// hier daten von hyperimu -}, -"serial": { -// hier Daten von M8U: -"timestamp": 37539672354 -"position": [0, 0, 0], -"orientation": [0, 0, 0] -} -} -*/ - type sourceId string const ( @@ -81,7 +68,7 @@ func (s Sensordata) Consolidate2(n Sensordata) Sensordata { if n.Position == null.Position { n.Position = s.Position } - if n.Orientation == null.Orientation{ + if n.Orientation == null.Orientation { n.Orientation = s.Orientation } return n @@ -144,10 +131,10 @@ func convertIPhoneSensorLog(jsonData []byte) (*Sensordata, error) { roll := gjson.Get(string(jsonData), "motionRoll").Float() * 180 / math.Pi yaw := gjson.Get(string(jsonData), "motionYaw").Float() * 180 / math.Pi sd := &Sensordata{ - SourceId: SOURCE_TCP, + SourceId: SOURCE_TCP, Timestamp: int64(timestamp * float64(time.Second)), //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), - Position: [3]float64{lat, lon, alt}, + Position: [3]float64{lat, lon, alt}, Orientation: [3]float64{pitch, roll, yaw}, } //log.Println(string(pretty.Pretty(jsonData))) @@ -155,7 +142,6 @@ func convertIPhoneSensorLog(jsonData []byte) (*Sensordata, error) { return sd, nil } - func convertAndroidHyperImu(jsonData []byte) (*Sensordata, error) { timestamp := gjson.Get(string(jsonData), "Timestamp").Int() lat := gjson.Get(string(jsonData), "GPS.0").Float() @@ -166,10 +152,10 @@ func convertAndroidHyperImu(jsonData []byte) (*Sensordata, error) { yaw := gjson.Get(string(jsonData), "orientation.2").Float() sd := &Sensordata{ - SourceId: SOURCE_TCP, + SourceId: SOURCE_TCP, Timestamp: timestamp * int64(time.Millisecond), //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), - Position: [3]float64{lat, lon, alt}, + Position: [3]float64{lat, lon, alt}, Orientation: [3]float64{pitch, roll, yaw}, } return sd, nil From b03d7fcfa26e34a6380f31436099e3a1b42e7858 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Thu, 10 Dec 2020 09:43:44 +0100 Subject: [PATCH 07/20] small changes in sensordata logic --- core/pipeline.go | 12 +++++------ core/sensordata.go | 50 ++++++++++++++++++++++------------------------ 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/core/pipeline.go b/core/pipeline.go index 4be7863..c677c5f 100644 --- a/core/pipeline.go +++ b/core/pipeline.go @@ -90,12 +90,12 @@ type aggregator struct { type UnixNanoTime int64 type synchronizer struct { - tcpSerialDelayMs int64 + tcpSerialDelayMs int64 //tcpBuffer map[UnixNanoTime]Sensordata //serialBuffer map[UnixNanoTime]Sensordata //bufferSize int - mutex *sync.Mutex - updateTicker *time.Ticker + mutex *sync.Mutex + updateTicker *time.Ticker // should run concurrently // // Methods: @@ -146,7 +146,7 @@ func (p *pipeline) Process(data *Sensordata) error { case SOURCE_SERIAL: go p.pushSerialDataToBuffer(*data) default: - return errors.New("invalid data source") + panic("pipeline: invalid data source") } return nil } @@ -156,7 +156,7 @@ func (p *pipeline) pushTcpDataToBuffer(data Sensordata) { time.Sleep(time.Duration(p.syn.tcpSerialDelayMs) * time.Millisecond) } p.agr.tcpMutex.Lock() - p.agr.tcpSensorData = p.agr.tcpSensorData.Consolidate2(data) + p.agr.tcpSensorData = p.agr.tcpSensorData.ConsolidateExTime(data) p.agr.tcpMutex.Unlock() } func (p *pipeline) pushSerialDataToBuffer(data Sensordata) { @@ -164,6 +164,6 @@ func (p *pipeline) pushSerialDataToBuffer(data Sensordata) { time.Sleep(time.Duration(-p.syn.tcpSerialDelayMs) * time.Millisecond) } p.agr.serialMutex.Lock() - p.agr.serialSensorData = p.agr.serialSensorData.Consolidate2(data) + p.agr.serialSensorData = p.agr.serialSensorData.ConsolidateEpochsOnly(data) p.agr.serialMutex.Unlock() } diff --git a/core/sensordata.go b/core/sensordata.go index a2cb8cc..c9a9d85 100644 --- a/core/sensordata.go +++ b/core/sensordata.go @@ -31,40 +31,30 @@ func (s Sensordata) isSameEpoch(n Sensordata) bool { return s.itow == n.itow } -func (s Sensordata) Consolidate(n Sensordata) Sensordata { - if (s.SourceId != n.SourceId && s != Sensordata{}) { - log.Println(s) - log.Println(n) - log.Fatalln("Do not consolidate Sensordata from different Sources") - } +// Consolidates two sensordata elements if they are in the same epoch +func (s Sensordata) ConsolidateEpochsOnly(n Sensordata) Sensordata { + s.checkSources(&n) if s.isSameEpoch(n) { null := Sensordata{} - //if s.Timestamp == null.Timestamp { s.Timestamp = n.Timestamp } - //if s.Position == null.Position { s.Position = n.Position } - //if s.Orientation == null.Orientation { s.Orientation = n.Orientation } - if n.Timestamp != null.Timestamp && s.Timestamp != n.Timestamp { - s.Timestamp = n.Timestamp - } - if n.Position != null.Position && s.Position != n.Position { - s.Position = n.Position - } - if n.Orientation != null.Orientation && s.Orientation != n.Orientation { - s.Orientation = n.Orientation - } - return s + if n.Timestamp == null.Timestamp { + n.Timestamp = s.Timestamp + } + if n.Position == null.Position { + n.Position = s.Position + } + if n.Orientation == null.Orientation { + n.Orientation = s.Orientation + } } return n } -func (s Sensordata) Consolidate2(n Sensordata) Sensordata { +// Consolidates two sensordata elements but ignores timestamps +func (s Sensordata) ConsolidateExTime(n Sensordata) Sensordata { + s.checkSources(&n) null := Sensordata{} - //if s.Timestamp == null.Timestamp { s.Timestamp = n.Timestamp } - //if s.Position == null.Position { s.Position = n.Position } - //if s.Orientation == null.Orientation { s.Orientation = n.Orientation } - //if n.Timestamp == null.Timestamp { - // n.Timestamp = s.Timestamp - //} + if n.Position == null.Position { n.Position = s.Position } @@ -74,6 +64,14 @@ func (s Sensordata) Consolidate2(n Sensordata) Sensordata { return n } +func (s *Sensordata) checkSources(n *Sensordata) { + if (s.SourceId != n.SourceId && *s != Sensordata{}) { + log.Println(s) + log.Println(n) + log.Fatalln("Do not consolidate Sensordata from different Sources") + } +} + var ( errNotImplemented = errors.New("message not implemented") errRawMessage = errors.New("raw message") From d16dd61863f87e3c74bec5e159fc10c514dacb2d Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Thu, 10 Dec 2020 13:20:20 +0100 Subject: [PATCH 08/20] changed serial read behavior --- cmd/server/server.go | 2 +- core/collectors.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/server/server.go b/cmd/server/server.go index c8ffed1..0ccdd93 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -8,7 +8,7 @@ import ( const ( TCP_PORT = ":3010" HTTP_PORT = ":3011" - SERIAL_PORT = "COM4" + SERIAL_PORT = "/dev/tty.usbmodem14201" ) func main() { diff --git a/core/collectors.go b/core/collectors.go index 923d6fe..c9e9bb1 100644 --- a/core/collectors.go +++ b/core/collectors.go @@ -75,6 +75,7 @@ func SerialCollector(proc Processor, serialPort string) { if err != nil { log.Fatalln(err.Error()) } + defer port.Close() decoder := ublox.NewDecoder(port) @@ -84,8 +85,8 @@ func SerialCollector(proc Processor, serialPort string) { if err.Error() == "NMEA not implemented" { continue } - log.Println(err) - continue + log.Println("serial read err:", err) + break } sd, err := ConvertUbxToSensorData(meas) if err != nil { From dfde29ed10fe20668acae24299617ecf8b2cadf3 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Thu, 10 Dec 2020 13:51:48 +0100 Subject: [PATCH 09/20] added synchronization check for invalid timestamps --- core/pipeline.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/core/pipeline.go b/core/pipeline.go index c677c5f..524a0f9 100644 --- a/core/pipeline.go +++ b/core/pipeline.go @@ -9,8 +9,6 @@ import ( "time" ) -// TODO: adapt HNR-INS data to continue orientation stream - type Processor interface { Process(data *Sensordata) error } @@ -125,11 +123,16 @@ func (p *pipeline) refreshDelay() error { p.agr.serialMutex.Unlock() p.agr.tcpMutex.Unlock() if tcpTime.UnixNano() == 0 || serTime.UnixNano() == 0 { - return errors.New("no sync possible. no data to compare") + return errors.New("no sync possible. check if both collectors running. otherwise check GPS fix") + } + currentDelay := tcpTime.Sub(serTime).Milliseconds() + if currentDelay > 5000 || currentDelay < -5000 { + p.syn.tcpSerialDelayMs = 0 + return errors.New("skipping synchronisation! time not properly configured or facing network problems.") } log.Println("TCP", tcpTime.String()) log.Println("SER", serTime.String()) - log.Println("Difference", tcpTime.Sub(serTime).Milliseconds()) + log.Println("Difference", tcpTime.Sub(serTime).Milliseconds(), "ms") delay := tcpTime.Sub(serTime).Milliseconds() p.syn.tcpSerialDelayMs += delay return nil From 2a9c3a1153dfc17334e50425ab70bbe99bc2ced4 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Thu, 10 Dec 2020 12:54:16 +0100 Subject: [PATCH 10/20] [WIP] add functionality --- .gitignore | 6 ++++- cmd/server/server.go | 11 +++++++-- core/collectors.go | 4 ++++ core/config.go | 11 +++++++++ core/http.go | 57 ++++++++++++++++++++++++++++++++++++++++++++ core/pipeline.go | 17 +++++++++---- example_config.yml | 12 ++++++++++ go.mod | 7 +++--- go.sum | 43 +++++++++++++++++++++++++++++---- 9 files changed, 153 insertions(+), 15 deletions(-) create mode 100644 core/config.go create mode 100644 example_config.yml diff --git a/.gitignore b/.gitignore index d96ad9b..7908a1f 100644 --- a/.gitignore +++ b/.gitignore @@ -155,4 +155,8 @@ Network Trash Folder Temporary Items .apdisk -# End of https://www.toptal.com/developers/gitignore/api/macos,go,intellij \ No newline at end of file +# End of https://www.toptal.com/developers/gitignore/api/macos,go,intellij + +# CUSTOM +.env +config.yml \ No newline at end of file diff --git a/cmd/server/server.go b/cmd/server/server.go index 0ccdd93..cd26270 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -3,6 +3,8 @@ package main import ( "git.timovolkmann.de/gyrogpsc/core" "log" + "os" + "runtime" ) const ( @@ -12,14 +14,19 @@ const ( ) func main() { + log.Println("GOROOT:", runtime.GOROOT()) + wd, _ := os.Getwd() + log.Println("WorkingDir:", wd) + log.Println("setup dispatcher") dispatcher := core.NewDispatcher() log.Println("initialize processing pipeline") processor := core.NewPipeline(dispatcher, 50, 494) processor.Run() - collectRoutines(processor) + //collectRoutines(processor) log.Println("start http server") - core.HttpListenAndServe(dispatcher, HTTP_PORT) + core.FiberListenAndServe(dispatcher) + //core.HttpListenAndServe(dispatcher, HTTP_PORT) } func collectRoutines(proc core.Processor) { diff --git a/core/collectors.go b/core/collectors.go index c9e9bb1..2f24378 100644 --- a/core/collectors.go +++ b/core/collectors.go @@ -104,3 +104,7 @@ func SerialCollector(proc Processor, serialPort string) { } } } + +func replayCollector(storage string, proc Processor, trackingId int) { + +} diff --git a/core/config.go b/core/config.go new file mode 100644 index 0000000..bf93f32 --- /dev/null +++ b/core/config.go @@ -0,0 +1,11 @@ +package core + +type Configuration struct { + TcpCollectorPort string + SerialCollectorPort string + HttpPort string +} + +func LoadConfigYaml() error { + return nil +} diff --git a/core/http.go b/core/http.go index 8e9468d..108304f 100644 --- a/core/http.go +++ b/core/http.go @@ -2,6 +2,8 @@ package core import ( "fmt" + "github.com/gofiber/fiber/v2" + ws "github.com/gofiber/websocket/v2" "github.com/gorilla/websocket" "html/template" "log" @@ -69,3 +71,58 @@ func HttpListenAndServe(sub Subscriber, httpPort string) { log.Println("start server") log.Fatal(http.ListenAndServe(httpPort, nil)) } + +func FiberListenAndServe(sub Subscriber) { + app := fiber.New() + app.Static("/static", "static") + + // Application Main Page + app.Get("/", fiberHomeHandler) + + // Websocket + app.Get("/ws", ws.New(fiberWebsocketHandler)) + + // Tracking persistence controls + trackings := app.Group("/trackings") + trackings.Get("/") // Get all trackings Metadata + trackings.Post("/") // Initialize new tracking by websocket and prepare for automatic recording. Toggle ?serial=true and ?tcp=true. Returns trackingId + trackings.Patch("/") // Starts recording + trackings.Put("/") // Stops current recording. Returns trackingId if record was successful + trackings.Delete("/") // Stops websocket connection, pipelines and collectors + + trackings.Get("/:trackingId") // Gets Tracking Metadata by Id. + trackings.Post("/:trackingId") // Starts Replay. + trackings.Patch("/:trackingId") // Pauses Replay. + trackings.Put("/:trackingId") // Stops Replay. + trackings.Delete("/:trackingId") // Deletes Tracking from database + + log.Fatal(app.Listen(":3011")) +} + +func fiberWebsocketHandler(c *ws.Conn) { + for { + mt, msg, err := c.ReadMessage() + if err != nil { + log.Println("read:", err) + break + } + log.Printf("recv: %s", msg) + err = c.WriteMessage(mt, msg) + if err != nil { + log.Println("write:", err) + break + } + } +} + +func fiberHomeHandler(c *fiber.Ctx) error { + tpl, err := template.ParseFiles("static/index.html") + if err != nil { + return err + } + err = tpl.Execute(c, "ws://"+c.Hostname()+"/ws") + if err != nil { + return err + } + return nil +} diff --git a/core/pipeline.go b/core/pipeline.go index 524a0f9..bd7efd1 100644 --- a/core/pipeline.go +++ b/core/pipeline.go @@ -13,19 +13,25 @@ type Processor interface { Process(data *Sensordata) error } +type Storer interface { + Persist() +} + type pipeline struct { - syn synchronizer - agr aggregator - pub Publisher + syn synchronizer + agr aggregator + pub Publisher + //stor Storer publishTicker *time.Ticker } -func NewPipeline(d Publisher, publishIntervalMs int, delayUpdateIntervalMs int) *pipeline { +func NewPipeline(d Publisher, publishIntervalMs int, syncUpdateIntervalMs int) *pipeline { + // TODO: replace timing params with config struct return &pipeline{ synchronizer{ //bufferSize: 100, mutex: &sync.Mutex{}, - updateTicker: time.NewTicker(time.Duration(delayUpdateIntervalMs) * time.Millisecond), + updateTicker: time.NewTicker(time.Duration(syncUpdateIntervalMs) * time.Millisecond), }, aggregator{ tcpMutex: &sync.Mutex{}, @@ -143,6 +149,7 @@ func (p *pipeline) Process(data *Sensordata) error { return errors.New("nil processing not allowed") } //log.Println(string(data.SourceId)) + // TODO: persist data here with current timestamp switch data.SourceId { case SOURCE_TCP: go p.pushTcpDataToBuffer(*data) diff --git a/example_config.yml b/example_config.yml new file mode 100644 index 0000000..9cd8ade --- /dev/null +++ b/example_config.yml @@ -0,0 +1,12 @@ +# Server configurations +server: + port: 8000 + +collectors: + port_tcp: + port_serial: + +# Database credentials +database: + user: "admin" + pass: "super-pedro-1980" \ No newline at end of file diff --git a/go.mod b/go.mod index 1b52ad7..3edfef5 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,11 @@ module git.timovolkmann.de/gyrogpsc go 1.15 require ( + github.com/gofiber/fiber/v2 v2.2.4 + github.com/gofiber/websocket/v2 v2.0.2 github.com/gorilla/websocket v1.4.2 - github.com/m7shapan/njson v1.0.1 + github.com/stretchr/testify v1.6.1 // indirect github.com/tidwall/gjson v1.6.0 - github.com/tidwall/pretty v1.0.2 + github.com/tidwall/pretty v1.0.2 // indirect go.bug.st/serial v1.1.1 - golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf // indirect ) diff --git a/go.sum b/go.sum index 6d0f678..5179a37 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,25 @@ +github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= +github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/creack/goselect v0.1.1 h1:tiSSgKE1eJtxs1h/VgGQWuXUP0YS4CDIFMp6vaI1ls0= github.com/creack/goselect v0.1.1/go.mod h1:a/NhLweNvqIYMuxcMOuWY516Cimucms3DglDzQP3hKY= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fasthttp/websocket v1.4.3 h1:qjhRJ/rTy4KB8oBxljEC00SDt6HUY9jLRfM601SUdS4= +github.com/fasthttp/websocket v1.4.3/go.mod h1:5r4oKssgS7W6Zn6mPWap3NWzNPJNzUUh3baWTOhcYQk= +github.com/gofiber/fiber/v2 v2.1.0/go.mod h1:aG+lMkwy3LyVit4CnmYUbUdgjpc3UYOltvlJZ78rgQ0= +github.com/gofiber/fiber/v2 v2.2.4 h1:t2V2SxlbQGdt8+SS/Mo+tQB0pDQn7OajKdA72qHcBVw= +github.com/gofiber/fiber/v2 v2.2.4/go.mod h1:Aso7/M+EQOinVkWp4LUYjdlTpKTBoCk2Qo4djnMsyHE= +github.com/gofiber/websocket/v2 v2.0.2 h1:UA/6NpyG+vmPGlvJvW8MJPJpRFuS7abinZ5HbLuV8u0= +github.com/gofiber/websocket/v2 v2.0.2/go.mod h1:7VBnzEVRK0K0eTIVc5GbXPF1JWUFnllY0X4cRtG2v78= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/m7shapan/njson v1.0.1 h1:s+odQrPkzcCCGRTp46cD0XCVYN3pvdoaVwbFVmjAvys= -github.com/m7shapan/njson v1.0.1/go.mod h1:4sidL3oRZO1KV5FkclRBPI7nqFzlIq3BwdxHRMlOa9U= +github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= +github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/savsgio/gotils v0.0.0-20200608150037-a5f6f5aef16c h1:2nF5+FZ4/qp7pZVL7fR6DEaSTzuDmNaFTyqp92/hwF8= +github.com/savsgio/gotils v0.0.0-20200608150037-a5f6f5aef16c/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -20,11 +32,34 @@ github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.14.0/go.mod h1:ol1PCaL0dX20wC0htZ7sYCsvCYmrouYra0zHzaclZhE= +github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA= +github.com/valyala/fasthttp v1.17.0 h1:P8/koH4aSnJ4xbd0cUUFEGQs3jQqIxoDDyRQrUiAkqg= +github.com/valyala/fasthttp v1.17.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= go.bug.st/serial v1.1.1 h1:5J1DpaIaSIruBi7jVnKXnhRS+YQ9+2PLJMtIZKoIgnc= go.bug.st/serial v1.1.1/go.mod h1:VmYBeyJWp5BnJ0tw2NUJHZdJTGl2ecBGABHlzRK1knY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf h1:kt3wY1Lu5MJAnKTfoMR52Cu4gwvna4VTzNOiT8tY73s= -golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= From 01d5feed7b66f418e47c6aeb82f2be1c7265d004 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Fri, 11 Dec 2020 01:18:05 +0100 Subject: [PATCH 11/20] [WIP] save point --- cmd/server/server.go | 4 +- core/collectors.go | 216 +++++++++++++++++++++++++++++++------------ core/config.go | 8 +- core/http.go | 46 +++++---- core/pipeline.go | 59 +++++++----- core/replay.go | 1 + core/repository.go | 46 +++++++++ core/trackings.go | 106 +++++++++++++++++++++ go.mod | 2 + go.sum | 50 ++++++++++ 10 files changed, 433 insertions(+), 105 deletions(-) create mode 100644 core/replay.go create mode 100644 core/repository.go create mode 100644 core/trackings.go diff --git a/cmd/server/server.go b/cmd/server/server.go index cd26270..feaa4f2 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -23,9 +23,9 @@ func main() { log.Println("initialize processing pipeline") processor := core.NewPipeline(dispatcher, 50, 494) processor.Run() - //collectRoutines(processor) + collectRoutines(processor) log.Println("start http server") - core.FiberListenAndServe(dispatcher) + core.FiberListen(dispatcher) //core.HttpListenAndServe(dispatcher, HTTP_PORT) } diff --git a/core/collectors.go b/core/collectors.go index 2f24378..71ca6e6 100644 --- a/core/collectors.go +++ b/core/collectors.go @@ -2,41 +2,162 @@ package core import ( "fmt" + "github.com/google/uuid" "log" "net" "os" + "time" "git.timovolkmann.de/gyrogpsc/ublox" "go.bug.st/serial" ) -func TcpCollector(proc Processor, tcpPort string) { - log.Println("start tcp collectors") +type Collector interface { + Collect() + Stop() +} - listener, err := net.Listen("tcp", tcpPort) - if err != nil { - fmt.Println("Error listening:", err.Error()) - os.Exit(1) - } - // Close the listener when the application closes. - defer listener.Close() +type CollectorType uint8 - for { - // Listen for an incoming connection. - conn, err := listener.Accept() - if err != nil { - fmt.Println("Error accepting: ", err.Error()) - os.Exit(1) +const ( + SERIAL CollectorType = iota + TCP +) + +var tcpSingleton *tcpCollector + +func NewCollector(typ CollectorType, proc Processor, config *Configuration) Collector { + var coll Collector + switch typ { + case SERIAL: + coll = newSerial(proc, config) + case TCP: + if tcpSingleton == nil { + tcpSingleton = newTcp(proc, config) + } else { + tcpSingleton.SetProcessor(proc) } - log.Println("...new incoming tcp connection...") + coll = tcpSingleton + //case DATABASE: + // if len(trackingId) != 1 { + // panic("only 1 tracking per collector") + // } + // coll = newDbReplay(proc, config, trackingId[0]) + default: + panic("selected collector type not implemented") + } + return coll +} - // Handle connections in a new goroutine. - go jsonHandler(conn, proc) +type serialCollector struct { + active bool + proc Processor + config *Configuration +} + +func (s *serialCollector) Collect() { + go func() { + log.Println("start serial collectors") + mode := &serial.Mode{ + BaudRate: 115200, + } + port, err := serial.Open(s.config.SerialCollectorPort, mode) + if err != nil { + log.Fatalln(err.Error()) + } + defer port.Close() + + decoder := ublox.NewDecoder(port) + + for s.active { + meas, err := decoder.Decode() + if err != nil { + if err.Error() == "NMEA not implemented" { + continue + } + log.Println("serial read err:", err) + break + } + sd, err := ConvertUbxToSensorData(meas) + if err != nil { + log.Println("convert err:", err, meas) + continue + } + if sd == nil { + continue + } + + err = s.proc.Push(sd) + if err != nil { + log.Println("process err:", err, *sd) + continue + } + } + + }() +} + +func (s *serialCollector) Stop() { + panic("implement me") +} + +func newSerial(proc Processor, config *Configuration) *serialCollector { + return &serialCollector{ + active: false, + proc: proc, + config: config, } } +type tcpCollector struct { + active bool + processor Processor + //config *Configuration +} + +func (t *tcpCollector) Collect() { + t.active = true +} + +func (t *tcpCollector) Stop() { + t.active = false +} + +func (t *tcpCollector) SetProcessor(p Processor) { + t.processor = p +} + +func newTcp(proc Processor, config *Configuration) *tcpCollector { + log.Println("start tcp collector") + + listener, err := net.Listen("tcp", config.TcpCollectorPort) + if err != nil { + fmt.Println("Error listening:", err.Error()) + //os.Exit(1) + } + coll := &tcpCollector{ + active: false, + processor: proc, + } + go func() { + for { + // Listen for an incoming connection. + conn, err := listener.Accept() + if err != nil { + fmt.Println("Error accepting: ", err.Error()) + os.Exit(1) + } + log.Println("...new incoming tcp connection...") + + // Handle connections in a new goroutine. + go coll.jsonHandler(conn) + } + }() + return coll +} + // handles incoming tcp connections with json payload. -func jsonHandler(conn net.Conn, proc Processor) { +func (c *tcpCollector) jsonHandler(conn net.Conn) { defer conn.Close() @@ -57,54 +178,27 @@ func jsonHandler(conn net.Conn, proc Processor) { log.Println(err) continue } - - err = proc.Process(sd) - if err != nil { - log.Println(err) + if !c.active { + time.Sleep(50 * time.Millisecond) continue } + err = c.processor.Push(sd) + if err != nil { + log.Fatalln(err) + } } } -func SerialCollector(proc Processor, serialPort string) { - log.Println("start serial collectors") - mode := &serial.Mode{ - BaudRate: 115200, - } - port, err := serial.Open(serialPort, mode) - if err != nil { - log.Fatalln(err.Error()) - } - defer port.Close() +type replayCollector struct{} - decoder := ublox.NewDecoder(port) - - for { - meas, err := decoder.Decode() - if err != nil { - if err.Error() == "NMEA not implemented" { - continue - } - log.Println("serial read err:", err) - break - } - sd, err := ConvertUbxToSensorData(meas) - if err != nil { - log.Println("convert err:", err, meas) - continue - } - if sd == nil { - continue - } - - err = proc.Process(sd) - if err != nil { - log.Println("process err:", err, *sd) - continue - } - } +func (r *replayCollector) Collect() { + panic("implement me") } -func replayCollector(storage string, proc Processor, trackingId int) { - +func (r *replayCollector) Stop() { + panic("implement me") +} + +func newDbReplay(proc Processor, config *Configuration, trackingId uuid.UUID) *replayCollector { + return nil } diff --git a/core/config.go b/core/config.go index bf93f32..b31e71f 100644 --- a/core/config.go +++ b/core/config.go @@ -1,9 +1,11 @@ package core type Configuration struct { - TcpCollectorPort string - SerialCollectorPort string - HttpPort string + TcpCollectorPort string + SerialCollectorPort string + HttpPort string + publishIntervalMs int + syncUpdateIntervalMs int } func LoadConfigYaml() error { diff --git a/core/http.go b/core/http.go index 108304f..b22c896 100644 --- a/core/http.go +++ b/core/http.go @@ -72,7 +72,7 @@ func HttpListenAndServe(sub Subscriber, httpPort string) { log.Fatal(http.ListenAndServe(httpPort, nil)) } -func FiberListenAndServe(sub Subscriber) { +func FiberListen(sub Subscriber) { app := fiber.New() app.Static("/static", "static") @@ -80,17 +80,20 @@ func FiberListenAndServe(sub Subscriber) { app.Get("/", fiberHomeHandler) // Websocket - app.Get("/ws", ws.New(fiberWebsocketHandler)) + app.Get("/ws", ws.New(createFiberWebsocketHandler(sub))) + + // TODO: Get all SerialPorts + app.Get("/serialports") // Tracking persistence controls trackings := app.Group("/trackings") trackings.Get("/") // Get all trackings Metadata - trackings.Post("/") // Initialize new tracking by websocket and prepare for automatic recording. Toggle ?serial=true and ?tcp=true. Returns trackingId + trackings.Post("/") // Initialize new tracking, open websocket and prepare for automatic recording. Toggle ?serial=true and ?tcp=true. Returns trackingId trackings.Patch("/") // Starts recording trackings.Put("/") // Stops current recording. Returns trackingId if record was successful trackings.Delete("/") // Stops websocket connection, pipelines and collectors - trackings.Get("/:trackingId") // Gets Tracking Metadata by Id. + trackings.Get("/:trackingId") // Gets Tracking Metadata and loads sensorRecords from database. trackings.Post("/:trackingId") // Starts Replay. trackings.Patch("/:trackingId") // Pauses Replay. trackings.Put("/:trackingId") // Stops Replay. @@ -99,20 +102,31 @@ func FiberListenAndServe(sub Subscriber) { log.Fatal(app.Listen(":3011")) } -func fiberWebsocketHandler(c *ws.Conn) { - for { - mt, msg, err := c.ReadMessage() - if err != nil { - log.Println("read:", err) - break - } - log.Printf("recv: %s", msg) - err = c.WriteMessage(mt, msg) - if err != nil { - log.Println("write:", err) - break +func createFiberWebsocketHandler(s Subscriber) func(conn *ws.Conn) { + return func(c *ws.Conn) { + // Handle and discard inbound messages + go func() { + for { + if _, _, err := c.NextReader(); err != nil { + c.Close() + break + } + } + }() + + dispatcherId, channel := s.Subscribe() + defer s.Unsubscribe(dispatcherId) + for { + cmsg := <-channel + err := c.WriteMessage(ws.TextMessage, []byte(cmsg)) + if err != nil { + log.Println("write:", err) + + break + } } } + } func fiberHomeHandler(c *fiber.Ctx) error { diff --git a/core/pipeline.go b/core/pipeline.go index bd7efd1..7ce3018 100644 --- a/core/pipeline.go +++ b/core/pipeline.go @@ -10,42 +10,59 @@ import ( ) type Processor interface { - Process(data *Sensordata) error + Push(data *Sensordata) error +} + +type Pipeline interface { + Run() + Stop() } type Storer interface { - Persist() + Store(data Sensordata) } type pipeline struct { - syn synchronizer - agr aggregator - pub Publisher - //stor Storer + active bool + syn synchronizer + agr aggregator + pub Publisher + stor Storer publishTicker *time.Ticker } -func NewPipeline(d Publisher, publishIntervalMs int, syncUpdateIntervalMs int) *pipeline { - // TODO: replace timing params with config struct +// pipe implements Runner & Processor +func NewPipeline(d Publisher, s Storer, conf *Configuration) *pipeline { return &pipeline{ + false, synchronizer{ //bufferSize: 100, mutex: &sync.Mutex{}, - updateTicker: time.NewTicker(time.Duration(syncUpdateIntervalMs) * time.Millisecond), + updateTicker: time.NewTicker(time.Duration(conf.syncUpdateIntervalMs) * time.Millisecond), }, aggregator{ tcpMutex: &sync.Mutex{}, serialMutex: &sync.Mutex{}, }, d, - time.NewTicker(time.Duration(publishIntervalMs) * time.Millisecond), + s, + time.NewTicker(time.Duration(conf.publishIntervalMs) * time.Millisecond), } } func (p *pipeline) Run() { - go p.scheduleSynchronizer() go func() { - for { + for p.active { + <-p.syn.updateTicker.C + err := p.refreshDelay() + if err != nil { + log.Println(err) + } + } + + }() + go func() { + for p.active { <-p.publishTicker.C err := p.Publish() if err != nil && err.Error() != "no data available" { @@ -53,7 +70,7 @@ func (p *pipeline) Run() { } } }() - log.Println("pipeline: processing service started") + log.Println("pipe: processing service started") } func (p *pipeline) Publish() error { @@ -109,14 +126,6 @@ type synchronizer struct { } func (p *pipeline) scheduleSynchronizer() { - log.Println("synchronizer: started") - for { - <-p.syn.updateTicker.C - err := p.refreshDelay() - if err != nil { - log.Println(err) - } - } } func (p *pipeline) refreshDelay() error { @@ -144,7 +153,7 @@ func (p *pipeline) refreshDelay() error { return nil } -func (p *pipeline) Process(data *Sensordata) error { +func (p *pipeline) Push(data *Sensordata) error { if data == nil { return errors.New("nil processing not allowed") } @@ -156,7 +165,7 @@ func (p *pipeline) Process(data *Sensordata) error { case SOURCE_SERIAL: go p.pushSerialDataToBuffer(*data) default: - panic("pipeline: invalid data source") + panic("pipe: invalid data source") } return nil } @@ -177,3 +186,7 @@ func (p *pipeline) pushSerialDataToBuffer(data Sensordata) { p.agr.serialSensorData = p.agr.serialSensorData.ConsolidateEpochsOnly(data) p.agr.serialMutex.Unlock() } + +func (p *pipeline) Stop() { + p.active = false +} diff --git a/core/replay.go b/core/replay.go new file mode 100644 index 0000000..9a8bc95 --- /dev/null +++ b/core/replay.go @@ -0,0 +1 @@ +package core diff --git a/core/repository.go b/core/repository.go new file mode 100644 index 0000000..d3c1fae --- /dev/null +++ b/core/repository.go @@ -0,0 +1,46 @@ +package core + +import ( + "github.com/google/uuid" + "sync" +) + +type writeRepo interface { + SaveTracking(tracking *Tracking) error +} + +type readRepo interface { + AllTrackings() ([]trackingMetadata, error) + LoadTracking(id uuid.UUID) (Tracking, error) +} + +type repository struct { + repoStubSave + loader repoStubLoad +} + +type repoStubSave struct { + sensordataBuffer +} +type repoStubLoad struct{} + +func (r *repoStubSave) SaveTracking(tracking *Tracking) error { + panic("implement me") +} + +func (r *repoStubLoad) AllTrackings() ([]trackingMetadata, error) { + panic("implement me") +} + +func (r *repoStubLoad) LoadTracking(id uuid.UUID) (Tracking, error) { + panic("implement me") +} + +type sensordataBuffer struct { + // TODO: database stub + data []Sensordata + mu sync.Mutex +} + +func (s *sensordataBuffer) Store(data Sensordata) { +} diff --git a/core/trackings.go b/core/trackings.go new file mode 100644 index 0000000..1d7a28b --- /dev/null +++ b/core/trackings.go @@ -0,0 +1,106 @@ +package core + +import ( + "github.com/google/uuid" + "log" + "time" +) + +type OpMode uint8 + +const ( + STOPPED OpMode = iota + LIVE + REPLAY +) + +type TrackingService struct { + Current *Tracking + config *Configuration + repo *repository + pipe *pipeline + opMode OpMode + collectors []Collector + //publish Publisher +} + +type Tracking struct { + trackingMetadata + data []*sensorRecord +} + +type trackingMetadata struct { + UUID uuid.UUID + TimeCreated time.Time +} + +type sensorRecord struct { + RecordTime time.Time + Sensordata +} + +func Service(c *Configuration) *TrackingService { + d := NewDispatcher() + r := &repository{} + return &TrackingService{ + Current: nil, + opMode: STOPPED, + config: c, + repo: r, + pipe: NewPipeline(d, r, c), + collectors: nil, + } +} + +func (t *TrackingService) NewTracking(cols ...CollectorType) { + t.opMode = LIVE + for _, col := range cols { + t.collectors = append(t.collectors, NewCollector(col, t.pipe, t.config)) + } + t.Current = emptyTracking() +} + +func (t *TrackingService) StartRecord() { + if t.opMode != LIVE { + log.Println("trackingservice: wrong mode of operation") + } + t.pipe.Run() + for _, e := range t.collectors { + e.Collect() + } +} + +func (t *TrackingService) StopRecord() { + if t.opMode != LIVE { + log.Println("trackingservice: wrong mode of operation") + } + t.pipe.Stop() + for _, e := range t.collectors { + e.Stop() + } + err := t.repo.SaveTracking(t.Current) + if err != nil { + log.Println(err) + } + t.Current = emptyTracking() +} + +func (t *TrackingService) Reset() { +} + +func (t *TrackingService) LoadTrackings() { + +} + +func (t *TrackingService) LoadTracking(trackingId uuid.UUID) { + +} + +func emptyTracking() *Tracking { + return &Tracking{ + trackingMetadata: trackingMetadata{ + UUID: uuid.New(), + }, + data: []*sensorRecord{}, + } +} diff --git a/go.mod b/go.mod index 3edfef5..1872bc9 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,11 @@ go 1.15 require ( github.com/gofiber/fiber/v2 v2.2.4 github.com/gofiber/websocket/v2 v2.0.2 + github.com/google/uuid v1.1.2 github.com/gorilla/websocket v1.4.2 github.com/stretchr/testify v1.6.1 // indirect github.com/tidwall/gjson v1.6.0 github.com/tidwall/pretty v1.0.2 // indirect + github.com/upper/db/v4 v4.0.1 // indirect go.bug.st/serial v1.1.1 ) diff --git a/go.sum b/go.sum index 5179a37..da3fa22 100644 --- a/go.sum +++ b/go.sum @@ -1,26 +1,44 @@ +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/creack/goselect v0.1.1 h1:tiSSgKE1eJtxs1h/VgGQWuXUP0YS4CDIFMp6vaI1ls0= github.com/creack/goselect v0.1.1/go.mod h1:a/NhLweNvqIYMuxcMOuWY516Cimucms3DglDzQP3hKY= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.0.0-20200620013148-b91950f658ec/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/fasthttp/websocket v1.4.3 h1:qjhRJ/rTy4KB8oBxljEC00SDt6HUY9jLRfM601SUdS4= github.com/fasthttp/websocket v1.4.3/go.mod h1:5r4oKssgS7W6Zn6mPWap3NWzNPJNzUUh3baWTOhcYQk= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/gofiber/fiber/v2 v2.1.0/go.mod h1:aG+lMkwy3LyVit4CnmYUbUdgjpc3UYOltvlJZ78rgQ0= github.com/gofiber/fiber/v2 v2.2.4 h1:t2V2SxlbQGdt8+SS/Mo+tQB0pDQn7OajKdA72qHcBVw= github.com/gofiber/fiber/v2 v2.2.4/go.mod h1:Aso7/M+EQOinVkWp4LUYjdlTpKTBoCk2Qo4djnMsyHE= github.com/gofiber/websocket/v2 v2.0.2 h1:UA/6NpyG+vmPGlvJvW8MJPJpRFuS7abinZ5HbLuV8u0= github.com/gofiber/websocket/v2 v2.0.2/go.mod h1:7VBnzEVRK0K0eTIVc5GbXPF1JWUFnllY0X4cRtG2v78= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.14.1 h1:AHx9Ra40wIzl+GelgX2X6AWxmT5tfxhI1PL0523HcSw= +github.com/mattn/go-sqlite3 v1.14.1/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/savsgio/gotils v0.0.0-20200608150037-a5f6f5aef16c h1:2nF5+FZ4/qp7pZVL7fR6DEaSTzuDmNaFTyqp92/hwF8= github.com/savsgio/gotils v0.0.0-20200608150037-a5f6f5aef16c/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= @@ -32,6 +50,10 @@ github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/upper/db v1.0.1 h1:AFzy0fFO7EZ5woZGE/LVl9BWLKqyAWlglu+ZyHQTlbU= +github.com/upper/db v3.7.1+incompatible h1:ehR+Q0msV25yZWXdFYD+3n4BvY9jefI336lmZy7FlAQ= +github.com/upper/db/v4 v4.0.1 h1:mHmGBffne8fdiJjmtYXCuHSghO9027q4TJA9oGP50OM= +github.com/upper/db/v4 v4.0.1/go.mod h1:pyAEIpPfnhpvO4zVbyUI+asgZjppZlq705fOFTyUOso= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.14.0/go.mod h1:ol1PCaL0dX20wC0htZ7sYCsvCYmrouYra0zHzaclZhE= @@ -40,16 +62,27 @@ github.com/valyala/fasthttp v1.17.0 h1:P8/koH4aSnJ4xbd0cUUFEGQs3jQqIxoDDyRQrUiAk github.com/valyala/fasthttp v1.17.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +gitlab.com/cznic/ebnf2y v1.0.0/go.mod h1:jx14dqOldV2pRvSi8HASTB/k5fkIv2TwjYAp5py0MTs= +gitlab.com/cznic/golex v1.0.0/go.mod h1:vkWdDgqbbThjRHoOLU7yNPgMxaubAkwnvF/4zeG8cvU= go.bug.st/serial v1.1.1 h1:5J1DpaIaSIruBi7jVnKXnhRS+YQ9+2PLJMtIZKoIgnc= go.bug.st/serial v1.1.1/go.mod h1:VmYBeyJWp5BnJ0tw2NUJHZdJTGl2ecBGABHlzRK1knY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190411193353-0480eff6dd7c/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -60,9 +93,26 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= +modernc.org/db v1.0.0/go.mod h1:kYD/cO29L/29RM0hXYl4i3+Q5VojL31kTUVpVJDw0s8= +modernc.org/ebnfutil v1.0.0/go.mod h1:+2n/OnQXoild9pzrPa/2wmVtR+ufWjB/0fYkc0BV9sc= +modernc.org/file v1.0.0/go.mod h1:uqEokAEn1u6e+J45e54dsEA/pw4o7zLrA2GwyntZzjw= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/internal v1.0.0/go.mod h1:VUD/+JAkhCpvkUitlEOnhpVxCgsBI90oTzSCRcqQVSM= +modernc.org/lex v1.0.0/go.mod h1:G6rxMTy3cH2iA0iXL/HRRv4Znu8MK4higxph/lE7ypk= +modernc.org/lexer v1.0.0/go.mod h1:F/Dld0YKYdZCLQ7bD0USbWL4YKCyTDRDHiDTOs0q0vk= +modernc.org/lldb v1.0.0/go.mod h1:jcRvJGWfCGodDZz8BPwiKMJxGJngQ/5DrRapkQnLob8= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/ql v1.1.0/go.mod h1:Fj1ylcVyzcu/fgWZTrvBO9j/aEUg/ixLFnGtmzh7quI= +modernc.org/sortutil v1.0.0/go.mod h1:1QO0q8IlIlmjBIwm6t/7sof874+xCfZouyqZMLIAtxM= +modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4= From ae9b08e0c4328f30cacf9d0d012baaf6715c1ed0 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Fri, 11 Dec 2020 17:34:31 +0100 Subject: [PATCH 12/20] [WIP] repository and service encapsulation --- .gitignore | 3 +- .idea/codeStyles/codeStyleConfig.xml | 5 + cmd/autostart/autostart.go | 55 ++++++ cmd/serial_only/serial_only.go | 21 --- cmd/server/server.go | 52 +++--- cmd/tcp_only/tcp_only.go | 27 --- core/collectors.go | 53 ++---- core/config.go | 24 ++- core/dispatcher.go | 10 +- core/http.go | 142 -------------- core/interfaces.go | 46 +++++ core/pipeline.go | 76 ++++---- core/repository.go | 46 ----- core/sensordata.go | 51 +++-- core/service.go | 107 +++++++++++ core/trackings.go | 143 +++++--------- database/repository.go | 27 +++ example_config.yml | 19 +- go.mod | 1 + go.sum | 270 +++++++++++++++++++++++++++ web/http.go | 85 +++++++++ 21 files changed, 780 insertions(+), 483 deletions(-) create mode 100644 .idea/codeStyles/codeStyleConfig.xml create mode 100644 cmd/autostart/autostart.go delete mode 100644 cmd/serial_only/serial_only.go delete mode 100644 cmd/tcp_only/tcp_only.go delete mode 100644 core/http.go create mode 100644 core/interfaces.go delete mode 100644 core/repository.go create mode 100644 core/service.go create mode 100644 database/repository.go create mode 100644 web/http.go diff --git a/.gitignore b/.gitignore index 7908a1f..e57b859 100644 --- a/.gitignore +++ b/.gitignore @@ -159,4 +159,5 @@ Temporary Items # CUSTOM .env -config.yml \ No newline at end of file +gpsconfig.yml +config.yml diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..a55e7a1 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/cmd/autostart/autostart.go b/cmd/autostart/autostart.go new file mode 100644 index 0000000..ad55990 --- /dev/null +++ b/cmd/autostart/autostart.go @@ -0,0 +1,55 @@ +package main + +import ( + "git.timovolkmann.de/gyrogpsc/core" + "git.timovolkmann.de/gyrogpsc/database" + "git.timovolkmann.de/gyrogpsc/web" + "github.com/spf13/viper" + "log" + "time" +) + +func main() { + conf := configurationFromFile() + + repo := database.NewRepository(conf) + disp := core.NewDispatcher() + + service := core.TrackingService(repo, disp, conf) + + go func() { + service.NewTracking(core.TCP, core.SERIAL) + time.Sleep(5 * time.Second) + service.StartRecord() + time.Sleep(15 * time.Second) + service.StopRecord() + }() + + web.CreateServer(service, disp, conf) +} + +func configurationFromFile() *core.Configuration { + viper.SetDefault("collectors.port_tcp", ":3010") + viper.SetDefault("collectors.port_serial", "/dev/tty.usbmodem14201") + viper.SetDefault("webserver.port", ":3011") + viper.SetDefault("pipeline.publishIntervalMs", 50) + viper.SetDefault("pipeline.syncUpdateIntervalMs", 494) + + viper.SetConfigName("gpsconfig") // name of config file (without extension) + viper.SetConfigType("yaml") + viper.AddConfigPath(".") + viper.AddConfigPath("./../../") + if err := viper.ReadInConfig(); err != nil { + log.Println("couldn't find config file. using standard configuration") + } + log.Println(viper.AllKeys()) + log.Println(viper.AllSettings()) + + c := core.Configuration{} + if err := viper.Unmarshal(&c); err != nil { + log.Println("couldn't load config...") + log.Println(err) + } + log.Println(c) + return &c +} \ No newline at end of file diff --git a/cmd/serial_only/serial_only.go b/cmd/serial_only/serial_only.go deleted file mode 100644 index 352585e..0000000 --- a/cmd/serial_only/serial_only.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import ( - "fmt" - "git.timovolkmann.de/gyrogpsc/core" -) - -const ( - SERIAL_PORT = "/dev/tty.usbmodem14201" -) - -func main() { - core.SerialCollector(&printer{}, SERIAL_PORT) -} - -type printer struct{} - -func (p *printer) Process(data *core.Sensordata) error { - fmt.Println(data) - return nil -} diff --git a/cmd/server/server.go b/cmd/server/server.go index feaa4f2..a5a9ef1 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -2,36 +2,36 @@ package main import ( "git.timovolkmann.de/gyrogpsc/core" + "git.timovolkmann.de/gyrogpsc/database" + "git.timovolkmann.de/gyrogpsc/web" + "github.com/spf13/viper" "log" - "os" - "runtime" -) - -const ( - TCP_PORT = ":3010" - HTTP_PORT = ":3011" - SERIAL_PORT = "/dev/tty.usbmodem14201" ) func main() { - log.Println("GOROOT:", runtime.GOROOT()) - wd, _ := os.Getwd() - log.Println("WorkingDir:", wd) + conf := &core.Configuration{} + configurationFromFile(conf) - log.Println("setup dispatcher") - dispatcher := core.NewDispatcher() - log.Println("initialize processing pipeline") - processor := core.NewPipeline(dispatcher, 50, 494) - processor.Run() - collectRoutines(processor) - log.Println("start http server") - core.FiberListen(dispatcher) - //core.HttpListenAndServe(dispatcher, HTTP_PORT) + repo := database.NewRepository(conf) + disp := core.NewDispatcher() + + service := core.TrackingService(repo, disp, conf) + + web.CreateServer(service, disp, conf) } -func collectRoutines(proc core.Processor) { - // collect Sensor data from Serial UBX in Goroutine - go core.SerialCollector(proc, SERIAL_PORT) - // collect Sensor data from JSON over TCP in Goroutine - go core.TcpCollector(proc, TCP_PORT) -} +func configurationFromFile(c *core.Configuration) { + viper.SetDefault("TcpCollectorPort", ":3010") + viper.SetDefault("SerialCollectorPort", "/dev/tty.usbmodem14201") + viper.SetDefault("HttpPort", "layouts") + viper.SetDefault("publishIntervalMs", 50) + viper.SetDefault("syncUpdateIntervalMs", 494) + + viper.SetConfigName("gpsconfig") // name of config file (without extension) + viper.SetConfigType("yaml") + viper.AddConfigPath(".") + viper.AddConfigPath("./../../") + + viper.Unmarshal(c) + log.Println(c) +} \ No newline at end of file diff --git a/cmd/tcp_only/tcp_only.go b/cmd/tcp_only/tcp_only.go deleted file mode 100644 index d535377..0000000 --- a/cmd/tcp_only/tcp_only.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -import ( - "git.timovolkmann.de/gyrogpsc/core" - "log" -) - -const ( - TCP_PORT = ":3010" - HTTP_PORT = ":3011" -) - -func main() { - log.Println("setup dispatcher") - dispatcher := core.NewDispatcher() - log.Println("initialize processing pipeline") - processor := core.NewPipeline(dispatcher, 50, 494) - processor.Run() - collectRoutines(processor) - log.Println("start http server") - core.HttpListenAndServe(dispatcher, HTTP_PORT) -} - -func collectRoutines(proc core.Processor) { - // collect Sensor data from JSON over TCP in Goroutine - go core.TcpCollector(proc, TCP_PORT) -} diff --git a/core/collectors.go b/core/collectors.go index 71ca6e6..d4bc595 100644 --- a/core/collectors.go +++ b/core/collectors.go @@ -2,7 +2,6 @@ package core import ( "fmt" - "github.com/google/uuid" "log" "net" "os" @@ -26,7 +25,7 @@ const ( var tcpSingleton *tcpCollector -func NewCollector(typ CollectorType, proc Processor, config *Configuration) Collector { +func NewCollector(typ CollectorType, proc Pusher, config *Configuration) Collector { var coll Collector switch typ { case SERIAL: @@ -38,11 +37,6 @@ func NewCollector(typ CollectorType, proc Processor, config *Configuration) Coll tcpSingleton.SetProcessor(proc) } coll = tcpSingleton - //case DATABASE: - // if len(trackingId) != 1 { - // panic("only 1 tracking per collector") - // } - // coll = newDbReplay(proc, config, trackingId[0]) default: panic("selected collector type not implemented") } @@ -51,17 +45,18 @@ func NewCollector(typ CollectorType, proc Processor, config *Configuration) Coll type serialCollector struct { active bool - proc Processor + proc Pusher config *Configuration } func (s *serialCollector) Collect() { + s.active = true go func() { - log.Println("start serial collectors") + log.Println("start serial collector") mode := &serial.Mode{ BaudRate: 115200, } - port, err := serial.Open(s.config.SerialCollectorPort, mode) + port, err := serial.Open(s.config.Collectors.SerialCollectorPort, mode) if err != nil { log.Fatalln(err.Error()) } @@ -79,29 +74,27 @@ func (s *serialCollector) Collect() { break } sd, err := ConvertUbxToSensorData(meas) - if err != nil { - log.Println("convert err:", err, meas) - continue - } - if sd == nil { + if err != nil || sd == nil { + log.Println("convert err:", err, meas, sd) continue } + err = s.proc.Push(sd) if err != nil { log.Println("process err:", err, *sd) continue } } - + log.Println("serial collector stopped") }() } func (s *serialCollector) Stop() { - panic("implement me") + s.active = false } -func newSerial(proc Processor, config *Configuration) *serialCollector { +func newSerial(proc Pusher, config *Configuration) *serialCollector { return &serialCollector{ active: false, proc: proc, @@ -111,7 +104,7 @@ func newSerial(proc Processor, config *Configuration) *serialCollector { type tcpCollector struct { active bool - processor Processor + processor Pusher //config *Configuration } @@ -123,14 +116,14 @@ func (t *tcpCollector) Stop() { t.active = false } -func (t *tcpCollector) SetProcessor(p Processor) { +func (t *tcpCollector) SetProcessor(p Pusher) { t.processor = p } -func newTcp(proc Processor, config *Configuration) *tcpCollector { +func newTcp(proc Pusher, config *Configuration) *tcpCollector { log.Println("start tcp collector") - listener, err := net.Listen("tcp", config.TcpCollectorPort) + listener, err := net.Listen("tcp", config.Collectors.TcpCollectorPort) if err != nil { fmt.Println("Error listening:", err.Error()) //os.Exit(1) @@ -187,18 +180,4 @@ func (c *tcpCollector) jsonHandler(conn net.Conn) { log.Fatalln(err) } } -} - -type replayCollector struct{} - -func (r *replayCollector) Collect() { - panic("implement me") -} - -func (r *replayCollector) Stop() { - panic("implement me") -} - -func newDbReplay(proc Processor, config *Configuration, trackingId uuid.UUID) *replayCollector { - return nil -} +} \ No newline at end of file diff --git a/core/config.go b/core/config.go index b31e71f..dad3510 100644 --- a/core/config.go +++ b/core/config.go @@ -1,13 +1,23 @@ package core type Configuration struct { - TcpCollectorPort string - SerialCollectorPort string - HttpPort string - publishIntervalMs int - syncUpdateIntervalMs int + Collectors struct{ + TcpCollectorPort string `mapstructure:"porttcp"` + SerialCollectorPort string `mapstructure:"portserial"` + } `mapstructure:"collectors"` + Webserver struct { + Port string `mapstructure:"port"` + } `mapstructure:"webserver"` + Pipeline struct { + PublishIntervalMs int `mapstructure:"publishintervalms"` + SyncUpdateIntervalMs int `mapstructure:"syncupdateintervalms"` + } `mapstructure:"pipeline"` } -func LoadConfigYaml() error { - return nil +type Configuration2 struct { + TcpCollectorPort string `mapstructure:"collectors.porttcp"` + SerialCollectorPort string `mapstructure:"collectors.portserial"` + HttpPort string `mapstructure:"webserver.port"` + PublishIntervalMs int `mapstructure:"pipeline.publishintervalms"` + SyncUpdateIntervalMs int `mapstructure:"pipeline.syncupdateintervalms"` } diff --git a/core/dispatcher.go b/core/dispatcher.go index e6a97b7..f2c7684 100644 --- a/core/dispatcher.go +++ b/core/dispatcher.go @@ -5,15 +5,6 @@ import ( "log" ) -type Subscriber interface { - Subscribe() (int16, <-chan string) - Unsubscribe(id int16) error -} - -type Publisher interface { - Publish(message string) -} - type dispatcher struct { listeners map[int16]chan string counter int16 @@ -26,6 +17,7 @@ func NewDispatcher() *dispatcher { } } + func (d *dispatcher) Publish(message string) { log.Printf("publish to %v listeners:\n%v\n", len(d.listeners), message) for _, ch := range d.listeners { diff --git a/core/http.go b/core/http.go deleted file mode 100644 index b22c896..0000000 --- a/core/http.go +++ /dev/null @@ -1,142 +0,0 @@ -package core - -import ( - "fmt" - "github.com/gofiber/fiber/v2" - ws "github.com/gofiber/websocket/v2" - "github.com/gorilla/websocket" - "html/template" - "log" - "net/http" -) - -func echo(sub Subscriber) func(w http.ResponseWriter, r *http.Request) { - var upgrader = websocket.Upgrader{} // use default options - return func(w http.ResponseWriter, r *http.Request) { - fmt.Println("upgrading to ws") - c, err := upgrader.Upgrade(w, r, nil) - if err != nil { - log.Print("upgrade:", err) - return - } - //defer c.Close() - go func() { - for { - if _, _, err := c.NextReader(); err != nil { - c.Close() - break - } - } - }() - - dispatcherId, channel := sub.Subscribe() - defer sub.Unsubscribe(dispatcherId) - for { - //log.Println("") - //if err != nil { - // log.Println("read:", err) - // break - //} - cmsg := <-channel - err = c.WriteMessage(websocket.TextMessage, []byte(cmsg)) - if err != nil { - log.Println("write:", err) - - break - } - } - } -} - -func home(w http.ResponseWriter, r *http.Request) { - //var homeTemplate = template.Must(template.NewDispatcher("").ParseFiles("index.html")) - tpl, err := template.ParseFiles("static/index.html") - if err != nil { - log.Fatalln(err) - } - err = tpl.Execute(w, "ws://"+r.Host+"/echo") - if err != nil { - log.Fatalln(err) - } -} - -func HttpListenAndServe(sub Subscriber, httpPort string) { - log.Println("register websocket handler") - http.HandleFunc("/echo", echo(sub)) - log.Println("register index handler") - http.HandleFunc("/", home) - log.Println("register static file handler") - http.Handle("/static/", http.FileServer(http.Dir("."))) - - log.Println("start server") - log.Fatal(http.ListenAndServe(httpPort, nil)) -} - -func FiberListen(sub Subscriber) { - app := fiber.New() - app.Static("/static", "static") - - // Application Main Page - app.Get("/", fiberHomeHandler) - - // Websocket - app.Get("/ws", ws.New(createFiberWebsocketHandler(sub))) - - // TODO: Get all SerialPorts - app.Get("/serialports") - - // Tracking persistence controls - trackings := app.Group("/trackings") - trackings.Get("/") // Get all trackings Metadata - trackings.Post("/") // Initialize new tracking, open websocket and prepare for automatic recording. Toggle ?serial=true and ?tcp=true. Returns trackingId - trackings.Patch("/") // Starts recording - trackings.Put("/") // Stops current recording. Returns trackingId if record was successful - trackings.Delete("/") // Stops websocket connection, pipelines and collectors - - trackings.Get("/:trackingId") // Gets Tracking Metadata and loads sensorRecords from database. - trackings.Post("/:trackingId") // Starts Replay. - trackings.Patch("/:trackingId") // Pauses Replay. - trackings.Put("/:trackingId") // Stops Replay. - trackings.Delete("/:trackingId") // Deletes Tracking from database - - log.Fatal(app.Listen(":3011")) -} - -func createFiberWebsocketHandler(s Subscriber) func(conn *ws.Conn) { - return func(c *ws.Conn) { - // Handle and discard inbound messages - go func() { - for { - if _, _, err := c.NextReader(); err != nil { - c.Close() - break - } - } - }() - - dispatcherId, channel := s.Subscribe() - defer s.Unsubscribe(dispatcherId) - for { - cmsg := <-channel - err := c.WriteMessage(ws.TextMessage, []byte(cmsg)) - if err != nil { - log.Println("write:", err) - - break - } - } - } - -} - -func fiberHomeHandler(c *fiber.Ctx) error { - tpl, err := template.ParseFiles("static/index.html") - if err != nil { - return err - } - err = tpl.Execute(c, "ws://"+c.Hostname()+"/ws") - if err != nil { - return err - } - return nil -} diff --git a/core/interfaces.go b/core/interfaces.go new file mode 100644 index 0000000..d2ce9af --- /dev/null +++ b/core/interfaces.go @@ -0,0 +1,46 @@ +package core + +import "github.com/google/uuid" + +type Subscriber interface { + Subscribe() (int16, <-chan string) + Unsubscribe(id int16) error +} + +type Publisher interface { + Publish(message string) +} + +type Pusher interface { + Push(data *sensorData) error +} + + +type Storer interface { + EnqueuePair(tcp sensorData, ser sensorData) + EnqueueRaw(data sensorData) +} + + + +type Repo interface { + Save(tracking Tracking) error + LoadAll() ([]TrackingMetadata, error) + Load(id uuid.UUID) (Tracking, error) +} + + +type Service interface { + AllTrackings() + NewTracking(cols ...CollectorType) + StartRecord() + StopRecord() + Reset() + + LoadTracking(trackingId uuid.UUID) + DeleteTracking(trackingId uuid.UUID) + + StartReplay() + PauseReplay() + StopReplay() +} diff --git a/core/pipeline.go b/core/pipeline.go index 7ce3018..23a9a5e 100644 --- a/core/pipeline.go +++ b/core/pipeline.go @@ -9,21 +9,9 @@ import ( "time" ) -type Processor interface { - Push(data *Sensordata) error -} - -type Pipeline interface { - Run() - Stop() -} - -type Storer interface { - Store(data Sensordata) -} - type pipeline struct { active bool + record bool syn synchronizer agr aggregator pub Publisher @@ -31,14 +19,15 @@ type pipeline struct { publishTicker *time.Ticker } -// pipe implements Runner & Processor +// pipe implements Runner & Pusher func NewPipeline(d Publisher, s Storer, conf *Configuration) *pipeline { return &pipeline{ + false, false, synchronizer{ //bufferSize: 100, mutex: &sync.Mutex{}, - updateTicker: time.NewTicker(time.Duration(conf.syncUpdateIntervalMs) * time.Millisecond), + updateTicker: time.NewTicker(time.Duration(conf.Pipeline.SyncUpdateIntervalMs) * time.Millisecond), }, aggregator{ tcpMutex: &sync.Mutex{}, @@ -46,11 +35,13 @@ func NewPipeline(d Publisher, s Storer, conf *Configuration) *pipeline { }, d, s, - time.NewTicker(time.Duration(conf.publishIntervalMs) * time.Millisecond), + time.NewTicker(time.Duration(conf.Pipeline.PublishIntervalMs) * time.Millisecond), } } func (p *pipeline) Run() { + p.active = true + log.Println("pipe: processing service started") go func() { for p.active { <-p.syn.updateTicker.C @@ -59,7 +50,7 @@ func (p *pipeline) Run() { log.Println(err) } } - + log.Println("pipe: updater stopped") }() go func() { for p.active { @@ -69,26 +60,34 @@ func (p *pipeline) Run() { log.Println(err) } } + log.Println("pipe: publisher stopped") }() - log.Println("pipe: processing service started") +} + +func (p *pipeline) Record() { + p.record = true +} +func (p *pipeline) Stop() { + p.record = false } func (p *pipeline) Publish() error { p.agr.tcpMutex.Lock() p.agr.serialMutex.Lock() - //log.Println(pub.tcpSensorData) - //log.Println(pub.serialSensorData) - if (p.agr.tcpSensorData == Sensordata{} && p.agr.serialSensorData == Sensordata{}) { + + if (p.agr.tcpSensorData == sensorData{} && p.agr.serialSensorData == sensorData{}) { p.agr.tcpMutex.Unlock() p.agr.serialMutex.Unlock() return errors.New("no data available") } - data := map[string]Sensordata{ + + p.stor.EnqueuePair(p.agr.tcpSensorData, p.agr.serialSensorData) + + data := map[string]sensorData{ string(SOURCE_TCP): p.agr.tcpSensorData, string(SOURCE_SERIAL): p.agr.serialSensorData, } - //p.agr.tcpSensorData = Sensordata{} - //p.agr.serialSensorData = Sensordata{} + p.agr.tcpMutex.Unlock() p.agr.serialMutex.Unlock() @@ -102,8 +101,8 @@ func (p *pipeline) Publish() error { } type aggregator struct { - tcpSensorData Sensordata - serialSensorData Sensordata + tcpSensorData sensorData + serialSensorData sensorData tcpMutex *sync.Mutex serialMutex *sync.Mutex } @@ -112,20 +111,8 @@ type UnixNanoTime int64 type synchronizer struct { tcpSerialDelayMs int64 - //tcpBuffer map[UnixNanoTime]Sensordata - //serialBuffer map[UnixNanoTime]Sensordata - //bufferSize int mutex *sync.Mutex updateTicker *time.Ticker - // should run concurrently - // - // Methods: - // pushSensordata(Sensordata), remove oldest if larger than bufferSize - // refreshDelay() - // Schedule() -} - -func (p *pipeline) scheduleSynchronizer() { } func (p *pipeline) refreshDelay() error { @@ -153,13 +140,14 @@ func (p *pipeline) refreshDelay() error { return nil } -func (p *pipeline) Push(data *Sensordata) error { +func (p *pipeline) Push(data *sensorData) error { if data == nil { return errors.New("nil processing not allowed") } - //log.Println(string(data.SourceId)) + //log.Println(string(data.source)) // TODO: persist data here with current timestamp - switch data.SourceId { + p.stor.EnqueueRaw(*data) + switch data.source { case SOURCE_TCP: go p.pushTcpDataToBuffer(*data) case SOURCE_SERIAL: @@ -170,7 +158,7 @@ func (p *pipeline) Push(data *Sensordata) error { return nil } -func (p *pipeline) pushTcpDataToBuffer(data Sensordata) { +func (p *pipeline) pushTcpDataToBuffer(data sensorData) { if p.syn.tcpSerialDelayMs > 0 { time.Sleep(time.Duration(p.syn.tcpSerialDelayMs) * time.Millisecond) } @@ -178,7 +166,7 @@ func (p *pipeline) pushTcpDataToBuffer(data Sensordata) { p.agr.tcpSensorData = p.agr.tcpSensorData.ConsolidateExTime(data) p.agr.tcpMutex.Unlock() } -func (p *pipeline) pushSerialDataToBuffer(data Sensordata) { +func (p *pipeline) pushSerialDataToBuffer(data sensorData) { if p.syn.tcpSerialDelayMs < 0 { time.Sleep(time.Duration(-p.syn.tcpSerialDelayMs) * time.Millisecond) } @@ -187,6 +175,6 @@ func (p *pipeline) pushSerialDataToBuffer(data Sensordata) { p.agr.serialMutex.Unlock() } -func (p *pipeline) Stop() { +func (p *pipeline) Close() { p.active = false } diff --git a/core/repository.go b/core/repository.go deleted file mode 100644 index d3c1fae..0000000 --- a/core/repository.go +++ /dev/null @@ -1,46 +0,0 @@ -package core - -import ( - "github.com/google/uuid" - "sync" -) - -type writeRepo interface { - SaveTracking(tracking *Tracking) error -} - -type readRepo interface { - AllTrackings() ([]trackingMetadata, error) - LoadTracking(id uuid.UUID) (Tracking, error) -} - -type repository struct { - repoStubSave - loader repoStubLoad -} - -type repoStubSave struct { - sensordataBuffer -} -type repoStubLoad struct{} - -func (r *repoStubSave) SaveTracking(tracking *Tracking) error { - panic("implement me") -} - -func (r *repoStubLoad) AllTrackings() ([]trackingMetadata, error) { - panic("implement me") -} - -func (r *repoStubLoad) LoadTracking(id uuid.UUID) (Tracking, error) { - panic("implement me") -} - -type sensordataBuffer struct { - // TODO: database stub - data []Sensordata - mu sync.Mutex -} - -func (s *sensordataBuffer) Store(data Sensordata) { -} diff --git a/core/sensordata.go b/core/sensordata.go index c9a9d85..f13dc31 100644 --- a/core/sensordata.go +++ b/core/sensordata.go @@ -16,15 +16,26 @@ const ( SOURCE_SERIAL sourceId = "SOURCE_SERIAL" ) -type Sensordata struct { +type sensorData struct { itow uint32 - SourceId sourceId + source sourceId Timestamp int64 Position [3]float64 Orientation [3]float64 } -func (s Sensordata) isSameEpoch(n Sensordata) bool { +type recordPair struct { + RecordTime time.Time + data map[sourceId]sensorData +} + +type rawRecord struct { + RecordTime time.Time + sensorData +} + + +func (s sensorData) isSameEpoch(n sensorData) bool { if n.itow == 0 { return false } @@ -32,10 +43,10 @@ func (s Sensordata) isSameEpoch(n Sensordata) bool { } // Consolidates two sensordata elements if they are in the same epoch -func (s Sensordata) ConsolidateEpochsOnly(n Sensordata) Sensordata { +func (s sensorData) ConsolidateEpochsOnly(n sensorData) sensorData { s.checkSources(&n) if s.isSameEpoch(n) { - null := Sensordata{} + null := sensorData{} if n.Timestamp == null.Timestamp { n.Timestamp = s.Timestamp @@ -51,9 +62,9 @@ func (s Sensordata) ConsolidateEpochsOnly(n Sensordata) Sensordata { } // Consolidates two sensordata elements but ignores timestamps -func (s Sensordata) ConsolidateExTime(n Sensordata) Sensordata { +func (s sensorData) ConsolidateExTime(n sensorData) sensorData { s.checkSources(&n) - null := Sensordata{} + null := sensorData{} if n.Position == null.Position { n.Position = s.Position @@ -64,11 +75,11 @@ func (s Sensordata) ConsolidateExTime(n Sensordata) Sensordata { return n } -func (s *Sensordata) checkSources(n *Sensordata) { - if (s.SourceId != n.SourceId && *s != Sensordata{}) { +func (s *sensorData) checkSources(n *sensorData) { + if (s.source != n.source && *s != sensorData{}) { log.Println(s) log.Println(n) - log.Fatalln("Do not consolidate Sensordata from different Sources") + log.Fatalln("Do not consolidate sensorData from different Sources") } } @@ -77,9 +88,9 @@ var ( errRawMessage = errors.New("raw message") ) -func ConvertUbxToSensorData(msg interface{}) (*Sensordata, error) { - sd := &Sensordata{ - SourceId: SOURCE_SERIAL, +func ConvertUbxToSensorData(msg interface{}) (*sensorData, error) { + sd := &sensorData{ + source: SOURCE_SERIAL, } switch v := msg.(type) { case *ublox.NavPvt: @@ -113,14 +124,14 @@ func ConvertUbxToSensorData(msg interface{}) (*Sensordata, error) { return sd, nil } -func ConvertSensorDataPhone(jsonData []byte) (*Sensordata, error) { +func ConvertSensorDataPhone(jsonData []byte) (*sensorData, error) { if gjson.Get(string(jsonData), "os").String() == "hyperimu" { return convertAndroidHyperImu(jsonData) } return convertIPhoneSensorLog(jsonData) } -func convertIPhoneSensorLog(jsonData []byte) (*Sensordata, error) { +func convertIPhoneSensorLog(jsonData []byte) (*sensorData, error) { timestamp := gjson.Get(string(jsonData), "locationTimestamp_since1970").Float() lat := gjson.Get(string(jsonData), "locationLatitude").Float() lon := gjson.Get(string(jsonData), "locationLongitude").Float() @@ -128,8 +139,8 @@ func convertIPhoneSensorLog(jsonData []byte) (*Sensordata, error) { pitch := gjson.Get(string(jsonData), "motionPitch").Float() * 180 / math.Pi roll := gjson.Get(string(jsonData), "motionRoll").Float() * 180 / math.Pi yaw := gjson.Get(string(jsonData), "motionYaw").Float() * 180 / math.Pi - sd := &Sensordata{ - SourceId: SOURCE_TCP, + sd := &sensorData{ + source: SOURCE_TCP, Timestamp: int64(timestamp * float64(time.Second)), //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), Position: [3]float64{lat, lon, alt}, @@ -140,7 +151,7 @@ func convertIPhoneSensorLog(jsonData []byte) (*Sensordata, error) { return sd, nil } -func convertAndroidHyperImu(jsonData []byte) (*Sensordata, error) { +func convertAndroidHyperImu(jsonData []byte) (*sensorData, error) { timestamp := gjson.Get(string(jsonData), "Timestamp").Int() lat := gjson.Get(string(jsonData), "GPS.0").Float() lon := gjson.Get(string(jsonData), "GPS.1").Float() @@ -149,8 +160,8 @@ func convertAndroidHyperImu(jsonData []byte) (*Sensordata, error) { roll := gjson.Get(string(jsonData), "orientation.1").Float() yaw := gjson.Get(string(jsonData), "orientation.2").Float() - sd := &Sensordata{ - SourceId: SOURCE_TCP, + sd := &sensorData{ + source: SOURCE_TCP, Timestamp: timestamp * int64(time.Millisecond), //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), Position: [3]float64{lat, lon, alt}, diff --git a/core/service.go b/core/service.go new file mode 100644 index 0000000..8df6a23 --- /dev/null +++ b/core/service.go @@ -0,0 +1,107 @@ +package core + +import ( + "github.com/google/uuid" + "log" + "time" +) + +type OpMode uint8 + +const ( + STOPPED OpMode = iota + LIVE + REPLAY +) + +type trackingService struct { + current *Tracking + config *Configuration + pipe *pipeline + repo Repo + opMode OpMode + collectors []Collector +} + +func TrackingService(r Repo, d Publisher, c *Configuration) *trackingService { + t := &Tracking{} + return &trackingService{ + current: t, + opMode: STOPPED, + config: c, + repo: r, + pipe: NewPipeline(d, t, c), + collectors: nil, + } +} + +func (t *trackingService) AllTrackings() { + panic("implement me") +} + +func (t *trackingService) NewTracking(cols ...CollectorType) { + log.Println("new tracking:", cols) + t.opMode = LIVE + t.collectors = nil + for _, col := range cols { + t.collectors = append(t.collectors, NewCollector(col, t.pipe, t.config)) + } + *t.current = emptyTracking() + t.current.collectors = cols + for _, e := range t.collectors { + e.Collect() + } + t.pipe.Run() + +} + +func (t *trackingService) StartRecord() { + if t.opMode != LIVE { + log.Println("trackingservice: wrong mode of operation") + } + t.current.TimeCreated = time.Now() + t.pipe.Record() +} + +func (t *trackingService) StopRecord() { + if t.opMode != LIVE { + log.Println("trackingservice: wrong mode of operation") + } + t.pipe.Stop() + for _, e := range t.collectors { + e.Stop() + } + err := t.repo.Save(*t.current) + if err != nil { + log.Println(err) + } + t.NewTracking(t.current.collectors...) +} + +func (t *trackingService) Reset() { + t.opMode = STOPPED + t.collectors = nil +} + + +func (t *trackingService) DeleteTracking(trackingId uuid.UUID) { + panic("implement me") +} + +func (t *trackingService) StartReplay() { + panic("implement me") +} + +func (t *trackingService) PauseReplay() { + panic("implement me") +} + +func (t *trackingService) StopReplay() { + panic("implement me") +} + +func (t *trackingService) LoadTracking(trackingId uuid.UUID) { + +} + + diff --git a/core/trackings.go b/core/trackings.go index 1d7a28b..1864d17 100644 --- a/core/trackings.go +++ b/core/trackings.go @@ -1,106 +1,61 @@ package core import ( - "github.com/google/uuid" - "log" - "time" + "github.com/google/uuid" + "log" + "sync" + "time" ) -type OpMode uint8 - -const ( - STOPPED OpMode = iota - LIVE - REPLAY -) - -type TrackingService struct { - Current *Tracking - config *Configuration - repo *repository - pipe *pipeline - opMode OpMode - collectors []Collector - //publish Publisher -} - type Tracking struct { - trackingMetadata - data []*sensorRecord -} - -type trackingMetadata struct { - UUID uuid.UUID - TimeCreated time.Time -} - -type sensorRecord struct { - RecordTime time.Time - Sensordata -} - -func Service(c *Configuration) *TrackingService { - d := NewDispatcher() - r := &repository{} - return &TrackingService{ - Current: nil, - opMode: STOPPED, - config: c, - repo: r, - pipe: NewPipeline(d, r, c), - collectors: nil, - } -} - -func (t *TrackingService) NewTracking(cols ...CollectorType) { - t.opMode = LIVE - for _, col := range cols { - t.collectors = append(t.collectors, NewCollector(col, t.pipe, t.config)) - } - t.Current = emptyTracking() -} - -func (t *TrackingService) StartRecord() { - if t.opMode != LIVE { - log.Println("trackingservice: wrong mode of operation") - } - t.pipe.Run() - for _, e := range t.collectors { - e.Collect() - } -} - -func (t *TrackingService) StopRecord() { - if t.opMode != LIVE { - log.Println("trackingservice: wrong mode of operation") - } - t.pipe.Stop() - for _, e := range t.collectors { - e.Stop() - } - err := t.repo.SaveTracking(t.Current) - if err != nil { - log.Println(err) - } - t.Current = emptyTracking() -} - -func (t *TrackingService) Reset() { -} - -func (t *TrackingService) LoadTrackings() { + TrackingMetadata + Records []recordPair + Rawdata []rawRecord + mu sync.Mutex } -func (t *TrackingService) LoadTracking(trackingId uuid.UUID) { - +type TrackingMetadata struct { + UUID uuid.UUID + TimeCreated time.Time + collectors []CollectorType } -func emptyTracking() *Tracking { - return &Tracking{ - trackingMetadata: trackingMetadata{ - UUID: uuid.New(), - }, - data: []*sensorRecord{}, - } +func (s *Tracking) EnqueuePair(tcp sensorData, ser sensorData) { + s.mu.Lock() + defer s.mu.Unlock() + rp := recordPair{ + RecordTime: time.Now(), + data: map[sourceId]sensorData{ + tcp.source: tcp, + ser.source: ser, + }, + } + s.Records = append(s.Records, rp) + log.Println("tracking Records: len->", len(s.Records)) +} + +func (s *Tracking) EnqueueRaw(data sensorData) { + s.mu.Lock() + defer s.mu.Unlock() + sr := rawRecord{ + time.Now(), + data, + } + s.Rawdata = append(s.Rawdata, sr) + log.Println("raw data points: len->", len(s.Records)) +} + +func emptyTracking() Tracking { + return Tracking{ + TrackingMetadata: TrackingMetadata{ + UUID: uuid.New(), + }, + Records: []recordPair{}, + Rawdata: []rawRecord{}, + } +} + +func (s *Tracking) isEmpty() bool { + return len(s.Rawdata) + len(s.Records) == 0 } diff --git a/database/repository.go b/database/repository.go new file mode 100644 index 0000000..1f9d413 --- /dev/null +++ b/database/repository.go @@ -0,0 +1,27 @@ +package database + +import ( + "git.timovolkmann.de/gyrogpsc/core" + "github.com/google/uuid" +) + +// Must implement Repo +type repository struct { +} + +func NewRepository(c *core.Configuration) *repository { + return &repository{} +} + +func (r *repository) Save(tracking core.Tracking) error { + panic("implement me") +} + +func (r *repository) LoadAll() ([]core.TrackingMetadata, error) { + panic("implement me") +} + +func (r *repository) Load(id uuid.UUID) (core.Tracking, error) { + panic("implement me") +} + diff --git a/example_config.yml b/example_config.yml index 9cd8ade..5d18e96 100644 --- a/example_config.yml +++ b/example_config.yml @@ -1,12 +1,13 @@ -# Server configurations -server: - port: 8000 +# server configurations +webserver: + port: ":3011" +# collector configurations collectors: - port_tcp: - port_serial: + porttcp: ":3010" + portserial: "/dev/tty.usbmodem14201" -# Database credentials -database: - user: "admin" - pass: "super-pedro-1980" \ No newline at end of file +# processing pipeline configurations +pipeline: + publishIntervalMs: 50 + syncUpdateIntervalMs: 494 \ No newline at end of file diff --git a/go.mod b/go.mod index 1872bc9..a034228 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/gofiber/websocket/v2 v2.0.2 github.com/google/uuid v1.1.2 github.com/gorilla/websocket v1.4.2 + github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.6.1 // indirect github.com/tidwall/gjson v1.6.0 github.com/tidwall/pretty v1.0.2 // indirect diff --git a/go.sum b/go.sum index da3fa22..321eef8 100644 --- a/go.sum +++ b/go.sum @@ -1,48 +1,203 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/creack/goselect v0.1.1 h1:tiSSgKE1eJtxs1h/VgGQWuXUP0YS4CDIFMp6vaI1ls0= github.com/creack/goselect v0.1.1/go.mod h1:a/NhLweNvqIYMuxcMOuWY516Cimucms3DglDzQP3hKY= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.0.0-20200620013148-b91950f658ec/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/fasthttp/websocket v1.4.3 h1:qjhRJ/rTy4KB8oBxljEC00SDt6HUY9jLRfM601SUdS4= github.com/fasthttp/websocket v1.4.3/go.mod h1:5r4oKssgS7W6Zn6mPWap3NWzNPJNzUUh3baWTOhcYQk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofiber/fiber/v2 v2.1.0/go.mod h1:aG+lMkwy3LyVit4CnmYUbUdgjpc3UYOltvlJZ78rgQ0= github.com/gofiber/fiber/v2 v2.2.4 h1:t2V2SxlbQGdt8+SS/Mo+tQB0pDQn7OajKdA72qHcBVw= github.com/gofiber/fiber/v2 v2.2.4/go.mod h1:Aso7/M+EQOinVkWp4LUYjdlTpKTBoCk2Qo4djnMsyHE= github.com/gofiber/websocket/v2 v2.0.2 h1:UA/6NpyG+vmPGlvJvW8MJPJpRFuS7abinZ5HbLuV8u0= github.com/gofiber/websocket/v2 v2.0.2/go.mod h1:7VBnzEVRK0K0eTIVc5GbXPF1JWUFnllY0X4cRtG2v78= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-sqlite3 v1.14.1 h1:AHx9Ra40wIzl+GelgX2X6AWxmT5tfxhI1PL0523HcSw= github.com/mattn/go-sqlite3 v1.14.1/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/savsgio/gotils v0.0.0-20200608150037-a5f6f5aef16c h1:2nF5+FZ4/qp7pZVL7fR6DEaSTzuDmNaFTyqp92/hwF8= github.com/savsgio/gotils v0.0.0-20200608150037-a5f6f5aef16c/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= +github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= @@ -50,6 +205,7 @@ github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/upper/db v1.0.1 h1:AFzy0fFO7EZ5woZGE/LVl9BWLKqyAWlglu+ZyHQTlbU= github.com/upper/db v3.7.1+incompatible h1:ehR+Q0msV25yZWXdFYD+3n4BvY9jefI336lmZy7FlAQ= github.com/upper/db/v4 v4.0.1 h1:mHmGBffne8fdiJjmtYXCuHSghO9027q4TJA9oGP50OM= @@ -62,27 +218,84 @@ github.com/valyala/fasthttp v1.17.0 h1:P8/koH4aSnJ4xbd0cUUFEGQs3jQqIxoDDyRQrUiAk github.com/valyala/fasthttp v1.17.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= gitlab.com/cznic/ebnf2y v1.0.0/go.mod h1:jx14dqOldV2pRvSi8HASTB/k5fkIv2TwjYAp5py0MTs= gitlab.com/cznic/golex v1.0.0/go.mod h1:vkWdDgqbbThjRHoOLU7yNPgMxaubAkwnvF/4zeG8cvU= go.bug.st/serial v1.1.1 h1:5J1DpaIaSIruBi7jVnKXnhRS+YQ9+2PLJMtIZKoIgnc= go.bug.st/serial v1.1.1/go.mod h1:VmYBeyJWp5BnJ0tw2NUJHZdJTGl2ecBGABHlzRK1knY= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190411193353-0480eff6dd7c/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -91,16 +304,72 @@ golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= modernc.org/db v1.0.0/go.mod h1:kYD/cO29L/29RM0hXYl4i3+Q5VojL31kTUVpVJDw0s8= modernc.org/ebnfutil v1.0.0/go.mod h1:+2n/OnQXoild9pzrPa/2wmVtR+ufWjB/0fYkc0BV9sc= @@ -116,3 +385,4 @@ modernc.org/ql v1.1.0/go.mod h1:Fj1ylcVyzcu/fgWZTrvBO9j/aEUg/ixLFnGtmzh7quI= modernc.org/sortutil v1.0.0/go.mod h1:1QO0q8IlIlmjBIwm6t/7sof874+xCfZouyqZMLIAtxM= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/web/http.go b/web/http.go new file mode 100644 index 0000000..3eb6033 --- /dev/null +++ b/web/http.go @@ -0,0 +1,85 @@ +package web + +import ( + "git.timovolkmann.de/gyrogpsc/core" + "github.com/gofiber/fiber/v2" + "github.com/gofiber/websocket/v2" + "html/template" + "log" +) + +func CreateServer(s core.Service, sub core.Subscriber, c *core.Configuration) { + app := fiber.New() + app.Static("/static", "static") + + // Application Main Page + app.Get("/", fiberHomeHandler) + + // Websocket + app.Get("/", websocket.New(createFiberWebsocketHandler(sub))) + + // TODO: Get all SerialPorts + // app.Get("/serialports") + + // Tracking persistence controls + trackings := app.Group("/trackings") + trackings.Get("/", stubhander()) // Get all trackings Metadata + trackings.Post("/", stubhander()) // Initialize new tracking, open websocket and prepare for automatic recording. Toggle ?serial=true and ?tcp=true. Returns trackingId + trackings.Patch("/", stubhander()) // Starts recording + trackings.Put("/", stubhander()) // Stops current recording. Returns trackingId if record was successful + trackings.Delete("/", stubhander()) // Stops websocket connection, pipelines and collectors + + trackings.Get("/:trackingId", stubhander()) // Gets Tracking Metadata and loads sensorRecords from database. + trackings.Delete("/:trackingId", stubhander()) // Deletes Tracking from database + + trackings.Post("/current", stubhander()) // Starts Replay. + trackings.Patch("/current", stubhander()) // Pauses Replay. + trackings.Put("/current", stubhander()) // Stops Replay. + + log.Fatal(app.Listen(c.Webserver.Port)) +} + +func stubhander() fiber.Handler { + return func(ctx *fiber.Ctx) error { + return nil + } +} + +func createFiberWebsocketHandler(s core.Subscriber) func(conn *websocket.Conn) { + return func(c *websocket.Conn) { + // Handle and discard inbound messages + go func() { + for { + if _, _, err := c.NextReader(); err != nil { + c.Close() + break + } + } + }() + + dispatcherId, channel := s.Subscribe() + defer s.Unsubscribe(dispatcherId) + for { + cmsg := <-channel + err := c.WriteMessage(websocket.TextMessage, []byte(cmsg)) + if err != nil { + log.Println("write:", err) + + break + } + } + } + +} + +func fiberHomeHandler(c *fiber.Ctx) error { + tpl, err := template.ParseFiles("static/index.html") + if err != nil { + return err + } + err = tpl.Execute(c, "://"+c.Hostname()+"/") + if err != nil { + return err + } + return nil +} From c3f6a9ff5b7dd918732d7ac3cafdafbf1c1500c2 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Fri, 11 Dec 2020 18:38:39 +0100 Subject: [PATCH 13/20] [WIP] bugfixes --- cmd/autostart/autostart.go | 4 +- core/collectors.go | 15 +- core/pipeline.go | 282 +++++++++++++++++++------------------ go.mod | 1 + go.sum | 1 + 5 files changed, 160 insertions(+), 143 deletions(-) diff --git a/cmd/autostart/autostart.go b/cmd/autostart/autostart.go index ad55990..0f4438a 100644 --- a/cmd/autostart/autostart.go +++ b/cmd/autostart/autostart.go @@ -18,11 +18,11 @@ func main() { service := core.TrackingService(repo, disp, conf) go func() { - service.NewTracking(core.TCP, core.SERIAL) + service.NewTracking(core.TCP) time.Sleep(5 * time.Second) service.StartRecord() time.Sleep(15 * time.Second) - service.StopRecord() + //service.StopRecord() }() web.CreateServer(service, disp, conf) diff --git a/core/collectors.go b/core/collectors.go index d4bc595..fb3ed1a 100644 --- a/core/collectors.go +++ b/core/collectors.go @@ -2,13 +2,11 @@ package core import ( "fmt" + "git.timovolkmann.de/gyrogpsc/ublox" + "go.bug.st/serial" "log" "net" "os" - "time" - - "git.timovolkmann.de/gyrogpsc/ublox" - "go.bug.st/serial" ) type Collector interface { @@ -74,11 +72,14 @@ func (s *serialCollector) Collect() { break } sd, err := ConvertUbxToSensorData(meas) - if err != nil || sd == nil { + if err != nil { log.Println("convert err:", err, meas, sd) continue } - + // skip irrelevant messages + if sd == nil { + continue + } err = s.proc.Push(sd) if err != nil { @@ -172,7 +173,7 @@ func (c *tcpCollector) jsonHandler(conn net.Conn) { continue } if !c.active { - time.Sleep(50 * time.Millisecond) + //time.Sleep(50 * time.Millisecond) continue } err = c.processor.Push(sd) diff --git a/core/pipeline.go b/core/pipeline.go index 23a9a5e..faf2032 100644 --- a/core/pipeline.go +++ b/core/pipeline.go @@ -1,180 +1,194 @@ package core import ( - "encoding/json" - "errors" - "fmt" - "log" - "sync" - "time" + "encoding/json" + "errors" + "fmt" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "log" + "sync" + "time" ) type pipeline struct { - active bool - record bool - syn synchronizer - agr aggregator - pub Publisher - stor Storer - publishTicker *time.Ticker + active bool + record bool + synchroniz synchronizer + buffer pipeBuffer + publisher Publisher + storer Storer + publishTicker *time.Ticker } // pipe implements Runner & Pusher func NewPipeline(d Publisher, s Storer, conf *Configuration) *pipeline { - return &pipeline{ - false, - false, - synchronizer{ - //bufferSize: 100, - mutex: &sync.Mutex{}, - updateTicker: time.NewTicker(time.Duration(conf.Pipeline.SyncUpdateIntervalMs) * time.Millisecond), - }, - aggregator{ - tcpMutex: &sync.Mutex{}, - serialMutex: &sync.Mutex{}, - }, - d, - s, - time.NewTicker(time.Duration(conf.Pipeline.PublishIntervalMs) * time.Millisecond), - } + return &pipeline{ + false, + false, + synchronizer{ + //bufferSize: 100, + mutex: &sync.Mutex{}, + updateTicker: time.NewTicker(time.Duration(conf.Pipeline.SyncUpdateIntervalMs) * time.Millisecond), + }, + pipeBuffer{ + tcpMutex: &sync.Mutex{}, + serialMutex: &sync.Mutex{}, + }, + d, + s, + time.NewTicker(time.Duration(conf.Pipeline.PublishIntervalMs) * time.Millisecond), + } } func (p *pipeline) Run() { - p.active = true - log.Println("pipe: processing service started") - go func() { - for p.active { - <-p.syn.updateTicker.C - err := p.refreshDelay() - if err != nil { - log.Println(err) - } - } - log.Println("pipe: updater stopped") - }() - go func() { - for p.active { - <-p.publishTicker.C - err := p.Publish() - if err != nil && err.Error() != "no data available" { - log.Println(err) - } - } - log.Println("pipe: publisher stopped") - }() + p.active = true + log.Println("pipe: processing service started") + go func() { + for p.active { + <-p.synchroniz.updateTicker.C + err := p.refreshDelay() + if err != nil { + log.Println(err) + } + } + log.Println("pipe: updater stopped") + }() + go func() { + for p.active { + <-p.publishTicker.C + err := p.publish() + if err != nil && err.Error() != "no data available" { + log.Println(err) + } + } + log.Println("pipe: publisher stopped") + }() } func (p *pipeline) Record() { - p.record = true + p.record = true } func (p *pipeline) Stop() { - p.record = false + p.record = false } -func (p *pipeline) Publish() error { - p.agr.tcpMutex.Lock() - p.agr.serialMutex.Lock() +func (p *pipeline) publish() error { + p.buffer.tcpMutex.Lock() + p.buffer.serialMutex.Lock() - if (p.agr.tcpSensorData == sensorData{} && p.agr.serialSensorData == sensorData{}) { - p.agr.tcpMutex.Unlock() - p.agr.serialMutex.Unlock() - return errors.New("no data available") - } + if (p.buffer.MeasTcp == sensorData{} && p.buffer.MeasSerial == sensorData{}) { + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() + return errors.New("no data available") + } + if cmp.Equal(p.buffer.MeasTcp, p.buffer.LastMeasTcp, cmpopts.IgnoreUnexported(sensorData{})) && + cmp.Equal(p.buffer.MeasSerial, p.buffer.LastMeasSerial, cmpopts.IgnoreUnexported(sensorData{})) { + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() + return errors.New("same data") + } + log.Println("––––––––––––––––––––––––––––––––––––") + log.Printf("MEAS old: %v", p.buffer.LastMeasTcp) + log.Printf("MEAS new: %v", p.buffer.MeasTcp) + log.Println("––––––––––––––––––––––––––––––––––––") + p.buffer.LastMeasTcp = p.buffer.MeasTcp + p.buffer.LastMeasSerial = p.buffer.MeasSerial + p.storer.EnqueuePair(p.buffer.MeasTcp, p.buffer.MeasSerial) - p.stor.EnqueuePair(p.agr.tcpSensorData, p.agr.serialSensorData) + data := map[string]sensorData{ + string(SOURCE_TCP): p.buffer.MeasTcp, + string(SOURCE_SERIAL): p.buffer.MeasSerial, + } - data := map[string]sensorData{ - string(SOURCE_TCP): p.agr.tcpSensorData, - string(SOURCE_SERIAL): p.agr.serialSensorData, - } + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() - p.agr.tcpMutex.Unlock() - p.agr.serialMutex.Unlock() - - jdata, err := json.Marshal(data) - //log.Println(string(pretty.Pretty(jdata))) - if err != nil { - return err - } - p.pub.Publish(string(jdata)) - return nil + jdata, err := json.Marshal(data) + //log.Println(string(pretty.Pretty(jdata))) + if err != nil { + return err + } + p.publisher.Publish(string(jdata)) + return nil } -type aggregator struct { - tcpSensorData sensorData - serialSensorData sensorData - tcpMutex *sync.Mutex - serialMutex *sync.Mutex +type pipeBuffer struct { + MeasTcp sensorData + MeasSerial sensorData + LastMeasTcp sensorData + LastMeasSerial sensorData + tcpMutex *sync.Mutex + serialMutex *sync.Mutex } type UnixNanoTime int64 type synchronizer struct { - tcpSerialDelayMs int64 - mutex *sync.Mutex - updateTicker *time.Ticker + tcpSerialDelayMs int64 + mutex *sync.Mutex + updateTicker *time.Ticker } func (p *pipeline) refreshDelay() error { - log.Println("refreshing delay....") - fmt.Println("Delay TCP/SERIAL", p.syn.tcpSerialDelayMs) - p.agr.serialMutex.Lock() - p.agr.tcpMutex.Lock() - tcpTime := time.Unix(0, p.agr.tcpSensorData.Timestamp) - serTime := time.Unix(0, p.agr.serialSensorData.Timestamp) - p.agr.serialMutex.Unlock() - p.agr.tcpMutex.Unlock() - if tcpTime.UnixNano() == 0 || serTime.UnixNano() == 0 { - return errors.New("no sync possible. check if both collectors running. otherwise check GPS fix") - } - currentDelay := tcpTime.Sub(serTime).Milliseconds() - if currentDelay > 5000 || currentDelay < -5000 { - p.syn.tcpSerialDelayMs = 0 - return errors.New("skipping synchronisation! time not properly configured or facing network problems.") - } - log.Println("TCP", tcpTime.String()) - log.Println("SER", serTime.String()) - log.Println("Difference", tcpTime.Sub(serTime).Milliseconds(), "ms") - delay := tcpTime.Sub(serTime).Milliseconds() - p.syn.tcpSerialDelayMs += delay - return nil + log.Println("refreshing delay....") + fmt.Println("Delay TCP/SERIAL", p.synchroniz.tcpSerialDelayMs) + p.buffer.serialMutex.Lock() + p.buffer.tcpMutex.Lock() + tcpTime := time.Unix(0, p.buffer.MeasTcp.Timestamp) + serTime := time.Unix(0, p.buffer.MeasSerial.Timestamp) + p.buffer.serialMutex.Unlock() + p.buffer.tcpMutex.Unlock() + if tcpTime.UnixNano() == 0 || serTime.UnixNano() == 0 { + return errors.New("no sync possible. check if both collectors running. otherwise check GPS fix") + } + currentDelay := tcpTime.Sub(serTime).Milliseconds() + if currentDelay > 5000 || currentDelay < -5000 { + p.synchroniz.tcpSerialDelayMs = 0 + return errors.New("skipping synchronisation! time not properly configured or facing network problems.") + } + log.Println("TCP", tcpTime.String()) + log.Println("SER", serTime.String()) + log.Println("Difference", tcpTime.Sub(serTime).Milliseconds(), "ms") + delay := tcpTime.Sub(serTime).Milliseconds() + p.synchroniz.tcpSerialDelayMs += delay + return nil } func (p *pipeline) Push(data *sensorData) error { - if data == nil { - return errors.New("nil processing not allowed") - } - //log.Println(string(data.source)) - // TODO: persist data here with current timestamp - p.stor.EnqueueRaw(*data) - switch data.source { - case SOURCE_TCP: - go p.pushTcpDataToBuffer(*data) - case SOURCE_SERIAL: - go p.pushSerialDataToBuffer(*data) - default: - panic("pipe: invalid data source") - } - return nil + if data == nil { + return errors.New("nil processing not allowed") + } + //log.Println("push data to pipe:", string(data.source)) + p.storer.EnqueueRaw(*data) + switch data.source { + case SOURCE_TCP: + go p.pushTcpDataToBuffer(*data) + case SOURCE_SERIAL: + go p.pushSerialDataToBuffer(*data) + default: + panic("pipe: invalid data source") + } + return nil } func (p *pipeline) pushTcpDataToBuffer(data sensorData) { - if p.syn.tcpSerialDelayMs > 0 { - time.Sleep(time.Duration(p.syn.tcpSerialDelayMs) * time.Millisecond) - } - p.agr.tcpMutex.Lock() - p.agr.tcpSensorData = p.agr.tcpSensorData.ConsolidateExTime(data) - p.agr.tcpMutex.Unlock() + if p.synchroniz.tcpSerialDelayMs > 0 { + time.Sleep(time.Duration(p.synchroniz.tcpSerialDelayMs) * time.Millisecond) + } + p.buffer.tcpMutex.Lock() + p.buffer.MeasTcp = p.buffer.MeasTcp.ConsolidateExTime(data) + p.buffer.tcpMutex.Unlock() } func (p *pipeline) pushSerialDataToBuffer(data sensorData) { - if p.syn.tcpSerialDelayMs < 0 { - time.Sleep(time.Duration(-p.syn.tcpSerialDelayMs) * time.Millisecond) - } - p.agr.serialMutex.Lock() - p.agr.serialSensorData = p.agr.serialSensorData.ConsolidateEpochsOnly(data) - p.agr.serialMutex.Unlock() + if p.synchroniz.tcpSerialDelayMs < 0 { + time.Sleep(time.Duration(-p.synchroniz.tcpSerialDelayMs) * time.Millisecond) + } + p.buffer.serialMutex.Lock() + p.buffer.MeasSerial = p.buffer.MeasSerial.ConsolidateEpochsOnly(data) + p.buffer.serialMutex.Unlock() } func (p *pipeline) Close() { - p.active = false + p.active = false } diff --git a/go.mod b/go.mod index a034228..201ab81 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.15 require ( github.com/gofiber/fiber/v2 v2.2.4 github.com/gofiber/websocket/v2 v2.0.2 + github.com/google/go-cmp v0.3.0 github.com/google/uuid v1.1.2 github.com/gorilla/websocket v1.4.2 github.com/spf13/viper v1.7.1 diff --git a/go.sum b/go.sum index 321eef8..de176c1 100644 --- a/go.sum +++ b/go.sum @@ -74,6 +74,7 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= From a4a739c64b1e8dfa97b6122e3997b463b43fc8ae Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Fri, 11 Dec 2020 20:42:19 +0100 Subject: [PATCH 14/20] [WIP] started with db implementation --- cmd/autostart/autostart.go | 40 +++-- cmd/server/server.go | 10 +- core/collectors.go | 24 +-- core/config.go | 19 +-- core/dispatcher.go | 5 +- core/pipeline.go | 302 +++++++++++++++++++------------------ core/sensordata.go | 41 ++--- core/service.go | 117 +++++++------- core/trackings.go | 77 +++++----- database/repository.go | 27 ---- example_config.yml | 21 ++- go.mod | 4 +- go.sum | 85 ++++++----- storage/kvstore.go | 33 ++++ web/http.go | 16 +- 15 files changed, 434 insertions(+), 387 deletions(-) delete mode 100644 database/repository.go create mode 100644 storage/kvstore.go diff --git a/cmd/autostart/autostart.go b/cmd/autostart/autostart.go index 0f4438a..a0bafeb 100644 --- a/cmd/autostart/autostart.go +++ b/cmd/autostart/autostart.go @@ -2,54 +2,64 @@ package main import ( "git.timovolkmann.de/gyrogpsc/core" - "git.timovolkmann.de/gyrogpsc/database" + "git.timovolkmann.de/gyrogpsc/storage" "git.timovolkmann.de/gyrogpsc/web" + "github.com/sirupsen/logrus" "github.com/spf13/viper" - "log" "time" ) func main() { conf := configurationFromFile() - repo := database.NewRepository(conf) + repo := storage.NewRepository(conf) disp := core.NewDispatcher() service := core.TrackingService(repo, disp, conf) go func() { - service.NewTracking(core.TCP) + service.NewTracking(core.TCP, core.SERIAL) time.Sleep(5 * time.Second) service.StartRecord() - time.Sleep(15 * time.Second) - //service.StopRecord() + time.Sleep(20 * time.Second) + service.StopRecord() + time.Sleep(5 * time.Second) + service.NewTracking(core.TCP) + service.NewTracking(core.SERIAL) + time.Sleep(5 * time.Second) + service.StartRecord() + time.Sleep(20 * time.Second) + service.StopRecord() }() web.CreateServer(service, disp, conf) } func configurationFromFile() *core.Configuration { - viper.SetDefault("collectors.port_tcp", ":3010") - viper.SetDefault("collectors.port_serial", "/dev/tty.usbmodem14201") + viper.SetDefault("collectors.porttcp", ":3010") + viper.SetDefault("collectors.portserial", "/dev/tty.usbmodem14201") viper.SetDefault("webserver.port", ":3011") viper.SetDefault("pipeline.publishIntervalMs", 50) viper.SetDefault("pipeline.syncUpdateIntervalMs", 494) + viper.SetDefault("debuglevel", "INFO") viper.SetConfigName("gpsconfig") // name of config file (without extension) viper.SetConfigType("yaml") viper.AddConfigPath(".") viper.AddConfigPath("./../../") if err := viper.ReadInConfig(); err != nil { - log.Println("couldn't find config file. using standard configuration") + logrus.Warn("couldn't find config file. using standard configuration") } - log.Println(viper.AllKeys()) - log.Println(viper.AllSettings()) c := core.Configuration{} if err := viper.Unmarshal(&c); err != nil { - log.Println("couldn't load config...") - log.Println(err) + logrus.Debug("couldn't load config...") + logrus.Error(err) } - log.Println(c) + lvl, err := logrus.ParseLevel(c.Debuglevel) + if err != nil { + logrus.Error(err) + } + logrus.SetLevel(lvl) return &c -} \ No newline at end of file +} diff --git a/cmd/server/server.go b/cmd/server/server.go index a5a9ef1..33d557d 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -2,17 +2,17 @@ package main import ( "git.timovolkmann.de/gyrogpsc/core" - "git.timovolkmann.de/gyrogpsc/database" + "git.timovolkmann.de/gyrogpsc/storage" "git.timovolkmann.de/gyrogpsc/web" + "github.com/sirupsen/logrus" "github.com/spf13/viper" - "log" ) func main() { conf := &core.Configuration{} configurationFromFile(conf) - repo := database.NewRepository(conf) + repo := storage.NewRepository(conf) disp := core.NewDispatcher() service := core.TrackingService(repo, disp, conf) @@ -33,5 +33,5 @@ func configurationFromFile(c *core.Configuration) { viper.AddConfigPath("./../../") viper.Unmarshal(c) - log.Println(c) -} \ No newline at end of file + logrus.Println(c) +} diff --git a/core/collectors.go b/core/collectors.go index fb3ed1a..27339ea 100644 --- a/core/collectors.go +++ b/core/collectors.go @@ -3,8 +3,8 @@ package core import ( "fmt" "git.timovolkmann.de/gyrogpsc/ublox" + "github.com/sirupsen/logrus" "go.bug.st/serial" - "log" "net" "os" ) @@ -50,13 +50,13 @@ type serialCollector struct { func (s *serialCollector) Collect() { s.active = true go func() { - log.Println("start serial collector") + logrus.Println("start serial collector") mode := &serial.Mode{ BaudRate: 115200, } port, err := serial.Open(s.config.Collectors.SerialCollectorPort, mode) if err != nil { - log.Fatalln(err.Error()) + logrus.Fatalln(err.Error()) } defer port.Close() @@ -68,12 +68,12 @@ func (s *serialCollector) Collect() { if err.Error() == "NMEA not implemented" { continue } - log.Println("serial read err:", err) + logrus.Println("serial read err:", err) break } sd, err := ConvertUbxToSensorData(meas) if err != nil { - log.Println("convert err:", err, meas, sd) + logrus.Println("convert err:", err, meas, sd) continue } // skip irrelevant messages @@ -83,11 +83,11 @@ func (s *serialCollector) Collect() { err = s.proc.Push(sd) if err != nil { - log.Println("process err:", err, *sd) + logrus.Println("process err:", err, *sd) continue } } - log.Println("serial collector stopped") + logrus.Println("serial collector stopped") }() } @@ -122,7 +122,7 @@ func (t *tcpCollector) SetProcessor(p Pusher) { } func newTcp(proc Pusher, config *Configuration) *tcpCollector { - log.Println("start tcp collector") + logrus.Println("start tcp collector") listener, err := net.Listen("tcp", config.Collectors.TcpCollectorPort) if err != nil { @@ -141,7 +141,7 @@ func newTcp(proc Pusher, config *Configuration) *tcpCollector { fmt.Println("Error accepting: ", err.Error()) os.Exit(1) } - log.Println("...new incoming tcp connection...") + logrus.Println("...new incoming tcp connection...") // Handle connections in a new goroutine. go coll.jsonHandler(conn) @@ -169,7 +169,7 @@ func (c *tcpCollector) jsonHandler(conn net.Conn) { //fmt.Println(string(buf[:n])) sd, err := ConvertSensorDataPhone(buf[:n]) if err != nil { - log.Println(err) + logrus.Println(err) continue } if !c.active { @@ -178,7 +178,7 @@ func (c *tcpCollector) jsonHandler(conn net.Conn) { } err = c.processor.Push(sd) if err != nil { - log.Fatalln(err) + logrus.Fatalln(err) } } -} \ No newline at end of file +} diff --git a/core/config.go b/core/config.go index dad3510..7d6255d 100644 --- a/core/config.go +++ b/core/config.go @@ -1,23 +1,16 @@ package core type Configuration struct { - Collectors struct{ - TcpCollectorPort string `mapstructure:"porttcp"` - SerialCollectorPort string `mapstructure:"portserial"` + Collectors struct { + TcpCollectorPort string `mapstructure:"porttcp"` + SerialCollectorPort string `mapstructure:"portserial"` } `mapstructure:"collectors"` Webserver struct { Port string `mapstructure:"port"` } `mapstructure:"webserver"` Pipeline struct { - PublishIntervalMs int `mapstructure:"publishintervalms"` - SyncUpdateIntervalMs int `mapstructure:"syncupdateintervalms"` + PublishIntervalMs int `mapstructure:"publishintervalms"` + SyncUpdateIntervalMs int `mapstructure:"syncupdateintervalms"` } `mapstructure:"pipeline"` -} - -type Configuration2 struct { - TcpCollectorPort string `mapstructure:"collectors.porttcp"` - SerialCollectorPort string `mapstructure:"collectors.portserial"` - HttpPort string `mapstructure:"webserver.port"` - PublishIntervalMs int `mapstructure:"pipeline.publishintervalms"` - SyncUpdateIntervalMs int `mapstructure:"pipeline.syncupdateintervalms"` + Debuglevel string `mapstructure:"debuglevel"` } diff --git a/core/dispatcher.go b/core/dispatcher.go index f2c7684..94d2197 100644 --- a/core/dispatcher.go +++ b/core/dispatcher.go @@ -2,7 +2,7 @@ package core import ( "errors" - "log" + "github.com/sirupsen/logrus" ) type dispatcher struct { @@ -17,9 +17,8 @@ func NewDispatcher() *dispatcher { } } - func (d *dispatcher) Publish(message string) { - log.Printf("publish to %v listeners:\n%v\n", len(d.listeners), message) + logrus.Debugf("publish to %v listeners:\n%v\n", len(d.listeners), message) for _, ch := range d.listeners { ch <- message } diff --git a/core/pipeline.go b/core/pipeline.go index faf2032..e77bbe3 100644 --- a/core/pipeline.go +++ b/core/pipeline.go @@ -1,194 +1,204 @@ package core import ( - "encoding/json" - "errors" - "fmt" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "log" - "sync" - "time" + "encoding/json" + "errors" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/sirupsen/logrus" + "sync" + "time" ) type pipeline struct { - active bool - record bool - synchroniz synchronizer - buffer pipeBuffer - publisher Publisher - storer Storer - publishTicker *time.Ticker + active bool + record bool + synchroniz synchronizer + buffer pipeBuffer + publisher Publisher + storer Storer + publishTicker *time.Ticker } // pipe implements Runner & Pusher func NewPipeline(d Publisher, s Storer, conf *Configuration) *pipeline { - return &pipeline{ - false, - false, - synchronizer{ - //bufferSize: 100, - mutex: &sync.Mutex{}, - updateTicker: time.NewTicker(time.Duration(conf.Pipeline.SyncUpdateIntervalMs) * time.Millisecond), - }, - pipeBuffer{ - tcpMutex: &sync.Mutex{}, - serialMutex: &sync.Mutex{}, - }, - d, - s, - time.NewTicker(time.Duration(conf.Pipeline.PublishIntervalMs) * time.Millisecond), - } + return &pipeline{ + false, + false, + synchronizer{ + //bufferSize: 100, + mutex: &sync.RWMutex{}, + updateTicker: time.NewTicker(time.Duration(conf.Pipeline.SyncUpdateIntervalMs) * time.Millisecond), + }, + pipeBuffer{ + tcpMutex: &sync.Mutex{}, + serialMutex: &sync.Mutex{}, + }, + d, + s, + time.NewTicker(time.Duration(conf.Pipeline.PublishIntervalMs) * time.Millisecond), + } } func (p *pipeline) Run() { - p.active = true - log.Println("pipe: processing service started") - go func() { - for p.active { - <-p.synchroniz.updateTicker.C - err := p.refreshDelay() - if err != nil { - log.Println(err) - } - } - log.Println("pipe: updater stopped") - }() - go func() { - for p.active { - <-p.publishTicker.C - err := p.publish() - if err != nil && err.Error() != "no data available" { - log.Println(err) - } - } - log.Println("pipe: publisher stopped") - }() + p.active = true + logrus.Println("pipe: processing service started") + go func() { + for p.active { + <-p.synchroniz.updateTicker.C + err := p.refreshDelay() + if err != nil { + logrus.Debugln(err) + } + } + logrus.Println("pipe: updater stopped") + }() + go func() { + for p.active { + <-p.publishTicker.C + err := p.publish() + if err != nil && err.Error() != "no data available" { + logrus.Debug(err) + } + } + logrus.Println("pipe: publisher stopped") + }() } func (p *pipeline) Record() { - p.record = true + p.record = true } func (p *pipeline) Stop() { - p.record = false + p.record = false } func (p *pipeline) publish() error { - p.buffer.tcpMutex.Lock() - p.buffer.serialMutex.Lock() + p.buffer.tcpMutex.Lock() + p.buffer.serialMutex.Lock() - if (p.buffer.MeasTcp == sensorData{} && p.buffer.MeasSerial == sensorData{}) { - p.buffer.tcpMutex.Unlock() - p.buffer.serialMutex.Unlock() - return errors.New("no data available") - } - if cmp.Equal(p.buffer.MeasTcp, p.buffer.LastMeasTcp, cmpopts.IgnoreUnexported(sensorData{})) && - cmp.Equal(p.buffer.MeasSerial, p.buffer.LastMeasSerial, cmpopts.IgnoreUnexported(sensorData{})) { - p.buffer.tcpMutex.Unlock() - p.buffer.serialMutex.Unlock() - return errors.New("same data") - } - log.Println("––––––––––––––––––––––––––––––––––––") - log.Printf("MEAS old: %v", p.buffer.LastMeasTcp) - log.Printf("MEAS new: %v", p.buffer.MeasTcp) - log.Println("––––––––––––––––––––––––––––––––––––") - p.buffer.LastMeasTcp = p.buffer.MeasTcp - p.buffer.LastMeasSerial = p.buffer.MeasSerial - p.storer.EnqueuePair(p.buffer.MeasTcp, p.buffer.MeasSerial) + if (p.buffer.MeasTcp == sensorData{} && p.buffer.MeasSerial == sensorData{}) { + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() + return errors.New("no data available") + } + if cmp.Equal(p.buffer.MeasTcp, p.buffer.LastMeasTcp, cmpopts.IgnoreUnexported(sensorData{})) && + cmp.Equal(p.buffer.MeasSerial, p.buffer.LastMeasSerial, cmpopts.IgnoreUnexported(sensorData{})) { + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() + return errors.New("same data") + } + logrus.Debug("––––––––––––––––––––––––––––––––––––") + logrus.Debugf("MEAS old: %v", p.buffer.LastMeasTcp) + logrus.Debugf("MEAS new: %v", p.buffer.MeasTcp) + logrus.Debug("––––––––––––––––––––––––––––––––––––") + p.buffer.LastMeasTcp = p.buffer.MeasTcp + p.buffer.LastMeasSerial = p.buffer.MeasSerial + p.storer.EnqueuePair(p.buffer.MeasTcp, p.buffer.MeasSerial) - data := map[string]sensorData{ - string(SOURCE_TCP): p.buffer.MeasTcp, - string(SOURCE_SERIAL): p.buffer.MeasSerial, - } + data := map[string]sensorData{ + string(SOURCE_TCP): p.buffer.MeasTcp, + string(SOURCE_SERIAL): p.buffer.MeasSerial, + } - p.buffer.tcpMutex.Unlock() - p.buffer.serialMutex.Unlock() + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() - jdata, err := json.Marshal(data) - //log.Println(string(pretty.Pretty(jdata))) - if err != nil { - return err - } - p.publisher.Publish(string(jdata)) - return nil + jdata, err := json.Marshal(data) + //logrus.Println(string(pretty.Pretty(jdata))) + if err != nil { + return err + } + p.publisher.Publish(string(jdata)) + return nil } type pipeBuffer struct { - MeasTcp sensorData - MeasSerial sensorData - LastMeasTcp sensorData - LastMeasSerial sensorData - tcpMutex *sync.Mutex - serialMutex *sync.Mutex + MeasTcp sensorData + MeasSerial sensorData + LastMeasTcp sensorData + LastMeasSerial sensorData + tcpMutex *sync.Mutex + serialMutex *sync.Mutex } type UnixNanoTime int64 type synchronizer struct { - tcpSerialDelayMs int64 - mutex *sync.Mutex - updateTicker *time.Ticker + tcpSerialDelayMs int64 + mutex *sync.RWMutex + updateTicker *time.Ticker } func (p *pipeline) refreshDelay() error { - log.Println("refreshing delay....") - fmt.Println("Delay TCP/SERIAL", p.synchroniz.tcpSerialDelayMs) - p.buffer.serialMutex.Lock() - p.buffer.tcpMutex.Lock() - tcpTime := time.Unix(0, p.buffer.MeasTcp.Timestamp) - serTime := time.Unix(0, p.buffer.MeasSerial.Timestamp) - p.buffer.serialMutex.Unlock() - p.buffer.tcpMutex.Unlock() - if tcpTime.UnixNano() == 0 || serTime.UnixNano() == 0 { - return errors.New("no sync possible. check if both collectors running. otherwise check GPS fix") - } - currentDelay := tcpTime.Sub(serTime).Milliseconds() - if currentDelay > 5000 || currentDelay < -5000 { - p.synchroniz.tcpSerialDelayMs = 0 - return errors.New("skipping synchronisation! time not properly configured or facing network problems.") - } - log.Println("TCP", tcpTime.String()) - log.Println("SER", serTime.String()) - log.Println("Difference", tcpTime.Sub(serTime).Milliseconds(), "ms") - delay := tcpTime.Sub(serTime).Milliseconds() - p.synchroniz.tcpSerialDelayMs += delay - return nil + p.synchroniz.mutex.RLock() + if p.synchroniz.tcpSerialDelayMs != 0 { + logrus.Println("Delay TCP/SERIAL", p.synchroniz.tcpSerialDelayMs) + } + p.synchroniz.mutex.RUnlock() + p.buffer.serialMutex.Lock() + p.buffer.tcpMutex.Lock() + tcpTime := time.Unix(0, p.buffer.MeasTcp.Timestamp) + serTime := time.Unix(0, p.buffer.MeasSerial.Timestamp) + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() + if tcpTime.UnixNano() == 0 || serTime.UnixNano() == 0 { + return errors.New("no sync possible. check if both collectors running. otherwise check GPS fix") + } + currentDelay := tcpTime.Sub(serTime).Milliseconds() + if currentDelay > 5000 || currentDelay < -5000 { + p.synchroniz.mutex.Lock() + p.synchroniz.tcpSerialDelayMs = 0 + p.synchroniz.mutex.Unlock() + return errors.New("skipping synchronisation! time not properly configured or facing network problems.") + } + logrus.Debug("TCP", tcpTime.String()) + logrus.Debug("SER", serTime.String()) + logrus.Debug("Difference", tcpTime.Sub(serTime).Milliseconds(), "ms") + delay := tcpTime.Sub(serTime).Milliseconds() + p.synchroniz.mutex.Lock() + p.synchroniz.tcpSerialDelayMs += delay + p.synchroniz.mutex.Unlock() + return nil } func (p *pipeline) Push(data *sensorData) error { - if data == nil { - return errors.New("nil processing not allowed") - } - //log.Println("push data to pipe:", string(data.source)) - p.storer.EnqueueRaw(*data) - switch data.source { - case SOURCE_TCP: - go p.pushTcpDataToBuffer(*data) - case SOURCE_SERIAL: - go p.pushSerialDataToBuffer(*data) - default: - panic("pipe: invalid data source") - } - return nil + if data == nil { + return errors.New("nil processing not allowed") + } + //logrus.Println("push data to pipe:", string(data.source)) + p.storer.EnqueueRaw(*data) + switch data.source { + case SOURCE_TCP: + go p.pushTcpDataToBuffer(*data) + case SOURCE_SERIAL: + go p.pushSerialDataToBuffer(*data) + default: + panic("pipe: invalid data source") + } + return nil } func (p *pipeline) pushTcpDataToBuffer(data sensorData) { - if p.synchroniz.tcpSerialDelayMs > 0 { - time.Sleep(time.Duration(p.synchroniz.tcpSerialDelayMs) * time.Millisecond) - } - p.buffer.tcpMutex.Lock() - p.buffer.MeasTcp = p.buffer.MeasTcp.ConsolidateExTime(data) - p.buffer.tcpMutex.Unlock() + p.synchroniz.mutex.RLock() + if p.synchroniz.tcpSerialDelayMs > 0 { + time.Sleep(time.Duration(p.synchroniz.tcpSerialDelayMs) * time.Millisecond) + } + p.synchroniz.mutex.RLock() + p.buffer.tcpMutex.Lock() + p.buffer.MeasTcp = p.buffer.MeasTcp.ConsolidateExTime(data) + p.buffer.tcpMutex.Unlock() } func (p *pipeline) pushSerialDataToBuffer(data sensorData) { - if p.synchroniz.tcpSerialDelayMs < 0 { - time.Sleep(time.Duration(-p.synchroniz.tcpSerialDelayMs) * time.Millisecond) - } - p.buffer.serialMutex.Lock() - p.buffer.MeasSerial = p.buffer.MeasSerial.ConsolidateEpochsOnly(data) - p.buffer.serialMutex.Unlock() + p.synchroniz.mutex.RLock() + if p.synchroniz.tcpSerialDelayMs < 0 { + time.Sleep(time.Duration(-p.synchroniz.tcpSerialDelayMs) * time.Millisecond) + } + p.synchroniz.mutex.RUnlock() + p.buffer.serialMutex.Lock() + p.buffer.MeasSerial = p.buffer.MeasSerial.ConsolidateEpochsOnly(data) + p.buffer.serialMutex.Unlock() } func (p *pipeline) Close() { - p.active = false + p.active = false } diff --git a/core/sensordata.go b/core/sensordata.go index f13dc31..e4d8ebf 100644 --- a/core/sensordata.go +++ b/core/sensordata.go @@ -3,8 +3,8 @@ package core import ( "errors" "git.timovolkmann.de/gyrogpsc/ublox" + "github.com/sirupsen/logrus" "github.com/tidwall/gjson" - "log" "math" "time" ) @@ -19,6 +19,7 @@ const ( type sensorData struct { itow uint32 source sourceId + ServerTime time.Time Timestamp int64 Position [3]float64 Orientation [3]float64 @@ -26,7 +27,7 @@ type sensorData struct { type recordPair struct { RecordTime time.Time - data map[sourceId]sensorData + data map[sourceId]sensorData } type rawRecord struct { @@ -34,7 +35,6 @@ type rawRecord struct { sensorData } - func (s sensorData) isSameEpoch(n sensorData) bool { if n.itow == 0 { return false @@ -77,9 +77,9 @@ func (s sensorData) ConsolidateExTime(n sensorData) sensorData { func (s *sensorData) checkSources(n *sensorData) { if (s.source != n.source && *s != sensorData{}) { - log.Println(s) - log.Println(n) - log.Fatalln("Do not consolidate sensorData from different Sources") + logrus.Println(s) + logrus.Println(n) + logrus.Fatalln("Do not consolidate sensorData from different Sources") } } @@ -90,25 +90,26 @@ var ( func ConvertUbxToSensorData(msg interface{}) (*sensorData, error) { sd := &sensorData{ - source: SOURCE_SERIAL, + ServerTime: time.Now(), + source: SOURCE_SERIAL, } switch v := msg.(type) { case *ublox.NavPvt: - //log.Println("NAV-PVT") + //logrus.Println("NAV-PVT") sd.itow = v.ITOW_ms sd.Timestamp = time.Date(int(v.Year_y), time.Month(v.Month_month), int(v.Day_d), int(v.Hour_h), int(v.Min_min), int(v.Sec_s), int(v.Nano_ns), time.UTC).UnixNano() sd.Position[0] = float64(v.Lat_dege7) / 1e+7 sd.Position[1] = float64(v.Lon_dege7) / 1e+7 sd.Position[2] = float64(v.HMSL_mm) / 1e+3 // mm in m case *ublox.HnrPvt: - //log.Println("HNR-PVT") + //logrus.Println("HNR-PVT") sd.itow = v.ITOW_ms sd.Timestamp = time.Date(int(v.Year_y), time.Month(v.Month_month), int(v.Day_d), int(v.Hour_h), int(v.Min_min), int(v.Sec_s), int(v.Nano_ns), time.UTC).UnixNano() sd.Position[0] = float64(v.Lat_dege7) / 1e+7 sd.Position[1] = float64(v.Lon_dege7) / 1e+7 sd.Position[2] = float64(v.HMSL_mm) / 1e+3 // mm in m case *ublox.NavAtt: - //log.Println("NAV-ATT") + //logrus.Println("NAV-ATT") sd.itow = v.ITOW_ms sd.Orientation[0] = float64(v.Pitch_deg) * 1e-5 sd.Orientation[1] = float64(v.Roll_deg) * 1e-5 @@ -116,7 +117,7 @@ func ConvertUbxToSensorData(msg interface{}) (*sensorData, error) { case *ublox.RawMessage: //class := make([]byte, 2) //binary.LittleEndian.PutUint16(class, v.ClassID()) - //log.Printf("%#v, %#v", class[0],class[1]) + //logrus.Printf("%#v, %#v", class[0],class[1]) return nil, nil default: return nil, errNotImplemented @@ -140,14 +141,15 @@ func convertIPhoneSensorLog(jsonData []byte) (*sensorData, error) { roll := gjson.Get(string(jsonData), "motionRoll").Float() * 180 / math.Pi yaw := gjson.Get(string(jsonData), "motionYaw").Float() * 180 / math.Pi sd := &sensorData{ - source: SOURCE_TCP, - Timestamp: int64(timestamp * float64(time.Second)), - //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), + ServerTime: time.Now(), + source: SOURCE_TCP, + Timestamp: int64(timestamp * float64(time.Second)), Position: [3]float64{lat, lon, alt}, Orientation: [3]float64{pitch, roll, yaw}, + //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), } - //log.Println(string(pretty.Pretty(jsonData))) - //log.Println(sd) + //logrus.Println(string(pretty.Pretty(jsonData))) + //logrus.Println(sd) return sd, nil } @@ -161,11 +163,12 @@ func convertAndroidHyperImu(jsonData []byte) (*sensorData, error) { yaw := gjson.Get(string(jsonData), "orientation.2").Float() sd := &sensorData{ - source: SOURCE_TCP, - Timestamp: timestamp * int64(time.Millisecond), - //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), + ServerTime: time.Now(), + source: SOURCE_TCP, + Timestamp: timestamp * int64(time.Millisecond), Position: [3]float64{lat, lon, alt}, Orientation: [3]float64{pitch, roll, yaw}, + //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), } return sd, nil } diff --git a/core/service.go b/core/service.go index 8df6a23..cbb6bb1 100644 --- a/core/service.go +++ b/core/service.go @@ -1,107 +1,104 @@ package core import ( - "github.com/google/uuid" - "log" - "time" + "github.com/google/uuid" + "github.com/sirupsen/logrus" + "time" ) type OpMode uint8 const ( - STOPPED OpMode = iota - LIVE - REPLAY + STOPPED OpMode = iota + LIVE + REPLAY ) type trackingService struct { - current *Tracking - config *Configuration - pipe *pipeline - repo Repo - opMode OpMode - collectors []Collector + current *Tracking + config *Configuration + pipe *pipeline + repo Repo + opMode OpMode + collectors []Collector } func TrackingService(r Repo, d Publisher, c *Configuration) *trackingService { - t := &Tracking{} - return &trackingService{ - current: t, - opMode: STOPPED, - config: c, - repo: r, - pipe: NewPipeline(d, t, c), - collectors: nil, - } + t := &Tracking{} + return &trackingService{ + current: t, + opMode: STOPPED, + config: c, + repo: r, + pipe: NewPipeline(d, t, c), + collectors: nil, + } } func (t *trackingService) AllTrackings() { - panic("implement me") + panic("implement me") } func (t *trackingService) NewTracking(cols ...CollectorType) { - log.Println("new tracking:", cols) - t.opMode = LIVE - t.collectors = nil - for _, col := range cols { - t.collectors = append(t.collectors, NewCollector(col, t.pipe, t.config)) - } - *t.current = emptyTracking() - t.current.collectors = cols - for _, e := range t.collectors { - e.Collect() - } - t.pipe.Run() + logrus.Debug("new tracking:", cols) + t.opMode = LIVE + t.collectors = nil + for _, col := range cols { + t.collectors = append(t.collectors, NewCollector(col, t.pipe, t.config)) + } + *t.current = emptyTracking() + t.current.collectors = cols + for _, e := range t.collectors { + e.Collect() + } + t.pipe.Run() } func (t *trackingService) StartRecord() { - if t.opMode != LIVE { - log.Println("trackingservice: wrong mode of operation") - } - t.current.TimeCreated = time.Now() - t.pipe.Record() + if t.opMode != LIVE { + logrus.Println("trackingservice: wrong mode of operation") + } + t.current.TimeCreated = time.Now() + t.pipe.Record() } func (t *trackingService) StopRecord() { - if t.opMode != LIVE { - log.Println("trackingservice: wrong mode of operation") - } - t.pipe.Stop() - for _, e := range t.collectors { - e.Stop() - } - err := t.repo.Save(*t.current) - if err != nil { - log.Println(err) - } - t.NewTracking(t.current.collectors...) + if t.opMode != LIVE { + logrus.Println("trackingservice: wrong mode of operation") + } + t.pipe.Stop() + for _, e := range t.collectors { + e.Stop() + } + err := t.repo.Save(*t.current) + if err != nil { + logrus.Println(err) + } + t.NewTracking(t.current.collectors...) } func (t *trackingService) Reset() { - t.opMode = STOPPED - t.collectors = nil + t.opMode = STOPPED + t.collectors = nil } - func (t *trackingService) DeleteTracking(trackingId uuid.UUID) { - panic("implement me") + panic("implement me") } func (t *trackingService) StartReplay() { - panic("implement me") + panic("implement me") } func (t *trackingService) PauseReplay() { - panic("implement me") + panic("implement me") } func (t *trackingService) StopReplay() { - panic("implement me") + panic("implement me") } func (t *trackingService) LoadTracking(trackingId uuid.UUID) { } - - diff --git a/core/trackings.go b/core/trackings.go index 1864d17..beb1f98 100644 --- a/core/trackings.go +++ b/core/trackings.go @@ -1,61 +1,60 @@ package core import ( - "github.com/google/uuid" - "log" - "sync" - "time" + "github.com/google/uuid" + "github.com/sirupsen/logrus" + "sync" + "time" ) type Tracking struct { - TrackingMetadata - Records []recordPair - Rawdata []rawRecord - mu sync.Mutex - + TrackingMetadata + Records []recordPair + Rawdata []rawRecord + mu sync.Mutex } type TrackingMetadata struct { - UUID uuid.UUID - TimeCreated time.Time - collectors []CollectorType + UUID uuid.UUID + TimeCreated time.Time + collectors []CollectorType } func (s *Tracking) EnqueuePair(tcp sensorData, ser sensorData) { - s.mu.Lock() - defer s.mu.Unlock() - rp := recordPair{ - RecordTime: time.Now(), - data: map[sourceId]sensorData{ - tcp.source: tcp, - ser.source: ser, - }, - } - s.Records = append(s.Records, rp) - log.Println("tracking Records: len->", len(s.Records)) + s.mu.Lock() + defer s.mu.Unlock() + rp := recordPair{ + RecordTime: time.Now(), + data: map[sourceId]sensorData{ + tcp.source: tcp, + ser.source: ser, + }, + } + s.Records = append(s.Records, rp) + logrus.Debugln("tracking Records: len->", len(s.Records)) } func (s *Tracking) EnqueueRaw(data sensorData) { - s.mu.Lock() - defer s.mu.Unlock() - sr := rawRecord{ - time.Now(), - data, - } - s.Rawdata = append(s.Rawdata, sr) - log.Println("raw data points: len->", len(s.Records)) + s.mu.Lock() + defer s.mu.Unlock() + sr := rawRecord{ + time.Now(), + data, + } + s.Rawdata = append(s.Rawdata, sr) + logrus.Debugln("raw data points: len->", len(s.Records)) } func emptyTracking() Tracking { - return Tracking{ - TrackingMetadata: TrackingMetadata{ - UUID: uuid.New(), - }, - Records: []recordPair{}, - Rawdata: []rawRecord{}, - } + return Tracking{ + TrackingMetadata: TrackingMetadata{ + UUID: uuid.New(), + }, + Records: []recordPair{}, + Rawdata: []rawRecord{}, + } } func (s *Tracking) isEmpty() bool { - return len(s.Rawdata) + len(s.Records) == 0 + return len(s.Rawdata)+len(s.Records) == 0 } diff --git a/database/repository.go b/database/repository.go deleted file mode 100644 index 1f9d413..0000000 --- a/database/repository.go +++ /dev/null @@ -1,27 +0,0 @@ -package database - -import ( - "git.timovolkmann.de/gyrogpsc/core" - "github.com/google/uuid" -) - -// Must implement Repo -type repository struct { -} - -func NewRepository(c *core.Configuration) *repository { - return &repository{} -} - -func (r *repository) Save(tracking core.Tracking) error { - panic("implement me") -} - -func (r *repository) LoadAll() ([]core.TrackingMetadata, error) { - panic("implement me") -} - -func (r *repository) Load(id uuid.UUID) (core.Tracking, error) { - panic("implement me") -} - diff --git a/example_config.yml b/example_config.yml index 5d18e96..872032e 100644 --- a/example_config.yml +++ b/example_config.yml @@ -1,3 +1,6 @@ +# !!! copy this file and rename it to gpsconfig.yml !!! +# !!! file must be located in working directory !!! + # server configurations webserver: port: ":3011" @@ -10,4 +13,20 @@ collectors: # processing pipeline configurations pipeline: publishIntervalMs: 50 - syncUpdateIntervalMs: 494 \ No newline at end of file + syncUpdateIntervalMs: 494 + +debuglevel: "INFO" + +#// ErrorLevel level. Logs. Used for errors that should definitely be noted. +#// Commonly used for hooks to send errors to an error tracking service. +#ErrorLevel + +#// WarnLevel level. Non-critical entries that deserve eyes. +#WarnLevel + +#// InfoLevel level. General operational entries about what's going on inside the +#// application. +#InfoLevel + +#// DebugLevel level. Usually only enabled when debugging. Very verbose logging. +#DebugLevel \ No newline at end of file diff --git a/go.mod b/go.mod index 201ab81..67522e3 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,15 @@ module git.timovolkmann.de/gyrogpsc go 1.15 require ( + github.com/dgraph-io/badger/v2 v2.2007.2 github.com/gofiber/fiber/v2 v2.2.4 github.com/gofiber/websocket/v2 v2.0.2 github.com/google/go-cmp v0.3.0 github.com/google/uuid v1.1.2 - github.com/gorilla/websocket v1.4.2 + github.com/sirupsen/logrus v1.6.0 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.6.1 // indirect github.com/tidwall/gjson v1.6.0 github.com/tidwall/pretty v1.0.2 // indirect - github.com/upper/db/v4 v4.0.1 // indirect go.bug.st/serial v1.1.1 ) diff --git a/go.sum b/go.sum index de176c1..9f012a5 100644 --- a/go.sum +++ b/go.sum @@ -11,38 +11,53 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/goselect v0.1.1 h1:tiSSgKE1eJtxs1h/VgGQWuXUP0YS4CDIFMp6vaI1ls0= github.com/creack/goselect v0.1.1/go.mod h1:a/NhLweNvqIYMuxcMOuWY516Cimucms3DglDzQP3hKY= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisenkom/go-mssqldb v0.0.0-20200620013148-b91950f658ec/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= +github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fasthttp/websocket v1.4.3 h1:qjhRJ/rTy4KB8oBxljEC00SDt6HUY9jLRfM601SUdS4= github.com/fasthttp/websocket v1.4.3/go.mod h1:5r4oKssgS7W6Zn6mPWap3NWzNPJNzUUh3baWTOhcYQk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -53,7 +68,6 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofiber/fiber/v2 v2.1.0/go.mod h1:aG+lMkwy3LyVit4CnmYUbUdgjpc3UYOltvlJZ78rgQ0= github.com/gofiber/fiber/v2 v2.2.4 h1:t2V2SxlbQGdt8+SS/Mo+tQB0pDQn7OajKdA72qHcBVw= @@ -62,7 +76,6 @@ github.com/gofiber/websocket/v2 v2.0.2 h1:UA/6NpyG+vmPGlvJvW8MJPJpRFuS7abinZ5HbL github.com/gofiber/websocket/v2 v2.0.2/go.mod h1:7VBnzEVRK0K0eTIVc5GbXPF1JWUFnllY0X4cRtG2v78= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -70,7 +83,10 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -80,11 +96,11 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -112,9 +128,11 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -123,22 +141,24 @@ github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-sqlite3 v1.14.1 h1:AHx9Ra40wIzl+GelgX2X6AWxmT5tfxhI1PL0523HcSw= -github.com/mattn/go-sqlite3 v1.14.1/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= @@ -153,6 +173,7 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -166,27 +187,34 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/savsgio/gotils v0.0.0-20200608150037-a5f6f5aef16c h1:2nF5+FZ4/qp7pZVL7fR6DEaSTzuDmNaFTyqp92/hwF8= github.com/savsgio/gotils v0.0.0-20200608150037-a5f6f5aef16c/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -207,10 +235,7 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/upper/db v1.0.1 h1:AFzy0fFO7EZ5woZGE/LVl9BWLKqyAWlglu+ZyHQTlbU= -github.com/upper/db v3.7.1+incompatible h1:ehR+Q0msV25yZWXdFYD+3n4BvY9jefI336lmZy7FlAQ= -github.com/upper/db/v4 v4.0.1 h1:mHmGBffne8fdiJjmtYXCuHSghO9027q4TJA9oGP50OM= -github.com/upper/db/v4 v4.0.1/go.mod h1:pyAEIpPfnhpvO4zVbyUI+asgZjppZlq705fOFTyUOso= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.14.0/go.mod h1:ol1PCaL0dX20wC0htZ7sYCsvCYmrouYra0zHzaclZhE= @@ -220,8 +245,7 @@ github.com/valyala/fasthttp v1.17.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8erm github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -gitlab.com/cznic/ebnf2y v1.0.0/go.mod h1:jx14dqOldV2pRvSi8HASTB/k5fkIv2TwjYAp5py0MTs= -gitlab.com/cznic/golex v1.0.0/go.mod h1:vkWdDgqbbThjRHoOLU7yNPgMxaubAkwnvF/4zeG8cvU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.bug.st/serial v1.1.1 h1:5J1DpaIaSIruBi7jVnKXnhRS+YQ9+2PLJMtIZKoIgnc= go.bug.st/serial v1.1.1/go.mod h1:VmYBeyJWp5BnJ0tw2NUJHZdJTGl2ecBGABHlzRK1knY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -232,14 +256,13 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190411193353-0480eff6dd7c/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= @@ -256,7 +279,6 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -271,9 +293,9 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0 h1:5kGOVHlq0euqwzgTC9Vu15p6fV1Wi0ArVi8da2urnVg= golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -289,6 +311,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -297,6 +320,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -353,11 +377,13 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -371,19 +397,4 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= -modernc.org/db v1.0.0/go.mod h1:kYD/cO29L/29RM0hXYl4i3+Q5VojL31kTUVpVJDw0s8= -modernc.org/ebnfutil v1.0.0/go.mod h1:+2n/OnQXoild9pzrPa/2wmVtR+ufWjB/0fYkc0BV9sc= -modernc.org/file v1.0.0/go.mod h1:uqEokAEn1u6e+J45e54dsEA/pw4o7zLrA2GwyntZzjw= -modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= -modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= -modernc.org/internal v1.0.0/go.mod h1:VUD/+JAkhCpvkUitlEOnhpVxCgsBI90oTzSCRcqQVSM= -modernc.org/lex v1.0.0/go.mod h1:G6rxMTy3cH2iA0iXL/HRRv4Znu8MK4higxph/lE7ypk= -modernc.org/lexer v1.0.0/go.mod h1:F/Dld0YKYdZCLQ7bD0USbWL4YKCyTDRDHiDTOs0q0vk= -modernc.org/lldb v1.0.0/go.mod h1:jcRvJGWfCGodDZz8BPwiKMJxGJngQ/5DrRapkQnLob8= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= -modernc.org/ql v1.1.0/go.mod h1:Fj1ylcVyzcu/fgWZTrvBO9j/aEUg/ixLFnGtmzh7quI= -modernc.org/sortutil v1.0.0/go.mod h1:1QO0q8IlIlmjBIwm6t/7sof874+xCfZouyqZMLIAtxM= -modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/storage/kvstore.go b/storage/kvstore.go new file mode 100644 index 0000000..c1fcae8 --- /dev/null +++ b/storage/kvstore.go @@ -0,0 +1,33 @@ +package storage + +import ( + "git.timovolkmann.de/gyrogpsc/core" + "github.com/dgraph-io/badger/v2" + "github.com/google/uuid" + "github.com/sirupsen/logrus" +) + +// Must implement Repo +type badgerStore struct { + db *badger.DB +} + +func NewRepository(c *core.Configuration) *badgerStore { + db, err := badger.Open(badger.DefaultOptions(".")) + if err != nil { + logrus.Warn(err) + } + return &badgerStore{db} +} + +func (r *badgerStore) Save(tracking core.Tracking) error { + panic("implement me") +} + +func (r *badgerStore) LoadAll() ([]core.TrackingMetadata, error) { + panic("implement me") +} + +func (r *badgerStore) Load(id uuid.UUID) (core.Tracking, error) { + panic("implement me") +} diff --git a/web/http.go b/web/http.go index 3eb6033..c68f38f 100644 --- a/web/http.go +++ b/web/http.go @@ -4,8 +4,8 @@ import ( "git.timovolkmann.de/gyrogpsc/core" "github.com/gofiber/fiber/v2" "github.com/gofiber/websocket/v2" + "github.com/sirupsen/logrus" "html/template" - "log" ) func CreateServer(s core.Service, sub core.Subscriber, c *core.Configuration) { @@ -29,14 +29,14 @@ func CreateServer(s core.Service, sub core.Subscriber, c *core.Configuration) { trackings.Put("/", stubhander()) // Stops current recording. Returns trackingId if record was successful trackings.Delete("/", stubhander()) // Stops websocket connection, pipelines and collectors - trackings.Get("/:trackingId", stubhander()) // Gets Tracking Metadata and loads sensorRecords from database. - trackings.Delete("/:trackingId", stubhander()) // Deletes Tracking from database + trackings.Get("/:trackingId", stubhander()) // Gets Tracking Metadata and loads sensorRecords from storage. + trackings.Delete("/:trackingId", stubhander()) // Deletes Tracking from storage - trackings.Post("/current", stubhander()) // Starts Replay. - trackings.Patch("/current", stubhander()) // Pauses Replay. - trackings.Put("/current", stubhander()) // Stops Replay. + trackings.Post("/current", stubhander()) // Starts Replay. + trackings.Patch("/current", stubhander()) // Pauses Replay. + trackings.Put("/current", stubhander()) // Stops Replay. - log.Fatal(app.Listen(c.Webserver.Port)) + logrus.Fatal(app.Listen(c.Webserver.Port)) } func stubhander() fiber.Handler { @@ -63,7 +63,7 @@ func createFiberWebsocketHandler(s core.Subscriber) func(conn *websocket.Conn) { cmsg := <-channel err := c.WriteMessage(websocket.TextMessage, []byte(cmsg)) if err != nil { - log.Println("write:", err) + logrus.Println("write:", err) break } From 09791727a46617bd225a3fda81e72e6b9eb40c4c Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Sat, 12 Dec 2020 02:29:22 +0100 Subject: [PATCH 15/20] [WIP] eliminated data races & formatted files --- .gitignore | 1 + cmd/autostart/autostart.go | 99 +++++------ cmd/server/server.go | 44 ++--- core/collectors.go | 277 +++++++++++++++--------------- core/config.go | 24 +-- core/dispatcher.go | 49 +++--- core/interfaces.go | 42 +++-- core/pipeline.go | 335 ++++++++++++++++++++----------------- core/sensordata.go | 248 +++++++++++++-------------- core/service.go | 165 +++++++++++------- core/trackings.go | 78 ++++----- go.mod | 1 + go.sum | 1 + storage/kvstore.go | 192 +++++++++++++++++++-- ublox/decode.go | 216 ++++++++++++------------ ublox/messages.go | 178 ++++++++++---------- web/http.go | 118 ++++++------- 17 files changed, 1158 insertions(+), 910 deletions(-) diff --git a/.gitignore b/.gitignore index e57b859..b861e7d 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,4 @@ Temporary Items .env gpsconfig.yml config.yml +_db diff --git a/cmd/autostart/autostart.go b/cmd/autostart/autostart.go index a0bafeb..8be4e32 100644 --- a/cmd/autostart/autostart.go +++ b/cmd/autostart/autostart.go @@ -1,65 +1,66 @@ package main import ( - "git.timovolkmann.de/gyrogpsc/core" - "git.timovolkmann.de/gyrogpsc/storage" - "git.timovolkmann.de/gyrogpsc/web" - "github.com/sirupsen/logrus" - "github.com/spf13/viper" - "time" + "git.timovolkmann.de/gyrogpsc/core" + "git.timovolkmann.de/gyrogpsc/storage" + "git.timovolkmann.de/gyrogpsc/web" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "time" ) func main() { - conf := configurationFromFile() + conf := configurationFromFile() - repo := storage.NewRepository(conf) - disp := core.NewDispatcher() + repo := storage.NewRepository(conf) + disp := core.NewDispatcher() - service := core.TrackingService(repo, disp, conf) + service := core.TrackingService(repo, disp, conf) - go func() { - service.NewTracking(core.TCP, core.SERIAL) - time.Sleep(5 * time.Second) - service.StartRecord() - time.Sleep(20 * time.Second) - service.StopRecord() - time.Sleep(5 * time.Second) - service.NewTracking(core.TCP) - service.NewTracking(core.SERIAL) - time.Sleep(5 * time.Second) - service.StartRecord() - time.Sleep(20 * time.Second) - service.StopRecord() - }() + go func() { + service.NewSetup(core.TCP, core.SERIAL) + time.Sleep(5 * time.Second) + service.StartRecord() + time.Sleep(5 * time.Second) + service.StopRecord() + time.Sleep(5 * time.Second) + service.NewSetup(core.TCP) + time.Sleep(5 * time.Second) + service.StartRecord() + time.Sleep(5 * time.Second) + service.StopRecord() + time.Sleep(5 * time.Second) + service.StopAll() + }() - web.CreateServer(service, disp, conf) + web.CreateServer(service, disp, conf) } func configurationFromFile() *core.Configuration { - viper.SetDefault("collectors.porttcp", ":3010") - viper.SetDefault("collectors.portserial", "/dev/tty.usbmodem14201") - viper.SetDefault("webserver.port", ":3011") - viper.SetDefault("pipeline.publishIntervalMs", 50) - viper.SetDefault("pipeline.syncUpdateIntervalMs", 494) - viper.SetDefault("debuglevel", "INFO") + viper.SetDefault("collectors.porttcp", ":3010") + viper.SetDefault("collectors.portserial", "/dev/tty.usbmodem14201") + viper.SetDefault("webserver.port", ":3011") + viper.SetDefault("pipeline.publishIntervalMs", 50) + viper.SetDefault("pipeline.syncUpdateIntervalMs", 494) + viper.SetDefault("debuglevel", "INFO") - viper.SetConfigName("gpsconfig") // name of config file (without extension) - viper.SetConfigType("yaml") - viper.AddConfigPath(".") - viper.AddConfigPath("./../../") - if err := viper.ReadInConfig(); err != nil { - logrus.Warn("couldn't find config file. using standard configuration") - } + viper.SetConfigName("gpsconfig") // name of config file (without extension) + viper.SetConfigType("yaml") + viper.AddConfigPath(".") + viper.AddConfigPath("./../../") + if err := viper.ReadInConfig(); err != nil { + logrus.Warn("couldn't find config file. using standard configuration") + } - c := core.Configuration{} - if err := viper.Unmarshal(&c); err != nil { - logrus.Debug("couldn't load config...") - logrus.Error(err) - } - lvl, err := logrus.ParseLevel(c.Debuglevel) - if err != nil { - logrus.Error(err) - } - logrus.SetLevel(lvl) - return &c + c := core.Configuration{} + if err := viper.Unmarshal(&c); err != nil { + logrus.Debug("couldn't load config...") + logrus.Error(err) + } + lvl, err := logrus.ParseLevel(c.Debuglevel) + if err != nil { + logrus.Error(err) + } + logrus.SetLevel(lvl) + return &c } diff --git a/cmd/server/server.go b/cmd/server/server.go index 33d557d..b7ebf58 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -1,37 +1,37 @@ package main import ( - "git.timovolkmann.de/gyrogpsc/core" - "git.timovolkmann.de/gyrogpsc/storage" - "git.timovolkmann.de/gyrogpsc/web" - "github.com/sirupsen/logrus" - "github.com/spf13/viper" + "git.timovolkmann.de/gyrogpsc/core" + "git.timovolkmann.de/gyrogpsc/storage" + "git.timovolkmann.de/gyrogpsc/web" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" ) func main() { - conf := &core.Configuration{} - configurationFromFile(conf) + conf := &core.Configuration{} + configurationFromFile(conf) - repo := storage.NewRepository(conf) - disp := core.NewDispatcher() + repo := storage.NewRepository(conf) + disp := core.NewDispatcher() - service := core.TrackingService(repo, disp, conf) + service := core.TrackingService(repo, disp, conf) - web.CreateServer(service, disp, conf) + web.CreateServer(service, disp, conf) } func configurationFromFile(c *core.Configuration) { - viper.SetDefault("TcpCollectorPort", ":3010") - viper.SetDefault("SerialCollectorPort", "/dev/tty.usbmodem14201") - viper.SetDefault("HttpPort", "layouts") - viper.SetDefault("publishIntervalMs", 50) - viper.SetDefault("syncUpdateIntervalMs", 494) + viper.SetDefault("TcpCollectorPort", ":3010") + viper.SetDefault("SerialCollectorPort", "/dev/tty.usbmodem14201") + viper.SetDefault("HttpPort", "layouts") + viper.SetDefault("publishIntervalMs", 50) + viper.SetDefault("syncUpdateIntervalMs", 494) - viper.SetConfigName("gpsconfig") // name of config file (without extension) - viper.SetConfigType("yaml") - viper.AddConfigPath(".") - viper.AddConfigPath("./../../") + viper.SetConfigName("gpsconfig") // name of config file (without extension) + viper.SetConfigType("yaml") + viper.AddConfigPath(".") + viper.AddConfigPath("./../../") - viper.Unmarshal(c) - logrus.Println(c) + viper.Unmarshal(c) + logrus.Println(c) } diff --git a/core/collectors.go b/core/collectors.go index 27339ea..cddf112 100644 --- a/core/collectors.go +++ b/core/collectors.go @@ -1,184 +1,199 @@ package core import ( - "fmt" - "git.timovolkmann.de/gyrogpsc/ublox" - "github.com/sirupsen/logrus" - "go.bug.st/serial" - "net" - "os" + "fmt" + "git.timovolkmann.de/gyrogpsc/ublox" + "github.com/sirupsen/logrus" + "go.bug.st/serial" + "net" + "os" + "sync" ) type Collector interface { - Collect() - Stop() + Collect() + Close() } type CollectorType uint8 const ( - SERIAL CollectorType = iota - TCP + SERIAL CollectorType = iota + TCP ) var tcpSingleton *tcpCollector func NewCollector(typ CollectorType, proc Pusher, config *Configuration) Collector { - var coll Collector - switch typ { - case SERIAL: - coll = newSerial(proc, config) - case TCP: - if tcpSingleton == nil { - tcpSingleton = newTcp(proc, config) - } else { - tcpSingleton.SetProcessor(proc) - } - coll = tcpSingleton - default: - panic("selected collector type not implemented") - } - return coll + var coll Collector + switch typ { + case SERIAL: + coll = newSerial(proc, config) + case TCP: + if tcpSingleton == nil { + tcpSingleton = newTcp(proc, config) + } else { + tcpSingleton.SetProcessor(proc) + } + coll = tcpSingleton + default: + panic("selected collector type not implemented") + } + return coll } type serialCollector struct { - active bool - proc Pusher - config *Configuration + active bool + proc Pusher + config *Configuration + mu sync.RWMutex } +func (s *serialCollector) isSerialCollActive() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return s.active +} + + func (s *serialCollector) Collect() { - s.active = true - go func() { - logrus.Println("start serial collector") - mode := &serial.Mode{ - BaudRate: 115200, - } - port, err := serial.Open(s.config.Collectors.SerialCollectorPort, mode) - if err != nil { - logrus.Fatalln(err.Error()) - } - defer port.Close() + s.mu.Lock() + s.active = true + s.mu.Unlock() + go func() { + logrus.Println("start serial collector") + mode := &serial.Mode{ + BaudRate: 115200, + } + port, err := serial.Open(s.config.Collectors.SerialCollectorPort, mode) + if err != nil { + logrus.Fatalln("can't open serial port:", err.Error()) + } + defer port.Close() - decoder := ublox.NewDecoder(port) + decoder := ublox.NewDecoder(port) - for s.active { - meas, err := decoder.Decode() - if err != nil { - if err.Error() == "NMEA not implemented" { - continue - } - logrus.Println("serial read err:", err) - break - } - sd, err := ConvertUbxToSensorData(meas) - if err != nil { - logrus.Println("convert err:", err, meas, sd) - continue - } - // skip irrelevant messages - if sd == nil { - continue - } + for s.isSerialCollActive() { + meas, err := decoder.Decode() + if err != nil { + if err.Error() == "NMEA not implemented" { + continue + } + logrus.Println("serial read err:", err) + break + } + sd, err := ConvertUbxToSensorData(meas) + if err != nil { + logrus.Println("convert err:", err, meas, sd) + continue + } + // skip irrelevant messages + if sd == nil { + continue + } - err = s.proc.Push(sd) - if err != nil { - logrus.Println("process err:", err, *sd) - continue - } - } - logrus.Println("serial collector stopped") - }() + err = s.proc.Push(sd) + if err != nil { + logrus.Println("process err:", err, *sd) + continue + } + } + logrus.Println("serial collector stopped") + }() } -func (s *serialCollector) Stop() { - s.active = false + + +func (s *serialCollector) Close() { + s.mu.Lock() + s.active = false + s.mu.Unlock() } func newSerial(proc Pusher, config *Configuration) *serialCollector { - return &serialCollector{ - active: false, - proc: proc, - config: config, - } + return &serialCollector{ + active: false, + proc: proc, + config: config, + } } type tcpCollector struct { - active bool - processor Pusher - //config *Configuration + active bool + processor Pusher + //config *Configuration } func (t *tcpCollector) Collect() { - t.active = true + t.active = true } -func (t *tcpCollector) Stop() { - t.active = false +func (t *tcpCollector) Close() { + t.active = false } func (t *tcpCollector) SetProcessor(p Pusher) { - t.processor = p + t.processor = p } func newTcp(proc Pusher, config *Configuration) *tcpCollector { - logrus.Println("start tcp collector") + logrus.Println("start tcp collector") - listener, err := net.Listen("tcp", config.Collectors.TcpCollectorPort) - if err != nil { - fmt.Println("Error listening:", err.Error()) - //os.Exit(1) - } - coll := &tcpCollector{ - active: false, - processor: proc, - } - go func() { - for { - // Listen for an incoming connection. - conn, err := listener.Accept() - if err != nil { - fmt.Println("Error accepting: ", err.Error()) - os.Exit(1) - } - logrus.Println("...new incoming tcp connection...") + listener, err := net.Listen("tcp", config.Collectors.TcpCollectorPort) + if err != nil { + fmt.Println("Error listening:", err) + //os.Exit(1) + } + coll := &tcpCollector{ + active: false, + processor: proc, + } + go func() { + for { + // Listen for an incoming connection. + conn, err := listener.Accept() + if err != nil { + fmt.Println("Error accepting: ", err.Error()) + os.Exit(1) + } + logrus.Println("...new incoming tcp connection...") - // Handle connections in a new goroutine. - go coll.jsonHandler(conn) - } - }() - return coll + // Handle connections in a new goroutine. + go coll.jsonHandler(conn) + } + }() + return coll } // handles incoming tcp connections with json payload. func (c *tcpCollector) jsonHandler(conn net.Conn) { - defer conn.Close() + defer conn.Close() - // TRY reader := bufio.NewReader(conn) OR NewScanner(conn) - buf := make([]byte, 2048) - for { - // Read the incoming connection into the buffer. - n, err := conn.Read(buf) - if err != nil { - fmt.Println("TCP error - reading from connection:", n, err.Error()) - break - } - //json := pretty.Pretty(buf[:n]) - //fmt.Println(string(json)) - //fmt.Println(string(buf[:n])) - sd, err := ConvertSensorDataPhone(buf[:n]) - if err != nil { - logrus.Println(err) - continue - } - if !c.active { - //time.Sleep(50 * time.Millisecond) - continue - } - err = c.processor.Push(sd) - if err != nil { - logrus.Fatalln(err) - } - } + // TRY reader := bufio.NewReader(conn) OR NewScanner(conn) + buf := make([]byte, 2048) + for { + // Read the incoming connection into the buffer. + n, err := conn.Read(buf) + if err != nil { + fmt.Println("TCP error - reading from connection:", n, err.Error()) + break + } + //json := pretty.Pretty(buf[:n]) + //fmt.Println(string(json)) + //fmt.Println(string(buf[:n])) + sd, err := ConvertSensorDataPhone(buf[:n]) + if err != nil { + logrus.Println(err) + continue + } + if !c.active { + //time.Sleep(50 * time.Millisecond) + continue + } + err = c.processor.Push(sd) + if err != nil { + logrus.Fatalln(err) + } + } } diff --git a/core/config.go b/core/config.go index 7d6255d..adabf79 100644 --- a/core/config.go +++ b/core/config.go @@ -1,16 +1,16 @@ package core type Configuration struct { - Collectors struct { - TcpCollectorPort string `mapstructure:"porttcp"` - SerialCollectorPort string `mapstructure:"portserial"` - } `mapstructure:"collectors"` - Webserver struct { - Port string `mapstructure:"port"` - } `mapstructure:"webserver"` - Pipeline struct { - PublishIntervalMs int `mapstructure:"publishintervalms"` - SyncUpdateIntervalMs int `mapstructure:"syncupdateintervalms"` - } `mapstructure:"pipeline"` - Debuglevel string `mapstructure:"debuglevel"` + Collectors struct { + TcpCollectorPort string `mapstructure:"porttcp"` + SerialCollectorPort string `mapstructure:"portserial"` + } `mapstructure:"Collectors"` + Webserver struct { + Port string `mapstructure:"port"` + } `mapstructure:"webserver"` + Pipeline struct { + PublishIntervalMs int `mapstructure:"publishintervalms"` + SyncUpdateIntervalMs int `mapstructure:"syncupdateintervalms"` + } `mapstructure:"pipeline"` + Debuglevel string `mapstructure:"debuglevel"` } diff --git a/core/dispatcher.go b/core/dispatcher.go index 94d2197..72e6429 100644 --- a/core/dispatcher.go +++ b/core/dispatcher.go @@ -1,43 +1,44 @@ package core import ( - "errors" - "github.com/sirupsen/logrus" + "errors" + "github.com/sirupsen/logrus" ) type dispatcher struct { - listeners map[int16]chan string - counter int16 + listeners map[int16]chan string + counter int16 } func NewDispatcher() *dispatcher { - return &dispatcher{ - listeners: make(map[int16]chan string), - counter: 0, - } + return &dispatcher{ + listeners: make(map[int16]chan string), + counter: 0, + } } func (d *dispatcher) Publish(message string) { - logrus.Debugf("publish to %v listeners:\n%v\n", len(d.listeners), message) - for _, ch := range d.listeners { - ch <- message - } + logrus.Debugf("publish to %v listeners:\n%v\n", len(d.listeners)) + logrus.Debug(message) + for _, ch := range d.listeners { + ch <- message + } } func (d *dispatcher) Subscribe() (id int16, receiver <-chan string) { - key := d.counter - d.counter++ - rec := make(chan string) - d.listeners[key] = rec - return key, rec + key := d.counter + d.counter++ + rec := make(chan string) + d.listeners[key] = rec + return key, rec } func (d *dispatcher) Unsubscribe(id int16) error { - receiver, ok := d.listeners[id] - if !ok { - return errors.New("no subscription with id") - } - delete(d.listeners, id) - close(receiver) - return nil + receiver, ok := d.listeners[id] + if !ok { + return errors.New("no subscription with id") + } + delete(d.listeners, id) + close(receiver) + return nil } diff --git a/core/interfaces.go b/core/interfaces.go index d2ce9af..ce10b0d 100644 --- a/core/interfaces.go +++ b/core/interfaces.go @@ -3,44 +3,40 @@ package core import "github.com/google/uuid" type Subscriber interface { - Subscribe() (int16, <-chan string) - Unsubscribe(id int16) error + Subscribe() (int16, <-chan string) + Unsubscribe(id int16) error } type Publisher interface { - Publish(message string) + Publish(message string) } type Pusher interface { - Push(data *sensorData) error + Push(data *sensorData) error } - type Storer interface { - EnqueuePair(tcp sensorData, ser sensorData) - EnqueueRaw(data sensorData) + EnqueuePair(tcp sensorData, ser sensorData) + EnqueueRaw(data sensorData) } - - type Repo interface { - Save(tracking Tracking) error - LoadAll() ([]TrackingMetadata, error) - Load(id uuid.UUID) (Tracking, error) + Save(tracking Tracking) error + LoadAll() ([]TrackingMetadata, error) + Load(id uuid.UUID) (Tracking, error) } - type Service interface { - AllTrackings() - NewTracking(cols ...CollectorType) - StartRecord() - StopRecord() - Reset() + AllTrackings() + NewSetup(cols ...CollectorType) + StartRecord() + StopRecord() + StopAll() - LoadTracking(trackingId uuid.UUID) - DeleteTracking(trackingId uuid.UUID) + LoadTracking(trackingId uuid.UUID) + DeleteTracking(trackingId uuid.UUID) - StartReplay() - PauseReplay() - StopReplay() + StartReplay() + PauseReplay() + StopReplay() } diff --git a/core/pipeline.go b/core/pipeline.go index e77bbe3..00c7f4e 100644 --- a/core/pipeline.go +++ b/core/pipeline.go @@ -1,204 +1,225 @@ package core import ( - "encoding/json" - "errors" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/sirupsen/logrus" - "sync" - "time" + "context" + "encoding/json" + "errors" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/sirupsen/logrus" + "golang.org/x/sync/semaphore" + "sync" + "time" ) type pipeline struct { - active bool - record bool - synchroniz synchronizer - buffer pipeBuffer - publisher Publisher - storer Storer - publishTicker *time.Ticker + active bool + record bool + synchroniz synchronizer + buffer pipeBuffer + publisher Publisher + storer Storer + publishTicker *time.Ticker + mu sync.RWMutex + sema *semaphore.Weighted } // pipe implements Runner & Pusher func NewPipeline(d Publisher, s Storer, conf *Configuration) *pipeline { - return &pipeline{ - false, - false, - synchronizer{ - //bufferSize: 100, - mutex: &sync.RWMutex{}, - updateTicker: time.NewTicker(time.Duration(conf.Pipeline.SyncUpdateIntervalMs) * time.Millisecond), - }, - pipeBuffer{ - tcpMutex: &sync.Mutex{}, - serialMutex: &sync.Mutex{}, - }, - d, - s, - time.NewTicker(time.Duration(conf.Pipeline.PublishIntervalMs) * time.Millisecond), - } + return &pipeline{ + false, + false, + synchronizer{ + //bufferSize: 100, + mutex: &sync.RWMutex{}, + updateTicker: time.NewTicker(time.Duration(conf.Pipeline.SyncUpdateIntervalMs) * time.Millisecond), + }, + pipeBuffer{ + tcpMutex: &sync.Mutex{}, + serialMutex: &sync.Mutex{}, + }, + d, + s, + time.NewTicker(time.Duration(conf.Pipeline.PublishIntervalMs) * time.Millisecond), + sync.RWMutex{}, + semaphore.NewWeighted(2), + } +} + +func (p *pipeline) isPipeActive() bool { + p.mu.RLock() + defer p.mu.RUnlock() + return p.active } func (p *pipeline) Run() { - p.active = true - logrus.Println("pipe: processing service started") - go func() { - for p.active { - <-p.synchroniz.updateTicker.C - err := p.refreshDelay() - if err != nil { - logrus.Debugln(err) - } - } - logrus.Println("pipe: updater stopped") - }() - go func() { - for p.active { - <-p.publishTicker.C - err := p.publish() - if err != nil && err.Error() != "no data available" { - logrus.Debug(err) - } - } - logrus.Println("pipe: publisher stopped") - }() + p.sema.Acquire(context.Background(), 2) + p.mu.Lock() + p.active = true + p.mu.Unlock() + logrus.Println("pipe: processing service started") + go func() { + for p.isPipeActive() { + <-p.synchroniz.updateTicker.C + err := p.refreshDelay() + if err != nil { + logrus.Debugln(err) + } + } + p.sema.Release(1) + logrus.Println("pipe: updater stopped") + }() + go func() { + for p.isPipeActive() { + <-p.publishTicker.C + err := p.publish() + if err != nil && err.Error() != "no data available" { + logrus.Debug(err) + } + } + p.sema.Release(1) + logrus.Println("pipe: publisher stopped") + }() } func (p *pipeline) Record() { - p.record = true + p.record = true } -func (p *pipeline) Stop() { - p.record = false +func (p *pipeline) StopRecord() { + p.record = false } func (p *pipeline) publish() error { - p.buffer.tcpMutex.Lock() - p.buffer.serialMutex.Lock() + p.buffer.tcpMutex.Lock() + p.buffer.serialMutex.Lock() - if (p.buffer.MeasTcp == sensorData{} && p.buffer.MeasSerial == sensorData{}) { - p.buffer.tcpMutex.Unlock() - p.buffer.serialMutex.Unlock() - return errors.New("no data available") - } - if cmp.Equal(p.buffer.MeasTcp, p.buffer.LastMeasTcp, cmpopts.IgnoreUnexported(sensorData{})) && - cmp.Equal(p.buffer.MeasSerial, p.buffer.LastMeasSerial, cmpopts.IgnoreUnexported(sensorData{})) { - p.buffer.tcpMutex.Unlock() - p.buffer.serialMutex.Unlock() - return errors.New("same data") - } - logrus.Debug("––––––––––––––––––––––––––––––––––––") - logrus.Debugf("MEAS old: %v", p.buffer.LastMeasTcp) - logrus.Debugf("MEAS new: %v", p.buffer.MeasTcp) - logrus.Debug("––––––––––––––––––––––––––––––––––––") - p.buffer.LastMeasTcp = p.buffer.MeasTcp - p.buffer.LastMeasSerial = p.buffer.MeasSerial - p.storer.EnqueuePair(p.buffer.MeasTcp, p.buffer.MeasSerial) + if (p.buffer.MeasTcp == sensorData{} && p.buffer.MeasSerial == sensorData{}) { + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() + return errors.New("no data available") + } + if cmp.Equal(p.buffer.MeasTcp, p.buffer.LastMeasTcp, cmpopts.IgnoreUnexported(sensorData{})) && + cmp.Equal(p.buffer.MeasSerial, p.buffer.LastMeasSerial, cmpopts.IgnoreUnexported(sensorData{})) { + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() + return errors.New("same data") + } + logrus.Debug("––––––––––––––––––––––––––––––––––––") + logrus.Debugf("SER old: %v", p.buffer.LastMeasSerial) + logrus.Debugf("SER new: %v", p.buffer.MeasSerial) + logrus.Debugf("TCP old: %v", p.buffer.LastMeasTcp) + logrus.Debugf("TCP new: %v", p.buffer.MeasTcp) + logrus.Debug("––––––––––––––––––––––––––––––––––––") + p.buffer.LastMeasTcp = p.buffer.MeasTcp + p.buffer.LastMeasSerial = p.buffer.MeasSerial + p.storer.EnqueuePair(p.buffer.MeasTcp, p.buffer.MeasSerial) - data := map[string]sensorData{ - string(SOURCE_TCP): p.buffer.MeasTcp, - string(SOURCE_SERIAL): p.buffer.MeasSerial, - } + data := map[string]sensorData{ + string(SOURCE_TCP): p.buffer.MeasTcp, + string(SOURCE_SERIAL): p.buffer.MeasSerial, + } - p.buffer.tcpMutex.Unlock() - p.buffer.serialMutex.Unlock() + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() - jdata, err := json.Marshal(data) - //logrus.Println(string(pretty.Pretty(jdata))) - if err != nil { - return err - } - p.publisher.Publish(string(jdata)) - return nil + jdata, err := json.Marshal(data) + //logrus.Println(string(pretty.Pretty(jdata))) + if err != nil { + return err + } + p.publisher.Publish(string(jdata)) + return nil } type pipeBuffer struct { - MeasTcp sensorData - MeasSerial sensorData - LastMeasTcp sensorData - LastMeasSerial sensorData - tcpMutex *sync.Mutex - serialMutex *sync.Mutex + MeasTcp sensorData + MeasSerial sensorData + LastMeasTcp sensorData + LastMeasSerial sensorData + tcpMutex *sync.Mutex + serialMutex *sync.Mutex } type UnixNanoTime int64 type synchronizer struct { - tcpSerialDelayMs int64 - mutex *sync.RWMutex - updateTicker *time.Ticker + tcpSerialDelayMs int64 + mutex *sync.RWMutex + updateTicker *time.Ticker } func (p *pipeline) refreshDelay() error { - p.synchroniz.mutex.RLock() - if p.synchroniz.tcpSerialDelayMs != 0 { - logrus.Println("Delay TCP/SERIAL", p.synchroniz.tcpSerialDelayMs) - } - p.synchroniz.mutex.RUnlock() - p.buffer.serialMutex.Lock() - p.buffer.tcpMutex.Lock() - tcpTime := time.Unix(0, p.buffer.MeasTcp.Timestamp) - serTime := time.Unix(0, p.buffer.MeasSerial.Timestamp) - p.buffer.tcpMutex.Unlock() - p.buffer.serialMutex.Unlock() - if tcpTime.UnixNano() == 0 || serTime.UnixNano() == 0 { - return errors.New("no sync possible. check if both collectors running. otherwise check GPS fix") - } - currentDelay := tcpTime.Sub(serTime).Milliseconds() - if currentDelay > 5000 || currentDelay < -5000 { - p.synchroniz.mutex.Lock() - p.synchroniz.tcpSerialDelayMs = 0 - p.synchroniz.mutex.Unlock() - return errors.New("skipping synchronisation! time not properly configured or facing network problems.") - } - logrus.Debug("TCP", tcpTime.String()) - logrus.Debug("SER", serTime.String()) - logrus.Debug("Difference", tcpTime.Sub(serTime).Milliseconds(), "ms") - delay := tcpTime.Sub(serTime).Milliseconds() - p.synchroniz.mutex.Lock() - p.synchroniz.tcpSerialDelayMs += delay - p.synchroniz.mutex.Unlock() - return nil + p.synchroniz.mutex.RLock() + if p.synchroniz.tcpSerialDelayMs != 0 { + logrus.Println("Delay TCP/SERIAL", p.synchroniz.tcpSerialDelayMs) + } + p.synchroniz.mutex.RUnlock() + p.buffer.serialMutex.Lock() + p.buffer.tcpMutex.Lock() + tcpTime := time.Unix(0, p.buffer.MeasTcp.Timestamp) + serTime := time.Unix(0, p.buffer.MeasSerial.Timestamp) + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() + if tcpTime.UnixNano() == 0 || serTime.UnixNano() == 0 { + return errors.New("no sync possible. check if both Collectors running. otherwise check GPS fix") + } + currentDelay := tcpTime.Sub(serTime).Milliseconds() + if currentDelay > 5000 || currentDelay < -5000 { + p.synchroniz.mutex.Lock() + p.synchroniz.tcpSerialDelayMs = 0 + p.synchroniz.mutex.Unlock() + return errors.New("skipping synchronisation! time not properly configured or facing network problems.") + } + logrus.Debug("TCP", tcpTime.String()) + logrus.Debug("SER", serTime.String()) + logrus.Debug("Difference", tcpTime.Sub(serTime).Milliseconds(), "ms") + delay := tcpTime.Sub(serTime).Milliseconds() + p.synchroniz.mutex.Lock() + p.synchroniz.tcpSerialDelayMs += delay + p.synchroniz.mutex.Unlock() + return nil } func (p *pipeline) Push(data *sensorData) error { - if data == nil { - return errors.New("nil processing not allowed") - } - //logrus.Println("push data to pipe:", string(data.source)) - p.storer.EnqueueRaw(*data) - switch data.source { - case SOURCE_TCP: - go p.pushTcpDataToBuffer(*data) - case SOURCE_SERIAL: - go p.pushSerialDataToBuffer(*data) - default: - panic("pipe: invalid data source") - } - return nil + if data == nil { + return errors.New("nil processing not allowed") + } + //logrus.Println("push data to pipe:", string(data.source)) + p.storer.EnqueueRaw(*data) + switch data.source { + case SOURCE_TCP: + go p.pushTcpDataToBuffer(*data) + case SOURCE_SERIAL: + go p.pushSerialDataToBuffer(*data) + default: + panic("pipe: invalid data source") + } + return nil } func (p *pipeline) pushTcpDataToBuffer(data sensorData) { - p.synchroniz.mutex.RLock() - if p.synchroniz.tcpSerialDelayMs > 0 { - time.Sleep(time.Duration(p.synchroniz.tcpSerialDelayMs) * time.Millisecond) - } - p.synchroniz.mutex.RLock() - p.buffer.tcpMutex.Lock() - p.buffer.MeasTcp = p.buffer.MeasTcp.ConsolidateExTime(data) - p.buffer.tcpMutex.Unlock() + p.synchroniz.mutex.RLock() + if p.synchroniz.tcpSerialDelayMs > 0 { + time.Sleep(time.Duration(p.synchroniz.tcpSerialDelayMs) * time.Millisecond) + } + p.synchroniz.mutex.RLock() + p.buffer.tcpMutex.Lock() + p.buffer.MeasTcp = p.buffer.MeasTcp.ConsolidateExTime(data) + p.buffer.tcpMutex.Unlock() } func (p *pipeline) pushSerialDataToBuffer(data sensorData) { - p.synchroniz.mutex.RLock() - if p.synchroniz.tcpSerialDelayMs < 0 { - time.Sleep(time.Duration(-p.synchroniz.tcpSerialDelayMs) * time.Millisecond) - } - p.synchroniz.mutex.RUnlock() - p.buffer.serialMutex.Lock() - p.buffer.MeasSerial = p.buffer.MeasSerial.ConsolidateEpochsOnly(data) - p.buffer.serialMutex.Unlock() + p.synchroniz.mutex.RLock() + if p.synchroniz.tcpSerialDelayMs < 0 { + time.Sleep(time.Duration(-p.synchroniz.tcpSerialDelayMs) * time.Millisecond) + } + p.synchroniz.mutex.RUnlock() + p.buffer.serialMutex.Lock() + p.buffer.MeasSerial = p.buffer.MeasSerial.ConsolidateEpochsOnly(data) + p.buffer.serialMutex.Unlock() } func (p *pipeline) Close() { - p.active = false + p.mu.Lock() + p.active = false + p.mu.Unlock() } diff --git a/core/sensordata.go b/core/sensordata.go index e4d8ebf..afd1575 100644 --- a/core/sensordata.go +++ b/core/sensordata.go @@ -1,174 +1,174 @@ package core import ( - "errors" - "git.timovolkmann.de/gyrogpsc/ublox" - "github.com/sirupsen/logrus" - "github.com/tidwall/gjson" - "math" - "time" + "errors" + "git.timovolkmann.de/gyrogpsc/ublox" + "github.com/sirupsen/logrus" + "github.com/tidwall/gjson" + "math" + "time" ) type sourceId string const ( - SOURCE_TCP sourceId = "SOURCE_TCP" - SOURCE_SERIAL sourceId = "SOURCE_SERIAL" + SOURCE_TCP sourceId = "SOURCE_TCP" + SOURCE_SERIAL sourceId = "SOURCE_SERIAL" ) type sensorData struct { - itow uint32 - source sourceId - ServerTime time.Time - Timestamp int64 - Position [3]float64 - Orientation [3]float64 + itow uint32 + source sourceId + ServerTime time.Time + Timestamp int64 + Position [3]float64 + Orientation [3]float64 } type recordPair struct { - RecordTime time.Time - data map[sourceId]sensorData + RecordTime time.Time + data map[sourceId]sensorData } type rawRecord struct { - RecordTime time.Time - sensorData + RecordTime time.Time + sensorData } func (s sensorData) isSameEpoch(n sensorData) bool { - if n.itow == 0 { - return false - } - return s.itow == n.itow + if n.itow == 0 { + return false + } + return s.itow == n.itow } // Consolidates two sensordata elements if they are in the same epoch func (s sensorData) ConsolidateEpochsOnly(n sensorData) sensorData { - s.checkSources(&n) - if s.isSameEpoch(n) { - null := sensorData{} + s.checkSources(&n) + if s.isSameEpoch(n) { + null := sensorData{} - if n.Timestamp == null.Timestamp { - n.Timestamp = s.Timestamp - } - if n.Position == null.Position { - n.Position = s.Position - } - if n.Orientation == null.Orientation { - n.Orientation = s.Orientation - } - } - return n + if n.Timestamp == null.Timestamp { + n.Timestamp = s.Timestamp + } + if n.Position == null.Position { + n.Position = s.Position + } + if n.Orientation == null.Orientation { + n.Orientation = s.Orientation + } + } + return n } // Consolidates two sensordata elements but ignores timestamps func (s sensorData) ConsolidateExTime(n sensorData) sensorData { - s.checkSources(&n) - null := sensorData{} + s.checkSources(&n) + null := sensorData{} - if n.Position == null.Position { - n.Position = s.Position - } - if n.Orientation == null.Orientation { - n.Orientation = s.Orientation - } - return n + if n.Position == null.Position { + n.Position = s.Position + } + if n.Orientation == null.Orientation { + n.Orientation = s.Orientation + } + return n } func (s *sensorData) checkSources(n *sensorData) { - if (s.source != n.source && *s != sensorData{}) { - logrus.Println(s) - logrus.Println(n) - logrus.Fatalln("Do not consolidate sensorData from different Sources") - } + if (s.source != n.source && *s != sensorData{}) { + logrus.Println(s) + logrus.Println(n) + logrus.Fatalln("Do not consolidate sensorData from different Sources") + } } var ( - errNotImplemented = errors.New("message not implemented") - errRawMessage = errors.New("raw message") + errNotImplemented = errors.New("message not implemented") + errRawMessage = errors.New("raw message") ) func ConvertUbxToSensorData(msg interface{}) (*sensorData, error) { - sd := &sensorData{ - ServerTime: time.Now(), - source: SOURCE_SERIAL, - } - switch v := msg.(type) { - case *ublox.NavPvt: - //logrus.Println("NAV-PVT") - sd.itow = v.ITOW_ms - sd.Timestamp = time.Date(int(v.Year_y), time.Month(v.Month_month), int(v.Day_d), int(v.Hour_h), int(v.Min_min), int(v.Sec_s), int(v.Nano_ns), time.UTC).UnixNano() - sd.Position[0] = float64(v.Lat_dege7) / 1e+7 - sd.Position[1] = float64(v.Lon_dege7) / 1e+7 - sd.Position[2] = float64(v.HMSL_mm) / 1e+3 // mm in m - case *ublox.HnrPvt: - //logrus.Println("HNR-PVT") - sd.itow = v.ITOW_ms - sd.Timestamp = time.Date(int(v.Year_y), time.Month(v.Month_month), int(v.Day_d), int(v.Hour_h), int(v.Min_min), int(v.Sec_s), int(v.Nano_ns), time.UTC).UnixNano() - sd.Position[0] = float64(v.Lat_dege7) / 1e+7 - sd.Position[1] = float64(v.Lon_dege7) / 1e+7 - sd.Position[2] = float64(v.HMSL_mm) / 1e+3 // mm in m - case *ublox.NavAtt: - //logrus.Println("NAV-ATT") - sd.itow = v.ITOW_ms - sd.Orientation[0] = float64(v.Pitch_deg) * 1e-5 - sd.Orientation[1] = float64(v.Roll_deg) * 1e-5 - sd.Orientation[2] = float64(v.Heading_deg) * 1e-5 - case *ublox.RawMessage: - //class := make([]byte, 2) - //binary.LittleEndian.PutUint16(class, v.ClassID()) - //logrus.Printf("%#v, %#v", class[0],class[1]) - return nil, nil - default: - return nil, errNotImplemented - } - return sd, nil + sd := &sensorData{ + ServerTime: time.Now(), + source: SOURCE_SERIAL, + } + switch v := msg.(type) { + case *ublox.NavPvt: + //logrus.Println("NAV-PVT") + sd.itow = v.ITOW_ms + sd.Timestamp = time.Date(int(v.Year_y), time.Month(v.Month_month), int(v.Day_d), int(v.Hour_h), int(v.Min_min), int(v.Sec_s), int(v.Nano_ns), time.UTC).UnixNano() + sd.Position[0] = float64(v.Lat_dege7) / 1e+7 + sd.Position[1] = float64(v.Lon_dege7) / 1e+7 + sd.Position[2] = float64(v.HMSL_mm) / 1e+3 // mm in m + case *ublox.HnrPvt: + //logrus.Println("HNR-PVT") + sd.itow = v.ITOW_ms + sd.Timestamp = time.Date(int(v.Year_y), time.Month(v.Month_month), int(v.Day_d), int(v.Hour_h), int(v.Min_min), int(v.Sec_s), int(v.Nano_ns), time.UTC).UnixNano() + sd.Position[0] = float64(v.Lat_dege7) / 1e+7 + sd.Position[1] = float64(v.Lon_dege7) / 1e+7 + sd.Position[2] = float64(v.HMSL_mm) / 1e+3 // mm in m + case *ublox.NavAtt: + //logrus.Println("NAV-ATT") + sd.itow = v.ITOW_ms + sd.Orientation[0] = float64(v.Pitch_deg) * 1e-5 + sd.Orientation[1] = float64(v.Roll_deg) * 1e-5 + sd.Orientation[2] = float64(v.Heading_deg) * 1e-5 + case *ublox.RawMessage: + //class := make([]byte, 2) + //binary.LittleEndian.PutUint16(class, v.ClassID()) + //logrus.Printf("%#v, %#v", class[0],class[1]) + return nil, nil + default: + return nil, errNotImplemented + } + return sd, nil } func ConvertSensorDataPhone(jsonData []byte) (*sensorData, error) { - if gjson.Get(string(jsonData), "os").String() == "hyperimu" { - return convertAndroidHyperImu(jsonData) - } - return convertIPhoneSensorLog(jsonData) + if gjson.Get(string(jsonData), "os").String() == "hyperimu" { + return convertAndroidHyperImu(jsonData) + } + return convertIPhoneSensorLog(jsonData) } func convertIPhoneSensorLog(jsonData []byte) (*sensorData, error) { - timestamp := gjson.Get(string(jsonData), "locationTimestamp_since1970").Float() - lat := gjson.Get(string(jsonData), "locationLatitude").Float() - lon := gjson.Get(string(jsonData), "locationLongitude").Float() - alt := gjson.Get(string(jsonData), "locationAltitude").Float() - pitch := gjson.Get(string(jsonData), "motionPitch").Float() * 180 / math.Pi - roll := gjson.Get(string(jsonData), "motionRoll").Float() * 180 / math.Pi - yaw := gjson.Get(string(jsonData), "motionYaw").Float() * 180 / math.Pi - sd := &sensorData{ - ServerTime: time.Now(), - source: SOURCE_TCP, - Timestamp: int64(timestamp * float64(time.Second)), - Position: [3]float64{lat, lon, alt}, - Orientation: [3]float64{pitch, roll, yaw}, - //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), - } - //logrus.Println(string(pretty.Pretty(jsonData))) - //logrus.Println(sd) - return sd, nil + timestamp := gjson.Get(string(jsonData), "locationTimestamp_since1970").Float() + lat := gjson.Get(string(jsonData), "locationLatitude").Float() + lon := gjson.Get(string(jsonData), "locationLongitude").Float() + alt := gjson.Get(string(jsonData), "locationAltitude").Float() + pitch := gjson.Get(string(jsonData), "motionPitch").Float() * 180 / math.Pi + roll := gjson.Get(string(jsonData), "motionRoll").Float() * 180 / math.Pi + yaw := gjson.Get(string(jsonData), "motionYaw").Float() * 180 / math.Pi + sd := &sensorData{ + ServerTime: time.Now(), + source: SOURCE_TCP, + Timestamp: int64(timestamp * float64(time.Second)), + Position: [3]float64{lat, lon, alt}, + Orientation: [3]float64{pitch, roll, yaw}, + //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), + } + //logrus.Println(string(pretty.Pretty(jsonData))) + //logrus.Println(sd) + return sd, nil } func convertAndroidHyperImu(jsonData []byte) (*sensorData, error) { - timestamp := gjson.Get(string(jsonData), "Timestamp").Int() - lat := gjson.Get(string(jsonData), "GPS.0").Float() - lon := gjson.Get(string(jsonData), "GPS.1").Float() - alt := gjson.Get(string(jsonData), "GPS.2").Float() - pitch := gjson.Get(string(jsonData), "orientation.0").Float() - roll := gjson.Get(string(jsonData), "orientation.1").Float() - yaw := gjson.Get(string(jsonData), "orientation.2").Float() + timestamp := gjson.Get(string(jsonData), "Timestamp").Int() + lat := gjson.Get(string(jsonData), "GPS.0").Float() + lon := gjson.Get(string(jsonData), "GPS.1").Float() + alt := gjson.Get(string(jsonData), "GPS.2").Float() + pitch := gjson.Get(string(jsonData), "orientation.0").Float() + roll := gjson.Get(string(jsonData), "orientation.1").Float() + yaw := gjson.Get(string(jsonData), "orientation.2").Float() - sd := &sensorData{ - ServerTime: time.Now(), - source: SOURCE_TCP, - Timestamp: timestamp * int64(time.Millisecond), - Position: [3]float64{lat, lon, alt}, - Orientation: [3]float64{pitch, roll, yaw}, - //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), - } - return sd, nil + sd := &sensorData{ + ServerTime: time.Now(), + source: SOURCE_TCP, + Timestamp: timestamp * int64(time.Millisecond), + Position: [3]float64{lat, lon, alt}, + Orientation: [3]float64{pitch, roll, yaw}, + //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), + } + return sd, nil } diff --git a/core/service.go b/core/service.go index cbb6bb1..d67c702 100644 --- a/core/service.go +++ b/core/service.go @@ -1,104 +1,149 @@ package core import ( - "github.com/google/uuid" - "github.com/sirupsen/logrus" - "time" + "github.com/google/uuid" + "github.com/sirupsen/logrus" + "time" ) type OpMode uint8 const ( - STOPPED OpMode = iota - LIVE - REPLAY + STOPPED OpMode = iota + LIVE + RECORDING + REPLAY ) type trackingService struct { - current *Tracking - config *Configuration - pipe *pipeline - repo Repo - opMode OpMode - collectors []Collector + current *Tracking + config *Configuration + pipe *pipeline + repo Repo + opMode OpMode + collectors []Collector } func TrackingService(r Repo, d Publisher, c *Configuration) *trackingService { - t := &Tracking{} - return &trackingService{ - current: t, - opMode: STOPPED, - config: c, - repo: r, - pipe: NewPipeline(d, t, c), - collectors: nil, - } + t := &Tracking{} + ts := &trackingService{ + current: t, + opMode: STOPPED, + config: c, + repo: r, + pipe: NewPipeline(d, t, c), + collectors: nil, + } + //ts.pipe.Run() + return ts } +//const( +// errA error = errors.New("A") +//) + func (t *trackingService) AllTrackings() { - panic("implement me") + panic("implement me") } -func (t *trackingService) NewTracking(cols ...CollectorType) { - logrus.Debug("new tracking:", cols) - t.opMode = LIVE - t.collectors = nil - for _, col := range cols { - t.collectors = append(t.collectors, NewCollector(col, t.pipe, t.config)) - } - *t.current = emptyTracking() - t.current.collectors = cols - for _, e := range t.collectors { - e.Collect() - } - t.pipe.Run() - +func (t *trackingService) NewSetup(cols ...CollectorType) { + logrus.Info("SERVICE: NEW SETUP") + if t.opMode == RECORDING { + logrus.Println("trackingservice: no reset while recording") + return + } + if t.opMode == LIVE { + logrus.Println("trackingservice: stop currently running setup before creating new one") + t.StopAll() + } + logrus.Debug("new tracking:", cols) + t.opMode = LIVE + t.collectors = nil + for _, col := range cols { + t.collectors = append(t.collectors, NewCollector(col, t.pipe, t.config)) + } + t.safelyReplaceTracking(emptyTracking()) + t.current.Collectors = cols + for _, e := range t.collectors { + e.Collect() + } + t.pipe.Run() + //time.Sleep(3 * time.Second) } func (t *trackingService) StartRecord() { - if t.opMode != LIVE { - logrus.Println("trackingservice: wrong mode of operation") - } - t.current.TimeCreated = time.Now() - t.pipe.Record() + logrus.Info("SERVICE: START RECORD") + if t.opMode != LIVE { + logrus.Println("trackingservice: wrong mode of operation") + return + } + t.opMode = RECORDING + t.current.TimeCreated = time.Now() + t.pipe.Record() } func (t *trackingService) StopRecord() { - if t.opMode != LIVE { - logrus.Println("trackingservice: wrong mode of operation") - } - t.pipe.Stop() - for _, e := range t.collectors { - e.Stop() - } - err := t.repo.Save(*t.current) - if err != nil { - logrus.Println(err) - } - t.NewTracking(t.current.collectors...) + logrus.Info("SERVICE: STOP RECORD") + if t.opMode != RECORDING { + logrus.Println("trackingservice: couldn't stop. not recording") + return + } + t.opMode = LIVE + t.pipe.StopRecord() + + m1.Lock() + m2.Lock() + err := t.repo.Save(*t.current) + m2.Unlock() + m1.Unlock() + + if err != nil { + logrus.Println(err) + } + t.safelyReplaceTracking(emptyTracking()) } -func (t *trackingService) Reset() { - t.opMode = STOPPED - t.collectors = nil +func (t *trackingService) StopAll() { + logrus.Info("SERVICE: STOP ALL") + if t.opMode == RECORDING { + logrus.Println("trackingservice: stop recording gracefully") + t.StopRecord() + } + t.opMode = STOPPED + t.pipe.Close() + for _, e := range t.collectors { + e.Close() + } + t.collectors = nil + t.safelyReplaceTracking(emptyTracking()) } func (t *trackingService) DeleteTracking(trackingId uuid.UUID) { - panic("implement me") + panic("implement me") } func (t *trackingService) StartReplay() { - panic("implement me") + panic("implement me") } func (t *trackingService) PauseReplay() { - panic("implement me") + panic("implement me") } func (t *trackingService) StopReplay() { - panic("implement me") + panic("implement me") } func (t *trackingService) LoadTracking(trackingId uuid.UUID) { } + + +func (t *trackingService) safelyReplaceTracking(tr Tracking) { + m1.Lock() + m2.Lock() + *t.current = tr + m2.Unlock() + m1.Unlock() +} + diff --git a/core/trackings.go b/core/trackings.go index beb1f98..12660de 100644 --- a/core/trackings.go +++ b/core/trackings.go @@ -1,60 +1,62 @@ package core import ( - "github.com/google/uuid" - "github.com/sirupsen/logrus" - "sync" - "time" + "github.com/google/uuid" + "github.com/sirupsen/logrus" + "sync" + "time" ) +var m1 sync.RWMutex +var m2 sync.RWMutex + type Tracking struct { - TrackingMetadata - Records []recordPair - Rawdata []rawRecord - mu sync.Mutex + TrackingMetadata + Records []recordPair + Rawdata []rawRecord } type TrackingMetadata struct { - UUID uuid.UUID - TimeCreated time.Time - collectors []CollectorType + UUID uuid.UUID + TimeCreated time.Time + Collectors []CollectorType } func (s *Tracking) EnqueuePair(tcp sensorData, ser sensorData) { - s.mu.Lock() - defer s.mu.Unlock() - rp := recordPair{ - RecordTime: time.Now(), - data: map[sourceId]sensorData{ - tcp.source: tcp, - ser.source: ser, - }, - } - s.Records = append(s.Records, rp) - logrus.Debugln("tracking Records: len->", len(s.Records)) + rp := recordPair{ + RecordTime: time.Now(), + data: map[sourceId]sensorData{ + tcp.source: tcp, + ser.source: ser, + }, + } + m1.Lock() + s.Records = append(s.Records, rp) + logrus.Debugln("tracking Records: len->", len(s.Records)) + m1.Unlock() } func (s *Tracking) EnqueueRaw(data sensorData) { - s.mu.Lock() - defer s.mu.Unlock() - sr := rawRecord{ - time.Now(), - data, - } - s.Rawdata = append(s.Rawdata, sr) - logrus.Debugln("raw data points: len->", len(s.Records)) + sr := rawRecord{ + time.Now(), + data, + } + m1.Lock() + s.Rawdata = append(s.Rawdata, sr) + logrus.Debugln("raw data points: len->", len(s.Rawdata)) + m1.Unlock() } func emptyTracking() Tracking { - return Tracking{ - TrackingMetadata: TrackingMetadata{ - UUID: uuid.New(), - }, - Records: []recordPair{}, - Rawdata: []rawRecord{}, - } + return Tracking{ + TrackingMetadata: TrackingMetadata{ + UUID: uuid.New(), + }, + Records: []recordPair{}, + Rawdata: []rawRecord{}, + } } func (s *Tracking) isEmpty() bool { - return len(s.Rawdata)+len(s.Records) == 0 + return len(s.Rawdata)+len(s.Records) == 0 } diff --git a/go.mod b/go.mod index 67522e3..4d61c0a 100644 --- a/go.mod +++ b/go.mod @@ -14,4 +14,5 @@ require ( github.com/tidwall/gjson v1.6.0 github.com/tidwall/pretty v1.0.2 // indirect go.bug.st/serial v1.1.1 + golang.org/x/sync v0.0.0-20190423024810-112230192c58 ) diff --git a/go.sum b/go.sum index 9f012a5..ac10ab6 100644 --- a/go.sum +++ b/go.sum @@ -304,6 +304,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/storage/kvstore.go b/storage/kvstore.go index c1fcae8..4be90ae 100644 --- a/storage/kvstore.go +++ b/storage/kvstore.go @@ -1,33 +1,197 @@ package storage import ( - "git.timovolkmann.de/gyrogpsc/core" - "github.com/dgraph-io/badger/v2" - "github.com/google/uuid" - "github.com/sirupsen/logrus" + "encoding/binary" + "encoding/json" + "git.timovolkmann.de/gyrogpsc/core" + "github.com/dgraph-io/badger/v2" + "github.com/google/uuid" + "github.com/sirupsen/logrus" + "os" + "path/filepath" ) // Must implement Repo type badgerStore struct { - db *badger.DB + trackings *badger.DB + records *badger.DB + rawdata *badger.DB } func NewRepository(c *core.Configuration) *badgerStore { - db, err := badger.Open(badger.DefaultOptions(".")) - if err != nil { - logrus.Warn(err) - } - return &badgerStore{db} + dir, _ := os.Getwd() + logrus.Debug(dir) + if _, err := os.Stat(filepath.Join(dir,"_db")); os.IsNotExist(err) { + os.Mkdir(filepath.Join(dir,"_db"), os.ModePerm) + } + + tr, err := badger.Open(badger.DefaultOptions("_db/trackings")) + dp, err := badger.Open(badger.DefaultOptions("_db/records")) + rd, err := badger.Open(badger.DefaultOptions("_db/raw")) + + if err != nil { + logrus.Error(err) + } + return &badgerStore{trackings: tr, records: dp, rawdata: rd} } -func (r *badgerStore) Save(tracking core.Tracking) error { - panic("implement me") +func (r *badgerStore) isDbAvailable() bool { + return r.trackings.IsClosed() || r.records.IsClosed() || r.rawdata.IsClosed() } +func (r *badgerStore) Save(tr core.Tracking) error { + if ok := r.isDbAvailable(); ok { + logrus.Error("unable to write to database. database closed!") + return badger.ErrDBClosed + } + ts, err := tr.TimeCreated.MarshalText() + if err != nil { + logrus.Error(err, tr) + } + logrus.Info("save tracking:", tr.TimeCreated) + meta, err := json.Marshal(tr.TrackingMetadata) + if err != nil { + logrus.Error(err, tr) + return err + } + err = r.records.Update(func(txn *badger.Txn) error { + for _, v := range tr.Records { + k := createDataKey(tr.UUID, v.RecordTime.UnixNano()) + j, err := json.Marshal(v) + if err != nil { + return err + } + txn.Set(k, j) + } + return nil + }) + if err != nil { + logrus.Error(err, tr) + return err + } + err = r.records.Update(func(txn *badger.Txn) error { + for _, v := range tr.Rawdata { + k := createDataKey(tr.UUID, v.Timestamp) + j, err := json.Marshal(v) + if err != nil { + return err + } + txn.Set(k, j) + } + return nil + }) + if err != nil { + logrus.Error(err, tr) + return err + } + err = r.trackings.Update(func(txn *badger.Txn) error { + err := txn.Set(ts, meta) + return err + }) + if err != nil { + logrus.Error(err, tr) + return err + } + logrus.Info("sucessfully saved tracking") + return nil +} + +//func (r *badgerStore) Save(tracking *core.Tracking) error { +// ts, err := tracking.TimeCreated.MarshalText() +// if err != nil { +// logrus.Error(err, tracking) +// } +// logrus.Info("save tracking:", ts) +// meta, err := json.Marshal(tracking.TrackingMetadata) +// if err != nil { +// logrus.Error(err, tracking) +// return err +// } +// wg := sync.WaitGroup{} +// wg.Add(3) +// ch := make(chan error, 3) +// go func() { +// defer wg.Done() +// err = r.records.Update(func(txn *badger.Txn) error { +// for _, v := range tracking.Records { +// k := createDataKey(tracking.UUID, v.RecordTime.UnixNano()) +// j, err := json.Marshal(v) +// if err != nil { +// return err +// } +// txn.Set(k, j) +// } +// return nil +// }) +// ch <- err +// }() +// go func() { +// defer wg.Done() +// err = r.records.Update(func(txn *badger.Txn) error { +// for _, v := range tracking.Rawdata { +// k := createDataKey(tracking.UUID, v.Timestamp) +// j, err := json.Marshal(v) +// if err != nil { +// return err +// } +// txn.Set(k, j) +// } +// return nil +// }) +// ch <- err +// }() +// go func() { +// defer wg.Done() +// err = r.trackings.Update(func(txn *badger.Txn) error { +// err := txn.Set(ts, meta) +// return err +// }) +// ch <- err +// }() +// wg.Wait() +// for { +// select { +// case err := <-ch: +// if err != nil { +// logrus.Error(err, tracking) +// return err +// } +// default: +// close(ch) +// break +// } +// } +// return nil +//} + func (r *badgerStore) LoadAll() ([]core.TrackingMetadata, error) { - panic("implement me") + panic("implement me") } func (r *badgerStore) Load(id uuid.UUID) (core.Tracking, error) { - panic("implement me") + panic("implement me") +} + +func createDataKey(uid uuid.UUID, timestamp int64) []byte { + prefix, err := uid.MarshalText() + if err != nil || timestamp < 0 { + logrus.Error("unable to create key", err) + } + suffix := make([]byte, 8) + binary.LittleEndian.PutUint64(suffix, uint64(timestamp)) + return append(prefix, suffix...) +} + +func unmarshalDataKey(key []byte) (uuid.UUID, int64) { + if len(key) != 24 { + panic("corrupted key") + } + prefix := key[0:15] + suffix := key[15:24] + uid, err := uuid.FromBytes(prefix) + if err != nil { + panic("corrupted key") + } + timestamp := int64(binary.LittleEndian.Uint64(suffix)) + return uid, timestamp } diff --git a/ublox/decode.go b/ublox/decode.go index 3e0ccab..5e98eba 100644 --- a/ublox/decode.go +++ b/ublox/decode.go @@ -6,152 +6,152 @@ package ublox import ( - "bufio" - "bytes" - "encoding/binary" - "errors" - "fmt" - "io" + "bufio" + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" ) // A decoder scans an io stream into UBX (0xB5-0x62 separated) or NMEA ("$xxx,,,,*FF\r\n") frames. // If you have an unmixed stream of NMEA-only data you can use nmea.Decode() on bufio.Scanner.Bytes() directly. type decoder struct { - s *bufio.Scanner + s *bufio.Scanner } // NewDecoder creates a new bufio Scanner with a splitfunc that can handle both UBX and NMEA frames. func NewDecoder(r io.Reader) *decoder { - d := bufio.NewScanner(r) - d.Split(splitFunc) - return &decoder{s: d} + d := bufio.NewScanner(r) + d.Split(splitFunc) + return &decoder{s: d} } // Assume we're either at the start of an NMEA sentence or at the start of a UBX message // if not, skip to the first $ or UBX SOM. func splitFunc(data []byte, atEOF bool) (advance int, token []byte, err error) { - if len(data) == 0 { - return 0, nil, nil - } + if len(data) == 0 { + return 0, nil, nil + } - switch data[0] { - case '$': - return bufio.ScanLines(data, atEOF) + switch data[0] { + case '$': + return bufio.ScanLines(data, atEOF) - case 0xB5: - if len(data) < 8 { - if atEOF { - return len(data), nil, io.ErrUnexpectedEOF - } - return 0, nil, nil - } + case 0xB5: + if len(data) < 8 { + if atEOF { + return len(data), nil, io.ErrUnexpectedEOF + } + return 0, nil, nil + } - sz := 8 + int(data[4]) + int(data[5])*256 - if data[1] == 0x62 { - if sz <= len(data) { - return sz, data[:sz], nil - } - if sz <= bufio.MaxScanTokenSize { - return 0, nil, nil - } - } - } + sz := 8 + int(data[4]) + int(data[5])*256 + if data[1] == 0x62 { + if sz <= len(data) { + return sz, data[:sz], nil + } + if sz <= bufio.MaxScanTokenSize { + return 0, nil, nil + } + } + } - // resync to SOM or $ - data = data[1:] - i1 := bytes.IndexByte(data, '$') - if i1 < 0 { - i1 = len(data) - } + // resync to SOM or $ + data = data[1:] + i1 := bytes.IndexByte(data, '$') + if i1 < 0 { + i1 = len(data) + } - i2 := bytes.IndexByte(data, 0xB5) - if i2 < 0 { - i2 = len(data) - } - if i1 > i2 { - i1 = i2 - } - return 1 + i1, nil, nil + i2 := bytes.IndexByte(data, 0xB5) + if i2 < 0 { + i2 = len(data) + } + if i1 > i2 { + i1 = i2 + } + return 1 + i1, nil, nil } // Decode reads on NMEA or UBX frame and calls decodeUbx accordingly to parse the message, while skipping NMEA. func (d *decoder) Decode() (msg interface{}, err error) { - if !d.s.Scan() { - if err = d.s.Err(); err == nil { - err = io.EOF - } - return nil, err - } + if !d.s.Scan() { + if err = d.s.Err(); err == nil { + err = io.EOF + } + return nil, err + } - switch d.s.Bytes()[0] { - case '$': - return nil, errors.New("NMEA not implemented") - //return nmea.Decode(d.s.Bytes()) - case 0xB5: - return decodeUbx(d.s.Bytes()) - } - panic("impossible frame") + switch d.s.Bytes()[0] { + case '$': + return nil, errors.New("NMEA not implemented") + //return nmea.Decode(d.s.Bytes()) + case 0xB5: + return decodeUbx(d.s.Bytes()) + } + panic("impossible frame") } var ( - errInvalidFrame = errors.New("invalid UBX frame") - errInvalidChkSum = errors.New("invalid UBX checksum") + errInvalidFrame = errors.New("invalid UBX frame") + errInvalidChkSum = errors.New("invalid UBX checksum") ) func decodeUbx(frame []byte) (msg Message, err error) { - buf := bytes.NewReader(frame) + buf := bytes.NewReader(frame) - var header struct { - Preamble uint16 - ClassID uint16 - Length uint16 - } + var header struct { + Preamble uint16 + ClassID uint16 + Length uint16 + } - if err := binary.Read(buf, binary.LittleEndian, &header); err != nil { - return nil, err - } + if err := binary.Read(buf, binary.LittleEndian, &header); err != nil { + return nil, err + } - if header.Preamble != 0x62B5 { - return nil, errInvalidFrame - } + if header.Preamble != 0x62B5 { + return nil, errInvalidFrame + } - if buf.Len()+2 < int(header.Length) { - return nil, io.ErrShortBuffer - } + if buf.Len()+2 < int(header.Length) { + return nil, io.ErrShortBuffer + } - var a, b byte - for _, v := range frame[2 : header.Length+6] { - a += byte(v) - b += a - } + var a, b byte + for _, v := range frame[2 : header.Length+6] { + a += byte(v) + b += a + } - if frame[header.Length+6] != a || frame[header.Length+7] != b { - return nil, errInvalidChkSum - } + if frame[header.Length+6] != a || frame[header.Length+7] != b { + return nil, errInvalidChkSum + } - switch header.ClassID { - case 0x0105: // ACK-ACK - fmt.Println("ACK-ACK not implemented") - //msg = &AckAck{} - case 0x0005: // ACK-NAK - fmt.Println("ACK-NAK not implemented") - //msg = &AckNak{} - case 0x0701: // NAV-PVT - msg = &NavPvt{} - case 0x0028: // HNR-PVT - msg = &HnrPvt{} - case 0x0501: // NAV-ATT - msg = &NavAtt{} - default: - } - if msg != nil { - err = binary.Read(buf, binary.LittleEndian, msg) - } else { - msg = &RawMessage{classID: header.ClassID, Data: append([]byte(nil), frame[6:len(frame)-2]...)} - } - //fmt.Println(msg) + switch header.ClassID { + case 0x0105: // ACK-ACK + fmt.Println("ACK-ACK not implemented") + //msg = &AckAck{} + case 0x0005: // ACK-NAK + fmt.Println("ACK-NAK not implemented") + //msg = &AckNak{} + case 0x0701: // NAV-PVT + msg = &NavPvt{} + case 0x0028: // HNR-PVT + msg = &HnrPvt{} + case 0x0501: // NAV-ATT + msg = &NavAtt{} + default: + } + if msg != nil { + err = binary.Read(buf, binary.LittleEndian, msg) + } else { + msg = &RawMessage{classID: header.ClassID, Data: append([]byte(nil), frame[6:len(frame)-2]...)} + } + //fmt.Println(msg) - return msg, err + return msg, err } diff --git a/ublox/messages.go b/ublox/messages.go index 610f7b2..fbebb09 100644 --- a/ublox/messages.go +++ b/ublox/messages.go @@ -1,7 +1,7 @@ package ublox type Message interface { - ClassID() uint16 + ClassID() uint16 } //type UbxMessage interface { @@ -11,90 +11,90 @@ type Message interface { //} type RawMessage struct { - classID uint16 - Data []byte + classID uint16 + Data []byte } func (msg *RawMessage) ClassID() uint16 { return msg.classID } type NavPvt struct { - ITOW_ms uint32 // - GPS time of week of the navigation epoch. See the description of iTOW for details. - Year_y uint16 // - Year (UTC) - Month_month byte // - Month, range 1..12 (UTC) - Day_d byte // - Day of month, range 1..31 (UTC) - Hour_h byte // - Hour of day, range 0..23 (UTC) - Min_min byte // - Minute of hour, range 0..59 (UTC) - Sec_s byte // - Seconds of minute, range 0..60 (UTC) - Valid NavPVTValid // - Validity flags (see graphic below) - TAcc_ns uint32 // - Time accuracy estimate (UTC) - Nano_ns int32 // - Fraction of second, range -1e9 .. 1e9 (UTC) - FixType NavPVTFixType // - GNSSfix Type - Flags NavPVTFlags // - Fix status flags (see graphic below) - Flags2 NavPVTFlags2 // - Additional flags (see graphic below) - NumSV byte // - Number of satellites used in Nav Solution - Lon_dege7 int32 // 1e-7 Longitude - Lat_dege7 int32 // 1e-7 Latitude - Height_mm int32 // - Height above ellipsoid - HMSL_mm int32 // - Height above mean sea level - HAcc_mm uint32 // - Horizontal accuracy estimate - VAcc_mm uint32 // - Vertical accuracy estimate - VelN_mm_s int32 // - NED north velocity - VelE_mm_s int32 // - NED east velocity - VelD_mm_s int32 // - NED down velocity - GSpeed_mm_s int32 // - Ground Speed (2-D) - HeadMot_dege5 int32 // 1e-5 Heading of motion (2-D) - SAcc_mm_s uint32 // - Speed accuracy estimate - HeadAcc_dege5 uint32 // 1e-5 Heading accuracy estimate (both motion and vehicle) - PDOPe2 uint16 // 0.01 Position DOP - Flags3 NavPVTFlags3 // - Additional flags (see graphic below) - Reserved1 [5]byte // - Reserved - HeadVeh_dege5 int32 // 1e-5 Heading of vehicle (2-D), this is only valid when headVehValid is set, otherwise the output is set to the heading of motion - MagDec_dege2 int16 // 1e-2 Magnetic declination. Only supported in ADR 4.10 and later. - MagAcc_deg2e uint16 // 1e-2 Magnetic declination accuracy. Only supported in ADR 4.10 and later. + ITOW_ms uint32 // - GPS time of week of the navigation epoch. See the description of iTOW for details. + Year_y uint16 // - Year (UTC) + Month_month byte // - Month, range 1..12 (UTC) + Day_d byte // - Day of month, range 1..31 (UTC) + Hour_h byte // - Hour of day, range 0..23 (UTC) + Min_min byte // - Minute of hour, range 0..59 (UTC) + Sec_s byte // - Seconds of minute, range 0..60 (UTC) + Valid NavPVTValid // - Validity flags (see graphic below) + TAcc_ns uint32 // - Time accuracy estimate (UTC) + Nano_ns int32 // - Fraction of second, range -1e9 .. 1e9 (UTC) + FixType NavPVTFixType // - GNSSfix Type + Flags NavPVTFlags // - Fix status flags (see graphic below) + Flags2 NavPVTFlags2 // - Additional flags (see graphic below) + NumSV byte // - Number of satellites used in Nav Solution + Lon_dege7 int32 // 1e-7 Longitude + Lat_dege7 int32 // 1e-7 Latitude + Height_mm int32 // - Height above ellipsoid + HMSL_mm int32 // - Height above mean sea level + HAcc_mm uint32 // - Horizontal accuracy estimate + VAcc_mm uint32 // - Vertical accuracy estimate + VelN_mm_s int32 // - NED north velocity + VelE_mm_s int32 // - NED east velocity + VelD_mm_s int32 // - NED down velocity + GSpeed_mm_s int32 // - Ground Speed (2-D) + HeadMot_dege5 int32 // 1e-5 Heading of motion (2-D) + SAcc_mm_s uint32 // - Speed accuracy estimate + HeadAcc_dege5 uint32 // 1e-5 Heading accuracy estimate (both motion and vehicle) + PDOPe2 uint16 // 0.01 Position DOP + Flags3 NavPVTFlags3 // - Additional flags (see graphic below) + Reserved1 [5]byte // - Reserved + HeadVeh_dege5 int32 // 1e-5 Heading of vehicle (2-D), this is only valid when headVehValid is set, otherwise the output is set to the heading of motion + MagDec_dege2 int16 // 1e-2 Magnetic declination. Only supported in ADR 4.10 and later. + MagAcc_deg2e uint16 // 1e-2 Magnetic declination accuracy. Only supported in ADR 4.10 and later. } func (NavPvt) ClassID() uint16 { return 0x0701 } type HnrPvt struct { - ITOW_ms uint32 // - GPS time of week of the navigation epoch. See the description of iTOW for details. - Year_y uint16 // - Year (UTC) - Month_month byte // - Month, range 1..12 (UTC) - Day_d byte // - Day of month, range 1..31 (UTC) - Hour_h byte // - Hour of day, range 0..23 (UTC) - Min_min byte // - Minute of hour, range 0..59 (UTC) - Sec_s byte // - Seconds of minute, range 0..60 (UTC) - Valid byte // - Validity flags (see graphic below) - Nano_ns int32 // - Fraction of second, range -1e9 .. 1e9 (UTC) - FixType byte // - GNSSfix Type - Flags byte // - Fix status flags (see graphic below) - Reserved [2]byte - Lon_dege7 int32 // 1e-7 Longitude - Lat_dege7 int32 // 1e-7 Latitude - Height_mm int32 // - Height above ellipsoid - HMSL_mm int32 // - Height above mean sea level - GSpeed_mm_s int32 // - Ground Speed (2-D) - Speed_mm_s int32 // Speed (3-D) - HeadMot_dege5 int32 // 1e-5 Heading of motion (2-D) - HeadVeh_dege5 int32 // 1e-5 Heading of vehicle (2-D), this is only valid when headVehValid is set, otherwise the output is set to the heading of motion - HAcc uint32 // 1e-5 Heading accuracy estimate (both motion and vehicle) - VAcc uint32 // 1e-5 Heading accuracy estimate (both motion and vehicle) - SAcc uint32 // 1e-5 Heading accuracy estimate (both motion and vehicle) - HeadAcc_dege5 uint32 // 1e-5 Heading accuracy estimate (both motion and vehicle) - Reserved1 [4]byte // - Reserved + ITOW_ms uint32 // - GPS time of week of the navigation epoch. See the description of iTOW for details. + Year_y uint16 // - Year (UTC) + Month_month byte // - Month, range 1..12 (UTC) + Day_d byte // - Day of month, range 1..31 (UTC) + Hour_h byte // - Hour of day, range 0..23 (UTC) + Min_min byte // - Minute of hour, range 0..59 (UTC) + Sec_s byte // - Seconds of minute, range 0..60 (UTC) + Valid byte // - Validity flags (see graphic below) + Nano_ns int32 // - Fraction of second, range -1e9 .. 1e9 (UTC) + FixType byte // - GNSSfix Type + Flags byte // - Fix status flags (see graphic below) + Reserved [2]byte + Lon_dege7 int32 // 1e-7 Longitude + Lat_dege7 int32 // 1e-7 Latitude + Height_mm int32 // - Height above ellipsoid + HMSL_mm int32 // - Height above mean sea level + GSpeed_mm_s int32 // - Ground Speed (2-D) + Speed_mm_s int32 // Speed (3-D) + HeadMot_dege5 int32 // 1e-5 Heading of motion (2-D) + HeadVeh_dege5 int32 // 1e-5 Heading of vehicle (2-D), this is only valid when headVehValid is set, otherwise the output is set to the heading of motion + HAcc uint32 // 1e-5 Heading accuracy estimate (both motion and vehicle) + VAcc uint32 // 1e-5 Heading accuracy estimate (both motion and vehicle) + SAcc uint32 // 1e-5 Heading accuracy estimate (both motion and vehicle) + HeadAcc_dege5 uint32 // 1e-5 Heading accuracy estimate (both motion and vehicle) + Reserved1 [4]byte // - Reserved } func (HnrPvt) ClassID() uint16 { return 0x0028 } type NavAtt struct { - ITOW_ms uint32 // - GPS time of week of the navigation epoch. See the description of iTOW for details. - Version byte - Reserved1 [3]byte - Roll_deg int32 - Pitch_deg int32 - Heading_deg int32 - AccRoll_deg uint32 - AccPitch_deg uint32 - AccHeading_deg uint32 + ITOW_ms uint32 // - GPS time of week of the navigation epoch. See the description of iTOW for details. + Version byte + Reserved1 [3]byte + Roll_deg int32 + Pitch_deg int32 + Heading_deg int32 + AccRoll_deg uint32 + AccPitch_deg uint32 + AccHeading_deg uint32 } func (NavAtt) ClassID() uint16 { return 0x0501 } @@ -104,43 +104,43 @@ func (NavAtt) ClassID() uint16 { return 0x0501 } type NavPVTFixType byte const ( - NavPVTNoFix NavPVTFixType = iota - NavPVTDeadReckoning - NavPVTFix2D - NavPVTFix3D - NavPVTGNSS - NavPVTTimeOnly + NavPVTNoFix NavPVTFixType = iota + NavPVTDeadReckoning + NavPVTFix2D + NavPVTFix3D + NavPVTGNSS + NavPVTTimeOnly ) type NavPVTValid byte const ( - NavPVTValidDate NavPVTValid = (1 << iota) // valid UTC Date (see Time Validity section for details) - NavPVTValidTime // valid UTC time of day (see Time Validity section for details) - NavPVTFullyResolved // UTC time of day has been fully resolved (no seconds uncertainty). Cannot be used to check if time is completely solved. - NavPVTValidMag // valid magnetic declination + NavPVTValidDate NavPVTValid = (1 << iota) // valid UTC Date (see Time Validity section for details) + NavPVTValidTime // valid UTC time of day (see Time Validity section for details) + NavPVTFullyResolved // UTC time of day has been fully resolved (no seconds uncertainty). Cannot be used to check if time is completely solved. + NavPVTValidMag // valid magnetic declination ) type NavPVTFlags byte const ( - NavPVTGnssFixOK NavPVTFlags = 1 << 0 // valid fix (i.e within DOP & accuracy masks) - NavPVTDiffSoln NavPVTFlags = 1 << 1 // differential corrections were applied - NavPVTHeadVehValid NavPVTFlags = 1 << 5 // heading of vehicle is valid, only set if the receiver is in sensor fusion mode - NavPVTCarrSolnFloat NavPVTFlags = 1 << 6 // carrier phase range solution with floating ambiguities - NavPVTCarrSolnFixed NavPVTFlags = 1 << 7 // carrier phase range solution with fixed ambiguities + NavPVTGnssFixOK NavPVTFlags = 1 << 0 // valid fix (i.e within DOP & accuracy masks) + NavPVTDiffSoln NavPVTFlags = 1 << 1 // differential corrections were applied + NavPVTHeadVehValid NavPVTFlags = 1 << 5 // heading of vehicle is valid, only set if the receiver is in sensor fusion mode + NavPVTCarrSolnFloat NavPVTFlags = 1 << 6 // carrier phase range solution with floating ambiguities + NavPVTCarrSolnFixed NavPVTFlags = 1 << 7 // carrier phase range solution with fixed ambiguities ) type NavPVTFlags2 byte const ( - NavPVTConfirmedAvai NavPVTFlags2 = 1 << 5 // information about UTC Date and Time of Day validity confirmation is available (see Time Validity section for details) - NavPVTConfirmedDate NavPVTFlags2 = 1 << 6 // UTC Date validity could be confirmed (see Time Validity section for details) - NavPVTConfirmedTime NavPVTFlags2 = 1 << 7 // UTC Time of Day could be confirmed (see Time Validity section for details) + NavPVTConfirmedAvai NavPVTFlags2 = 1 << 5 // information about UTC Date and Time of Day validity confirmation is available (see Time Validity section for details) + NavPVTConfirmedDate NavPVTFlags2 = 1 << 6 // UTC Date validity could be confirmed (see Time Validity section for details) + NavPVTConfirmedTime NavPVTFlags2 = 1 << 7 // UTC Time of Day could be confirmed (see Time Validity section for details) ) type NavPVTFlags3 byte const ( - NavPVTInvalidLlh NavPVTFlags3 = (1 << iota) // 1 = Invalid lon, lat, height and hMSL + NavPVTInvalidLlh NavPVTFlags3 = (1 << iota) // 1 = Invalid lon, lat, height and hMSL ) diff --git a/web/http.go b/web/http.go index c68f38f..9478fe4 100644 --- a/web/http.go +++ b/web/http.go @@ -1,85 +1,85 @@ package web import ( - "git.timovolkmann.de/gyrogpsc/core" - "github.com/gofiber/fiber/v2" - "github.com/gofiber/websocket/v2" - "github.com/sirupsen/logrus" - "html/template" + "git.timovolkmann.de/gyrogpsc/core" + "github.com/gofiber/fiber/v2" + "github.com/gofiber/websocket/v2" + "github.com/sirupsen/logrus" + "html/template" ) func CreateServer(s core.Service, sub core.Subscriber, c *core.Configuration) { - app := fiber.New() - app.Static("/static", "static") + app := fiber.New() + app.Static("/static", "static") - // Application Main Page - app.Get("/", fiberHomeHandler) + // Application Main Page + app.Get("/", fiberHomeHandler) - // Websocket - app.Get("/", websocket.New(createFiberWebsocketHandler(sub))) + // Websocket + app.Get("/", websocket.New(createFiberWebsocketHandler(sub))) - // TODO: Get all SerialPorts - // app.Get("/serialports") + // TODO: Get all SerialPorts + // app.Get("/serialports") - // Tracking persistence controls - trackings := app.Group("/trackings") - trackings.Get("/", stubhander()) // Get all trackings Metadata - trackings.Post("/", stubhander()) // Initialize new tracking, open websocket and prepare for automatic recording. Toggle ?serial=true and ?tcp=true. Returns trackingId - trackings.Patch("/", stubhander()) // Starts recording - trackings.Put("/", stubhander()) // Stops current recording. Returns trackingId if record was successful - trackings.Delete("/", stubhander()) // Stops websocket connection, pipelines and collectors + // Tracking persistence controls + trackings := app.Group("/trackings") + trackings.Get("/", stubhander()) // Get all trackings Metadata + trackings.Post("/", stubhander()) // Initialize new tracking, open websocket and prepare for automatic recording. Toggle ?serial=true and ?tcp=true. Returns trackingId + trackings.Patch("/", stubhander()) // Starts recording + trackings.Put("/", stubhander()) // Stops current recording. Returns trackingId if record was successful + trackings.Delete("/", stubhander()) // Stops websocket connection, pipelines and collectors - trackings.Get("/:trackingId", stubhander()) // Gets Tracking Metadata and loads sensorRecords from storage. - trackings.Delete("/:trackingId", stubhander()) // Deletes Tracking from storage + trackings.Get("/:trackingId", stubhander()) // Gets Tracking Metadata and loads sensorRecords from storage. + trackings.Delete("/:trackingId", stubhander()) // Deletes Tracking from storage - trackings.Post("/current", stubhander()) // Starts Replay. - trackings.Patch("/current", stubhander()) // Pauses Replay. - trackings.Put("/current", stubhander()) // Stops Replay. + trackings.Post("/current", stubhander()) // Starts Replay. + trackings.Patch("/current", stubhander()) // Pauses Replay. + trackings.Put("/current", stubhander()) // Stops Replay. - logrus.Fatal(app.Listen(c.Webserver.Port)) + logrus.Fatal(app.Listen(c.Webserver.Port)) } func stubhander() fiber.Handler { - return func(ctx *fiber.Ctx) error { - return nil - } + return func(ctx *fiber.Ctx) error { + return nil + } } func createFiberWebsocketHandler(s core.Subscriber) func(conn *websocket.Conn) { - return func(c *websocket.Conn) { - // Handle and discard inbound messages - go func() { - for { - if _, _, err := c.NextReader(); err != nil { - c.Close() - break - } - } - }() + return func(c *websocket.Conn) { + // Handle and discard inbound messages + go func() { + for { + if _, _, err := c.NextReader(); err != nil { + c.Close() + break + } + } + }() - dispatcherId, channel := s.Subscribe() - defer s.Unsubscribe(dispatcherId) - for { - cmsg := <-channel - err := c.WriteMessage(websocket.TextMessage, []byte(cmsg)) - if err != nil { - logrus.Println("write:", err) + dispatcherId, channel := s.Subscribe() + defer s.Unsubscribe(dispatcherId) + for { + cmsg := <-channel + err := c.WriteMessage(websocket.TextMessage, []byte(cmsg)) + if err != nil { + logrus.Println("write:", err) - break - } - } - } + break + } + } + } } func fiberHomeHandler(c *fiber.Ctx) error { - tpl, err := template.ParseFiles("static/index.html") - if err != nil { - return err - } - err = tpl.Execute(c, "://"+c.Hostname()+"/") - if err != nil { - return err - } - return nil + tpl, err := template.ParseFiles("static/index.html") + if err != nil { + return err + } + err = tpl.Execute(c, "://"+c.Hostname()+"/") + if err != nil { + return err + } + return nil } From 1ea68222029e831316f57de6ca72acde740387a7 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Sat, 12 Dec 2020 03:23:00 +0100 Subject: [PATCH 16/20] implemented automatic persistence for trackings --- cmd/autostart/autostart.go | 102 ++++++------- cmd/server/server.go | 58 +++++--- core/collectors.go | 293 +++++++++++++++++++------------------ core/service.go | 187 ++++++++++++----------- 4 files changed, 332 insertions(+), 308 deletions(-) diff --git a/cmd/autostart/autostart.go b/cmd/autostart/autostart.go index 8be4e32..6d57603 100644 --- a/cmd/autostart/autostart.go +++ b/cmd/autostart/autostart.go @@ -1,66 +1,68 @@ package main import ( - "git.timovolkmann.de/gyrogpsc/core" - "git.timovolkmann.de/gyrogpsc/storage" - "git.timovolkmann.de/gyrogpsc/web" - "github.com/sirupsen/logrus" - "github.com/spf13/viper" - "time" + "git.timovolkmann.de/gyrogpsc/core" + "git.timovolkmann.de/gyrogpsc/storage" + "git.timovolkmann.de/gyrogpsc/web" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "os" + "time" ) func main() { - conf := configurationFromFile() + conf := configurationFromFile() - repo := storage.NewRepository(conf) - disp := core.NewDispatcher() + repo := storage.NewRepository(conf) + disp := core.NewDispatcher() - service := core.TrackingService(repo, disp, conf) + service := core.TrackingService(repo, disp, conf) - go func() { - service.NewSetup(core.TCP, core.SERIAL) - time.Sleep(5 * time.Second) - service.StartRecord() - time.Sleep(5 * time.Second) - service.StopRecord() - time.Sleep(5 * time.Second) - service.NewSetup(core.TCP) - time.Sleep(5 * time.Second) - service.StartRecord() - time.Sleep(5 * time.Second) - service.StopRecord() - time.Sleep(5 * time.Second) - service.StopAll() - }() + go func() { + service.NewSetup(core.TCP, core.SERIAL) + time.Sleep(5 * time.Second) + service.StartRecord() + time.Sleep(5 * time.Second) + service.StopRecord() + time.Sleep(5 * time.Second) + service.NewSetup(core.TCP, core.SERIAL) + time.Sleep(5 * time.Second) + service.StartRecord() + time.Sleep(120 * time.Second) + service.StopRecord() + time.Sleep(5 * time.Second) + service.StopAll() + os.Exit(0) + }() - web.CreateServer(service, disp, conf) + web.CreateServer(service, disp, conf) } func configurationFromFile() *core.Configuration { - viper.SetDefault("collectors.porttcp", ":3010") - viper.SetDefault("collectors.portserial", "/dev/tty.usbmodem14201") - viper.SetDefault("webserver.port", ":3011") - viper.SetDefault("pipeline.publishIntervalMs", 50) - viper.SetDefault("pipeline.syncUpdateIntervalMs", 494) - viper.SetDefault("debuglevel", "INFO") + viper.SetDefault("collectors.porttcp", ":3010") + viper.SetDefault("collectors.portserial", "/dev/tty.usbmodem14201") + viper.SetDefault("webserver.port", ":3011") + viper.SetDefault("pipeline.publishIntervalMs", 50) + viper.SetDefault("pipeline.syncUpdateIntervalMs", 494) + viper.SetDefault("debuglevel", "INFO") - viper.SetConfigName("gpsconfig") // name of config file (without extension) - viper.SetConfigType("yaml") - viper.AddConfigPath(".") - viper.AddConfigPath("./../../") - if err := viper.ReadInConfig(); err != nil { - logrus.Warn("couldn't find config file. using standard configuration") - } + viper.SetConfigName("gpsconfig") // name of config file (without extension) + viper.SetConfigType("yaml") + viper.AddConfigPath(".") + viper.AddConfigPath("./../../") + if err := viper.ReadInConfig(); err != nil { + logrus.Warn("couldn't find config file. using standard configuration") + } - c := core.Configuration{} - if err := viper.Unmarshal(&c); err != nil { - logrus.Debug("couldn't load config...") - logrus.Error(err) - } - lvl, err := logrus.ParseLevel(c.Debuglevel) - if err != nil { - logrus.Error(err) - } - logrus.SetLevel(lvl) - return &c + c := core.Configuration{} + if err := viper.Unmarshal(&c); err != nil { + logrus.Debug("couldn't load config...") + logrus.Error(err) + } + lvl, err := logrus.ParseLevel(c.Debuglevel) + if err != nil { + logrus.Error(err) + } + logrus.SetLevel(lvl) + return &c } diff --git a/cmd/server/server.go b/cmd/server/server.go index b7ebf58..f69458e 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -1,37 +1,49 @@ package main import ( - "git.timovolkmann.de/gyrogpsc/core" - "git.timovolkmann.de/gyrogpsc/storage" - "git.timovolkmann.de/gyrogpsc/web" - "github.com/sirupsen/logrus" - "github.com/spf13/viper" + "git.timovolkmann.de/gyrogpsc/core" + "git.timovolkmann.de/gyrogpsc/storage" + "git.timovolkmann.de/gyrogpsc/web" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" ) func main() { - conf := &core.Configuration{} - configurationFromFile(conf) + conf := configurationFromFile() - repo := storage.NewRepository(conf) - disp := core.NewDispatcher() + repo := storage.NewRepository(conf) + disp := core.NewDispatcher() - service := core.TrackingService(repo, disp, conf) + service := core.TrackingService(repo, disp, conf) - web.CreateServer(service, disp, conf) + web.CreateServer(service, disp, conf) } -func configurationFromFile(c *core.Configuration) { - viper.SetDefault("TcpCollectorPort", ":3010") - viper.SetDefault("SerialCollectorPort", "/dev/tty.usbmodem14201") - viper.SetDefault("HttpPort", "layouts") - viper.SetDefault("publishIntervalMs", 50) - viper.SetDefault("syncUpdateIntervalMs", 494) +func configurationFromFile() *core.Configuration { + viper.SetDefault("collectors.porttcp", ":3010") + viper.SetDefault("collectors.portserial", "/dev/tty.usbmodem14201") + viper.SetDefault("webserver.port", ":3011") + viper.SetDefault("pipeline.publishIntervalMs", 50) + viper.SetDefault("pipeline.syncUpdateIntervalMs", 494) + viper.SetDefault("debuglevel", "INFO") - viper.SetConfigName("gpsconfig") // name of config file (without extension) - viper.SetConfigType("yaml") - viper.AddConfigPath(".") - viper.AddConfigPath("./../../") + viper.SetConfigName("gpsconfig") // name of config file (without extension) + viper.SetConfigType("yaml") + viper.AddConfigPath(".") + viper.AddConfigPath("./../../") + if err := viper.ReadInConfig(); err != nil { + logrus.Warn("couldn't find config file. using standard configuration") + } - viper.Unmarshal(c) - logrus.Println(c) + c := core.Configuration{} + if err := viper.Unmarshal(&c); err != nil { + logrus.Debug("couldn't load config...") + logrus.Error(err) + } + lvl, err := logrus.ParseLevel(c.Debuglevel) + if err != nil { + logrus.Error(err) + } + logrus.SetLevel(lvl) + return &c } diff --git a/core/collectors.go b/core/collectors.go index cddf112..f85be11 100644 --- a/core/collectors.go +++ b/core/collectors.go @@ -1,199 +1,210 @@ package core import ( - "fmt" - "git.timovolkmann.de/gyrogpsc/ublox" - "github.com/sirupsen/logrus" - "go.bug.st/serial" - "net" - "os" - "sync" + "fmt" + "git.timovolkmann.de/gyrogpsc/ublox" + "github.com/sirupsen/logrus" + "go.bug.st/serial" + "net" + "os" + "sync" + "time" ) type Collector interface { - Collect() - Close() + Collect() + Close() } type CollectorType uint8 const ( - SERIAL CollectorType = iota - TCP + SERIAL CollectorType = iota + TCP ) var tcpSingleton *tcpCollector func NewCollector(typ CollectorType, proc Pusher, config *Configuration) Collector { - var coll Collector - switch typ { - case SERIAL: - coll = newSerial(proc, config) - case TCP: - if tcpSingleton == nil { - tcpSingleton = newTcp(proc, config) - } else { - tcpSingleton.SetProcessor(proc) - } - coll = tcpSingleton - default: - panic("selected collector type not implemented") - } - return coll + var coll Collector + switch typ { + case SERIAL: + coll = newSerial(proc, config) + case TCP: + if tcpSingleton == nil { + tcpSingleton = newTcp(proc, config) + } else { + tcpSingleton.SetProcessor(proc) + } + coll = tcpSingleton + default: + panic("selected collector type not implemented") + } + return coll } type serialCollector struct { - active bool - proc Pusher - config *Configuration - mu sync.RWMutex + active bool + proc Pusher + config *Configuration + mu sync.RWMutex } func (s *serialCollector) isSerialCollActive() bool { - s.mu.RLock() - defer s.mu.RUnlock() - return s.active + s.mu.RLock() + defer s.mu.RUnlock() + return s.active } - func (s *serialCollector) Collect() { - s.mu.Lock() - s.active = true - s.mu.Unlock() - go func() { - logrus.Println("start serial collector") - mode := &serial.Mode{ - BaudRate: 115200, - } - port, err := serial.Open(s.config.Collectors.SerialCollectorPort, mode) - if err != nil { - logrus.Fatalln("can't open serial port:", err.Error()) - } - defer port.Close() + s.mu.Lock() + s.active = true + s.mu.Unlock() + go func() { + logrus.Println("start serial collector") + mode := &serial.Mode{ + BaudRate: 115200, + } + port, err := serial.Open(s.config.Collectors.SerialCollectorPort, mode) + if err != nil { + logrus.Warn("can't open serial port:", err) + //if e, ok := err.(serial.PortError); ok && e.Code() == serial.PortBusy { + for i := 3; i < 20; i = i + 2 { + logrus.Warnf("try again in -> %vms", i*i) + time.Sleep(time.Millisecond * time.Duration(i*i)) + port, err = serial.Open(s.config.Collectors.SerialCollectorPort, mode) + if err == nil { + break + } + } + if err != nil { + logrus.Fatal(err) + } + //} + } + defer port.Close() - decoder := ublox.NewDecoder(port) + decoder := ublox.NewDecoder(port) - for s.isSerialCollActive() { - meas, err := decoder.Decode() - if err != nil { - if err.Error() == "NMEA not implemented" { - continue - } - logrus.Println("serial read err:", err) - break - } - sd, err := ConvertUbxToSensorData(meas) - if err != nil { - logrus.Println("convert err:", err, meas, sd) - continue - } - // skip irrelevant messages - if sd == nil { - continue - } + for s.isSerialCollActive() { + meas, err := decoder.Decode() + if err != nil { + if err.Error() == "NMEA not implemented" { + continue + } + logrus.Println("serial read err:", err) + break + } + sd, err := ConvertUbxToSensorData(meas) + if err != nil { + logrus.Println("convert err:", err, meas, sd) + continue + } + // skip irrelevant messages + if sd == nil { + continue + } - err = s.proc.Push(sd) - if err != nil { - logrus.Println("process err:", err, *sd) - continue - } - } - logrus.Println("serial collector stopped") - }() + err = s.proc.Push(sd) + if err != nil { + logrus.Println("process err:", err, *sd) + continue + } + } + logrus.Println("serial collector stopped") + }() } - - func (s *serialCollector) Close() { - s.mu.Lock() - s.active = false - s.mu.Unlock() + s.mu.Lock() + s.active = false + s.mu.Unlock() } func newSerial(proc Pusher, config *Configuration) *serialCollector { - return &serialCollector{ - active: false, - proc: proc, - config: config, - } + return &serialCollector{ + active: false, + proc: proc, + config: config, + } } type tcpCollector struct { - active bool - processor Pusher - //config *Configuration + active bool + processor Pusher + //config *Configuration } func (t *tcpCollector) Collect() { - t.active = true + t.active = true } func (t *tcpCollector) Close() { - t.active = false + t.active = false } func (t *tcpCollector) SetProcessor(p Pusher) { - t.processor = p + t.processor = p } func newTcp(proc Pusher, config *Configuration) *tcpCollector { - logrus.Println("start tcp collector") + logrus.Println("start tcp collector") - listener, err := net.Listen("tcp", config.Collectors.TcpCollectorPort) - if err != nil { - fmt.Println("Error listening:", err) - //os.Exit(1) - } - coll := &tcpCollector{ - active: false, - processor: proc, - } - go func() { - for { - // Listen for an incoming connection. - conn, err := listener.Accept() - if err != nil { - fmt.Println("Error accepting: ", err.Error()) - os.Exit(1) - } - logrus.Println("...new incoming tcp connection...") + listener, err := net.Listen("tcp", config.Collectors.TcpCollectorPort) + if err != nil { + fmt.Println("Error listening:", err) + //os.Exit(1) + } + coll := &tcpCollector{ + active: false, + processor: proc, + } + go func() { + for { + // Listen for an incoming connection. + conn, err := listener.Accept() + if err != nil { + fmt.Println("Error accepting: ", err.Error()) + os.Exit(1) + } + logrus.Println("...new incoming tcp connection...") - // Handle connections in a new goroutine. - go coll.jsonHandler(conn) - } - }() - return coll + // Handle connections in a new goroutine. + go coll.jsonHandler(conn) + } + }() + return coll } // handles incoming tcp connections with json payload. func (c *tcpCollector) jsonHandler(conn net.Conn) { - defer conn.Close() + defer conn.Close() - // TRY reader := bufio.NewReader(conn) OR NewScanner(conn) - buf := make([]byte, 2048) - for { - // Read the incoming connection into the buffer. - n, err := conn.Read(buf) - if err != nil { - fmt.Println("TCP error - reading from connection:", n, err.Error()) - break - } - //json := pretty.Pretty(buf[:n]) - //fmt.Println(string(json)) - //fmt.Println(string(buf[:n])) - sd, err := ConvertSensorDataPhone(buf[:n]) - if err != nil { - logrus.Println(err) - continue - } - if !c.active { - //time.Sleep(50 * time.Millisecond) - continue - } - err = c.processor.Push(sd) - if err != nil { - logrus.Fatalln(err) - } - } + // TRY reader := bufio.NewReader(conn) OR NewScanner(conn) + buf := make([]byte, 2048) + for { + // Read the incoming connection into the buffer. + n, err := conn.Read(buf) + if err != nil { + fmt.Println("TCP error - reading from connection:", n, err.Error()) + break + } + //json := pretty.Pretty(buf[:n]) + //fmt.Println(string(json)) + //fmt.Println(string(buf[:n])) + sd, err := ConvertSensorDataPhone(buf[:n]) + if err != nil { + logrus.Println(err) + continue + } + if !c.active { + //time.Sleep(50 * time.Millisecond) + continue + } + err = c.processor.Push(sd) + if err != nil { + logrus.Fatalln(err) + } + } } diff --git a/core/service.go b/core/service.go index d67c702..d650cd9 100644 --- a/core/service.go +++ b/core/service.go @@ -1,41 +1,41 @@ package core import ( - "github.com/google/uuid" - "github.com/sirupsen/logrus" - "time" + "github.com/google/uuid" + "github.com/sirupsen/logrus" + "time" ) type OpMode uint8 const ( - STOPPED OpMode = iota - LIVE - RECORDING - REPLAY + STOPPED OpMode = iota + LIVE + RECORDING + REPLAY ) type trackingService struct { - current *Tracking - config *Configuration - pipe *pipeline - repo Repo - opMode OpMode - collectors []Collector + current *Tracking + config *Configuration + pipe *pipeline + repo Repo + opMode OpMode + collectors []Collector } func TrackingService(r Repo, d Publisher, c *Configuration) *trackingService { - t := &Tracking{} - ts := &trackingService{ - current: t, - opMode: STOPPED, - config: c, - repo: r, - pipe: NewPipeline(d, t, c), - collectors: nil, - } - //ts.pipe.Run() - return ts + t := &Tracking{} + ts := &trackingService{ + current: t, + opMode: STOPPED, + config: c, + repo: r, + pipe: NewPipeline(d, t, c), + collectors: nil, + } + //ts.pipe.Run() + return ts } //const( @@ -43,107 +43,106 @@ func TrackingService(r Repo, d Publisher, c *Configuration) *trackingService { //) func (t *trackingService) AllTrackings() { - panic("implement me") + panic("implement me") } func (t *trackingService) NewSetup(cols ...CollectorType) { - logrus.Info("SERVICE: NEW SETUP") - if t.opMode == RECORDING { - logrus.Println("trackingservice: no reset while recording") - return - } - if t.opMode == LIVE { - logrus.Println("trackingservice: stop currently running setup before creating new one") - t.StopAll() - } - logrus.Debug("new tracking:", cols) - t.opMode = LIVE - t.collectors = nil - for _, col := range cols { - t.collectors = append(t.collectors, NewCollector(col, t.pipe, t.config)) - } - t.safelyReplaceTracking(emptyTracking()) - t.current.Collectors = cols - for _, e := range t.collectors { - e.Collect() - } - t.pipe.Run() - //time.Sleep(3 * time.Second) + logrus.Info("SERVICE: NEW SETUP") + if t.opMode == RECORDING { + logrus.Println("trackingservice: no reset while recording") + return + } + if t.opMode == LIVE { + logrus.Println("trackingservice: stop currently running setup before creating new one") + t.StopAll() + time.Sleep(20 * time.Millisecond) + } + logrus.Debug("new tracking:", cols) + t.opMode = LIVE + t.collectors = nil + for _, col := range cols { + t.collectors = append(t.collectors, NewCollector(col, t.pipe, t.config)) + } + t.safelyReplaceTracking(emptyTracking()) + t.current.Collectors = cols + for _, e := range t.collectors { + e.Collect() + } + t.pipe.Run() + //time.Sleep(3 * time.Second) } func (t *trackingService) StartRecord() { - logrus.Info("SERVICE: START RECORD") - if t.opMode != LIVE { - logrus.Println("trackingservice: wrong mode of operation") - return - } - t.opMode = RECORDING - t.current.TimeCreated = time.Now() - t.pipe.Record() + logrus.Info("SERVICE: START RECORD") + if t.opMode != LIVE { + logrus.Println("trackingservice: wrong mode of operation") + return + } + t.opMode = RECORDING + t.current.TimeCreated = time.Now() + t.pipe.Record() } func (t *trackingService) StopRecord() { - logrus.Info("SERVICE: STOP RECORD") - if t.opMode != RECORDING { - logrus.Println("trackingservice: couldn't stop. not recording") - return - } - t.opMode = LIVE - t.pipe.StopRecord() + logrus.Info("SERVICE: STOP RECORD") + if t.opMode != RECORDING { + logrus.Println("trackingservice: couldn't stop. not recording") + return + } + t.opMode = LIVE + t.pipe.StopRecord() - m1.Lock() - m2.Lock() - err := t.repo.Save(*t.current) - m2.Unlock() - m1.Unlock() + m1.Lock() + m2.Lock() + err := t.repo.Save(*t.current) + m2.Unlock() + m1.Unlock() - if err != nil { - logrus.Println(err) - } - t.safelyReplaceTracking(emptyTracking()) + if err != nil { + logrus.Println(err) + } + t.safelyReplaceTracking(emptyTracking()) } func (t *trackingService) StopAll() { - logrus.Info("SERVICE: STOP ALL") - if t.opMode == RECORDING { - logrus.Println("trackingservice: stop recording gracefully") - t.StopRecord() - } - t.opMode = STOPPED - t.pipe.Close() - for _, e := range t.collectors { - e.Close() - } - t.collectors = nil - t.safelyReplaceTracking(emptyTracking()) + logrus.Info("SERVICE: STOP ALL") + if t.opMode == RECORDING { + logrus.Println("trackingservice: stop recording gracefully") + t.StopRecord() + } + t.opMode = STOPPED + t.pipe.Close() + for _, e := range t.collectors { + e.Close() + } + t.collectors = nil + t.safelyReplaceTracking(emptyTracking()) } func (t *trackingService) DeleteTracking(trackingId uuid.UUID) { - panic("implement me") + panic("implement me") } func (t *trackingService) StartReplay() { - panic("implement me") + panic("implement me") } func (t *trackingService) PauseReplay() { - panic("implement me") + panic("implement me") } func (t *trackingService) StopReplay() { - panic("implement me") + panic("implement me") } func (t *trackingService) LoadTracking(trackingId uuid.UUID) { } - func (t *trackingService) safelyReplaceTracking(tr Tracking) { - m1.Lock() - m2.Lock() - *t.current = tr - m2.Unlock() - m1.Unlock() + m1.Lock() + m2.Lock() + *t.current = tr + m2.Unlock() + m1.Unlock() } - From 391020ec47fdd829130256735bd1fdd9931398d0 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Sat, 12 Dec 2020 16:34:35 +0100 Subject: [PATCH 17/20] some improvements in logging and naming/structuring --- core/dispatcher.go | 50 ++++----- core/sensordata.go | 250 ++++++++++++++++++++++----------------------- core/service.go | 16 +-- core/trackings.go | 93 +++++++++-------- go.mod | 2 +- storage/kvstore.go | 226 +++++++++++++++++++++------------------- 6 files changed, 329 insertions(+), 308 deletions(-) diff --git a/core/dispatcher.go b/core/dispatcher.go index 72e6429..a705d67 100644 --- a/core/dispatcher.go +++ b/core/dispatcher.go @@ -1,44 +1,44 @@ package core import ( - "errors" - "github.com/sirupsen/logrus" + "errors" + "github.com/sirupsen/logrus" ) type dispatcher struct { - listeners map[int16]chan string - counter int16 + listeners map[int16]chan string + counter int16 } func NewDispatcher() *dispatcher { - return &dispatcher{ - listeners: make(map[int16]chan string), - counter: 0, - } + return &dispatcher{ + listeners: make(map[int16]chan string), + counter: 0, + } } func (d *dispatcher) Publish(message string) { - logrus.Debugf("publish to %v listeners:\n%v\n", len(d.listeners)) - logrus.Debug(message) - for _, ch := range d.listeners { - ch <- message - } + logrus.Debugf("publish to %v listeners\n", len(d.listeners)) + logrus.Debug(message) + for _, ch := range d.listeners { + ch <- message + } } func (d *dispatcher) Subscribe() (id int16, receiver <-chan string) { - key := d.counter - d.counter++ - rec := make(chan string) - d.listeners[key] = rec - return key, rec + key := d.counter + d.counter++ + rec := make(chan string) + d.listeners[key] = rec + return key, rec } func (d *dispatcher) Unsubscribe(id int16) error { - receiver, ok := d.listeners[id] - if !ok { - return errors.New("no subscription with id") - } - delete(d.listeners, id) - close(receiver) - return nil + receiver, ok := d.listeners[id] + if !ok { + return errors.New("no subscription with id") + } + delete(d.listeners, id) + close(receiver) + return nil } diff --git a/core/sensordata.go b/core/sensordata.go index afd1575..1114f37 100644 --- a/core/sensordata.go +++ b/core/sensordata.go @@ -1,174 +1,164 @@ package core import ( - "errors" - "git.timovolkmann.de/gyrogpsc/ublox" - "github.com/sirupsen/logrus" - "github.com/tidwall/gjson" - "math" - "time" + "errors" + "git.timovolkmann.de/gyrogpsc/ublox" + "github.com/sirupsen/logrus" + "github.com/tidwall/gjson" + "math" + "time" ) type sourceId string const ( - SOURCE_TCP sourceId = "SOURCE_TCP" - SOURCE_SERIAL sourceId = "SOURCE_SERIAL" + SOURCE_TCP sourceId = "SOURCE_TCP" + SOURCE_SERIAL sourceId = "SOURCE_SERIAL" ) type sensorData struct { - itow uint32 - source sourceId - ServerTime time.Time - Timestamp int64 - Position [3]float64 - Orientation [3]float64 -} - -type recordPair struct { - RecordTime time.Time - data map[sourceId]sensorData -} - -type rawRecord struct { - RecordTime time.Time - sensorData + itow uint32 + source sourceId + ServerTime time.Time + Timestamp int64 + Position [3]float64 + Orientation [3]float64 } func (s sensorData) isSameEpoch(n sensorData) bool { - if n.itow == 0 { - return false - } - return s.itow == n.itow + if n.itow == 0 { + return false + } + return s.itow == n.itow } // Consolidates two sensordata elements if they are in the same epoch func (s sensorData) ConsolidateEpochsOnly(n sensorData) sensorData { - s.checkSources(&n) - if s.isSameEpoch(n) { - null := sensorData{} + s.checkSources(&n) + if s.isSameEpoch(n) { + null := sensorData{} - if n.Timestamp == null.Timestamp { - n.Timestamp = s.Timestamp - } - if n.Position == null.Position { - n.Position = s.Position - } - if n.Orientation == null.Orientation { - n.Orientation = s.Orientation - } - } - return n + if n.Timestamp == null.Timestamp { + n.Timestamp = s.Timestamp + } + if n.Position == null.Position { + n.Position = s.Position + } + if n.Orientation == null.Orientation { + n.Orientation = s.Orientation + } + } + return n } // Consolidates two sensordata elements but ignores timestamps func (s sensorData) ConsolidateExTime(n sensorData) sensorData { - s.checkSources(&n) - null := sensorData{} + s.checkSources(&n) + null := sensorData{} - if n.Position == null.Position { - n.Position = s.Position - } - if n.Orientation == null.Orientation { - n.Orientation = s.Orientation - } - return n + if n.Position == null.Position { + n.Position = s.Position + } + if n.Orientation == null.Orientation { + n.Orientation = s.Orientation + } + return n } func (s *sensorData) checkSources(n *sensorData) { - if (s.source != n.source && *s != sensorData{}) { - logrus.Println(s) - logrus.Println(n) - logrus.Fatalln("Do not consolidate sensorData from different Sources") - } + if (s.source != n.source && *s != sensorData{}) { + logrus.Println(s) + logrus.Println(n) + logrus.Fatalln("Do not consolidate sensorData from different Sources") + } } var ( - errNotImplemented = errors.New("message not implemented") - errRawMessage = errors.New("raw message") + errNotImplemented = errors.New("message not implemented") + errRawMessage = errors.New("raw message") ) func ConvertUbxToSensorData(msg interface{}) (*sensorData, error) { - sd := &sensorData{ - ServerTime: time.Now(), - source: SOURCE_SERIAL, - } - switch v := msg.(type) { - case *ublox.NavPvt: - //logrus.Println("NAV-PVT") - sd.itow = v.ITOW_ms - sd.Timestamp = time.Date(int(v.Year_y), time.Month(v.Month_month), int(v.Day_d), int(v.Hour_h), int(v.Min_min), int(v.Sec_s), int(v.Nano_ns), time.UTC).UnixNano() - sd.Position[0] = float64(v.Lat_dege7) / 1e+7 - sd.Position[1] = float64(v.Lon_dege7) / 1e+7 - sd.Position[2] = float64(v.HMSL_mm) / 1e+3 // mm in m - case *ublox.HnrPvt: - //logrus.Println("HNR-PVT") - sd.itow = v.ITOW_ms - sd.Timestamp = time.Date(int(v.Year_y), time.Month(v.Month_month), int(v.Day_d), int(v.Hour_h), int(v.Min_min), int(v.Sec_s), int(v.Nano_ns), time.UTC).UnixNano() - sd.Position[0] = float64(v.Lat_dege7) / 1e+7 - sd.Position[1] = float64(v.Lon_dege7) / 1e+7 - sd.Position[2] = float64(v.HMSL_mm) / 1e+3 // mm in m - case *ublox.NavAtt: - //logrus.Println("NAV-ATT") - sd.itow = v.ITOW_ms - sd.Orientation[0] = float64(v.Pitch_deg) * 1e-5 - sd.Orientation[1] = float64(v.Roll_deg) * 1e-5 - sd.Orientation[2] = float64(v.Heading_deg) * 1e-5 - case *ublox.RawMessage: - //class := make([]byte, 2) - //binary.LittleEndian.PutUint16(class, v.ClassID()) - //logrus.Printf("%#v, %#v", class[0],class[1]) - return nil, nil - default: - return nil, errNotImplemented - } - return sd, nil + sd := &sensorData{ + ServerTime: time.Now(), + source: SOURCE_SERIAL, + } + switch v := msg.(type) { + case *ublox.NavPvt: + //logrus.Println("NAV-PVT") + sd.itow = v.ITOW_ms + sd.Timestamp = time.Date(int(v.Year_y), time.Month(v.Month_month), int(v.Day_d), int(v.Hour_h), int(v.Min_min), int(v.Sec_s), int(v.Nano_ns), time.UTC).UnixNano() + sd.Position[0] = float64(v.Lat_dege7) / 1e+7 + sd.Position[1] = float64(v.Lon_dege7) / 1e+7 + sd.Position[2] = float64(v.HMSL_mm) / 1e+3 // mm in m + case *ublox.HnrPvt: + //logrus.Println("HNR-PVT") + sd.itow = v.ITOW_ms + sd.Timestamp = time.Date(int(v.Year_y), time.Month(v.Month_month), int(v.Day_d), int(v.Hour_h), int(v.Min_min), int(v.Sec_s), int(v.Nano_ns), time.UTC).UnixNano() + sd.Position[0] = float64(v.Lat_dege7) / 1e+7 + sd.Position[1] = float64(v.Lon_dege7) / 1e+7 + sd.Position[2] = float64(v.HMSL_mm) / 1e+3 // mm in m + case *ublox.NavAtt: + //logrus.Println("NAV-ATT") + sd.itow = v.ITOW_ms + sd.Orientation[0] = float64(v.Pitch_deg) * 1e-5 + sd.Orientation[1] = float64(v.Roll_deg) * 1e-5 + sd.Orientation[2] = float64(v.Heading_deg) * 1e-5 + case *ublox.RawMessage: + //class := make([]byte, 2) + //binary.LittleEndian.PutUint16(class, v.ClassID()) + //logrus.Printf("%#v, %#v", class[0],class[1]) + return nil, nil + default: + return nil, errNotImplemented + } + return sd, nil } func ConvertSensorDataPhone(jsonData []byte) (*sensorData, error) { - if gjson.Get(string(jsonData), "os").String() == "hyperimu" { - return convertAndroidHyperImu(jsonData) - } - return convertIPhoneSensorLog(jsonData) + if gjson.Get(string(jsonData), "os").String() == "hyperimu" { + return convertAndroidHyperImu(jsonData) + } + return convertIPhoneSensorLog(jsonData) } func convertIPhoneSensorLog(jsonData []byte) (*sensorData, error) { - timestamp := gjson.Get(string(jsonData), "locationTimestamp_since1970").Float() - lat := gjson.Get(string(jsonData), "locationLatitude").Float() - lon := gjson.Get(string(jsonData), "locationLongitude").Float() - alt := gjson.Get(string(jsonData), "locationAltitude").Float() - pitch := gjson.Get(string(jsonData), "motionPitch").Float() * 180 / math.Pi - roll := gjson.Get(string(jsonData), "motionRoll").Float() * 180 / math.Pi - yaw := gjson.Get(string(jsonData), "motionYaw").Float() * 180 / math.Pi - sd := &sensorData{ - ServerTime: time.Now(), - source: SOURCE_TCP, - Timestamp: int64(timestamp * float64(time.Second)), - Position: [3]float64{lat, lon, alt}, - Orientation: [3]float64{pitch, roll, yaw}, - //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), - } - //logrus.Println(string(pretty.Pretty(jsonData))) - //logrus.Println(sd) - return sd, nil + timestamp := gjson.Get(string(jsonData), "locationTimestamp_since1970").Float() + lat := gjson.Get(string(jsonData), "locationLatitude").Float() + lon := gjson.Get(string(jsonData), "locationLongitude").Float() + alt := gjson.Get(string(jsonData), "locationAltitude").Float() + pitch := gjson.Get(string(jsonData), "motionPitch").Float() * 180 / math.Pi + roll := gjson.Get(string(jsonData), "motionRoll").Float() * 180 / math.Pi + yaw := gjson.Get(string(jsonData), "motionYaw").Float() * 180 / math.Pi + sd := &sensorData{ + ServerTime: time.Now(), + source: SOURCE_TCP, + Timestamp: int64(timestamp * float64(time.Second)), + Position: [3]float64{lat, lon, alt}, + Orientation: [3]float64{pitch, roll, yaw}, + //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), + } + //logrus.Println(string(pretty.Pretty(jsonData))) + //logrus.Println(sd) + return sd, nil } func convertAndroidHyperImu(jsonData []byte) (*sensorData, error) { - timestamp := gjson.Get(string(jsonData), "Timestamp").Int() - lat := gjson.Get(string(jsonData), "GPS.0").Float() - lon := gjson.Get(string(jsonData), "GPS.1").Float() - alt := gjson.Get(string(jsonData), "GPS.2").Float() - pitch := gjson.Get(string(jsonData), "orientation.0").Float() - roll := gjson.Get(string(jsonData), "orientation.1").Float() - yaw := gjson.Get(string(jsonData), "orientation.2").Float() + timestamp := gjson.Get(string(jsonData), "Timestamp").Int() + lat := gjson.Get(string(jsonData), "GPS.0").Float() + lon := gjson.Get(string(jsonData), "GPS.1").Float() + alt := gjson.Get(string(jsonData), "GPS.2").Float() + pitch := gjson.Get(string(jsonData), "orientation.0").Float() + roll := gjson.Get(string(jsonData), "orientation.1").Float() + yaw := gjson.Get(string(jsonData), "orientation.2").Float() - sd := &sensorData{ - ServerTime: time.Now(), - source: SOURCE_TCP, - Timestamp: timestamp * int64(time.Millisecond), - Position: [3]float64{lat, lon, alt}, - Orientation: [3]float64{pitch, roll, yaw}, - //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), - } - return sd, nil + sd := &sensorData{ + ServerTime: time.Now(), + source: SOURCE_TCP, + Timestamp: timestamp * int64(time.Millisecond), + Position: [3]float64{lat, lon, alt}, + Orientation: [3]float64{pitch, roll, yaw}, + //Timestamp: time.Unix(0, prep.Timestamp * int64(time.Millisecond)), + } + return sd, nil } diff --git a/core/service.go b/core/service.go index d650cd9..c9a0ebd 100644 --- a/core/service.go +++ b/core/service.go @@ -92,11 +92,11 @@ func (t *trackingService) StopRecord() { t.opMode = LIVE t.pipe.StopRecord() - m1.Lock() - m2.Lock() + mRec.Lock() + mRaw.Lock() err := t.repo.Save(*t.current) - m2.Unlock() - m1.Unlock() + mRaw.Unlock() + mRec.Unlock() if err != nil { logrus.Println(err) @@ -140,9 +140,9 @@ func (t *trackingService) LoadTracking(trackingId uuid.UUID) { } func (t *trackingService) safelyReplaceTracking(tr Tracking) { - m1.Lock() - m2.Lock() + mRec.Lock() + mRaw.Lock() *t.current = tr - m2.Unlock() - m1.Unlock() + mRaw.Unlock() + mRec.Unlock() } diff --git a/core/trackings.go b/core/trackings.go index 12660de..6df3672 100644 --- a/core/trackings.go +++ b/core/trackings.go @@ -1,62 +1,75 @@ package core import ( - "github.com/google/uuid" - "github.com/sirupsen/logrus" - "sync" - "time" + "github.com/google/uuid" + "github.com/sirupsen/logrus" + "sync" + "time" ) -var m1 sync.RWMutex -var m2 sync.RWMutex - type Tracking struct { - TrackingMetadata - Records []recordPair - Rawdata []rawRecord + TrackingMetadata + Records []recordPair + Rawdata []rawRecord } +var mRec sync.RWMutex +var mRaw sync.RWMutex + type TrackingMetadata struct { - UUID uuid.UUID - TimeCreated time.Time - Collectors []CollectorType + UUID uuid.UUID + TimeCreated time.Time + Collectors []CollectorType } +// persistence wrapper for sensordata +type recordPair struct { + RecordTimeKey time.Time // uniqueness ensured through mutex + DataPair map[sourceId]sensorData +} + +type rawRecord struct { + RecordTimeKey time.Time // uniqueness ensured through mutex + Data sensorData +} + +// END persistence wrapper for sensordata + func (s *Tracking) EnqueuePair(tcp sensorData, ser sensorData) { - rp := recordPair{ - RecordTime: time.Now(), - data: map[sourceId]sensorData{ - tcp.source: tcp, - ser.source: ser, - }, - } - m1.Lock() - s.Records = append(s.Records, rp) - logrus.Debugln("tracking Records: len->", len(s.Records)) - m1.Unlock() + rp := recordPair{ + RecordTimeKey: time.Now(), + DataPair: map[sourceId]sensorData{ + tcp.source: tcp, + ser.source: ser, + }, + } + mRec.Lock() + s.Records = append(s.Records, rp) + logrus.Debugln("tracking Records: len->", len(s.Records)) + mRec.Unlock() } func (s *Tracking) EnqueueRaw(data sensorData) { - sr := rawRecord{ - time.Now(), - data, - } - m1.Lock() - s.Rawdata = append(s.Rawdata, sr) - logrus.Debugln("raw data points: len->", len(s.Rawdata)) - m1.Unlock() + sr := rawRecord{ + time.Now(), + data, + } + mRec.Lock() + s.Rawdata = append(s.Rawdata, sr) + logrus.Debugln("raw data points: len->", len(s.Rawdata)) + mRec.Unlock() } func emptyTracking() Tracking { - return Tracking{ - TrackingMetadata: TrackingMetadata{ - UUID: uuid.New(), - }, - Records: []recordPair{}, - Rawdata: []rawRecord{}, - } + return Tracking{ + TrackingMetadata: TrackingMetadata{ + UUID: uuid.New(), + }, + Records: []recordPair{}, + Rawdata: []rawRecord{}, + } } func (s *Tracking) isEmpty() bool { - return len(s.Rawdata)+len(s.Records) == 0 + return len(s.Rawdata)+len(s.Records) == 0 } diff --git a/go.mod b/go.mod index 4d61c0a..3f06f55 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.6.1 // indirect github.com/tidwall/gjson v1.6.0 - github.com/tidwall/pretty v1.0.2 // indirect + github.com/tidwall/pretty v1.0.2 go.bug.st/serial v1.1.1 golang.org/x/sync v0.0.0-20190423024810-112230192c58 ) diff --git a/storage/kvstore.go b/storage/kvstore.go index 4be90ae..87e28f0 100644 --- a/storage/kvstore.go +++ b/storage/kvstore.go @@ -1,99 +1,116 @@ package storage import ( - "encoding/binary" - "encoding/json" - "git.timovolkmann.de/gyrogpsc/core" - "github.com/dgraph-io/badger/v2" - "github.com/google/uuid" - "github.com/sirupsen/logrus" - "os" - "path/filepath" + "encoding/binary" + "encoding/json" + "git.timovolkmann.de/gyrogpsc/core" + "github.com/dgraph-io/badger/v2" + "github.com/google/uuid" + "github.com/sirupsen/logrus" + "github.com/tidwall/pretty" + "os" + "path/filepath" + "strconv" ) // Must implement Repo type badgerStore struct { - trackings *badger.DB - records *badger.DB - rawdata *badger.DB + trackingsDb *badger.DB + recordsDb *badger.DB + rawdataDb *badger.DB } func NewRepository(c *core.Configuration) *badgerStore { - dir, _ := os.Getwd() - logrus.Debug(dir) - if _, err := os.Stat(filepath.Join(dir,"_db")); os.IsNotExist(err) { - os.Mkdir(filepath.Join(dir,"_db"), os.ModePerm) - } + dir, _ := os.Getwd() + logrus.Debug(dir) + if _, err := os.Stat(filepath.Join(dir, "_db")); os.IsNotExist(err) { + os.Mkdir(filepath.Join(dir, "_db"), os.ModePerm) + } - tr, err := badger.Open(badger.DefaultOptions("_db/trackings")) - dp, err := badger.Open(badger.DefaultOptions("_db/records")) - rd, err := badger.Open(badger.DefaultOptions("_db/raw")) + tr, err := badger.Open(badger.DefaultOptions("_db/trackings")) + dp, err := badger.Open(badger.DefaultOptions("_db/records")) + rd, err := badger.Open(badger.DefaultOptions("_db/raw")) - if err != nil { - logrus.Error(err) - } - return &badgerStore{trackings: tr, records: dp, rawdata: rd} + if err != nil { + logrus.Error(err) + } + return &badgerStore{trackingsDb: tr, recordsDb: dp, rawdataDb: rd} } func (r *badgerStore) isDbAvailable() bool { - return r.trackings.IsClosed() || r.records.IsClosed() || r.rawdata.IsClosed() + return r.trackingsDb.IsClosed() || r.recordsDb.IsClosed() || r.rawdataDb.IsClosed() } func (r *badgerStore) Save(tr core.Tracking) error { - if ok := r.isDbAvailable(); ok { - logrus.Error("unable to write to database. database closed!") - return badger.ErrDBClosed - } - ts, err := tr.TimeCreated.MarshalText() - if err != nil { - logrus.Error(err, tr) - } - logrus.Info("save tracking:", tr.TimeCreated) - meta, err := json.Marshal(tr.TrackingMetadata) - if err != nil { - logrus.Error(err, tr) - return err - } - err = r.records.Update(func(txn *badger.Txn) error { - for _, v := range tr.Records { - k := createDataKey(tr.UUID, v.RecordTime.UnixNano()) - j, err := json.Marshal(v) - if err != nil { - return err - } - txn.Set(k, j) - } - return nil - }) - if err != nil { - logrus.Error(err, tr) - return err - } - err = r.records.Update(func(txn *badger.Txn) error { - for _, v := range tr.Rawdata { - k := createDataKey(tr.UUID, v.Timestamp) - j, err := json.Marshal(v) - if err != nil { - return err - } - txn.Set(k, j) - } - return nil - }) - if err != nil { - logrus.Error(err, tr) - return err - } - err = r.trackings.Update(func(txn *badger.Txn) error { - err := txn.Set(ts, meta) - return err - }) - if err != nil { - logrus.Error(err, tr) - return err - } - logrus.Info("sucessfully saved tracking") - return nil + if ok := r.isDbAvailable(); ok { + logrus.Error("unable to write to database. database closed!") + return badger.ErrDBClosed + } + ts, err := tr.TimeCreated.MarshalText() + if err != nil { + logrus.Error(err, tr) + } + logrus.Info("save tracking:", tr.TimeCreated) + meta, err := json.Marshal(tr.TrackingMetadata) + if err != nil { + logrus.Error(err, tr) + return err + } + err = r.recordsDb.Update(func(txn *badger.Txn) error { + for _, v := range tr.Records { + k := createRecordKey(tr.UUID, v.RecordTimeKey.UnixNano()) + j, err := json.Marshal(v.DataPair) + logrus.Debugln("save record k/v:\n", tr.UUID.String(), strconv.FormatInt(v.RecordTimeKey.UnixNano(), 10)) + logrus.Debugln(string(pretty.Pretty(j))) + if err != nil { + return err + } + txn.Set(k, j) + } + return nil + }) + if err != nil { + logrus.Error(err, tr) + return err + } + err = r.rawdataDb.Update(func(txn *badger.Txn) error { + for _, v := range tr.Rawdata { + k := createRecordKey(tr.UUID, v.RecordTimeKey.UnixNano()) + j, err := json.Marshal(v) + logrus.Debugln("save raw k/v:\n", tr.UUID.String(), strconv.FormatInt(v.RecordTimeKey.UnixNano(), 10)) + logrus.Debugln(string(pretty.Pretty(j))) + if err != nil { + return err + } + txn.Set(k, j) + } + return nil + }) + if err != nil { + logrus.Error(err, tr) + return err + } + err = r.trackingsDb.Update(func(txn *badger.Txn) error { + logrus.Debug("save tracking meta k/v:\n", string(ts), string(meta)) + err := txn.Set(ts, meta) + return err + }) + if err != nil { + logrus.Error(err, tr) + return err + } + + r.trackingsDb.PrintHistogram(nil) + dr := 0.5 + err = r.trackingsDb.RunValueLogGC(dr) + logrus.Debug("DB GC:", err) + err = r.recordsDb.RunValueLogGC(dr) + logrus.Debug("DB GC:", err) + err = r.rawdataDb.RunValueLogGC(dr) + logrus.Debug("DB GC:", err) + r.trackingsDb.PrintHistogram(nil) + logrus.Info("sucessfully saved tracking") + return nil } //func (r *badgerStore) Save(tracking *core.Tracking) error { @@ -112,9 +129,9 @@ func (r *badgerStore) Save(tr core.Tracking) error { // ch := make(chan error, 3) // go func() { // defer wg.Done() -// err = r.records.Update(func(txn *badger.Txn) error { +// err = r.recordsDb.Update(func(txn *badger.Txn) error { // for _, v := range tracking.Records { -// k := createDataKey(tracking.UUID, v.RecordTime.UnixNano()) +// k := createRecordKey(tracking.UUID, v.RecordTime.UnixNano()) // j, err := json.Marshal(v) // if err != nil { // return err @@ -127,9 +144,9 @@ func (r *badgerStore) Save(tr core.Tracking) error { // }() // go func() { // defer wg.Done() -// err = r.records.Update(func(txn *badger.Txn) error { +// err = r.recordsDb.Update(func(txn *badger.Txn) error { // for _, v := range tracking.Rawdata { -// k := createDataKey(tracking.UUID, v.Timestamp) +// k := createRecordKey(tracking.UUID, v.Timestamp) // j, err := json.Marshal(v) // if err != nil { // return err @@ -142,7 +159,7 @@ func (r *badgerStore) Save(tr core.Tracking) error { // }() // go func() { // defer wg.Done() -// err = r.trackings.Update(func(txn *badger.Txn) error { +// err = r.trackingsDb.Update(func(txn *badger.Txn) error { // err := txn.Set(ts, meta) // return err // }) @@ -165,33 +182,34 @@ func (r *badgerStore) Save(tr core.Tracking) error { //} func (r *badgerStore) LoadAll() ([]core.TrackingMetadata, error) { - panic("implement me") + panic("implement me") } func (r *badgerStore) Load(id uuid.UUID) (core.Tracking, error) { - panic("implement me") + panic("implement me") } -func createDataKey(uid uuid.UUID, timestamp int64) []byte { - prefix, err := uid.MarshalText() - if err != nil || timestamp < 0 { - logrus.Error("unable to create key", err) - } - suffix := make([]byte, 8) - binary.LittleEndian.PutUint64(suffix, uint64(timestamp)) - return append(prefix, suffix...) +func createRecordKey(uid uuid.UUID, timestamp int64) []byte { + prefix, err := uid.MarshalText() + if err != nil || timestamp < 0 { + logrus.Error("unable to create key", err) + } + suffix := make([]byte, 8) + binary.BigEndian.PutUint64(suffix, uint64(timestamp)) + + return append(prefix, suffix...) } func unmarshalDataKey(key []byte) (uuid.UUID, int64) { - if len(key) != 24 { - panic("corrupted key") - } - prefix := key[0:15] - suffix := key[15:24] - uid, err := uuid.FromBytes(prefix) - if err != nil { - panic("corrupted key") - } - timestamp := int64(binary.LittleEndian.Uint64(suffix)) - return uid, timestamp + if len(key) != 24 { + panic("corrupted key") + } + prefix := key[0:15] + suffix := key[15:24] + uid, err := uuid.FromBytes(prefix) + if err != nil { + panic("corrupted key") + } + timestamp := int64(binary.BigEndian.Uint64(suffix)) + return uid, timestamp } From 40216f11b38aeb810a50b72002578fd4d922a0c5 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Sun, 13 Dec 2020 02:52:19 +0100 Subject: [PATCH 18/20] wired webhandler up --- .../autostart.go => test/autorun.go} | 25 +- core/collectors.go | 6 +- core/interfaces.go | 33 +- core/pipeline.go | 350 +++++++++--------- core/service.go | 97 +++-- go.mod | 7 +- go.sum | 192 ++++++++++ static/scripts/websocket.js | 2 +- storage/kvstore.go | 2 - {static => templates}/index.html | 2 +- web/http.go | 268 +++++++++++--- 11 files changed, 677 insertions(+), 307 deletions(-) rename cmd/{autostart/autostart.go => test/autorun.go} (75%) rename {static => templates}/index.html (98%) diff --git a/cmd/autostart/autostart.go b/cmd/test/autorun.go similarity index 75% rename from cmd/autostart/autostart.go rename to cmd/test/autorun.go index 6d57603..ea54994 100644 --- a/cmd/autostart/autostart.go +++ b/cmd/test/autorun.go @@ -6,11 +6,23 @@ import ( "git.timovolkmann.de/gyrogpsc/web" "github.com/sirupsen/logrus" "github.com/spf13/viper" + "net/http" + _ "net/http/pprof" "os" "time" ) func main() { + //f, err := os.Create("_profile.pprof") + //if err != nil { + // logrus.Fatal(err) + //} + //pprof.StartCPUProfile(f) + go func() { + logrus.Println(http.ListenAndServe("localhost:6060", nil)) + }() + //defer pprof.StopCPUProfile() + conf := configurationFromFile() repo := storage.NewRepository(conf) @@ -19,19 +31,22 @@ func main() { service := core.TrackingService(repo, disp, conf) go func() { - service.NewSetup(core.TCP, core.SERIAL) + service.StartPipeline(core.TCP, core.SERIAL) time.Sleep(5 * time.Second) service.StartRecord() - time.Sleep(5 * time.Second) + time.Sleep(10 * time.Second) service.StopRecord() time.Sleep(5 * time.Second) - service.NewSetup(core.TCP, core.SERIAL) + service.StartPipeline(core.TCP, core.SERIAL) time.Sleep(5 * time.Second) service.StartRecord() - time.Sleep(120 * time.Second) + time.Sleep(60 * time.Second) service.StopRecord() - time.Sleep(5 * time.Second) + time.Sleep(2 * time.Second) service.StopAll() + time.Sleep(1 * time.Second) + + //pprof.StopCPUProfile() os.Exit(0) }() diff --git a/core/collectors.go b/core/collectors.go index f85be11..0cac1d9 100644 --- a/core/collectors.go +++ b/core/collectors.go @@ -16,11 +16,11 @@ type Collector interface { Close() } -type CollectorType uint8 +type CollectorType string const ( - SERIAL CollectorType = iota - TCP + SERIAL CollectorType = "SERIAL_COLLECTOR" //"\x00SERIAL?/" + TCP CollectorType = "TCP_COLLECTOR" //"\x00TCP?/" ) var tcpSingleton *tcpCollector diff --git a/core/interfaces.go b/core/interfaces.go index ce10b0d..4beafe9 100644 --- a/core/interfaces.go +++ b/core/interfaces.go @@ -3,40 +3,25 @@ package core import "github.com/google/uuid" type Subscriber interface { - Subscribe() (int16, <-chan string) - Unsubscribe(id int16) error + Subscribe() (int16, <-chan string) + Unsubscribe(id int16) error } type Publisher interface { - Publish(message string) + Publish(message string) } type Pusher interface { - Push(data *sensorData) error + Push(data *sensorData) error } type Storer interface { - EnqueuePair(tcp sensorData, ser sensorData) - EnqueueRaw(data sensorData) + EnqueuePair(tcp sensorData, ser sensorData) + EnqueueRaw(data sensorData) } type Repo interface { - Save(tracking Tracking) error - LoadAll() ([]TrackingMetadata, error) - Load(id uuid.UUID) (Tracking, error) -} - -type Service interface { - AllTrackings() - NewSetup(cols ...CollectorType) - StartRecord() - StopRecord() - StopAll() - - LoadTracking(trackingId uuid.UUID) - DeleteTracking(trackingId uuid.UUID) - - StartReplay() - PauseReplay() - StopReplay() + Save(tracking Tracking) error + LoadAll() ([]TrackingMetadata, error) + Load(id uuid.UUID) (Tracking, error) } diff --git a/core/pipeline.go b/core/pipeline.go index 00c7f4e..36cfc58 100644 --- a/core/pipeline.go +++ b/core/pipeline.go @@ -1,225 +1,227 @@ package core import ( - "context" - "encoding/json" - "errors" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/sirupsen/logrus" - "golang.org/x/sync/semaphore" - "sync" - "time" + "context" + "encoding/json" + "errors" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/sirupsen/logrus" + "golang.org/x/sync/semaphore" + "sync" + "time" ) type pipeline struct { - active bool - record bool - synchroniz synchronizer - buffer pipeBuffer - publisher Publisher - storer Storer - publishTicker *time.Ticker - mu sync.RWMutex - sema *semaphore.Weighted + active bool + record bool + synchroniz synchronizer + buffer pipeBuffer + publisher Publisher + storer Storer + publishTicker *time.Ticker + mu sync.RWMutex + sema *semaphore.Weighted } // pipe implements Runner & Pusher func NewPipeline(d Publisher, s Storer, conf *Configuration) *pipeline { - return &pipeline{ - false, - false, - synchronizer{ - //bufferSize: 100, - mutex: &sync.RWMutex{}, - updateTicker: time.NewTicker(time.Duration(conf.Pipeline.SyncUpdateIntervalMs) * time.Millisecond), - }, - pipeBuffer{ - tcpMutex: &sync.Mutex{}, - serialMutex: &sync.Mutex{}, - }, - d, - s, - time.NewTicker(time.Duration(conf.Pipeline.PublishIntervalMs) * time.Millisecond), - sync.RWMutex{}, - semaphore.NewWeighted(2), - } + return &pipeline{ + false, + false, + synchronizer{ + //bufferSize: 100, + mutex: &sync.RWMutex{}, + updateTicker: time.NewTicker(time.Duration(conf.Pipeline.SyncUpdateIntervalMs) * time.Millisecond), + }, + pipeBuffer{ + tcpMutex: &sync.Mutex{}, + serialMutex: &sync.Mutex{}, + }, + d, + s, + time.NewTicker(time.Duration(conf.Pipeline.PublishIntervalMs) * time.Millisecond), + sync.RWMutex{}, + semaphore.NewWeighted(2), + } } func (p *pipeline) isPipeActive() bool { - p.mu.RLock() - defer p.mu.RUnlock() - return p.active + p.mu.RLock() + defer p.mu.RUnlock() + return p.active } func (p *pipeline) Run() { - p.sema.Acquire(context.Background(), 2) - p.mu.Lock() - p.active = true - p.mu.Unlock() - logrus.Println("pipe: processing service started") - go func() { - for p.isPipeActive() { - <-p.synchroniz.updateTicker.C - err := p.refreshDelay() - if err != nil { - logrus.Debugln(err) - } - } - p.sema.Release(1) - logrus.Println("pipe: updater stopped") - }() - go func() { - for p.isPipeActive() { - <-p.publishTicker.C - err := p.publish() - if err != nil && err.Error() != "no data available" { - logrus.Debug(err) - } - } - p.sema.Release(1) - logrus.Println("pipe: publisher stopped") - }() + p.sema.Acquire(context.Background(), 2) + p.mu.Lock() + p.active = true + p.mu.Unlock() + logrus.Println("pipe: processing service started") + go func() { + for p.isPipeActive() { + <-p.synchroniz.updateTicker.C + err := p.refreshDelay() + if err != nil { + logrus.Debugln(err) + } + } + p.sema.Release(1) + logrus.Println("pipe: updater stopped") + }() + go func() { + for p.isPipeActive() { + <-p.publishTicker.C + err := p.publish() + if err != nil && err.Error() != "no data available" { + logrus.Debug(err) + } + } + p.sema.Release(1) + logrus.Println("pipe: publisher stopped") + }() } func (p *pipeline) Record() { - p.record = true + p.record = true } func (p *pipeline) StopRecord() { - p.record = false + p.record = false } func (p *pipeline) publish() error { - p.buffer.tcpMutex.Lock() - p.buffer.serialMutex.Lock() + p.buffer.tcpMutex.Lock() + p.buffer.serialMutex.Lock() - if (p.buffer.MeasTcp == sensorData{} && p.buffer.MeasSerial == sensorData{}) { - p.buffer.tcpMutex.Unlock() - p.buffer.serialMutex.Unlock() - return errors.New("no data available") - } - if cmp.Equal(p.buffer.MeasTcp, p.buffer.LastMeasTcp, cmpopts.IgnoreUnexported(sensorData{})) && - cmp.Equal(p.buffer.MeasSerial, p.buffer.LastMeasSerial, cmpopts.IgnoreUnexported(sensorData{})) { - p.buffer.tcpMutex.Unlock() - p.buffer.serialMutex.Unlock() - return errors.New("same data") - } - logrus.Debug("––––––––––––––––––––––––––––––––––––") - logrus.Debugf("SER old: %v", p.buffer.LastMeasSerial) - logrus.Debugf("SER new: %v", p.buffer.MeasSerial) - logrus.Debugf("TCP old: %v", p.buffer.LastMeasTcp) - logrus.Debugf("TCP new: %v", p.buffer.MeasTcp) - logrus.Debug("––––––––––––––––––––––––––––––––––––") - p.buffer.LastMeasTcp = p.buffer.MeasTcp - p.buffer.LastMeasSerial = p.buffer.MeasSerial - p.storer.EnqueuePair(p.buffer.MeasTcp, p.buffer.MeasSerial) + if (p.buffer.MeasTcp == sensorData{} && p.buffer.MeasSerial == sensorData{}) { + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() + return errors.New("no data available") + } + if cmp.Equal(p.buffer.MeasTcp, p.buffer.LastMeasTcp, cmpopts.IgnoreUnexported(sensorData{})) && + cmp.Equal(p.buffer.MeasSerial, p.buffer.LastMeasSerial, cmpopts.IgnoreUnexported(sensorData{})) { + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() + return errors.New("same data") + } + logrus.Debug("––––––––––––––––––––––––––––––––––––") + logrus.Debugf("SER old: %v", p.buffer.LastMeasSerial) + logrus.Debugf("SER new: %v", p.buffer.MeasSerial) + logrus.Debugf("TCP old: %v", p.buffer.LastMeasTcp) + logrus.Debugf("TCP new: %v", p.buffer.MeasTcp) + logrus.Debug("––––––––––––––––––––––––––––––––––––") + p.buffer.LastMeasTcp = p.buffer.MeasTcp + p.buffer.LastMeasSerial = p.buffer.MeasSerial + p.storer.EnqueuePair(p.buffer.MeasTcp, p.buffer.MeasSerial) - data := map[string]sensorData{ - string(SOURCE_TCP): p.buffer.MeasTcp, - string(SOURCE_SERIAL): p.buffer.MeasSerial, - } + data := map[string]sensorData{ + string(SOURCE_TCP): p.buffer.MeasTcp, + string(SOURCE_SERIAL): p.buffer.MeasSerial, + } - p.buffer.tcpMutex.Unlock() - p.buffer.serialMutex.Unlock() + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() - jdata, err := json.Marshal(data) - //logrus.Println(string(pretty.Pretty(jdata))) - if err != nil { - return err - } - p.publisher.Publish(string(jdata)) - return nil + jdata, err := json.Marshal(data) + //logrus.Println(string(pretty.Pretty(jdata))) + if err != nil { + return err + } + p.publisher.Publish(string(jdata)) + return nil } type pipeBuffer struct { - MeasTcp sensorData - MeasSerial sensorData - LastMeasTcp sensorData - LastMeasSerial sensorData - tcpMutex *sync.Mutex - serialMutex *sync.Mutex + MeasTcp sensorData + MeasSerial sensorData + LastMeasTcp sensorData + LastMeasSerial sensorData + tcpMutex *sync.Mutex + serialMutex *sync.Mutex } type UnixNanoTime int64 type synchronizer struct { - tcpSerialDelayMs int64 - mutex *sync.RWMutex - updateTicker *time.Ticker + tcpSerialDelayMs int64 + mutex *sync.RWMutex + updateTicker *time.Ticker } func (p *pipeline) refreshDelay() error { - p.synchroniz.mutex.RLock() - if p.synchroniz.tcpSerialDelayMs != 0 { - logrus.Println("Delay TCP/SERIAL", p.synchroniz.tcpSerialDelayMs) - } - p.synchroniz.mutex.RUnlock() - p.buffer.serialMutex.Lock() - p.buffer.tcpMutex.Lock() - tcpTime := time.Unix(0, p.buffer.MeasTcp.Timestamp) - serTime := time.Unix(0, p.buffer.MeasSerial.Timestamp) - p.buffer.tcpMutex.Unlock() - p.buffer.serialMutex.Unlock() - if tcpTime.UnixNano() == 0 || serTime.UnixNano() == 0 { - return errors.New("no sync possible. check if both Collectors running. otherwise check GPS fix") - } - currentDelay := tcpTime.Sub(serTime).Milliseconds() - if currentDelay > 5000 || currentDelay < -5000 { - p.synchroniz.mutex.Lock() - p.synchroniz.tcpSerialDelayMs = 0 - p.synchroniz.mutex.Unlock() - return errors.New("skipping synchronisation! time not properly configured or facing network problems.") - } - logrus.Debug("TCP", tcpTime.String()) - logrus.Debug("SER", serTime.String()) - logrus.Debug("Difference", tcpTime.Sub(serTime).Milliseconds(), "ms") - delay := tcpTime.Sub(serTime).Milliseconds() - p.synchroniz.mutex.Lock() - p.synchroniz.tcpSerialDelayMs += delay - p.synchroniz.mutex.Unlock() - return nil + p.synchroniz.mutex.RLock() + if p.synchroniz.tcpSerialDelayMs != 0 { + logrus.Println("Delay TCP/SERIAL", p.synchroniz.tcpSerialDelayMs) + } + p.synchroniz.mutex.RUnlock() + p.buffer.serialMutex.Lock() + p.buffer.tcpMutex.Lock() + tcpTime := time.Unix(0, p.buffer.MeasTcp.Timestamp) + serTime := time.Unix(0, p.buffer.MeasSerial.Timestamp) + p.buffer.tcpMutex.Unlock() + p.buffer.serialMutex.Unlock() + if tcpTime.UnixNano() == 0 || serTime.UnixNano() == 0 { + return errors.New("no sync possible. check if both Collectors running. otherwise check GPS fix") + } + currentDelay := tcpTime.Sub(serTime).Milliseconds() + if currentDelay > 5000 || currentDelay < -5000 { + p.synchroniz.mutex.Lock() + p.synchroniz.tcpSerialDelayMs = 0 + p.synchroniz.mutex.Unlock() + return errors.New("skipping synchronisation! time not properly configured or facing network problems.") + } + logrus.Debug("TCP", tcpTime.String()) + logrus.Debug("SER", serTime.String()) + logrus.Debug("Difference", tcpTime.Sub(serTime).Milliseconds(), "ms") + delay := tcpTime.Sub(serTime).Milliseconds() + p.synchroniz.mutex.Lock() + p.synchroniz.tcpSerialDelayMs += delay + p.synchroniz.mutex.Unlock() + return nil } func (p *pipeline) Push(data *sensorData) error { - if data == nil { - return errors.New("nil processing not allowed") - } - //logrus.Println("push data to pipe:", string(data.source)) - p.storer.EnqueueRaw(*data) - switch data.source { - case SOURCE_TCP: - go p.pushTcpDataToBuffer(*data) - case SOURCE_SERIAL: - go p.pushSerialDataToBuffer(*data) - default: - panic("pipe: invalid data source") - } - return nil + if data == nil { + return errors.New("nil processing not allowed") + } + //logrus.Println("push data to pipe:", string(data.source)) + if p.isPipeActive() { + p.storer.EnqueueRaw(*data) + } + switch data.source { + case SOURCE_TCP: + go p.pushTcpDataToBuffer(*data) + case SOURCE_SERIAL: + go p.pushSerialDataToBuffer(*data) + default: + panic("pipe: invalid data source") + } + return nil } func (p *pipeline) pushTcpDataToBuffer(data sensorData) { - p.synchroniz.mutex.RLock() - if p.synchroniz.tcpSerialDelayMs > 0 { - time.Sleep(time.Duration(p.synchroniz.tcpSerialDelayMs) * time.Millisecond) - } - p.synchroniz.mutex.RLock() - p.buffer.tcpMutex.Lock() - p.buffer.MeasTcp = p.buffer.MeasTcp.ConsolidateExTime(data) - p.buffer.tcpMutex.Unlock() + p.synchroniz.mutex.RLock() + if p.synchroniz.tcpSerialDelayMs > 0 { + time.Sleep(time.Duration(p.synchroniz.tcpSerialDelayMs) * time.Millisecond) + } + p.synchroniz.mutex.RLock() + p.buffer.tcpMutex.Lock() + p.buffer.MeasTcp = p.buffer.MeasTcp.ConsolidateExTime(data) + p.buffer.tcpMutex.Unlock() } func (p *pipeline) pushSerialDataToBuffer(data sensorData) { - p.synchroniz.mutex.RLock() - if p.synchroniz.tcpSerialDelayMs < 0 { - time.Sleep(time.Duration(-p.synchroniz.tcpSerialDelayMs) * time.Millisecond) - } - p.synchroniz.mutex.RUnlock() - p.buffer.serialMutex.Lock() - p.buffer.MeasSerial = p.buffer.MeasSerial.ConsolidateEpochsOnly(data) - p.buffer.serialMutex.Unlock() + p.synchroniz.mutex.RLock() + if p.synchroniz.tcpSerialDelayMs < 0 { + time.Sleep(time.Duration(-p.synchroniz.tcpSerialDelayMs) * time.Millisecond) + } + p.synchroniz.mutex.RUnlock() + p.buffer.serialMutex.Lock() + p.buffer.MeasSerial = p.buffer.MeasSerial.ConsolidateEpochsOnly(data) + p.buffer.serialMutex.Unlock() } func (p *pipeline) Close() { - p.mu.Lock() - p.active = false - p.mu.Unlock() + p.mu.Lock() + p.active = false + p.mu.Unlock() } diff --git a/core/service.go b/core/service.go index c9a0ebd..f6d3b75 100644 --- a/core/service.go +++ b/core/service.go @@ -1,6 +1,7 @@ package core import ( + "errors" "github.com/google/uuid" "github.com/sirupsen/logrus" "time" @@ -15,6 +16,21 @@ const ( REPLAY ) +type Service interface { + AllTrackings() ([]TrackingMetadata, error) + StartPipeline(cols ...CollectorType) (string, error) + StartRecord() (string, error) + StopRecord() (*TrackingMetadata, error) + StopAll() (*TrackingMetadata, error) + + LoadTracking(trackingId uuid.UUID) + DeleteTracking(trackingId uuid.UUID) + + StartReplay() + PauseReplay() + StopReplay() +} + type trackingService struct { current *Tracking config *Configuration @@ -38,24 +54,17 @@ func TrackingService(r Repo, d Publisher, c *Configuration) *trackingService { return ts } -//const( -// errA error = errors.New("A") -//) - -func (t *trackingService) AllTrackings() { - panic("implement me") -} - -func (t *trackingService) NewSetup(cols ...CollectorType) { +func (t *trackingService) StartPipeline(cols ...CollectorType) (string, error) { logrus.Info("SERVICE: NEW SETUP") if t.opMode == RECORDING { - logrus.Println("trackingservice: no reset while recording") - return + txt := "trackingservice: please stop recording before resetting pipeline" + logrus.Warn(txt) + return "RECORDING", errors.New(txt) } if t.opMode == LIVE { - logrus.Println("trackingservice: stop currently running setup before creating new one") + logrus.Warn("trackingservice: stop current running stream/record before creating new one") t.StopAll() - time.Sleep(20 * time.Millisecond) + //time.Sleep(1000 * time.Millisecond) } logrus.Debug("new tracking:", cols) t.opMode = LIVE @@ -70,53 +79,75 @@ func (t *trackingService) NewSetup(cols ...CollectorType) { } t.pipe.Run() //time.Sleep(3 * time.Second) + return "LIVE", nil } -func (t *trackingService) StartRecord() { +func (t *trackingService) AllTrackings() ([]TrackingMetadata, error) { + panic("implement me") + return nil, nil +} + +func (t *trackingService) StartRecord() (string, error) { logrus.Info("SERVICE: START RECORD") if t.opMode != LIVE { - logrus.Println("trackingservice: wrong mode of operation") - return + if t.opMode == RECORDING { + txt := "trackingservice: already recording" + logrus.Warn(txt) + return "record already running since: " + t.current.TimeCreated.String(), errors.New(txt) + } else { + txt := "trackingservice: start collector pipeline to record data" + logrus.Warn(txt) + return "record already running since: " + t.current.TimeCreated.String(), errors.New(txt) + } } t.opMode = RECORDING t.current.TimeCreated = time.Now() t.pipe.Record() + return "record started at: " + t.current.TimeCreated.String(), nil } -func (t *trackingService) StopRecord() { +func (t *trackingService) StopRecord() (*TrackingMetadata, error) { logrus.Info("SERVICE: STOP RECORD") if t.opMode != RECORDING { - logrus.Println("trackingservice: couldn't stop. not recording") - return + txt := "trackingservice: couldn't stop. not recording" + logrus.Info(txt) + return nil, errors.New(txt) } - t.opMode = LIVE t.pipe.StopRecord() - mRec.Lock() mRaw.Lock() err := t.repo.Save(*t.current) mRaw.Unlock() mRec.Unlock() - if err != nil { - logrus.Println(err) + logrus.Error(err) } + t.opMode = LIVE + //time.Sleep(20 * time.Millisecond) + tm := t.current.TrackingMetadata t.safelyReplaceTracking(emptyTracking()) + t.current.Collectors = tm.Collectors + return &tm, err } -func (t *trackingService) StopAll() { +func (t *trackingService) StopAll() (*TrackingMetadata, error) { logrus.Info("SERVICE: STOP ALL") - if t.opMode == RECORDING { - logrus.Println("trackingservice: stop recording gracefully") - t.StopRecord() - } - t.opMode = STOPPED + var tm *TrackingMetadata = nil + var err error t.pipe.Close() for _, e := range t.collectors { e.Close() } - t.collectors = nil - t.safelyReplaceTracking(emptyTracking()) + if t.opMode == RECORDING { + logrus.Warn("trackingservice: stop recording gracefully") + tm, err = t.StopRecord() + } + t.opMode = STOPPED + return tm, err +} + +func (t *trackingService) LoadTracking(trackingId uuid.UUID) { + panic("implement me") } func (t *trackingService) DeleteTracking(trackingId uuid.UUID) { @@ -135,10 +166,6 @@ func (t *trackingService) StopReplay() { panic("implement me") } -func (t *trackingService) LoadTracking(trackingId uuid.UUID) { - -} - func (t *trackingService) safelyReplaceTracking(tr Tracking) { mRec.Lock() mRaw.Lock() diff --git a/go.mod b/go.mod index 3f06f55..2483a21 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,15 @@ go 1.15 require ( github.com/dgraph-io/badger/v2 v2.2007.2 github.com/gofiber/fiber/v2 v2.2.4 + github.com/gofiber/template v1.6.6 github.com/gofiber/websocket/v2 v2.0.2 - github.com/google/go-cmp v0.3.0 + github.com/google/go-cmp v0.4.0 github.com/google/uuid v1.1.2 github.com/sirupsen/logrus v1.6.0 github.com/spf13/viper v1.7.1 - github.com/stretchr/testify v1.6.1 // indirect github.com/tidwall/gjson v1.6.0 github.com/tidwall/pretty v1.0.2 go.bug.st/serial v1.1.1 - golang.org/x/sync v0.0.0-20190423024810-112230192c58 + golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e + golang.org/x/sys v0.0.0-20201211090839-8ad439b19e0f // indirect ) diff --git a/go.sum b/go.sum index ac10ab6..15133b6 100644 --- a/go.sum +++ b/go.sum @@ -14,10 +14,19 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= +github.com/CloudyKit/jet/v3 v3.0.1/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= +github.com/Joker/hpp v0.0.0-20180418125244-6893e659854a/go.mod h1:MzD2WMdSxvbHw5fM/OXOFily/lipJWRc9C1px0Mt0ZE= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/Joker/jade v1.0.0/go.mod h1:efZIdO0py/LtcJRSa/j2WEklMSAw84WV0zZVMxNToB8= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= @@ -26,10 +35,13 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc= +github.com/cbroglie/mustache v1.2.0/go.mod h1:gomHsVlF4zTcsY2H8d7U9SipCYbbrAks5breARbqAM0= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -42,12 +54,15 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/goselect v0.1.1 h1:tiSSgKE1eJtxs1h/VgGQWuXUP0YS4CDIFMp6vaI1ls0= github.com/creack/goselect v0.1.1/go.mod h1:a/NhLweNvqIYMuxcMOuWY516Cimucms3DglDzQP3hKY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denis-tingajkin/go-header v0.3.1/go.mod h1:sq/2IxMhaZX+RRcgHfCRx/m0M5na0fBt4/CRe7Lrji0= github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= @@ -58,22 +73,49 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/fasthttp/websocket v1.4.3 h1:qjhRJ/rTy4KB8oBxljEC00SDt6HUY9jLRfM601SUdS4= github.com/fasthttp/websocket v1.4.3/go.mod h1:5r4oKssgS7W6Zn6mPWap3NWzNPJNzUUh3baWTOhcYQk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/flosch/pongo2/v4 v4.0.1/go.mod h1:B5ObFANs/36VwxxlgKpdchIJHMvHB562PW+BWPhwZD8= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-critic/go-critic v0.5.0/go.mod h1:4jeRh3ZAVnRYhuWdOEvwzVqLUpxMSoAT0xZ74JsTPlo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= +github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= +github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= +github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= +github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= +github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= +github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= +github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= +github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= +github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/go-toolsmith/typep v1.0.2/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gofiber/fiber/v2 v2.1.0/go.mod h1:aG+lMkwy3LyVit4CnmYUbUdgjpc3UYOltvlJZ78rgQ0= +github.com/gofiber/fiber/v2 v2.2.2/go.mod h1:Aso7/M+EQOinVkWp4LUYjdlTpKTBoCk2Qo4djnMsyHE= github.com/gofiber/fiber/v2 v2.2.4 h1:t2V2SxlbQGdt8+SS/Mo+tQB0pDQn7OajKdA72qHcBVw= github.com/gofiber/fiber/v2 v2.2.4/go.mod h1:Aso7/M+EQOinVkWp4LUYjdlTpKTBoCk2Qo4djnMsyHE= +github.com/gofiber/template v1.6.6 h1:mZ0sMRomxndpzVOSez6jKPaMO9UlUDfmBfHFmkFnb/A= +github.com/gofiber/template v1.6.6/go.mod h1:WmzkzTh6QWwrGysWFU8bwNsmHmqRbxHmWZcApIT1ORo= github.com/gofiber/websocket/v2 v2.0.2 h1:UA/6NpyG+vmPGlvJvW8MJPJpRFuS7abinZ5HbLuV8u0= github.com/gofiber/websocket/v2 v2.0.2/go.mod h1:7VBnzEVRK0K0eTIVc5GbXPF1JWUFnllY0X4cRtG2v78= +github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -87,23 +129,45 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= +github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= +github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= +github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= +github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= +github.com/golangci/golangci-lint v1.28.3/go.mod h1:JlLqleIwwgLVJtjKtrB37OKp3LGLrUhEx9tWY4VKWSY= +github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= +github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= +github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= +github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= +github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gookit/color v1.2.4/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= +github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -128,7 +192,12 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s= +github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -138,6 +207,7 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -149,16 +219,32 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kyoh86/exportloopref v0.1.4/go.mod h1:h1rDl2Kdj97+Kwh4gdz3ujE7XHmH51Q0lUiZ1z4NLj8= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU= +github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-slim v0.0.0-20200618151855-bde33eecb5ee/go.mod h1:ma9TUJeni8LGZMJvOwbAv/FOwiwqIMQN570LnpqCBSM= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= @@ -167,14 +253,28 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c= +github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nishanths/exhaustive v0.0.0-20200525081945-8e46705b6132/go.mod h1:wBEpHwM2OdmeNpdCvRPUlkEbBuaFmcK4Wv8Q7FuGW3c= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -187,13 +287,26 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= +github.com/quasilyte/go-ruleguard v0.1.2-0.20200318202121-b00d7a75d3d8/go.mod h1:CGFX09Ci3pq9QZdj86B+VGIdNj4VyCo2iPOGS9esB/k= +github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryancurrah/gomodguard v1.1.0/go.mod h1:4O8tr7hBODaGE6VIhfJDHcwzh5GUccKSJBU0UMXJFVM= +github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/savsgio/gotils v0.0.0-20200608150037-a5f6f5aef16c h1:2nF5+FZ4/qp7pZVL7fR6DEaSTzuDmNaFTyqp92/hwF8= github.com/savsgio/gotils v0.0.0-20200608150037-a5f6f5aef16c/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/securego/gosec/v2 v2.3.0/go.mod h1:UzeVyUXbxukhLeHKV3VVqo7HdoQR9MrRfFmZYotn8ME= +github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -202,6 +315,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sonatard/noctx v0.0.1/go.mod h1:9D2D/EoULe8Yy2joDHJj7bv3sZoq9AaSb8B4lqBjiZI= +github.com/sourcegraph/go-diff v0.5.3/go.mod h1:v9JDtjCE4HHHCZGId75rg8gkKKa98RVjBcBGsVmMmak= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -210,11 +325,16 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -223,10 +343,13 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= +github.com/tetafro/godot v0.4.2/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0= github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= @@ -234,18 +357,29 @@ github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= +github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= +github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.12.0/go.mod h1:229t1eWu9UXTPmoUkbpN/fctKPBY4IJoFXQnxHGXy6E= github.com/valyala/fasthttp v1.14.0/go.mod h1:ol1PCaL0dX20wC0htZ7sYCsvCYmrouYra0zHzaclZhE= github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA= github.com/valyala/fasthttp v1.17.0 h1:P8/koH4aSnJ4xbd0cUUFEGQs3jQqIxoDDyRQrUiAkqg= github.com/valyala/fasthttp v1.17.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= +github.com/valyala/quicktemplate v1.5.0/go.mod h1:v7yYWpBEiutDyNfVaph6oC/yKwejzVyTX/2cwwHxyok= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.bug.st/serial v1.1.1 h1:5J1DpaIaSIruBi7jVnKXnhRS+YQ9+2PLJMtIZKoIgnc= go.bug.st/serial v1.1.1/go.mod h1:VmYBeyJWp5BnJ0tw2NUJHZdJTGl2ecBGABHlzRK1knY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -260,6 +394,7 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -279,8 +414,12 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -288,11 +427,14 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0 h1:5kGOVHlq0euqwzgTC9Vu15p6fV1Wi0ArVi8da2urnVg= @@ -306,14 +448,18 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -322,6 +468,10 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -329,6 +479,8 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201211090839-8ad439b19e0f h1:QdHQnPce6K4XQewki9WNbG5KOROuDzqO3NaYjI1cXJ0= +golang.org/x/sys v0.0.0-20201211090839-8ad439b19e0f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -337,23 +489,48 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200117220505-0cba7a3a9ee9/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200321224714-0d839f3cf2ed/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200331202046-9d5940d49312/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200428185508-e9a00ec82136/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200519015757-0d0afa43d58a/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200625211823-6506e20df31f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -374,6 +551,7 @@ google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBr google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= @@ -382,20 +560,34 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +mvdan.cc/gofumpt v0.0.0-20200513141252-abc0db2c416a/go.mod h1:4q/PlrZKQLU5MowSvCKM3U4xJUPtJ8vKWx7vsWFJ3MI= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= +mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/static/scripts/websocket.js b/static/scripts/websocket.js index e065b38..30d1891 100644 --- a/static/scripts/websocket.js +++ b/static/scripts/websocket.js @@ -22,7 +22,7 @@ window.addEventListener("load", function(evt) { print("Websocket already open") return false; } - ws = new WebSocket("ws://localhost:3011/echo"); + ws = new WebSocket("ws://localhost:3011/ws"); ws.onopen = function(evt) { print("OPEN"); } diff --git a/storage/kvstore.go b/storage/kvstore.go index 87e28f0..e4ee5e8 100644 --- a/storage/kvstore.go +++ b/storage/kvstore.go @@ -100,7 +100,6 @@ func (r *badgerStore) Save(tr core.Tracking) error { return err } - r.trackingsDb.PrintHistogram(nil) dr := 0.5 err = r.trackingsDb.RunValueLogGC(dr) logrus.Debug("DB GC:", err) @@ -108,7 +107,6 @@ func (r *badgerStore) Save(tr core.Tracking) error { logrus.Debug("DB GC:", err) err = r.rawdataDb.RunValueLogGC(dr) logrus.Debug("DB GC:", err) - r.trackingsDb.PrintHistogram(nil) logrus.Info("sucessfully saved tracking") return nil } diff --git a/static/index.html b/templates/index.html similarity index 98% rename from static/index.html rename to templates/index.html index 0f6a538..49827f4 100644 --- a/static/index.html +++ b/templates/index.html @@ -77,7 +77,7 @@
-

Click "Open" to create a connection to the server. +

{{.}} Click "Open" to create a connection to the server.

diff --git a/web/http.go b/web/http.go index 9478fe4..dbc7fd5 100644 --- a/web/http.go +++ b/web/http.go @@ -1,85 +1,235 @@ package web import ( - "git.timovolkmann.de/gyrogpsc/core" - "github.com/gofiber/fiber/v2" - "github.com/gofiber/websocket/v2" - "github.com/sirupsen/logrus" - "html/template" + "git.timovolkmann.de/gyrogpsc/core" + "github.com/gofiber/fiber/v2" + "github.com/gofiber/template/html" + "github.com/gofiber/websocket/v2" + "github.com/sirupsen/logrus" ) func CreateServer(s core.Service, sub core.Subscriber, c *core.Configuration) { - app := fiber.New() - app.Static("/static", "static") + app := fiber.New(fiber.Config{ + Views: fiberTemplateEngine(c), + }) + app.Static("/static", "static") - // Application Main Page - app.Get("/", fiberHomeHandler) + // Application Main Page + app.Get("/", fiberHomeHandler) - // Websocket - app.Get("/", websocket.New(createFiberWebsocketHandler(sub))) + // Websocket + app.Get("/ws", websocket.New(createFiberWebsocketHandler(sub))) - // TODO: Get all SerialPorts - // app.Get("/serialports") + // TODO: Get all SerialPorts + // app.Get("/serialports") - // Tracking persistence controls - trackings := app.Group("/trackings") - trackings.Get("/", stubhander()) // Get all trackings Metadata - trackings.Post("/", stubhander()) // Initialize new tracking, open websocket and prepare for automatic recording. Toggle ?serial=true and ?tcp=true. Returns trackingId - trackings.Patch("/", stubhander()) // Starts recording - trackings.Put("/", stubhander()) // Stops current recording. Returns trackingId if record was successful - trackings.Delete("/", stubhander()) // Stops websocket connection, pipelines and collectors + // Tracking persistence controls HTTP JSON RPC API + trackings := app.Group("/trackings") + trackings.Get("/", allTrackingsHandler(s, c)) // Get all trackings Metadata + trackings.Post("/", startPipelineHandler(s, c)) // Initialize new tracking, open websocket and prepare for automatic recording. Toggle ?serial=true and ?tcp=true. Returns trackingId + trackings.Patch("/", startRecordingHandler(s, c)) // Starts recording + trackings.Put("/", stopRecordingHandler(s, c)) // Stops current recording. Returns trackingId if record was successful + trackings.Delete("/", stopAllHandler(s, c)) // Stops websocket connection, pipelines and collectors - trackings.Get("/:trackingId", stubhander()) // Gets Tracking Metadata and loads sensorRecords from storage. - trackings.Delete("/:trackingId", stubhander()) // Deletes Tracking from storage + trackings.Get("/:trackingId", stubhander()) // Gets Tracking Metadata and loads sensorRecords from storage. + trackings.Delete("/:trackingId", stubhander()) // Deletes Tracking from storage - trackings.Post("/current", stubhander()) // Starts Replay. - trackings.Patch("/current", stubhander()) // Pauses Replay. - trackings.Put("/current", stubhander()) // Stops Replay. + trackings.Post("/current", stubhander()) // Starts Replay. + trackings.Patch("/current", stubhander()) // Pauses Replay. + trackings.Put("/current", stubhander()) // Stops Replay. - logrus.Fatal(app.Listen(c.Webserver.Port)) + logrus.Fatal(app.Listen(c.Webserver.Port)) } func stubhander() fiber.Handler { - return func(ctx *fiber.Ctx) error { - return nil - } + return func(ctx *fiber.Ctx) error { + return nil + } +} +func allTrackingsHandler(s core.Service, c *core.Configuration) fiber.Handler { + return func(ctx *fiber.Ctx) error { + trackings, err := s.AllTrackings() + if err != nil { + //ctx.Status(500).JSON(err) + //return err + } + prepres := map[string]interface{}{} + prepres["data"] = trackings + if err != nil { + prepres["error"] = err.Error() + + } + + err2 := ctx.JSON(prepres) + if err2 != nil { + ctx.Status(500).JSON(err2) + return err2 + } + return nil + } +} + +func startPipelineHandler(s core.Service, c *core.Configuration) fiber.Handler { + return func(ctx *fiber.Ctx) error { + var collecs []core.CollectorType + ser := ctx.Params("serial", "true") + if ser == "true" { + collecs = append(collecs, core.SERIAL) + } else if ser != "false" && ser != "" { + collecs = append(collecs, core.CollectorType(ser)) // TODO: allow passing serial port as url parameter + } + tcp := ctx.Params("tcp", "true") + if tcp == "true" { + collecs = append(collecs, core.TCP) + } + res, err := s.StartPipeline(collecs...) + if err != nil { + //ctx.Status(500).JSON(err) + //return err + } + prepres := map[string]interface{}{} + prepres["tracking_state"] = res + //prepres["data"] = res + if err != nil { + prepres["error"] = err.Error() + + } + err2 := ctx.JSON(prepres) + if err2 != nil { + ctx.Status(500).JSON(err2) + return err2 + } + return nil + } +} + +func startRecordingHandler(s core.Service, c *core.Configuration) fiber.Handler { + return func(ctx *fiber.Ctx) error { + rec, err := s.StartRecord() + if err != nil { + //ctx.Status(500).JSON(err) + //return err + } + prepres := map[string]interface{}{} + prepres["tracking_state"] = "RECORD" + prepres["data"] = rec + if err != nil { + prepres["error"] = err.Error() + + } + err2 := ctx.JSON(prepres) + if err2 != nil { + ctx.Status(500).JSON(err2) + return err2 + } + return nil + } +} + +func stopRecordingHandler(s core.Service, c *core.Configuration) fiber.Handler { + return func(ctx *fiber.Ctx) error { + rec, err := s.StopRecord() + if err != nil { + //ctx.Status(500).JSON(err) + //return err + } + prepres := map[string]interface{}{} + prepres["tracking_state"] = "LIVE" + prepres["data"] = rec + if err != nil { + prepres["error"] = err.Error() + + } + err2 := ctx.JSON(prepres) + if err2 != nil { + ctx.Status(500).JSON(err2) + return err2 + } + return nil + } +} + +func stopAllHandler(s core.Service, c *core.Configuration) fiber.Handler { + return func(ctx *fiber.Ctx) error { + rec, err := s.StopAll() + if err != nil { + //ctx.Status(500).JSON(err) + //return err + } + prepres := map[string]interface{}{} + prepres["tracking_state"] = "STOPPED" + prepres["data"] = rec + if err != nil { + prepres["error"] = err.Error() + + } + err2 := ctx.JSON(prepres) + if err2 != nil { + ctx.Status(500).JSON(err2) + return err2 + } + return nil + } } func createFiberWebsocketHandler(s core.Subscriber) func(conn *websocket.Conn) { - return func(c *websocket.Conn) { - // Handle and discard inbound messages - go func() { - for { - if _, _, err := c.NextReader(); err != nil { - c.Close() - break - } - } - }() + return func(c *websocket.Conn) { + logrus.Info("new websocket client") + // Handle and discard inbound messages + go func() { + for { + if _, _, err := c.NextReader(); err != nil { + c.Close() + break + } + } + }() - dispatcherId, channel := s.Subscribe() - defer s.Unsubscribe(dispatcherId) - for { - cmsg := <-channel - err := c.WriteMessage(websocket.TextMessage, []byte(cmsg)) - if err != nil { - logrus.Println("write:", err) + dispatcherId, channel := s.Subscribe() + defer s.Unsubscribe(dispatcherId) + for { + cmsg := <-channel + err := c.WriteMessage(websocket.TextMessage, []byte(cmsg)) + if err != nil { + logrus.Info("close websocket connection") + c.Close() + break + } + } + } +} - break - } - } - } +func fiberTemplateEngine(c *core.Configuration) *html.Engine { + // Create a new engine by passing the template folder + // and template extension using .New(dir, ext string) + engine := html.New("./templates", ".html") + // We also support the http.FileSystem interface + // See examples below to load templates from embedded files + // engine := html.NewFileSystem(http.Dir("./views"), ".html") + + // Reload the templates on each render, good for development + //engine.Reload(strings.ToLower(c.Debuglevel) == "debug") // Optional. Default: false + engine.Reload(true) // Optional. Default: false + + // Debug will print each template that is parsed, good for debugging + //engine.Debug(strings.ToLower(c.Debuglevel) == "debug") // Optional. Default: false + engine.Debug(true) // Optional. Default: false + + // Layout defines the variable name that is used to yield templates within layouts + //engine.Layout("embed") // Optional. Default: "embed" + + // Delims sets the action delimiters to the specified strings + //engine.Delims("{{", "}}") // Optional. Default: engine delimiters + + // AddFunc adds a function to the template's global function map. + //engine.AddFunc("greet", func(name string) string { + // return "Hello, " + name + "!" + //}) + return engine } func fiberHomeHandler(c *fiber.Ctx) error { - tpl, err := template.ParseFiles("static/index.html") - if err != nil { - return err - } - err = tpl.Execute(c, "://"+c.Hostname()+"/") - if err != nil { - return err - } - return nil + // Render index template + return c.Render("index", "ws://"+c.Hostname()+"/ws") } From f223bbec8f042a952ab8459286923cc65ce994c4 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Sun, 13 Dec 2020 03:23:18 +0100 Subject: [PATCH 19/20] get all trackings --- core/service.go | 5 +-- storage/kvstore.go | 90 +++++++++++----------------------------------- web/http.go | 1 - 3 files changed, 24 insertions(+), 72 deletions(-) diff --git a/core/service.go b/core/service.go index f6d3b75..b05db75 100644 --- a/core/service.go +++ b/core/service.go @@ -83,8 +83,9 @@ func (t *trackingService) StartPipeline(cols ...CollectorType) (string, error) { } func (t *trackingService) AllTrackings() ([]TrackingMetadata, error) { - panic("implement me") - return nil, nil + logrus.Info("SERVICE: GET ALL TRACKINGS") + data, err := t.repo.LoadAll() + return data, err } func (t *trackingService) StartRecord() (string, error) { diff --git a/storage/kvstore.go b/storage/kvstore.go index e4ee5e8..3997b9b 100644 --- a/storage/kvstore.go +++ b/storage/kvstore.go @@ -111,76 +111,28 @@ func (r *badgerStore) Save(tr core.Tracking) error { return nil } -//func (r *badgerStore) Save(tracking *core.Tracking) error { -// ts, err := tracking.TimeCreated.MarshalText() -// if err != nil { -// logrus.Error(err, tracking) -// } -// logrus.Info("save tracking:", ts) -// meta, err := json.Marshal(tracking.TrackingMetadata) -// if err != nil { -// logrus.Error(err, tracking) -// return err -// } -// wg := sync.WaitGroup{} -// wg.Add(3) -// ch := make(chan error, 3) -// go func() { -// defer wg.Done() -// err = r.recordsDb.Update(func(txn *badger.Txn) error { -// for _, v := range tracking.Records { -// k := createRecordKey(tracking.UUID, v.RecordTime.UnixNano()) -// j, err := json.Marshal(v) -// if err != nil { -// return err -// } -// txn.Set(k, j) -// } -// return nil -// }) -// ch <- err -// }() -// go func() { -// defer wg.Done() -// err = r.recordsDb.Update(func(txn *badger.Txn) error { -// for _, v := range tracking.Rawdata { -// k := createRecordKey(tracking.UUID, v.Timestamp) -// j, err := json.Marshal(v) -// if err != nil { -// return err -// } -// txn.Set(k, j) -// } -// return nil -// }) -// ch <- err -// }() -// go func() { -// defer wg.Done() -// err = r.trackingsDb.Update(func(txn *badger.Txn) error { -// err := txn.Set(ts, meta) -// return err -// }) -// ch <- err -// }() -// wg.Wait() -// for { -// select { -// case err := <-ch: -// if err != nil { -// logrus.Error(err, tracking) -// return err -// } -// default: -// close(ch) -// break -// } -// } -// return nil -//} - func (r *badgerStore) LoadAll() ([]core.TrackingMetadata, error) { - panic("implement me") + var result []core.TrackingMetadata + err := r.trackingsDb.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + it := txn.NewIterator(opts) + defer it.Close() + for it.Rewind(); it.Valid(); it.Next() { + item := it.Item() + el := core.TrackingMetadata{} + item.Value(func(val []byte) error { + logrus.Debugln(string(val)) + err := json.Unmarshal(val, &el) + return err + }) + result = append(result, el) + } + return nil + }) + if err != nil { + return nil, err + } + return result, nil } func (r *badgerStore) Load(id uuid.UUID) (core.Tracking, error) { diff --git a/web/http.go b/web/http.go index dbc7fd5..a371652 100644 --- a/web/http.go +++ b/web/http.go @@ -59,7 +59,6 @@ func allTrackingsHandler(s core.Service, c *core.Configuration) fiber.Handler { prepres["error"] = err.Error() } - err2 := ctx.JSON(prepres) if err2 != nil { ctx.Status(500).JSON(err2) From 145642c6e93641aa8b619982b440faf6236734d2 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Sun, 13 Dec 2020 03:38:00 +0100 Subject: [PATCH 20/20] fixed wrong param type in http handler --- web/http.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/web/http.go b/web/http.go index a371652..daee7fe 100644 --- a/web/http.go +++ b/web/http.go @@ -1,6 +1,7 @@ package web import ( + "errors" "git.timovolkmann.de/gyrogpsc/core" "github.com/gofiber/fiber/v2" "github.com/gofiber/template/html" @@ -71,13 +72,14 @@ func allTrackingsHandler(s core.Service, c *core.Configuration) fiber.Handler { func startPipelineHandler(s core.Service, c *core.Configuration) fiber.Handler { return func(ctx *fiber.Ctx) error { var collecs []core.CollectorType - ser := ctx.Params("serial", "true") + ser := ctx.Query("serial", "true") if ser == "true" { collecs = append(collecs, core.SERIAL) } else if ser != "false" && ser != "" { collecs = append(collecs, core.CollectorType(ser)) // TODO: allow passing serial port as url parameter } - tcp := ctx.Params("tcp", "true") + tcp := ctx.Query("tcp", "true") + logrus.Debugln("query values: serial/tcp", ser, tcp) if tcp == "true" { collecs = append(collecs, core.TCP) } @@ -88,10 +90,13 @@ func startPipelineHandler(s core.Service, c *core.Configuration) fiber.Handler { } prepres := map[string]interface{}{} prepres["tracking_state"] = res - //prepres["data"] = res + prepres["data"] = collecs if err != nil { prepres["error"] = err.Error() - + } + if len(collecs) == 0 { + e := errors.New("attention! no collectors running. start a new pipeline") + prepres["error"] = e.Error() } err2 := ctx.JSON(prepres) if err2 != nil {