michal/oko-server
1
0
Fork 0
mirror of https://github.com/Cernobor/oko-server.git synced 2025-02-24 08:27:17 +00:00
Code Issues Projects Releases Wiki Activity

Collecting usage info, change sqlite lib.
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source 
* X-User-ID header is processed to get user ID.
* Time of last request for a user is saved into DB.
* Time of last upload and download is stored for a user.
* Added DB migration to add columns into users table to store the times and app version.
* Backward fix of datatype of the deadline column in features table.
* Switched from crawshaw.io/sqlite to zombiezen.com/go/sqlite.
  * Refactored DB handling.
  * Used migration routine from zombiezen in favour of manual one.
  * Runtime DB reinit simply deletes the db file and initializes the db anew.

Fix #6
This commit is contained in:
zegkljan 2023-06-11 18:06:33 +02:00
parent 63e79c657c
commit 8440e3b7d7
11 changed files with 740 additions and 475 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
docs/api.json
View File

@ -125,6 +125,37 @@
"required": ["id", "name"],
"description": "A user in the system."
},
"UserInfo": {
"type": "object",
"properties": {
"id": {
"type": "integer",
"description": "User ID."
},
"name": {
"type": "string",
"description": "User name."
},
"app_version": {
"type": "string",
"description": "Version of the app last used by the user."
},
"last_seen_time": {
"anyOf": [{"$ref": "#/components/schemas/LocalDateTime"}],
"description": "Time of last contact (any) with the server."
},
"last_upload_time": {
"anyOf": [{"$ref": "#/components/schemas/LocalDateTime"}],
"description": "Time of last data upload."
},
"last_download_time": {
"anyOf": [{"$ref": "#/components/schemas/LocalDateTime"}],
"description": "Time of last data download."
}
},
"required": ["id", "name", "app_version", "last_seen_time", "last_upload_time", "last_download_time"],
"description": "Extended info about a user."
},
"Feature": {
"type": "object",
"properties": {
@ -312,11 +343,22 @@
"required": ["version_hash", "build_time"],
"description": "Server build info."
}
},
"parameters": {
"UserID": {
"in": "header",
"name": "X-User-ID",
"schema": {
"type": "integer"
},
"required": false
}
}
},
"security": [],
"paths": {
"/ping" : {
"parameters": [{"$ref": "#/components/parameters/UserID"}],
"get": {
"operationId": "ping",
"tags": ["app"],
@ -342,6 +384,7 @@
}
},
"/handshake": {
"parameters": [{"$ref": "#/components/parameters/UserID"}],
"post": {
"operationId": "handshake",
"tags": ["app"],
@ -384,6 +427,7 @@
}
},
"/data": {
"parameters": [{"$ref": "#/components/parameters/UserID"}],
"get": {
"operationId": "getData",
"tags": ["app"],
@ -500,6 +544,7 @@
}
},
"/data/people": {
"parameters": [{"$ref": "#/components/parameters/UserID"}],
"get": {
"operationId": "getPeople",
"tags": ["app"],
@ -523,6 +568,7 @@
}
},
"/data/features": {
"parameters": [{"$ref": "#/components/parameters/UserID"}],
"get": {
"operationId": "getFeatures",
"tags": ["app"],
@ -547,6 +593,7 @@
},
"/data/features/{featureID}/photos/{photoID}": {
"parameters": [
{"$ref": "#/components/parameters/UserID"},
{
"in": "path",
"name": "featureID",
@ -599,6 +646,7 @@
}
},
"/data/proposals": {
"parameters": [{"$ref": "#/components/parameters/UserID"}],
"get": {
"operationId": "getProposals",
"tags": ["app"],
@ -622,6 +670,7 @@
}
},
"/mappack": {
"parameters": [{"$ref": "#/components/parameters/UserID"}],
"get": {
"operationId": "getTilePack",
"tags": ["app"],
@ -634,6 +683,7 @@
}
},
"/build-info": {
"parameters": [{"$ref": "#/components/parameters/UserID"}],
"get": {
"operationId": "getBuildInfo",
"tags": ["debug", "utils"],
@ -653,6 +703,7 @@
}
},
"/hard-fail": {
"parameters": [{"$ref": "#/components/parameters/UserID"}],
"get": {
"operationId": "hardFail",
"tags": ["debug"],
@ -665,6 +716,7 @@
}
},
"/soft-fail": {
"parameters": [{"$ref": "#/components/parameters/UserID"}],
"get": {
"operationId": "softFail",
"tags": ["debug"],
@ -689,6 +741,7 @@
}
},
"/app-versions": {
"parameters": [{"$ref": "#/components/parameters/UserID"}],
"get": {
"operationId": "getAppVersions",
"tags": ["utils", "debug"],
@ -736,6 +789,7 @@
},
"/app-versions/{version}": {
"parameters": [
{"$ref": "#/components/parameters/UserID"},
{
"in": "path",
"name": "version",
@ -786,6 +840,7 @@
}
},
"/reinit": {
"parameters": [{"$ref": "#/components/parameters/UserID"}],
"post": {
"operationId": "reinitDb",
"tags": ["debug", "utils"],
@ -796,6 +851,28 @@
}
}
}
},
"/usage-info": {
"get": {
"operationId": "getUsageInfo",
"tags": ["utils", "debug"],
"summary": "Similar to /data/people, but used app version and times of last contact, upload, and download are retreived as well.",
"responses": {
"200": {
"description": "User usage info.",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/UserInfo"
}
}
}
}
}
}
}
}
}
}

10
go.mod
View File

@ -3,24 +3,32 @@ module cernobor.cz/oko-server
go 1.20
require (
crawshaw.io/sqlite v0.3.3-0.20211227050848-2cdb5c1a86a1
github.com/consbio/mbtileserver v0.9.0
github.com/gin-gonic/gin v1.9.1
github.com/paulmach/go.geojson v1.4.0
github.com/sirupsen/logrus v1.9.3
zombiezen.com/go/sqlite v0.13.0
)
require (
crawshaw.io/sqlite v0.3.3-0.20211227050848-2cdb5c1a86a1 // indirect
github.com/bytedance/sonic v1.9.1 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/net v0.10.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/libc v1.24.1 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.6.0 // indirect
modernc.org/sqlite v1.23.0 // indirect
)
require (

20
go.sum
View File

@ -17,6 +17,8 @@ github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03V
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
@ -33,9 +35,11 @@ github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QX
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@ -58,6 +62,9 @@ github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZ
github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -120,7 +127,6 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
@ -130,4 +136,14 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM=
modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/memory v1.6.0 h1:i6mzavxrE9a30whzMfwf7XWVODx2r5OYXvU46cirX7o=
modernc.org/memory v1.6.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/sqlite v1.23.0 h1:MWTFBI5H1WLnXpNBh/BTruBVqzzoh28DA0iOnlkkRaM=
modernc.org/sqlite v1.23.0/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
zombiezen.com/go/sqlite v0.13.0 h1:iEeyVqcm3fk5PCA8OQBhBxPnqrP4yYuVJBF+XZpSnOE=
zombiezen.com/go/sqlite v0.13.0/go.mod h1:Ht/5Rg3Ae2hoyh1I7gbWtWAl89CNocfqeb/aAMTkJr4=

8
models/models.go
View File

@ -19,6 +19,14 @@ type User struct {
Name string `json:"name"`
}
type UserInfo struct {
User
AppVersion *semver.Version `json:"app_version"`
LastSeenTime *time.Time `json:"last_seen_time"`
LastUploadTime *time.Time `json:"last_upload_time"`
LastDownloadTime *time.Time `json:"last_download_time"`
}
type Feature struct {
// ID is an ID of the feature.
// When the feature is submitted by a client for creation (i.e. in Update.Create) it is considered a 'local' ID which must be unique across all submitted features.

1
server/constants.go
View File

@ -21,4 +21,5 @@ const (
URITileTemplate = URITileserverRoot + "/map/tiles/{z}/{x}/{y}.pbf"
AppName = "OKO"
UserIDHeader = "X-User-ID"
)

229
server/db.go Normal file
View File

@ -0,0 +1,229 @@
package server
import (
"embed"
"fmt"
"os"
"path"
"regexp"
"sort"
"strconv"
"time"
"zombiezen.com/go/sqlite"
"zombiezen.com/go/sqlite/sqlitemigration"
"zombiezen.com/go/sqlite/sqlitex"
)
func (s *Server) cleanupDb() {
close(s.checkpointNotice)
s.log.Info("Closing db connection pool...")
s.dbpool.Close()
// manually force truncate checkpoint
conn, err := sqlite.OpenConn(fmt.Sprintf("file:%s", s.config.DbPath), 0)
if err != nil {
s.log.WithError(err).Error("Failed to open connection for final checkpoint.")
return
}
err = sqlitex.Execute(conn, "vacuum", nil)
if err != nil {
s.log.WithError(err).Error("Failed to vacuum db.")
}
s.checkpointDb(conn, true)
conn.Close()
}
func (s *Server) getDbConn() *sqlite.Conn {
conn, err := s.dbpool.Get(s.ctx)
if err != nil {
panic(err)
}
return conn
}
func (s *Server) returnDbConn(conn *sqlite.Conn) {
s.dbpool.Put(conn)
}
func withDbConn[T any](s *Server, f func(conn *sqlite.Conn) T) T {
conn := s.getDbConn()
defer s.returnDbConn(conn)
return f(conn)
}
func (s *Server) checkpointDb(conn *sqlite.Conn, truncate bool) {
var query string
if truncate {
query = "PRAGMA wal_checkpoint(TRUNCATE)"
} else {
query = "PRAGMA wal_checkpoint(RESTART)"
}
stmt, _, err := conn.PrepareTransient(query)
if err != nil {
s.log.WithError(err).Error("Failed to prepare checkpoint query.")
return
}
defer stmt.Finalize()
has, err := stmt.Step()
if err != nil {
s.log.WithError(err).Error("Failed to step through checkpoint query.")
return
}
if !has {
s.log.Error("Checkpoint query returned no rows.")
return
}
blocked := stmt.ColumnInt(0)
noWalPages := stmt.ColumnInt(1)
noReclaimedPages := stmt.ColumnInt(2)
if blocked == 1 {
s.log.Warn("Checkpoint query was blocked.")
}
s.log.Debugf("Checkpoint complete. %d pages written to WAL, %d pages written back to DB.", noWalPages, noReclaimedPages)
}
func (s *Server) setupDB() error {
s.dbAvailable.Store(false)
s.log.Debugf("Using db %s", s.config.DbPath)
ready := make(chan struct{})
migErr := make(chan error)
s.dbpool = sqlitemigration.NewPool(fmt.Sprintf("file:%s", s.config.DbPath), sqlSchema, sqlitemigration.Options{
PoolSize: 10,
PrepareConn: func(conn *sqlite.Conn) error {
return sqlitex.ExecuteTransient(conn, "PRAGMA foreign_keys = ON;", nil)
},
OnReady: func() {
close(ready)
},
OnError: func(err error) {
migErr <- err
},
})
select {
case <-ready:
case err := <-migErr:
return fmt.Errorf("error during db migration: %w", err)
}
s.checkpointNotice = make(chan struct{})
// aggressively checkpoint the database on idle times
go func() {
s.log.Debug("Starting manual restart checkpointing.")
defer s.log.Debug("Manual restart checkpointing stopped.")
delay := time.Minute * 15
var (
timer <-chan time.Time
ok bool
)
for {
select {
case _, ok = <-s.checkpointNotice:
if !ok {
return
}
timer = time.After(delay)
case <-timer:
withDbConn(s, func(conn *sqlite.Conn) any {
s.checkpointDb(conn, false)
return nil
})
timer = nil
}
}
}()
s.dbAvailable.Store(true)
return nil
}
func (s *Server) requestCheckpoint() {
go func() {
s.checkpointNotice <- struct{}{}
}()
}
func (s *Server) reinitDb() error {
s.log.Debug("Reinitializing db.")
s.dbAvailable.Store(false)
defer s.dbAvailable.Store(true)
close(s.checkpointNotice)
err := s.dbpool.Close()
if err != nil {
return fmt.Errorf("failed to close db: %w", err)
}
s.log.Debug("Removing main db file.")
err = os.Remove(s.config.DbPath)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to remove db file %s: %w", s.config.DbPath, err)
}
s.log.Debug("Removing WAL file.")
err = os.Remove(s.config.DbPath + "-wal")
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to remove db-wal file %s-wal: %w", s.config.DbPath, err)
}
s.log.Debug("Initializing db.")
err = s.setupDB()
if err != nil {
return fmt.Errorf("failed to setup db during reinit")
}
s.log.Debug("DB reinitialized.")
return nil
}
// SQL schema
//go:embed sql_schema/V*.sql
var sqlSchemaFiles embed.FS
var sqlSchema sqlitemigration.Schema = func() sqlitemigration.Schema {
type migration struct {
content string
version int
name string
}
entries, err := sqlSchemaFiles.ReadDir("sql_schema")
if err != nil {
panic(fmt.Errorf("failed to read sql_schema migrations: %w", err))
}
pattern := regexp.MustCompile("^V([0-9]+)_(.*)[.][sS][qQ][lL]$")
migrations := []*migration{}
for _, entry := range entries {
name := entry.Name()
if entry.IsDir() {
panic(fmt.Errorf("embedded sql migration '%s' is a directory", name))
}
matches := pattern.FindStringSubmatch(name)
if matches == nil {
panic(fmt.Errorf("embedded sql migration '%s' does not match the filename pattern", name))
}
if len(matches) != 3 {
panic(fmt.Errorf("embedded sql migration '%s' does not have the correct number of submatches", name))
}
version, err := strconv.Atoi(matches[1])
if err != nil {
panic(fmt.Errorf("failed to parse version number of migration '%s': %w", name, err))
}
migName := matches[2]
file := path.Join("sql_schema", name)
content, err := sqlSchemaFiles.ReadFile(file)
if err != nil {
panic(fmt.Errorf("failed to read embedded migration %s", entry.Name()))
}
migrations = append(migrations, &migration{
content: string(content),
version: version,
name: migName,
})
}
sort.Slice(migrations, func(i, j int) bool {
return migrations[i].version < migrations[j].version
})
return sqlitemigration.Schema{
Migrations: Map(func(m *migration) string { return m.content }, migrations),
}
}()

117
server/handling.go
View File

@ -17,6 +17,8 @@ import (
"github.com/gin-gonic/gin"
"github.com/mssola/user_agent"
"github.com/sirupsen/logrus"
"zombiezen.com/go/sqlite"
"zombiezen.com/go/sqlite/sqlitex"
)
func internalError(gc *gin.Context, err error) {
@ -39,7 +41,8 @@ func (s *Server) setupRouter() *gin.Engine {
if err != nil {
hostname = "unknown"
}
router.Use(func(gc *gin.Context) {
router.Use(
func(gc *gin.Context) {
path := gc.Request.URL.Path
start := time.Now()
gc.Next()
@ -48,6 +51,7 @@ func (s *Server) setupRouter() *gin.Engine {
statusCode := gc.Writer.Status()
clientIP := gc.ClientIP()
clientUserAgent := gc.Request.UserAgent()
userId := gc.GetHeader(UserIDHeader)
referer := gc.Request.Referer()
dataLength := gc.Writer.Size()
if dataLength < 0 {
@ -63,6 +67,7 @@ func (s *Server) setupRouter() *gin.Engine {
"referer": referer,
"dataLength": dataLength,
"userAgent": clientUserAgent,
"userID": userId,
})
if len(gc.Errors) > 0 {
@ -82,7 +87,58 @@ func (s *Server) setupRouter() *gin.Engine {
entry.Info(msg)
}
}
},
func(gc *gin.Context) {
if !s.dbAvailable.Load() {
gc.AbortWithError(http.StatusServiceUnavailable, fmt.Errorf("server database is not ready/available"))
gc.Header("Retry-After", "60")
return
}
gc.Next()
},
func(gc *gin.Context) {
userIdStr := gc.GetHeader(UserIDHeader)
userIdPresent := userIdStr != ""
var userId int64
var userIdErr error
if userIdPresent {
userId, userIdErr = strconv.ParseInt(userIdStr, 10, 0)
}
appVersion, appVersionErr := extractAppVersion(gc)
if userIdErr != nil {
gc.Error(fmt.Errorf("malformed %s: %w", UserIDHeader, userIdErr))
} else if userIdPresent {
gc.Set("uid", userId)
var err error
if appVersion == nil {
err = withDbConn(s, func(conn *sqlite.Conn) error {
err := sqlitex.Execute(conn, "update users set last_seen_time = ? where id = ?", &sqlitex.ExecOptions{
Args: []interface{}{time.Now().Unix(), userId},
})
return err
})
} else {
err = withDbConn(s, func(conn *sqlite.Conn) error {
err := sqlitex.Execute(conn, "update users set app_version = ?, last_seen_time = ? where id = ?", &sqlitex.ExecOptions{
Args: []interface{}{appVersion, time.Now().Unix(), userId},
})
return err
})
}
if err != nil {
gc.AbortWithError(http.StatusInternalServerError, fmt.Errorf("failed to store user data into db: %w", err))
}
}
if appVersion != nil {
gc.Set("app-version", appVersion)
}
if appVersionErr != nil {
gc.Error(appVersionErr)
gc.Set("app-version-error", appVersionErr.Error())
}
gc.Next()
},
)
// tileserver
router.GET(URITileserver, gin.WrapH(s.tileserverSvSet.Handler()))
@ -118,6 +174,7 @@ func (s *Server) setupRouter() *gin.Engine {
router.GET(URIAppVersion, s.handleGETAppVersion)
router.DELETE(URIAppVersion, s.handleDELETEAppVersion)
router.POST(URIReinit, s.handlePOSTReset)
router.GET(URIUsageInfo, s.handleGETUsageInfo)
return router
}
@ -137,9 +194,19 @@ func extractAppVersion(gc *gin.Context) (*semver.Version, error) {
}
func (s *Server) handleGETPing(gc *gin.Context) {
version, err := extractAppVersion(gc)
if err != nil {
badRequest(gc, err)
versionRaw, exists := gc.Get("app-version")
if !exists {
versionErr := gc.GetString("app-version-error")
if versionErr != "" {
badRequest(gc, fmt.Errorf("malformed app version: %v", &versionErr))
} else {
badRequest(gc, fmt.Errorf("app version not specified"))
}
return
}
version, ok := versionRaw.(*semver.Version)
if !ok {
internalError(gc, fmt.Errorf("malformed app version extracted"))
return
}
@ -220,12 +287,12 @@ func (s *Server) handleDELETEAppVersion(gc *gin.Context) {
}
func (s *Server) handlePOSTReset(gc *gin.Context) {
err := s.initDB(true)
err := s.reinitDb()
if err != nil {
internalError(gc, err)
return
}
gc.Status(http.StatusOK)
gc.Status(http.StatusNoContent)
}
func (s *Server) handleGETTilepack(gc *gin.Context) {
@ -268,6 +335,8 @@ func (s *Server) handlePOSTHandshake(gc *gin.Context) {
}
func (s *Server) handleGETData(gc *gin.Context) {
uidRaw, uidExists := gc.Get("uid")
uid, _ := uidRaw.(int64)
accept := gc.GetHeader("Accept")
if accept == "application/json" {
data, err := s.getDataOnly()
@ -276,7 +345,6 @@ func (s *Server) handleGETData(gc *gin.Context) {
return
}
gc.JSON(http.StatusOK, data)
return
} else if accept == "application/zip" {
file, err := s.getDataWithPhotos()
defer func() {
@ -299,12 +367,25 @@ func (s *Server) handleGETData(gc *gin.Context) {
return
}
gc.DataFromReader(http.StatusOK, size, "application/zip", file, nil)
} else {
gc.String(http.StatusNotAcceptable, "%s is not acceptable", accept)
return
}
gc.String(http.StatusNotAcceptable, "%s is not acceptable", accept)
if uidExists {
err := withDbConn(s, func(conn *sqlite.Conn) error {
return sqlitex.Execute(conn, "update users set last_download_time = ? where id = ?", &sqlitex.ExecOptions{
Args: []interface{}{time.Now().Unix(), uid},
})
})
if err != nil {
gc.Error(fmt.Errorf("failed to store last download time: %w", err))
}
}
}
func (s *Server) handlePOSTData(gc *gin.Context) {
uidRaw, uidExists := gc.Get("uid")
uid, _ := uidRaw.(int64)
switch gc.ContentType() {
case "application/json":
s.handlePOSTDataJSON(gc)
@ -312,6 +393,17 @@ func (s *Server) handlePOSTData(gc *gin.Context) {
s.handlePOSTDataMultipart(gc)
default:
badRequest(gc, fmt.Errorf("unsupported Content-Type"))
return
}
if uidExists {
err := withDbConn(s, func(conn *sqlite.Conn) error {
return sqlitex.Execute(conn, "update users set last_upload_time = ? where id = ?", &sqlitex.ExecOptions{
Args: []interface{}{time.Now().Unix(), uid},
})
})
if err != nil {
gc.Error(fmt.Errorf("failed to store last upload time: %w", err))
}
}
}
@ -470,3 +562,12 @@ func (s *Server) handleGETDataProposals(gc *gin.Context) {
}
gc.JSON(http.StatusOK, proposals)
}
func (s *Server) handleGETUsageInfo(gc *gin.Context) {
usage, err := s.getUsageInfo(nil)
if err != nil {
internalError(gc, err)
return
}
gc.JSON(http.StatusOK, usage)
}

397
server/server.go
View File

@ -4,35 +4,30 @@ import (
"archive/zip"
"bytes"
"context"
"embed"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path"
"regexp"
"sort"
"strconv"
"sync/atomic"
"time"
"cernobor.cz/oko-server/errs"
"cernobor.cz/oko-server/models"
"crawshaw.io/sqlite"
"crawshaw.io/sqlite/sqlitex"
mbsh "github.com/consbio/mbtileserver/handlers"
"github.com/coreos/go-semver/semver"
geojson "github.com/paulmach/go.geojson"
"github.com/sirupsen/logrus"
"zombiezen.com/go/sqlite"
"zombiezen.com/go/sqlite/sqlitemigration"
"zombiezen.com/go/sqlite/sqlitex"
)
//go:embed sql_schema/V*.sql
var sqlSchema embed.FS
type Server struct {
config ServerConfig
dbpool *sqlitex.Pool
dbpool *sqlitemigration.Pool
dbAvailable atomic.Bool
checkpointNotice chan struct{}
log *logrus.Logger
ctx context.Context
@ -66,8 +61,13 @@ func (s *Server) Run(ctx context.Context) {
s.log.SetLevel(logrus.DebugLevel)
}
s.dbAvailable.Store(false)
s.ctx = ctx
s.setupDB()
err := s.setupDB()
if err != nil {
panic(err)
}
defer s.cleanupDb()
s.setupTiles()
@ -96,116 +96,6 @@ func (s *Server) Run(ctx context.Context) {
}
}
func (s *Server) cleanupDb() {
close(s.checkpointNotice)
s.log.Info("Closing db connection pool...")
s.dbpool.Close()
// manually force truncate checkpoint
conn, err := sqlite.OpenConn(fmt.Sprintf("file:%s", s.config.DbPath), 0)
if err != nil {
s.log.WithError(err).Error("Failed to open connection for final checkpoint.")
return
}
err = sqlitex.Exec(conn, "vacuum", nil)
if err != nil {
s.log.WithError(err).Error("Failed to vacuum db.")
}
s.checkpointDb(conn, true)
conn.Close()
}
func (s *Server) getDbConn() *sqlite.Conn {
conn := s.dbpool.Get(s.ctx)
_, err := conn.Prep("PRAGMA foreign_keys = ON").Step()
if err != nil {
panic(err)
}
return conn
}
func (s *Server) checkpointDb(conn *sqlite.Conn, truncate bool) {
var query string
if truncate {
query = "PRAGMA wal_checkpoint(TRUNCATE)"
} else {
query = "PRAGMA wal_checkpoint(RESTART)"
}
stmt, _, err := conn.PrepareTransient(query)
if err != nil {
s.log.WithError(err).Error("Failed to prepare checkpoint query.")
return
}
defer stmt.Finalize()
has, err := stmt.Step()
if err != nil {
s.log.WithError(err).Error("Failed to step through checkpoint query.")
return
}
if !has {
s.log.Error("Checkpoint query returned no rows.")
return
}
blocked := stmt.ColumnInt(0)
noWalPages := stmt.ColumnInt(1)
noReclaimedPages := stmt.ColumnInt(2)
if blocked == 1 {
s.log.Warn("Checkpoint query was blocked.")
}
s.log.Debugf("Checkpoint complete. %d pages written to WAL, %d pages written back to DB.", noWalPages, noReclaimedPages)
}
func (s *Server) setupDB() {
sqlitex.PoolCloseTimeout = time.Second * 10
s.log.Debugf("Using db %s", s.config.DbPath)
dbpool, err := sqlitex.Open(fmt.Sprintf("file:%s", s.config.DbPath), 0, 10)
if err != nil {
s.log.WithError(err).Fatal("Failed to open/create DB.")
}
s.dbpool = dbpool
s.checkpointNotice = make(chan struct{})
err = s.initDB(s.config.ReinitDB)
if err != nil {
s.log.WithError(err).Fatal("init DB transaction failed")
}
// aggressively checkpoint the database on idle times
go func() {
s.log.Debug("Starting manual restart checkpointing.")
defer s.log.Debug("Manual restart checkpointing stopped.")
delay := time.Minute * 15
var (
timer <-chan time.Time
ok bool
)
for {
select {
case _, ok = <-s.checkpointNotice:
if !ok {
return
}
timer = time.After(delay)
case <-timer:
func() {
conn := s.dbpool.Get(s.ctx)
defer s.dbpool.Put(conn)
s.checkpointDb(conn, false)
timer = nil
}()
}
}
}()
}
func (s *Server) requestCheckpoint() {
go func() {
s.checkpointNotice <- struct{}{}
}()
}
func (s *Server) setupTiles() {
tsRootURL, err := url.Parse(URITileserverRoot)
if err != nil {
@ -234,146 +124,6 @@ func (s *Server) setupTiles() {
s.mapPackSize = info.Size()
}
func (s *Server) initDB(reinit bool) error {
s.log.Info("Initializing DB.")
conn := s.getDbConn()
defer s.dbpool.Put(conn)
defer s.requestCheckpoint()
if reinit {
s.log.Warn("Reinitializing DB.")
tables := []string{}
err := sqlitex.Exec(conn, "select name from sqlite_master where type = 'table'", func(stmt *sqlite.Stmt) error {
tables = append(tables, stmt.ColumnText(0))
return nil
})
if err != nil {
return fmt.Errorf("failed to get table names: %w", err)
}
for _, table := range tables {
err = sqlitex.Exec(conn, "drop table "+table, nil)
if err != nil {
return fmt.Errorf("failed to drop tables: %w", err)
}
}
err = sqlitex.Exec(conn, "PRAGMA user_version = 0", nil)
if err != nil {
return fmt.Errorf("failed to reset user version: %w", err)
}
}
err := s.migrateDb(conn)
if err != nil {
return fmt.Errorf("failed to migrate db: %w", err)
}
return nil
}
func (s *Server) migrateDb(conn *sqlite.Conn) error {
var version int
err := sqlitex.Exec(conn, "PRAGMA user_version", func(stmt *sqlite.Stmt) error {
version = stmt.ColumnInt(0)
return nil
})
if err != nil {
return fmt.Errorf("failed to get user version: %w", err)
}
s.log.Debugf("Current db version: %d", version)
entries, err := sqlSchema.ReadDir("sql_schema")
if err != nil {
return fmt.Errorf("failed to read sql_schema migrations: %w", err)
}
type migration struct {
file string
version int
name string
}
pattern := regexp.MustCompile("^V([0-9]+)_(.*)[.][sS][qQ][lL]$")
migrations := []migration{}
for _, entry := range entries {
name := entry.Name()
if entry.IsDir() {
return fmt.Errorf("embedded sql migration '%s' is a directory", name)
}
matches := pattern.FindStringSubmatch(name)
if matches == nil {
return fmt.Errorf("embedded sql migration '%s' does not match the filename pattern", name)
}
if len(matches) != 3 {
return fmt.Errorf("embedded sql migration '%s' does not have the correct number of submatches", name)
}
version, err := strconv.Atoi(matches[1])
if err != nil {
return fmt.Errorf("failed to parse version number of migration '%s': %w", name, err)
}
migName := matches[2]
file := path.Join("sql_schema", name)
migrations = append(migrations, migration{
file: file,
version: version,
name: migName,
})
}
sort.Slice(migrations, func(i, j int) bool {
return migrations[i].version < migrations[j].version
})
for _, migration := range migrations {
if version >= migration.version {
s.log.Debugf("Skipping migration version %d because current version %d is not smaller.", migration.version, version)
continue
}
migContent, err := sqlSchema.ReadFile(migration.file)
if err != nil {
return fmt.Errorf("failed to read embedded migration '%s': %w", migration.file, err)
}
err = func() (err error) {
rollback := sqlitex.Save(conn)
defer func() {
if err != nil {
s.log.Info("Rolling back last migration attempt.")
}
rollback(&err)
}()
s.log.Infof("Executing migration V%d - %s", migration.version, migration.name)
err = sqlitex.ExecScript(conn, string(migContent))
if err != nil {
return fmt.Errorf("failed to execute migration '%s': %w", migration.name, err)
}
err = sqlitex.Exec(conn, fmt.Sprintf("PRAGMA user_version = %d", migration.version), nil)
if err != nil {
return fmt.Errorf("failed to set user_version in db: %w", err)
}
err = sqlitex.Exec(conn, "PRAGMA user_version", func(stmt *sqlite.Stmt) error {
version = stmt.ColumnInt(0)
return nil
})
if err != nil {
return fmt.Errorf("failed to get user_version: %w", err)
}
s.log.Infof("Migrated db to version: %d", version)
return nil
}()
if err != nil {
return err
}
}
return nil
}
func (s *Server) getLatestVersion(v *semver.Version) (*models.AppVersionInfo, error) {
conn := s.getDbConn()
defer s.dbpool.Put(conn)
@ -398,7 +148,8 @@ func (s *Server) getAppVersions() ([]*models.AppVersionInfo, error) {
defer s.dbpool.Put(conn)
versions := []*models.AppVersionInfo{}
err := sqlitex.Exec(conn, "select version, address from app_versions", func(stmt *sqlite.Stmt) error {
err := sqlitex.Execute(conn, "select version, address from app_versions", &sqlitex.ExecOptions{
ResultFunc: func(stmt *sqlite.Stmt) error {
verStr := stmt.ColumnText(0)
addr := stmt.ColumnText(1)
@ -412,6 +163,7 @@ func (s *Server) getAppVersions() ([]*models.AppVersionInfo, error) {
Address: addr,
})
return nil
},
})
if err != nil {
return nil, fmt.Errorf("failed to insert/retrieve user from db: %w", err)
@ -424,7 +176,9 @@ func (s *Server) putAppVersion(versionInfo *models.AppVersionInfo) error {
conn := s.getDbConn()
defer s.dbpool.Put(conn)
err := sqlitex.Exec(conn, "insert into app_versions(version, address) values(?, ?) on conflict(version) do update set address = excluded.address", nil, versionInfo.Version.String(), versionInfo.Address)
err := sqlitex.Execute(conn, "insert into app_versions(version, address) values(?, ?) on conflict(version) do update set address = excluded.address", &sqlitex.ExecOptions{
Args: []interface{}{versionInfo.Version.String(), versionInfo.Address},
})
if err != nil {
return fmt.Errorf("failed to insert app version into db: %w", err)
}
@ -437,7 +191,9 @@ func (s *Server) getAppVersion(version string) (*models.AppVersionInfo, error) {
defer s.dbpool.Put(conn)
var v *models.AppVersionInfo
err := sqlitex.Exec(conn, "select version, address from app_versions where version = ?", func(stmt *sqlite.Stmt) error {
err := sqlitex.Execute(conn, "select version, address from app_versions where version = ?", &sqlitex.ExecOptions{
Args: []interface{}{version},
ResultFunc: func(stmt *sqlite.Stmt) error {
verStr := stmt.ColumnText(0)
ver, err := semver.NewVersion(verStr)
if err != nil {
@ -449,7 +205,8 @@ func (s *Server) getAppVersion(version string) (*models.AppVersionInfo, error) {
Address: addr,
}
return nil
}, version)
},
})
if err != nil {
return nil, fmt.Errorf("failed to retrieve app version %s from db: %w", version, err)
}
@ -461,7 +218,9 @@ func (s *Server) deleteAppVersion(version string) (bool, error) {
conn := s.getDbConn()
defer s.dbpool.Put(conn)
err := sqlitex.Exec(conn, "delete from app_versions where version = ?", nil, version)
err := sqlitex.Execute(conn, "delete from app_versions where version = ?", &sqlitex.ExecOptions{
Args: []interface{}{version},
})
if err != nil {
return false, fmt.Errorf("failed to delete app version %s from db: %w", version, err)
}
@ -479,11 +238,14 @@ func (s *Server) handshake(hc models.HandshakeChallenge) (models.UserID, error)
var id *int64
if hc.Exists {
err = sqlitex.Exec(conn, "select id from users where name = ?", func(stmt *sqlite.Stmt) error {
err = sqlitex.Execute(conn, "select id from users where name = ?", &sqlitex.ExecOptions{
Args: []interface{}{hc.Name},
ResultFunc: func(stmt *sqlite.Stmt) error {
id = ptr(stmt.ColumnInt64(0))
return nil
}, hc.Name)
if sqlite.ErrCode(err) != sqlite.SQLITE_OK {
},
})
if sqlite.ErrCode(err) != sqlite.ResultOK {
return 0, err
}
if id == nil {
@ -493,14 +255,17 @@ func (s *Server) handshake(hc models.HandshakeChallenge) (models.UserID, error)
return 0, errs.ErrAttemptedSystemUser
}
} else {
err = sqlitex.Exec(conn, "insert into users(name) values(?)", func(stmt *sqlite.Stmt) error {
err = sqlitex.Execute(conn, "insert into users(name) values(?)", &sqlitex.ExecOptions{
Args: []interface{}{hc.Name},
ResultFunc: func(stmt *sqlite.Stmt) error {
id = ptr(stmt.ColumnInt64(0))
return nil
}, hc.Name)
if sqlite.ErrCode(err) == sqlite.SQLITE_CONSTRAINT_UNIQUE {
},
})
if sqlite.ErrCode(err) == sqlite.ResultConstraintUnique {
return 0, errs.ErrUserAlreadyExists
}
if sqlite.ErrCode(err) != sqlite.SQLITE_OK {
if sqlite.ErrCode(err) != sqlite.ResultOK {
return 0, err
}
id = ptr(conn.LastInsertRowID())
@ -566,7 +331,8 @@ func (s *Server) getDataWithPhotos() (file *os.File, err error) {
}
data.PhotoMetadata = make(map[string]models.PhotoMetadata, 100)
err = sqlitex.Exec(conn, "select id, content_type, length(contents) from feature_photos", func(stmt *sqlite.Stmt) error {
err = sqlitex.Execute(conn, "select id, content_type, length(contents) from feature_photos", &sqlitex.ExecOptions{
ResultFunc: func(stmt *sqlite.Stmt) error {
id := models.FeaturePhotoID(stmt.ColumnInt64(0))
contentType := stmt.ColumnText(1)
fileSize := stmt.ColumnInt64(2)
@ -577,6 +343,7 @@ func (s *Server) getDataWithPhotos() (file *os.File, err error) {
ThumbnailFilename: makeThumbnailFilename(id),
}
return nil
},
})
if err != nil {
return nil, fmt.Errorf("failed to collect photo metadata: %w", err)
@ -603,7 +370,8 @@ func (s *Server) getDataWithPhotos() (file *os.File, err error) {
return nil, fmt.Errorf("failed to write data zip entry: %w", err)
}
err = sqlitex.Exec(conn, "select id from feature_photos fp where exists (select 1 from features f where f.id = fp.feature_id)", func(stmt *sqlite.Stmt) error {
err = sqlitex.Execute(conn, "select id from feature_photos fp where exists (select 1 from features f where f.id = fp.feature_id)", &sqlitex.ExecOptions{
ResultFunc: func(stmt *sqlite.Stmt) error {
id := stmt.ColumnInt64(0)
blob, err := conn.OpenBlob("", "feature_photos", "thumbnail_contents", id, false)
@ -647,6 +415,7 @@ func (s *Server) getDataWithPhotos() (file *os.File, err error) {
}
return nil
},
})
if err != nil {
return nil, fmt.Errorf("failed to collect photo files: %w", err)
@ -661,7 +430,7 @@ func (s *Server) update(data models.Update, photos map[string]models.Photo) erro
defer s.dbpool.Put(conn)
return func() (err error) {
defer sqlitex.Save(conn)(&err)
defer sqlitex.Transaction(conn)(&err)
var createdIDMapping map[models.FeatureID]models.FeatureID
if data.Create != nil {
@ -716,14 +485,10 @@ func (s *Server) update(data models.Update, photos map[string]models.Photo) erro
}
func (s *Server) deleteExpiredFeatures(conn *sqlite.Conn) error {
stmt, err := conn.Prepare("delete from features where deadline < ?")
if err != nil {
return fmt.Errorf("failed to prepare statement: %w", err)
}
defer stmt.Finalize()
now := time.Now().Unix()
err = sqlitex.Exec(conn, "delete from features where deadline < ?", func(stmt *sqlite.Stmt) error { return nil }, now)
err := sqlitex.Execute(conn, "delete from features where deadline < ?", &sqlitex.ExecOptions{
Args: []interface{}{now},
})
if err != nil {
return fmt.Errorf("failed to delete expired features: %w", err)
}
@ -734,16 +499,18 @@ func (s *Server) deleteExpiredFeatures(conn *sqlite.Conn) error {
func (s *Server) getPeople(conn *sqlite.Conn) ([]models.User, error) {
if conn == nil {
conn = s.getDbConn()
defer s.dbpool.Put(conn)
defer s.returnDbConn(conn)
}
users := make([]models.User, 0, 50)
err := sqlitex.Exec(conn, "select id, name from users", func(stmt *sqlite.Stmt) error {
err := sqlitex.Execute(conn, "select id, name from users", &sqlitex.ExecOptions{
ResultFunc: func(stmt *sqlite.Stmt) error {
users = append(users, models.User{
ID: models.UserID(stmt.ColumnInt(0)),
Name: stmt.ColumnText(1),
})
return nil
},
})
if err != nil {
return nil, fmt.Errorf("failed to get users from db: %w", err)
@ -758,10 +525,11 @@ func (s *Server) getFeatures(conn *sqlite.Conn) ([]models.Feature, error) {
}
features := make([]models.Feature, 0, 100)
err := sqlitex.Exec(conn, `select f.id, f.owner_id, f.name, f.deadline, f.properties, f.geom, '[' || coalesce(group_concat(p.id, ', '), '') || ']'
err := sqlitex.Execute(conn, `select f.id, f.owner_id, f.name, f.deadline, f.properties, f.geom, '[' || coalesce(group_concat(p.id, ', '), '') || ']'
from features f
left join feature_photos p on f.id = p.feature_id
group by f.id, f.owner_id, f.name, f.properties, f.geom`, func(stmt *sqlite.Stmt) error {
group by f.id, f.owner_id, f.name, f.properties, f.geom`, &sqlitex.ExecOptions{
ResultFunc: func(stmt *sqlite.Stmt) error {
id := stmt.ColumnInt64(0)
@ -770,7 +538,7 @@ func (s *Server) getFeatures(conn *sqlite.Conn) ([]models.Feature, error) {
name := stmt.ColumnText(2)
var deadline *time.Time
if stmt.ColumnType(3) != sqlite.SQLITE_NULL {
if stmt.ColumnType(3) != sqlite.TypeNull {
dl := time.Unix(stmt.ColumnInt64(3), 0)
deadline = &dl
}
@ -808,6 +576,7 @@ func (s *Server) getFeatures(conn *sqlite.Conn) ([]models.Feature, error) {
features = append(features, feature)
return nil
},
})
if err != nil {
return nil, fmt.Errorf("failed to get users from db: %w", err)
@ -1101,7 +870,9 @@ func (s *Server) getPhoto(featureID models.FeatureID, photoID models.FeaturePhot
var contentType *string = nil
var data []byte = nil
found := false
err := sqlitex.Exec(conn, "select content_type, contents from feature_photos where id = ? and feature_id = ?", func(stmt *sqlite.Stmt) error {
err := sqlitex.Execute(conn, "select content_type, contents from feature_photos where id = ? and feature_id = ?", &sqlitex.ExecOptions{
Args: []interface{}{photoID, featureID},
ResultFunc: func(stmt *sqlite.Stmt) error {
if found {
return fmt.Errorf("multiple photos returned for feature id %d, photo id %d", featureID, photoID)
}
@ -1110,7 +881,8 @@ func (s *Server) getPhoto(featureID models.FeatureID, photoID models.FeaturePhot
stmt.ColumnBytes(1, data)
found = true
return nil
}, photoID, featureID)
},
})
if err != nil {
return nil, "", fmt.Errorf("photo db query failed: %w", err)
}
@ -1159,16 +931,57 @@ func (s *Server) getProposals(conn *sqlite.Conn) ([]models.Proposal, error) {
}
proposals := make([]models.Proposal, 0, 100)
err := sqlitex.Exec(conn, "select owner_id, description, how from proposals", func(stmt *sqlite.Stmt) error {
err := sqlitex.Execute(conn, "select owner_id, description, how from proposals", &sqlitex.ExecOptions{
ResultFunc: func(stmt *sqlite.Stmt) error {
proposals = append(proposals, models.Proposal{
OwnerID: models.UserID(stmt.ColumnInt(0)),
Description: stmt.ColumnText(1),
How: stmt.ColumnText(2),
})
return nil
},
})
if err != nil {
return nil, fmt.Errorf("failed to get proposals from db: %w", err)
}
return proposals, nil
}
func (s *Server) getUsageInfo(conn *sqlite.Conn) ([]models.UserInfo, error) {
if conn == nil {
conn = s.getDbConn()
defer s.returnDbConn(conn)
}
users := make([]models.UserInfo, 0, 50)
err := sqlitex.Execute(conn, "select id, name, app_version, last_seen_time, last_upload_time, last_download_time from users", &sqlitex.ExecOptions{
ResultFunc: func(stmt *sqlite.Stmt) error {
ver, _ := semver.NewVersion(stmt.ColumnText(2))
var lstp, lutp, ldtp *time.Time
if stmt.ColumnType(3) != sqlite.TypeNull {
lstp = ptr(time.Unix(stmt.ColumnInt64(3), 0))
}
if stmt.ColumnType(4) != sqlite.TypeNull {
lutp = ptr(time.Unix(stmt.ColumnInt64(3), 0))
}
if stmt.ColumnType(5) != sqlite.TypeNull {
ldtp = ptr(time.Unix(stmt.ColumnInt64(3), 0))
}
users = append(users, models.UserInfo{
User: models.User{
ID: models.UserID(stmt.ColumnInt(0)),
Name: stmt.ColumnText(1),
},
AppVersion: ver,
LastSeenTime: lstp,
LastUploadTime: lutp,
LastDownloadTime: ldtp,
})
return nil
},
})
if err != nil {
return nil, fmt.Errorf("failed to get users from db: %w", err)
}
return users, nil
}

2
server/sql_schema/V1_init.sql
View File

@ -8,7 +8,7 @@ CREATE TABLE features (
id integer PRIMARY KEY AUTOINCREMENT,
owner_id integer NOT NULL REFERENCES users(id) ON DELETE CASCADE,
name text NOT NULL,
deadline text,
deadline integer,
properties text NOT NULL,
geom text NOT NULL
);

4
server/sql_schema/V4_user_info.sql Normal file
View File

@ -0,0 +1,4 @@
ALTER TABLE users ADD COLUMN app_version text;
ALTER TABLE users ADD COLUMN last_seen_time integer;
ALTER TABLE users ADD COLUMN last_upload_time integer;
ALTER TABLE users ADD COLUMN last_download_time integer;

8
server/utils.go
View File

@ -17,6 +17,14 @@ func ptr[T any](x T) *T {
return &x
}
func Map[T any, U any](f func(T) U, x []T) []U {
res := make([]U, len(x))
for i, e := range x {
res[i] = f(e)
}
return res
}
var contentTypes map[string]struct{} = map[string]struct{}{"image/jpeg": {}, "image/png": {}}
func checkImageContentType(contentType string) bool {