App version management.

* DB migration adds app_versions table for managing known app versions.
* /ping reads app version from User-Agent header and responds with latest version if the app is OKO and its version is older than the latest one stored.
* Added endpoint /app-versions which
  * lists all known versions via GET
  * adds/updates a version via POST

#3
Fix #4

.dockerignore

@@ -38,4 +38,5 @@ go.work
 oko-server
 *.sqlite*
 .vscode
-.git
+data
+local-testing

.gitignore

@@ -37,4 +37,5 @@ go.work
 # Project-specific
 oko-server
 *.sqlite*
-.vscode
+.vscode
+__debug_bin

Dockerfile

@@ -1,19 +1,19 @@
-FROM alpine:3.15.0 AS build
+FROM alpine:3.16 AS build
 
+ARG CAPROVER_GIT_COMMIT_SHA=""
 VOLUME ["/data"]
 COPY . /oko-server/git
 
-RUN apk add --no-cache go && \
+RUN apk update && \
+    apk add go && \
     cd /oko-server/git && \
-    go build
+    go build -ldflags "-X \"main.sha1ver=${CAPROVER_GIT_COMMIT_SHA:-$(cat .git/$(cat .git/HEAD | sed 's|ref: ||g'))}\" -X \"main.buildTime=$(date -Iseconds)\""
 
-FROM alpine:3.15.0
+FROM alpine:3.16
 WORKDIR /oko-server
 VOLUME [ "/data" ]
 
-RUN echo -e '#!/bin/sh\n/oko-server/oko-server "$@"' > /oko-server/entrypoint.sh && \
-    chmod +x /oko-server/entrypoint.sh
 COPY --from=build /oko-server/git/oko-server/ /oko-server/
 
-ENTRYPOINT ["/oko-server/entrypoint.sh"]
+ENTRYPOINT ["/oko-server/oko-server"]
 

errs/errors.go

@@ -34,6 +34,22 @@ func (e *ErrPhotoNotProvided) Error() string {
 	return fmt.Sprintf("referenced photo %s which was not provided", e.Reference)
 }
 
+type EErrPhotoThumbnailNotProvided *ErrPhotoThumbnailNotProvided
+
+type ErrPhotoThumbnailNotProvided struct {
+	Reference string
+}
+
+func NewErrPhotoThumbnailNotProvided(reference string) *ErrPhotoThumbnailNotProvided {
+	return &ErrPhotoThumbnailNotProvided{
+		Reference: reference,
+	}
+}
+
+func (e *ErrPhotoThumbnailNotProvided) Error() string {
+	return fmt.Sprintf("referenced photo %s the thumbnail of which was not provided", e.Reference)
+}
+
 type ErrFeatureForPhotoNotExists struct {
 	PhotoFeatureReference int64
 }

go.mod

@@ -1,17 +1,18 @@
 module cernobor.cz/oko-server
 
-go 1.17
+go 1.19
 
 require (
 	crawshaw.io/sqlite v0.3.2
-	github.com/consbio/mbtileserver v0.8.1
+	github.com/consbio/mbtileserver v0.8.2
 	github.com/gin-gonic/gin v1.7.7
 	github.com/paulmach/go.geojson v1.4.0
 	github.com/sirupsen/logrus v1.8.1
 )
 
 require (
-	github.com/brendan-ward/mbtiles-go v0.0.0-20211210015813-553bc514bbdf // indirect
+	github.com/brendan-ward/mbtiles-go v0.1.1-0.20220129145719-67a7dabdbaab // indirect
+	github.com/coreos/go-semver v0.3.0
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-playground/locales v0.14.0 // indirect
 	github.com/go-playground/universal-translator v0.18.0 // indirect
@@ -22,8 +23,10 @@ require (
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/mssola/user_agent v0.5.3
 	github.com/ugorji/go/codec v1.2.6 // indirect
 	golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce // indirect
+	golang.org/x/image v0.0.0-20220722155232-062f8c9fd539
 	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect

go.sum

@@ -68,8 +68,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/brendan-ward/mbtiles-go v0.0.0-20211210015813-553bc514bbdf h1:RT/qOIiM0yuaELiwbyli7g6bHFARWlU8B27Ff1r8K6g=
+github.com/brendan-ward/mbtiles-go v0.1.1-0.20220129145719-67a7dabdbaab h1:KC2TqbUmfkIrTwVpkH1ysJiZicPoSyb6chH3JxKHqBM=
-github.com/brendan-ward/mbtiles-go v0.0.0-20211210015813-553bc514bbdf/go.mod h1:rlKkU0/sjOysMB+dvVI+b90UyzmD5alQK7KlhJplVrg=
+github.com/brendan-ward/mbtiles-go v0.1.1-0.20220129145719-67a7dabdbaab/go.mod h1:rlKkU0/sjOysMB+dvVI+b90UyzmD5alQK7KlhJplVrg=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
@@ -92,8 +92,9 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/consbio/mbtileserver v0.8.1 h1:Q1x3Vf4Wbb92uNQ9ZAIn2bkS+nzMbcnH2qidq81lxOs=
+github.com/consbio/mbtileserver v0.8.2 h1:iXOBkCp4r/7Z7wC4sw6dYkFAJnuyVpQMu0dbT+RHSxE=
-github.com/consbio/mbtileserver v0.8.1/go.mod h1:efp5wvQOhIhElvcXrFhXUw+k43dwn9EfdACvXBwHy7o=
+github.com/consbio/mbtileserver v0.8.2/go.mod h1:lVBCXeAL6OzooRXWbQ/MZFGFtwm/0A1fkeNTf9T4gzA=
+github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -317,6 +318,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/mssola/user_agent v0.5.3 h1:lBRPML9mdFuIZgI2cmlQ+atbpJdLdeVl2IDodjBR578=
+github.com/mssola/user_agent v0.5.3/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -377,7 +380,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
-github.com/ugorji/go v1.2.6 h1:tGiWC9HENWE2tqYycIqFTNorMmFRVhNwCpDOpWqnk8E=
 github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
@@ -430,6 +432,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 h1:/eM0PCrQI2xd471rI+snWuu251/+/jpBpZqir2mPdnU=
+golang.org/x/image v0.0.0-20220722155232-062f8c9fd539/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

main.go

@@ -7,12 +7,18 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"cernobor.cz/oko-server/models"
 	"cernobor.cz/oko-server/server"
 	"github.com/sirupsen/logrus"
 )
 
+var (
+	sha1ver   string
+	buildTime string
+)
+
 func main() {
 	tilepackFileArg := flag.String("tilepack", "", "File that will be sent to clients when they request a tile pack, also used to serve tiles in online mode. Required.")
 	portArg := flag.Int("port", 8080, "Port where the server will listen to. Default is 8080.")
@@ -22,6 +28,9 @@ func main() {
 	minZoomArg := flag.Int("min-zoom", 1, "Minimum zoom that will be sent to clients.")
 	defaultCenterLatArg := flag.Float64("default-center-lat", 0, "Latitude of the default map center.")
 	defaultCenterLngArg := flag.Float64("default-center-lng", 0, "Longitude of the default map center.")
+	maxPhotoXArg := flag.Int("max-photo-width", 0, "Maximum width of photos. 0 means no limit.")
+	maxPhotoYArg := flag.Int("max-photo-height", 0, "Maximum height of photos. 0 means no limit.")
+	photoQualityArg := flag.Int("photo-quality", 90, "Photo JPEG quality.")
 
 	flag.Parse()
 
@@ -31,7 +40,19 @@ func main() {
 		os.Exit(1)
 	}
 
+	t, err := time.Parse(time.RFC3339, buildTime)
+	if err != nil {
+		t = time.Now()
+	}
+
+	if *maxPhotoXArg < 0 || *maxPhotoYArg < 0 {
+		fmt.Fprintln(os.Stderr, "Max photo width and height cannot be less than 0.")
+		os.Exit(1)
+	}
+
 	s := server.New(server.ServerConfig{
+		VersionHash:  sha1ver,
+		BuildTime:    &t,
 		Port:         *portArg,
 		DbPath:       *dbFileArg,
 		TilepackPath: *tilepackFileArg,
@@ -42,6 +63,9 @@ func main() {
 			Lat: *defaultCenterLatArg,
 			Lng: *defaultCenterLngArg,
 		},
+		MaxPhotoX:    *maxPhotoXArg,
+		MaxPhotoY:    *maxPhotoYArg,
+		PhotoQuality: *photoQualityArg,
 	})
 
 	sigs := make(chan os.Signal, 1)

models/models.go

@@ -4,6 +4,7 @@ import (
 	"io"
 	"time"
 
+	"github.com/coreos/go-semver/semver"
 	geojson "github.com/paulmach/go.geojson"
 )
 
@@ -33,8 +34,18 @@ type Feature struct {
 	PhotoIDs []FeaturePhotoID `json:"photo_ids"`
 }
 
+type BuildInfo struct {
+	VersionHash string     `json:"version_hash"`
+	BuildTime   *time.Time `json:"build_time"`
+}
+
 // transport objects
 
+type AppVersionInfo struct {
+	Version semver.Version `json:"version"`
+	Address string         `json:"address"`
+}
+
 type Coords struct {
 	Lat float64 `json:"lat"`
 	Lng float64 `json:"lng"`
@@ -55,6 +66,7 @@ type Update struct {
 	Update        []Feature              `json:"update"`
 	Delete        []FeatureID            `json:"delete"`
 	DeletePhotos  []FeaturePhotoID       `json:"delete_photos"`
+	Proposals     []Proposal             `json:"proposals"`
 }
 
 type HandshakeChallenge struct {
@@ -69,8 +81,9 @@ type HandshakeResponse struct {
 }
 
 type Data struct {
-	Users    []User    `json:"users"`
+	Users         []User                   `json:"users"`
-	Features []Feature `json:"features"`
+	Features      []Feature                `json:"features"`
+	PhotoMetadata map[string]PhotoMetadata `json:"photo_metadata,omitempty"`
 }
 
 type Photo struct {
@@ -78,3 +91,17 @@ type Photo struct {
 	File        io.ReadCloser
 	Size        int64
 }
+
+type PhotoMetadata struct {
+	ContentType          string         `json:"content_type"`
+	ThumbnailContentType string         `json:"thumbnail_content_type"`
+	Size                 int64          `json:"size"`
+	ID                   FeaturePhotoID `json:"id"`
+	ThumbnailFilename    string         `json:"thumbnail_filename"`
+}
+
+type Proposal struct {
+	OwnerID     UserID `json:"owner_id"`
+	Description string `json:"description"`
+	How         string `json:"how"`
+}

server/constants.go

@@ -2,8 +2,10 @@ package server
 
 const (
 	URIPing              = "/ping"
+	URIBuildInfo         = "/build-info"
 	URIHardFail          = "/hard-fail"
 	URISoftFail          = "/soft-fail"
+	URIAppVersions       = "/app-versions"
 	URIReinit            = "/reinit"
 	URIMapPack           = "/mappack"
 	URIHandshake         = "/handshake"
@@ -11,7 +13,10 @@ const (
 	URIDataPeople        = "/data/people"
 	URIDataFeatures      = "/data/features"
 	URIDataFeaturesPhoto = "/data/features/:feature/photos/:photo"
+	URIDataProposals     = "/data/proposals"
 	URITileserverRoot    = "/tileserver"
 	URITileserver        = URITileserverRoot + "/*x"
 	URITileTemplate      = URITileserverRoot + "/map/tiles/{z}/{x}/{y}.pbf"
+
+	AppName = "OKO"
 )

server/handling.go

@@ -4,7 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"math"
 	"net/http"
 	"os"
@@ -13,14 +13,22 @@ import (
 
 	"cernobor.cz/oko-server/errs"
 	"cernobor.cz/oko-server/models"
+	"github.com/coreos/go-semver/semver"
 	"github.com/gin-gonic/gin"
+	"github.com/mssola/user_agent"
 	"github.com/sirupsen/logrus"
 )
 
 func internalError(gc *gin.Context, err error) {
+	gc.Error(err)
 	gc.String(http.StatusInternalServerError, "%v", err)
 }
 
+func badRequest(gc *gin.Context, err error) {
+	gc.Error(err)
+	gc.String(http.StatusBadRequest, "%v", err)
+}
+
 func (s *Server) setupRouter() *gin.Engine {
 	gin.SetMode(gin.ReleaseMode)
 	router := gin.New()
@@ -77,9 +85,28 @@ func (s *Server) setupRouter() *gin.Engine {
 		}
 	})
 
+	// tileserver
+	router.GET(URITileserver, gin.WrapH(s.tileserverSvSet.Handler()))
+
+	/*** API ***/
+	router.GET(URIPing, s.handleGETPing)
+	router.POST(URIHandshake, s.handlePOSTHandshake)
+	router.GET(URIData, s.handleGETData)
+	router.POST(URIData, s.handlePOSTData)
+	router.GET(URIDataPeople, s.handleGETDataPeople)
+	router.GET(URIDataFeatures, s.handleGETDataFeatures)
+	router.GET(URIDataFeaturesPhoto, s.handleGETDataFeaturesPhoto)
+	router.GET(URIDataProposals, s.handleGETDataProposals)
+
+	// resources
+	router.GET(URIMapPack, s.handleGETTilepack)
+
 	// utility/debug paths
-	router.GET(URIPing, func(gc *gin.Context) {
+	router.GET(URIBuildInfo, func(gc *gin.Context) {
-		gc.Status(http.StatusNoContent)
+		gc.JSON(http.StatusOK, models.BuildInfo{
+			VersionHash: s.config.VersionHash,
+			BuildTime:   s.config.BuildTime,
+		})
 	})
 	router.GET(URIHardFail, func(gc *gin.Context) {
 		gc.Status(http.StatusNotImplemented)
@@ -87,27 +114,74 @@ func (s *Server) setupRouter() *gin.Engine {
 	router.GET(URISoftFail, func(gc *gin.Context) {
 		gc.JSON(http.StatusOK, map[string]string{"error": "artificial fail"})
 	})
+	router.GET(URIAppVersions, s.handleGETAppVersions)
+	router.POST(URIAppVersions, s.handlePOSTAppVersions)
 	router.POST(URIReinit, s.handlePOSTReset)
 
-	// resources
-	router.GET(URIMapPack, s.handleGETTilepack)
-
-	// API
-	router.POST(URIHandshake, s.handlePOSTHandshake)
-	router.GET(URIData, s.handleGETData)
-	router.POST(URIData, s.handlePOSTData)
-	router.GET(URIDataPeople, s.handleGETDataPeople)
-	router.GET(URIDataFeatures, s.handleGETDataFeatures)
-	router.GET(URIDataFeaturesPhoto, s.handleGETDataFeaturesPhoto)
-
-	// tileserver
-	router.GET(URITileserver, gin.WrapH(s.tileserverSvSet.Handler()))
-
 	return router
 }
 
+func extractAppVersion(gc *gin.Context) (*semver.Version, error) {
+	ua := user_agent.New(gc.Request.UserAgent())
+	n, v := ua.Browser()
+	if n != AppName {
+		return nil, nil
+	}
+
+	version, err := semver.NewVersion(v)
+	if err != nil {
+		return nil, fmt.Errorf("malformed version in User-Agent header: %w", err)
+	}
+	return version, nil
+}
+
+func (s *Server) handleGETPing(gc *gin.Context) {
+	version, err := extractAppVersion(gc)
+	if err != nil {
+		badRequest(gc, err)
+		return
+	}
+
+	res, err := s.getLatestVersion(version)
+	if err != nil {
+		internalError(gc, err)
+		return
+	}
+
+	if res == nil {
+		gc.Status(http.StatusNoContent)
+	} else {
+		gc.JSON(http.StatusOK, res)
+	}
+}
+
+func (s *Server) handleGETAppVersions(gc *gin.Context) {
+	versions, err := s.getAppVersions()
+	if err != nil {
+		internalError(gc, err)
+		return
+	}
+	gc.JSON(http.StatusOK, versions)
+}
+
+func (s *Server) handlePOSTAppVersions(gc *gin.Context) {
+	var versionInfo models.AppVersionInfo
+	err := gc.ShouldBindJSON(&versionInfo)
+	if err != nil {
+		badRequest(gc, fmt.Errorf("malformed version info: %w", err))
+		return
+	}
+
+	err = s.putAppVersion(&versionInfo)
+	if err != nil {
+		internalError(gc, err)
+		return
+	}
+	gc.Status(http.StatusNoContent)
+}
+
 func (s *Server) handlePOSTReset(gc *gin.Context) {
-	err := s.reinitDB()
+	err := s.initDB(true)
 	if err != nil {
 		internalError(gc, err)
 		return
@@ -123,7 +197,7 @@ func (s *Server) handlePOSTHandshake(gc *gin.Context) {
 	var hs models.HandshakeChallenge
 	err := gc.ShouldBindJSON(&hs)
 	if err != nil {
-		gc.String(http.StatusBadRequest, fmt.Sprintf("malformed handshake challenge: %v", err))
+		badRequest(gc, fmt.Errorf("malformed handshake challenge: %w", err))
 		return
 	}
 
@@ -155,12 +229,40 @@ func (s *Server) handlePOSTHandshake(gc *gin.Context) {
 }
 
 func (s *Server) handleGETData(gc *gin.Context) {
-	data, err := s.getData()
+	accept := gc.GetHeader("Accept")
-	if err != nil {
+	if accept == "application/json" {
-		internalError(gc, err)
+		data, err := s.getDataOnly()
+		if err != nil {
+			internalError(gc, err)
+			return
+		}
+		gc.JSON(http.StatusOK, data)
+		return
+	} else if accept == "application/zip" {
+		file, err := s.getDataWithPhotos()
+		defer func() {
+			file.Close()
+			os.Remove(file.Name())
+		}()
+		if err != nil {
+			internalError(gc, err)
+			return
+		}
+		fi, err := file.Stat()
+		if err != nil {
+			internalError(gc, err)
+			return
+		}
+		size := fi.Size()
+		_, err = file.Seek(0, 0)
+		if err != nil {
+			internalError(gc, err)
+			return
+		}
+		gc.DataFromReader(http.StatusOK, size, "application/zip", file, nil)
 		return
 	}
-	gc.JSON(http.StatusOK, data)
+	gc.String(http.StatusNotAcceptable, "%s is not acceptable", accept)
 }
 
 func (s *Server) handlePOSTData(gc *gin.Context) {
@@ -170,7 +272,7 @@ func (s *Server) handlePOSTData(gc *gin.Context) {
 	case "multipart/form-data":
 		s.handlePOSTDataMultipart(gc)
 	default:
-		gc.String(http.StatusBadRequest, "unsupported Content-Type")
+		badRequest(gc, fmt.Errorf("unsupported Content-Type"))
 	}
 }
 
@@ -178,17 +280,17 @@ func (s *Server) handlePOSTDataJSON(gc *gin.Context) {
 	var data models.Update
 	err := gc.ShouldBindJSON(&data)
 	if err != nil {
-		gc.String(http.StatusBadRequest, fmt.Sprintf("malformed data: %v", err))
+		badRequest(gc, fmt.Errorf("malformed data: %w", err))
 		return
 	}
 
 	if !isUniqueFeatureID(data.Create) {
-		gc.String(http.StatusBadRequest, "created features do not have unique IDs")
+		badRequest(gc, fmt.Errorf("created features do not have unique IDs"))
 		return
 	}
 
 	if data.CreatedPhotos != nil || data.AddPhotos != nil {
-		gc.String(http.StatusBadRequest, "created_photos and/or add_photos present, but Content-Type is application/json")
+		badRequest(gc, fmt.Errorf("created_photos and/or add_photos present, but Content-Type is application/json"))
 		return
 	}
 
@@ -203,7 +305,7 @@ func (s *Server) handlePOSTDataJSON(gc *gin.Context) {
 func (s *Server) handlePOSTDataMultipart(gc *gin.Context) {
 	form, err := gc.MultipartForm()
 	if err != nil {
-		gc.String(http.StatusBadRequest, "malformed multipart/form-data content")
+		badRequest(gc, fmt.Errorf("malformed multipart/form-data content"))
 		return
 	}
 
@@ -211,11 +313,11 @@ func (s *Server) handlePOSTDataMultipart(gc *gin.Context) {
 	if !ok {
 		dataFile, ok := form.File["data"]
 		if !ok {
-			gc.String(http.StatusBadRequest, "value 'data' is missing from the content")
+			badRequest(gc, fmt.Errorf("value 'data' is missing from the content"))
 			return
 		}
 		if len(dataFile) != 1 {
-			gc.String(http.StatusBadRequest, "value 'data' does not contain exactly 1 item")
+			badRequest(gc, fmt.Errorf("value 'data' does not contain exactly 1 item"))
 			return
 		}
 		df, err := dataFile[0].Open()
@@ -223,7 +325,7 @@ func (s *Server) handlePOSTDataMultipart(gc *gin.Context) {
 			internalError(gc, fmt.Errorf("failed to open 'data' 'file': %w", err))
 			return
 		}
-		dataBytes, err := ioutil.ReadAll(df)
+		dataBytes, err := io.ReadAll(df)
 		if err != nil {
 			internalError(gc, fmt.Errorf("failed to open 'data' 'file': %w", err))
 			return
@@ -231,26 +333,26 @@ func (s *Server) handlePOSTDataMultipart(gc *gin.Context) {
 		dataStr = []string{string(dataBytes)}
 	}
 	if len(dataStr) != 1 {
-		gc.String(http.StatusBadRequest, "value 'data' does not contain exactly 1 item")
+		badRequest(gc, fmt.Errorf("value 'data' does not contain exactly 1 item"))
 		return
 	}
 
 	var data models.Update
 	err = json.Unmarshal([]byte(dataStr[0]), &data)
 	if err != nil {
-		gc.String(http.StatusBadRequest, "malformed 'data' value: %v", err)
+		badRequest(gc, fmt.Errorf("malformed 'data' value: %w", err))
 		return
 	}
 
 	if !isUniqueFeatureID(data.Create) {
-		gc.String(http.StatusBadRequest, "created features do not have unique IDs")
+		badRequest(gc, fmt.Errorf("created features do not have unique IDs"))
 		return
 	}
 
 	photos := make(map[string]models.Photo, len(form.File))
 	for name, fh := range form.File {
 		if len(fh) != 1 {
-			gc.String(http.StatusBadRequest, "file item %s does not contain exactly 1 file", name)
+			badRequest(gc, fmt.Errorf("file item %s does not contain exactly 1 file", name))
 			return
 		}
 		var photo models.Photo
@@ -268,7 +370,7 @@ func (s *Server) handlePOSTDataMultipart(gc *gin.Context) {
 	if err != nil {
 		var e *errs.ErrUnsupportedContentType
 		if errors.As(err, &e) {
-			gc.String(http.StatusBadRequest, e.Error())
+			badRequest(gc, e)
 			return
 		}
 		internalError(gc, fmt.Errorf("failed to update data: %w", err))
@@ -299,12 +401,12 @@ func (s *Server) handleGETDataFeatures(gc *gin.Context) {
 func (s *Server) handleGETDataFeaturesPhoto(gc *gin.Context) {
 	reqFeatureID, err := strconv.Atoi(gc.Param("feature"))
 	if err != nil {
-		gc.String(http.StatusBadRequest, "malformed feature ID")
+		badRequest(gc, fmt.Errorf("malformed feature ID"))
 		return
 	}
 	reqPhotoID, err := strconv.Atoi(gc.Param("photo"))
 	if err != nil {
-		gc.String(http.StatusBadRequest, "malformed photo ID")
+		badRequest(gc, fmt.Errorf("malformed photo ID"))
 		return
 	}
 
@@ -320,3 +422,12 @@ func (s *Server) handleGETDataFeaturesPhoto(gc *gin.Context) {
 
 	gc.Data(http.StatusOK, contentType, photoBytes)
 }
+
+func (s *Server) handleGETDataProposals(gc *gin.Context) {
+	proposals, err := s.getProposals(nil)
+	if err != nil {
+		internalError(gc, err)
+		return
+	}
+	gc.JSON(http.StatusOK, proposals)
+}

server/server.go

@@ -1,39 +1,48 @@
 package server
 
 import (
+	"archive/zip"
+	"bytes"
 	"context"
+	"embed"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"os"
+	"path"
+	"regexp"
+	"sort"
+	"strconv"
 	"time"
 
-	_ "embed"
-
 	"cernobor.cz/oko-server/errs"
 	"cernobor.cz/oko-server/models"
 	"crawshaw.io/sqlite"
 	"crawshaw.io/sqlite/sqlitex"
 	mbsh "github.com/consbio/mbtileserver/handlers"
+	"github.com/coreos/go-semver/semver"
 	geojson "github.com/paulmach/go.geojson"
 	"github.com/sirupsen/logrus"
 )
 
-//go:embed initdb.sql
+//go:embed sql_schema/V*.sql
-var initDB string
+var sqlSchema embed.FS
 
 type Server struct {
-	config          ServerConfig
+	config           ServerConfig
-	dbpool          *sqlitex.Pool
+	dbpool           *sqlitex.Pool
-	log             *logrus.Logger
+	checkpointNotice chan struct{}
-	ctx             context.Context
+	log              *logrus.Logger
-	tileserverSvSet *mbsh.ServiceSet
+	ctx              context.Context
-	mapPackSize     int64
+	tileserverSvSet  *mbsh.ServiceSet
+	mapPackSize      int64
 }
 
 type ServerConfig struct {
+	VersionHash   string
+	BuildTime     *time.Time
 	Port          int
 	DbPath        string
 	TilepackPath  string
@@ -41,6 +50,9 @@ type ServerConfig struct {
 	ReinitDB      bool
 	MinZoom       int
 	DefaultCenter models.Coords
+	MaxPhotoX     int
+	MaxPhotoY     int
+	PhotoQuality  int
 }
 
 func New(config ServerConfig) *Server {
@@ -55,6 +67,8 @@ func (s *Server) Run(ctx context.Context) {
 
 	s.ctx = ctx
 	s.setupDB()
+	defer s.cleanupDb()
+
 	s.setupTiles()
 
 	router := s.setupRouter()
@@ -72,15 +86,32 @@ func (s *Server) Run(ctx context.Context) {
 
 	<-s.ctx.Done()
 	s.log.Info("Shutting down server...")
+	defer s.log.Info("Server exitting.")
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
 	if err := server.Shutdown(ctx); err != nil {
 		s.log.WithError(err).Fatal("Server forced to shutdown.")
 	}
+}
+
+func (s *Server) cleanupDb() {
+	close(s.checkpointNotice)
+	s.log.Info("Closing db connection pool...")
 	s.dbpool.Close()
 
-	s.log.Info("Server exitting.")
+	// manually force truncate checkpoint
+	conn, err := sqlite.OpenConn(fmt.Sprintf("file:%s", s.config.DbPath), 0)
+	if err != nil {
+		s.log.WithError(err).Error("Failed to open connection for final checkpoint.")
+		return
+	}
+	err = sqlitex.Exec(conn, "vacuum", nil)
+	if err != nil {
+		s.log.WithError(err).Error("Failed to vacuum db.")
+	}
+	s.checkpointDb(conn, true)
+	conn.Close()
 }
 
 func (s *Server) getDbConn() *sqlite.Conn {
@@ -92,19 +123,86 @@ func (s *Server) getDbConn() *sqlite.Conn {
 	return conn
 }
 
+func (s *Server) checkpointDb(conn *sqlite.Conn, truncate bool) {
+	var query string
+	if truncate {
+		query = "PRAGMA wal_checkpoint(TRUNCATE)"
+	} else {
+		query = "PRAGMA wal_checkpoint(RESTART)"
+	}
+	stmt, _, err := conn.PrepareTransient(query)
+	if err != nil {
+		s.log.WithError(err).Error("Failed to prepare checkpoint query.")
+		return
+	}
+	defer stmt.Finalize()
+
+	has, err := stmt.Step()
+	if err != nil {
+		s.log.WithError(err).Error("Failed to step through checkpoint query.")
+		return
+	}
+	if !has {
+		s.log.Error("Checkpoint query returned no rows.")
+		return
+	}
+
+	blocked := stmt.ColumnInt(0)
+	noWalPages := stmt.ColumnInt(1)
+	noReclaimedPages := stmt.ColumnInt(2)
+	if blocked == 1 {
+		s.log.Warn("Checkpoint query was blocked.")
+	}
+	s.log.Debugf("Checkpoint complete. %d pages written to WAL, %d pages written back to DB.", noWalPages, noReclaimedPages)
+}
+
 func (s *Server) setupDB() {
+	sqlitex.PoolCloseTimeout = time.Second * 10
+	s.log.Debugf("Using db %s", s.config.DbPath)
 	dbpool, err := sqlitex.Open(fmt.Sprintf("file:%s", s.config.DbPath), 0, 10)
 	if err != nil {
 		s.log.WithError(err).Fatal("Failed to open/create DB.")
 	}
 	s.dbpool = dbpool
+	s.checkpointNotice = make(chan struct{})
 
-	if s.config.ReinitDB {
+	err = s.initDB(s.config.ReinitDB)
-		err = s.reinitDB()
+	if err != nil {
-		if err != nil {
+		s.log.WithError(err).Fatal("init DB transaction failed")
-			s.log.WithError(err).Fatal("init DB transaction failed")
-		}
 	}
+
+	// aggressively checkpoint the database on idle times
+	go func() {
+		s.log.Debug("Starting manual restart checkpointing.")
+		defer s.log.Debug("Manual restart checkpointing stopped.")
+		delay := time.Minute * 15
+		var (
+			timer <-chan time.Time
+			ok    bool
+		)
+		for {
+			select {
+			case _, ok = <-s.checkpointNotice:
+				if !ok {
+					return
+				}
+				timer = time.After(delay)
+			case <-timer:
+				func() {
+					conn := s.dbpool.Get(s.ctx)
+					defer s.dbpool.Put(conn)
+					s.checkpointDb(conn, false)
+					timer = nil
+				}()
+			}
+		}
+	}()
+}
+
+func (s *Server) requestCheckpoint() {
+	go func() {
+		s.checkpointNotice <- struct{}{}
+	}()
 }
 
 func (s *Server) setupTiles() {
@@ -135,12 +233,202 @@ func (s *Server) setupTiles() {
 	s.mapPackSize = info.Size()
 }
 
-func (s *Server) reinitDB() error {
+func (s *Server) initDB(reinit bool) error {
-	s.log.Info("Reinitializing DB.")
+	s.log.Info("Initializing DB.")
 	conn := s.getDbConn()
 	defer s.dbpool.Put(conn)
 
-	return sqlitex.ExecScript(conn, initDB)
+	defer s.requestCheckpoint()
+
+	if reinit {
+		s.log.Warn("Reinitializing DB.")
+		tables := []string{}
+		err := sqlitex.Exec(conn, "select name from sqlite_master where type = 'table'", func(stmt *sqlite.Stmt) error {
+			tables = append(tables, stmt.ColumnText(0))
+			return nil
+		})
+		if err != nil {
+			return fmt.Errorf("failed to get table names: %w", err)
+		}
+
+		for _, table := range tables {
+			err = sqlitex.Exec(conn, "drop table "+table, nil)
+			if err != nil {
+				return fmt.Errorf("failed to drop tables: %w", err)
+			}
+		}
+
+		err = sqlitex.Exec(conn, "PRAGMA user_version = 0", nil)
+		if err != nil {
+			return fmt.Errorf("failed to reset user version: %w", err)
+		}
+	}
+
+	err := s.migrateDb(conn)
+	if err != nil {
+		return fmt.Errorf("failed to migrate db: %w", err)
+	}
+
+	return nil
+}
+
+func (s *Server) migrateDb(conn *sqlite.Conn) error {
+	var version int
+	err := sqlitex.Exec(conn, "PRAGMA user_version", func(stmt *sqlite.Stmt) error {
+		version = stmt.ColumnInt(0)
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to get user version: %w", err)
+	}
+	s.log.Debugf("Current db version: %d", version)
+
+	entries, err := sqlSchema.ReadDir("sql_schema")
+	if err != nil {
+		return fmt.Errorf("failed to read sql_schema migrations: %w", err)
+	}
+
+	type migration struct {
+		file    string
+		version int
+		name    string
+	}
+
+	pattern := regexp.MustCompile("^V([0-9]+)_(.*)[.][sS][qQ][lL]$")
+	migrations := []migration{}
+	for _, entry := range entries {
+		name := entry.Name()
+		if entry.IsDir() {
+			return fmt.Errorf("embedded sql migration '%s' is a directory", name)
+		}
+		matches := pattern.FindStringSubmatch(name)
+		if matches == nil {
+			return fmt.Errorf("embedded sql migration '%s' does not match the filename pattern", name)
+		}
+		if len(matches) != 3 {
+			return fmt.Errorf("embedded sql migration '%s' does not have the correct number of submatches", name)
+		}
+		version, err := strconv.Atoi(matches[1])
+		if err != nil {
+			return fmt.Errorf("failed to parse version number of migration '%s': %w", name, err)
+		}
+		migName := matches[2]
+		file := path.Join("sql_schema", name)
+		migrations = append(migrations, migration{
+			file:    file,
+			version: version,
+			name:    migName,
+		})
+	}
+	sort.Slice(migrations, func(i, j int) bool {
+		return migrations[i].version < migrations[j].version
+	})
+
+	for _, migration := range migrations {
+		if version >= migration.version {
+			s.log.Debugf("Skipping migration version %d because current version %d is not smaller.", migration.version, version)
+			continue
+		}
+
+		migContent, err := sqlSchema.ReadFile(migration.file)
+		if err != nil {
+			return fmt.Errorf("failed to read embedded migration '%s': %w", migration.file, err)
+		}
+
+		err = func() (err error) {
+			rollback := sqlitex.Save(conn)
+			defer func() {
+				if err != nil {
+					s.log.Info("Rolling back last migration attempt.")
+				}
+				rollback(&err)
+			}()
+
+			s.log.Infof("Executing migration V%d - %s", migration.version, migration.name)
+			err = sqlitex.ExecScript(conn, string(migContent))
+			if err != nil {
+				return fmt.Errorf("failed to execute migration '%s': %w", migration.name, err)
+			}
+
+			err = sqlitex.Exec(conn, fmt.Sprintf("PRAGMA user_version = %d", migration.version), nil)
+			if err != nil {
+				return fmt.Errorf("failed to set user_version in db: %w", err)
+			}
+
+			err = sqlitex.Exec(conn, "PRAGMA user_version", func(stmt *sqlite.Stmt) error {
+				version = stmt.ColumnInt(0)
+				return nil
+			})
+			if err != nil {
+				return fmt.Errorf("failed to get user_version: %w", err)
+			}
+
+			s.log.Infof("Migrated db to version: %d", version)
+			return nil
+		}()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Server) getLatestVersion(v *semver.Version) (*models.AppVersionInfo, error) {
+	conn := s.getDbConn()
+	defer s.dbpool.Put(conn)
+
+	var latest *models.AppVersionInfo
+	versions, err := s.getAppVersions()
+	if err != nil {
+		return nil, fmt.Errorf("failed to retrieve app versions from db: %w", err)
+	}
+
+	for _, ver := range versions {
+		if (v == nil || v.LessThan(ver.Version)) && (latest == nil || latest.Version.LessThan(ver.Version)) {
+			latest = ver
+		}
+	}
+
+	return latest, nil
+}
+
+func (s *Server) getAppVersions() ([]*models.AppVersionInfo, error) {
+	conn := s.getDbConn()
+	defer s.dbpool.Put(conn)
+
+	versions := []*models.AppVersionInfo{}
+	err := sqlitex.Exec(conn, "select version, address from app_versions", func(stmt *sqlite.Stmt) error {
+		verStr := stmt.ColumnText(0)
+		addr := stmt.ColumnText(1)
+
+		ver, err := semver.NewVersion(verStr)
+		if err != nil {
+			return fmt.Errorf("failed to parse version: %w", err)
+		}
+
+		versions = append(versions, &models.AppVersionInfo{
+			Version: *ver,
+			Address: addr,
+		})
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to insert/retrieve user from db: %w", err)
+	}
+
+	return versions, nil
+}
+
+func (s *Server) putAppVersion(versionInfo *models.AppVersionInfo) error {
+	conn := s.getDbConn()
+	defer s.dbpool.Put(conn)
+
+	err := sqlitex.Exec(conn, "insert into app_versions(version, address) values(?, ?) on conflict(version) do update set address = excluded.address", nil, versionInfo.Version.String(), versionInfo.Address)
+	if err != nil {
+		return fmt.Errorf("failed to insert app version into db: %w", err)
+	}
+
+	return nil
 }
 
 func (s *Server) handshake(hc models.HandshakeChallenge) (models.UserID, error) {
@@ -153,7 +441,7 @@ func (s *Server) handshake(hc models.HandshakeChallenge) (models.UserID, error)
 		var id *int64
 		if hc.Exists {
 			err = sqlitex.Exec(conn, "select id from users where name = ?", func(stmt *sqlite.Stmt) error {
-				id = ptrInt64(stmt.ColumnInt64(0))
+				id = ptr(stmt.ColumnInt64(0))
 				return nil
 			}, hc.Name)
 			if sqlite.ErrCode(err) != sqlite.SQLITE_OK {
@@ -167,7 +455,7 @@ func (s *Server) handshake(hc models.HandshakeChallenge) (models.UserID, error)
 			}
 		} else {
 			err = sqlitex.Exec(conn, "insert into users(name) values(?)", func(stmt *sqlite.Stmt) error {
-				id = ptrInt64(stmt.ColumnInt64(0))
+				id = ptr(stmt.ColumnInt64(0))
 				return nil
 			}, hc.Name)
 			if sqlite.ErrCode(err) == sqlite.SQLITE_CONSTRAINT_UNIQUE {
@@ -176,7 +464,8 @@ func (s *Server) handshake(hc models.HandshakeChallenge) (models.UserID, error)
 			if sqlite.ErrCode(err) != sqlite.SQLITE_OK {
 				return 0, err
 			}
-			id = ptrInt64(conn.LastInsertRowID())
+			id = ptr(conn.LastInsertRowID())
+			s.requestCheckpoint()
 		}
 		return *id, nil
 	}()
@@ -187,35 +476,148 @@ func (s *Server) handshake(hc models.HandshakeChallenge) (models.UserID, error)
 	return models.UserID(userID), nil
 }
 
-func (s *Server) getData() (models.Data, error) {
+func (s *Server) getData(conn *sqlite.Conn) (data models.Data, err error) {
+	err = s.deleteExpiredFeatures(conn)
+	if err != nil {
+		return models.Data{}, fmt.Errorf("failed to delete expired featured: %w", err)
+	}
+	people, err := s.getPeople(conn)
+	if err != nil {
+		return models.Data{}, fmt.Errorf("failed to retreive people: %w", err)
+	}
+	features, err := s.getFeatures(conn)
+	if err != nil {
+		return models.Data{}, fmt.Errorf("failed to retreive features: %w", err)
+	}
+
+	return models.Data{
+		Users:    people,
+		Features: features,
+	}, nil
+}
+
+func (s *Server) getDataOnly() (data models.Data, err error) {
+	conn := s.getDbConn()
+	defer s.dbpool.Put(conn)
+	defer sqlitex.Save(conn)(&err)
+
+	data, err = s.getData(conn)
+	if err != nil {
+		return models.Data{}, fmt.Errorf("failed to get json data: %w", err)
+	}
+	return data, err
+}
+
+func (s *Server) getDataWithPhotos() (file *os.File, err error) {
 	conn := s.getDbConn()
 	defer s.dbpool.Put(conn)
 
-	return func() (data models.Data, err error) {
+	defer sqlitex.Save(conn)(&err)
-		defer sqlitex.Save(conn)(&err)
 
-		err = s.deleteExpiredFeatures(conn)
+	makePhotoFilename := func(id models.FeaturePhotoID) string {
-		if err != nil {
+		return fmt.Sprintf("img%d", id)
-			return models.Data{}, fmt.Errorf("failed to delete expired featured: %w", err)
+	}
+	makeThumbnailFilename := func(id models.FeaturePhotoID) string {
+		return fmt.Sprintf("thumb_%s", makePhotoFilename(id))
+	}
+
+	data, err := s.getData(conn)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get json data: %w", err)
+	}
+
+	data.PhotoMetadata = make(map[string]models.PhotoMetadata, 100)
+	err = sqlitex.Exec(conn, "select id, content_type, length(contents) from feature_photos", func(stmt *sqlite.Stmt) error {
+		id := models.FeaturePhotoID(stmt.ColumnInt64(0))
+		contentType := stmt.ColumnText(1)
+		fileSize := stmt.ColumnInt64(2)
+		data.PhotoMetadata[makePhotoFilename(id)] = models.PhotoMetadata{
+			ContentType:       contentType,
+			Size:              fileSize,
+			ID:                id,
+			ThumbnailFilename: makeThumbnailFilename(id),
 		}
-		people, err := s.getPeople(conn)
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to collect photo metadata: %w", err)
+	}
+
+	dataBytes, err := json.MarshalIndent(&data, "", "  ")
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal json data: %w", err)
+	}
+
+	f, err := os.CreateTemp("", "")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temporary file: %w", err)
+	}
+
+	zw := zip.NewWriter(f)
+	defer zw.Close()
+	w, err := zw.Create("data.json")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create data zip entry: %w", err)
+	}
+	_, err = w.Write(dataBytes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to write data zip entry: %w", err)
+	}
+
+	err = sqlitex.Exec(conn, "select id from feature_photos fp where exists (select 1 from features f where f.id = fp.feature_id)", func(stmt *sqlite.Stmt) error {
+		id := stmt.ColumnInt64(0)
+
+		blob, err := conn.OpenBlob("", "feature_photos", "thumbnail_contents", id, false)
 		if err != nil {
-			return models.Data{}, fmt.Errorf("failed to retreive people: %w", err)
+			return fmt.Errorf("failed to open photo ID %d thumbnail content blob: %w", id, err)
 		}
-		features, err := s.getFeatures(conn)
+		err = func() error {
+			defer blob.Close()
+			w, err := zw.Create(makeThumbnailFilename(models.FeaturePhotoID(id)))
+			if err != nil {
+				return fmt.Errorf("failed to create zip entry: %w", err)
+			}
+			_, err = io.Copy(w, blob)
+			if err != nil {
+				return fmt.Errorf("failed to write zip entry: %w", err)
+			}
+			return nil
+		}()
 		if err != nil {
-			return models.Data{}, fmt.Errorf("failed to retreive features: %w", err)
+			return fmt.Errorf("failed to write photo ID %d thumbnail: %w", id, err)
 		}
 
-		return models.Data{
+		blob, err = conn.OpenBlob("", "feature_photos", "contents", id, false)
-			Users:    people,
+		if err != nil {
-			Features: features,
+			return fmt.Errorf("failed to open photo ID %d photo content blob: %w", id, err)
-		}, nil
+		}
-	}()
+		err = func() error {
+			defer blob.Close()
+			w, err := zw.Create(makePhotoFilename(models.FeaturePhotoID(id)))
+			if err != nil {
+				return fmt.Errorf("failed to create zip entry: %w", err)
+			}
+			_, err = io.Copy(w, blob)
+			if err != nil {
+				return fmt.Errorf("failed to write zip entry: %w", err)
+			}
+			return nil
+		}()
+		if err != nil {
+			return fmt.Errorf("failed to write photo ID %d photo: %w", id, err)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to collect photo files: %w", err)
+	}
+
+	return f, nil
 }
 
 func (s *Server) update(data models.Update, photos map[string]models.Photo) error {
-	s.log.Debugf("Updating data: %d created, %d created photos, %d updated, %d deleted, %d deleted photos, %d photo files", len(data.Create), len(data.CreatedPhotos), len(data.Update), len(data.Delete), len(data.DeletePhotos), len(photos))
+	s.log.Debugf("Updating data: %d created, %d created photos, %d updated, %d deleted, %d deleted photos, %d photo files, %d proposals", len(data.Create), len(data.CreatedPhotos), len(data.Update), len(data.Delete), len(data.DeletePhotos), len(photos), len(data.Proposals))
 	conn := s.getDbConn()
 	defer s.dbpool.Put(conn)
 
@@ -262,6 +664,13 @@ func (s *Server) update(data models.Update, photos map[string]models.Photo) erro
 				return
 			}
 		}
+		if data.Proposals != nil {
+			err = s.addProposals(conn, data.Proposals)
+			if err != nil {
+				err = fmt.Errorf("failed to add proposals: %w", err)
+				return
+			}
+		}
 
 		return nil
 	}()
@@ -279,6 +688,7 @@ func (s *Server) deleteExpiredFeatures(conn *sqlite.Conn) error {
 	if err != nil {
 		return fmt.Errorf("failed to delete expired features: %w", err)
 	}
+	s.requestCheckpoint()
 	return nil
 }
 
@@ -420,11 +830,12 @@ func (s *Server) addFeatures(conn *sqlite.Conn, features []models.Feature) (map[
 
 		localIDMapping[feature.ID] = models.FeatureID(conn.LastInsertRowID())
 	}
+	s.requestCheckpoint()
 	return localIDMapping, nil
 }
 
 func (s *Server) addPhotos(conn *sqlite.Conn, createdFeatureMapping, addedFeatureMapping map[models.FeatureID][]string, createdIDMapping map[models.FeatureID]models.FeatureID, photos map[string]models.Photo) error {
-	stmt, err := conn.Prepare("insert into feature_photos(feature_id, content_type, file_contents) values(?, ?, ?)")
+	stmt, err := conn.Prepare("insert into feature_photos(feature_id, thumbnail_content_type, content_type, thumbnail_contents, contents) values(?, ?, ?, ?, ?)")
 	if err != nil {
 		return fmt.Errorf("failed to prepare statement: %w", err)
 	}
@@ -436,6 +847,10 @@ func (s *Server) addPhotos(conn *sqlite.Conn, createdFeatureMapping, addedFeatur
 			if !ok {
 				return errs.NewErrPhotoNotProvided(photoName)
 			}
+			thumbnail, ok := photos[fmt.Sprintf("thumb_%s", photoName)]
+			if !ok {
+				return errs.NewErrPhotoThumbnailNotProvided(photoName)
+			}
 			if !checkImageContentType(photo.ContentType) {
 				return errs.NewErrUnsupportedContentType(photoName)
 			}
@@ -449,24 +864,51 @@ func (s *Server) addPhotos(conn *sqlite.Conn, createdFeatureMapping, addedFeatur
 				return fmt.Errorf("failed to clear bindings of prepared statement: %w", err)
 			}
 
+			resizedPhoto, err := func() ([]byte, error) {
+				defer photo.File.Close()
+
+				resized, err := resizePhoto(s.config.MaxPhotoX, s.config.MaxPhotoY, s.config.PhotoQuality, photo.File)
+				if err != nil {
+					return nil, err
+				}
+
+				return resized, nil
+			}()
+
 			stmt.BindInt64(1, int64(featureID))
-			stmt.BindText(2, photo.ContentType)
+			stmt.BindText(2, thumbnail.ContentType)
-			stmt.BindZeroBlob(3, photo.Size)
+			stmt.BindText(3, "image/jpeg")
+			stmt.BindZeroBlob(4, thumbnail.Size)
+			stmt.BindZeroBlob(5, int64(len(resizedPhoto)))
 
 			_, err = stmt.Step()
 			if err != nil {
 				return fmt.Errorf("failed to evaluate prepared statement: %w", err)
 			}
 
-			blob, err := conn.OpenBlob("", "feature_photos", "file_contents", conn.LastInsertRowID(), true)
+			blob, err := conn.OpenBlob("", "feature_photos", "thumbnail_contents", conn.LastInsertRowID(), true)
+			if err != nil {
+				return fmt.Errorf("failed to open thumbnail content blob: %w", err)
+			}
+			err = func() error {
+				defer blob.Close()
+				defer thumbnail.File.Close()
+
+				_, err := io.Copy(blob, thumbnail.File)
+				return err
+			}()
+			if err != nil {
+				return fmt.Errorf("failed to write to thumbnail content blob: %w", err)
+			}
+
+			blob, err = conn.OpenBlob("", "feature_photos", "contents", conn.LastInsertRowID(), true)
 			if err != nil {
 				return fmt.Errorf("failed to open photo content blob: %w", err)
 			}
 			err = func() error {
 				defer blob.Close()
-				defer photo.File.Close()
 
-				_, err := io.Copy(blob, photo.File)
+				_, err = io.Copy(blob, bytes.NewBuffer(resizedPhoto))
 				return err
 			}()
 			if err != nil {
@@ -476,18 +918,26 @@ func (s *Server) addPhotos(conn *sqlite.Conn, createdFeatureMapping, addedFeatur
 		return nil
 	}
 
+	changed := false
+	defer func() {
+		if changed {
+			s.requestCheckpoint()
+		}
+	}()
 	for localFeatureID, photoNames := range createdFeatureMapping {
 		featureID, ok := createdIDMapping[localFeatureID]
 		if !ok {
 			return errs.NewErrFeatureForPhotoNotExists(int64(localFeatureID))
 		}
 		err = uploadPhotos(featureID, photoNames)
+		changed = true
 		if err != nil {
 			return fmt.Errorf("failed to upload photos for created features: %w", err)
 		}
 	}
 	for featureID, photoNames := range addedFeatureMapping {
 		err = uploadPhotos(featureID, photoNames)
+		changed = true
 		if err != nil {
 			return fmt.Errorf("failed to upload photos for existing features: %w", err)
 		}
@@ -545,6 +995,7 @@ func (s *Server) updateFeatures(conn *sqlite.Conn, features []models.Feature) er
 			return fmt.Errorf("failed to evaluate prepared statement: %w", err)
 		}
 	}
+	s.requestCheckpoint()
 	return nil
 }
 
@@ -572,6 +1023,7 @@ func (s *Server) deleteFeatures(conn *sqlite.Conn, featureIDs []models.FeatureID
 			return fmt.Errorf("failed to evaluate prepared statement: %w", err)
 		}
 	}
+	s.requestCheckpoint()
 	return nil
 }
 
@@ -599,6 +1051,7 @@ func (s *Server) deletePhotos(conn *sqlite.Conn, photoIDs []models.FeaturePhotoI
 			return fmt.Errorf("failed to evaluate prepared statement: %w", err)
 		}
 	}
+	s.requestCheckpoint()
 	return nil
 }
 
@@ -609,11 +1062,11 @@ func (s *Server) getPhoto(featureID models.FeatureID, photoID models.FeaturePhot
 	var contentType *string = nil
 	var data []byte = nil
 	found := false
-	err := sqlitex.Exec(conn, "select content_type, file_contents from feature_photos where id = ? and feature_id = ?", func(stmt *sqlite.Stmt) error {
+	err := sqlitex.Exec(conn, "select content_type, contents from feature_photos where id = ? and feature_id = ?", func(stmt *sqlite.Stmt) error {
 		if found {
 			return fmt.Errorf("multiple photos returned for feature id %d, photo id %d", featureID, photoID)
 		}
-		contentType = ptrString(stmt.ColumnText(0))
+		contentType = ptr(stmt.ColumnText(0))
 		data = make([]byte, stmt.ColumnLen(1))
 		stmt.ColumnBytes(1, data)
 		found = true
@@ -629,3 +1082,54 @@ func (s *Server) getPhoto(featureID models.FeatureID, photoID models.FeaturePhot
 
 	return data, *contentType, nil
 }
+
+func (s *Server) addProposals(conn *sqlite.Conn, proposals []models.Proposal) error {
+	stmt, err := conn.Prepare("insert into proposals(owner_id, description, how) values(?, ?, ?)")
+	if err != nil {
+		return fmt.Errorf("failed to prepare statement: %w", err)
+	}
+	defer stmt.Finalize()
+
+	for _, proposal := range proposals {
+		err := stmt.Reset()
+		if err != nil {
+			return fmt.Errorf("failed to reset prepared statement: %w", err)
+		}
+		err = stmt.ClearBindings()
+		if err != nil {
+			return fmt.Errorf("failed to clear bindings of prepared statement: %w", err)
+		}
+
+		stmt.BindInt64(1, int64(proposal.OwnerID))
+		stmt.BindText(2, proposal.Description)
+		stmt.BindText(3, proposal.How)
+
+		_, err = stmt.Step()
+		if err != nil {
+			return fmt.Errorf("failed to evaluate prepared statement: %w", err)
+		}
+	}
+	s.requestCheckpoint()
+	return nil
+}
+
+func (s *Server) getProposals(conn *sqlite.Conn) ([]models.Proposal, error) {
+	if conn == nil {
+		conn = s.getDbConn()
+		defer s.dbpool.Put(conn)
+	}
+
+	proposals := make([]models.Proposal, 0, 100)
+	err := sqlitex.Exec(conn, "select owner_id, description, how from proposals", func(stmt *sqlite.Stmt) error {
+		proposals = append(proposals, models.Proposal{
+			OwnerID:     models.UserID(stmt.ColumnInt(0)),
+			Description: stmt.ColumnText(1),
+			How:         stmt.ColumnText(2),
+		})
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get proposals from db: %w", err)
+	}
+	return proposals, nil
+}

server/sql_schema/V1_init.sql

@@ -1,12 +1,10 @@
-DROP TABLE IF EXISTS users;
+CREATE TABLE users (
-CREATE TABLE IF NOT EXISTS users (
   id integer PRIMARY KEY AUTOINCREMENT,
   name text NOT NULL UNIQUE
 );
 INSERT INTO users(id, name) VALUES(0, 'system');
 
-DROP TABLE IF EXISTS features;
+CREATE TABLE features (
-CREATE TABLE IF NOT EXISTS features (
   id integer PRIMARY KEY AUTOINCREMENT,
   owner_id integer NOT NULL REFERENCES users(id) ON DELETE CASCADE,
   name text NOT NULL,
@@ -15,10 +13,11 @@ CREATE TABLE IF NOT EXISTS features (
   geom text NOT NULL
 );
 
-DROP TABLE IF EXISTS feature_photos;
+CREATE TABLE feature_photos (
-CREATE TABLE IF NOT EXISTS feature_photos (
   id integer PRIMARY KEY AUTOINCREMENT,
   feature_id integer NOT NULL REFERENCES features(id) ON DELETE CASCADE,
+  thumbnail_content_type text NOT NULL,
   content_type text NOT NULL,
-  file_contents blob NOT NULL
+  thumbnail_contents blob NOT NULL,
-);
+  contents blob NOT NULL
+);

server/sql_schema/V2_proposals.sql

@@ -0,0 +1,5 @@
+CREATE TABLE proposals (
+  owner_id integer NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
+  description text NOT NULL,
+  how text NOT NULL
+);

server/sql_schema/V3_app_versions.sql

@@ -0,0 +1,4 @@
+CREATE TABLE app_versions (
+  version text NOT NULL PRIMARY KEY,
+  address text NOT NULL
+);

server/utils.go

@@ -1,18 +1,19 @@
 package server
 
 import (
+	"bytes"
+	"fmt"
+	"image"
+	"image/jpeg"
+	"io"
+	"math"
+
+	"golang.org/x/image/draw"
+
 	"cernobor.cz/oko-server/models"
 )
 
-func ptrInt(x int) *int {
+func ptr[T any](x T) *T {
-	return &x
-}
-
-func ptrInt64(x int64) *int64 {
-	return &x
-}
-
-func ptrString(x string) *string {
 	return &x
 }
 
@@ -32,3 +33,57 @@ func isUniqueFeatureID(features []models.Feature) bool {
 	}
 	return true
 }
+
+func resizePhoto(maxX, maxY, quality int, data io.Reader) ([]byte, error) {
+	src, _, err := image.Decode(data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode image: %w", err)
+	}
+
+	if maxX == 0 && maxY == 0 {
+		output := &bytes.Buffer{}
+		jpeg.Encode(output, src, &jpeg.Options{
+			Quality: 90,
+		})
+
+		return output.Bytes(), nil
+	}
+
+	var dst draw.Image
+	srcX := src.Bounds().Max.X
+	srcY := src.Bounds().Max.Y
+	srcRatio := float64(srcX) / float64(srcY)
+	if maxX == 0 && srcY > maxY {
+		newX := int(math.Round(float64(maxY) * srcRatio))
+		newY := maxY
+		dst = image.NewRGBA(image.Rect(0, 0, newX, newY))
+		draw.CatmullRom.Scale(dst, dst.Bounds(), src, src.Bounds(), draw.Over, nil)
+	} else if maxY == 0 && srcX > maxX {
+		newX := maxX
+		newY := int(math.Round(float64(maxX) / srcRatio))
+		dst = image.NewRGBA(image.Rect(0, 0, newX, newY))
+		draw.CatmullRom.Scale(dst, dst.Bounds(), src, src.Bounds(), draw.Over, nil)
+	} else if srcX > maxX || srcY > maxY {
+		tgtRatio := float64(maxX) / float64(maxY)
+		var newX, newY int
+		if srcRatio > tgtRatio {
+			newX = maxX
+			newY = int(math.Round(float64(maxX) / srcRatio))
+		} else {
+			newX = int(math.Round(float64(maxY) * srcRatio))
+			newY = maxY
+		}
+		dst = image.NewRGBA(image.Rect(0, 0, newX, newY))
+		draw.CatmullRom.Scale(dst, dst.Bounds(), src, src.Bounds(), draw.Over, nil)
+	} else {
+		dst = image.NewRGBA(image.Rect(0, 0, srcX, srcY))
+		draw.Copy(dst, image.Point{X: 0, Y: 0}, src, src.Bounds(), draw.Over, nil)
+	}
+
+	output := &bytes.Buffer{}
+	jpeg.Encode(output, dst, &jpeg.Options{
+		Quality: quality,
+	})
+
+	return output.Bytes(), nil
+}