Retrieving proposals.

* Added GET /data/proposals endpoint which returns all submitted enhancement proposals.

Fix #2

.dockerignore

@@ -38,4 +38,5 @@ go.work
 oko-server
 *.sqlite*
 .vscode
-.git
+data
+local-testing

.gitignore

@@ -37,4 +37,5 @@ go.work
 # Project-specific
 oko-server
 *.sqlite*
-.vscode
+.vscode
+__debug_bin

Dockerfile

@@ -1,19 +1,19 @@
-FROM alpine:3.15.0 AS build
+FROM alpine:3.16 AS build
 
+ARG CAPROVER_GIT_COMMIT_SHA=""
 VOLUME ["/data"]
 COPY . /oko-server/git
 
-RUN apk add --no-cache go && \
+RUN apk update && \
+    apk add go && \
     cd /oko-server/git && \
-    go build
+    go build -ldflags "-X \"main.sha1ver=${CAPROVER_GIT_COMMIT_SHA:-$(cat .git/$(cat .git/HEAD | sed 's|ref: ||g'))}\" -X \"main.buildTime=$(date -Iseconds)\""
 
-FROM alpine:3.15.0
+FROM alpine:3.16
 WORKDIR /oko-server
 VOLUME [ "/data" ]
 
-RUN echo -e '#!/bin/sh\n/oko-server/oko-server "$@"' > /oko-server/entrypoint.sh && \
-    chmod +x /oko-server/entrypoint.sh
 COPY --from=build /oko-server/git/oko-server/ /oko-server/
 
-ENTRYPOINT ["/oko-server/entrypoint.sh"]
+ENTRYPOINT ["/oko-server/oko-server"]
 

errs/errors.go

@@ -34,6 +34,22 @@ func (e *ErrPhotoNotProvided) Error() string {
 	return fmt.Sprintf("referenced photo %s which was not provided", e.Reference)
 }
 
+type EErrPhotoThumbnailNotProvided *ErrPhotoThumbnailNotProvided
+
+type ErrPhotoThumbnailNotProvided struct {
+	Reference string
+}
+
+func NewErrPhotoThumbnailNotProvided(reference string) *ErrPhotoThumbnailNotProvided {
+	return &ErrPhotoThumbnailNotProvided{
+		Reference: reference,
+	}
+}
+
+func (e *ErrPhotoThumbnailNotProvided) Error() string {
+	return fmt.Sprintf("referenced photo %s the thumbnail of which was not provided", e.Reference)
+}
+
 type ErrFeatureForPhotoNotExists struct {
 	PhotoFeatureReference int64
 }

go.mod

@@ -1,6 +1,6 @@
 module cernobor.cz/oko-server
 
-go 1.17
+go 1.19
 
 require (
 	crawshaw.io/sqlite v0.3.2
@@ -24,6 +24,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/ugorji/go/codec v1.2.6 // indirect
 	golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce // indirect
+	golang.org/x/image v0.0.0-20220722155232-062f8c9fd539
 	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect

go.sum

@@ -377,7 +377,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
-github.com/ugorji/go v1.2.6 h1:tGiWC9HENWE2tqYycIqFTNorMmFRVhNwCpDOpWqnk8E=
 github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn0=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
@@ -430,6 +429,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20220722155232-062f8c9fd539 h1:/eM0PCrQI2xd471rI+snWuu251/+/jpBpZqir2mPdnU=
+golang.org/x/image v0.0.0-20220722155232-062f8c9fd539/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

main.go

@@ -7,12 +7,18 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"cernobor.cz/oko-server/models"
 	"cernobor.cz/oko-server/server"
 	"github.com/sirupsen/logrus"
 )
 
+var (
+	sha1ver   string
+	buildTime string
+)
+
 func main() {
 	tilepackFileArg := flag.String("tilepack", "", "File that will be sent to clients when they request a tile pack, also used to serve tiles in online mode. Required.")
 	portArg := flag.Int("port", 8080, "Port where the server will listen to. Default is 8080.")
@@ -22,6 +28,9 @@ func main() {
 	minZoomArg := flag.Int("min-zoom", 1, "Minimum zoom that will be sent to clients.")
 	defaultCenterLatArg := flag.Float64("default-center-lat", 0, "Latitude of the default map center.")
 	defaultCenterLngArg := flag.Float64("default-center-lng", 0, "Longitude of the default map center.")
+	maxPhotoXArg := flag.Int("max-photo-width", 0, "Maximum width of photos. 0 means no limit.")
+	maxPhotoYArg := flag.Int("max-photo-height", 0, "Maximum height of photos. 0 means no limit.")
+	photoQualityArg := flag.Int("photo-quality", 90, "Photo JPEG quality.")
 
 	flag.Parse()
 
@@ -31,7 +40,19 @@ func main() {
 		os.Exit(1)
 	}
 
+	t, err := time.Parse(time.RFC3339, buildTime)
+	if err != nil {
+		t = time.Now()
+	}
+
+	if *maxPhotoXArg < 0 || *maxPhotoYArg < 0 {
+		fmt.Fprintln(os.Stderr, "Max photo width and height cannot be less than 0.")
+		os.Exit(1)
+	}
+
 	s := server.New(server.ServerConfig{
+		VersionHash:  sha1ver,
+		BuildTime:    &t,
 		Port:         *portArg,
 		DbPath:       *dbFileArg,
 		TilepackPath: *tilepackFileArg,
@@ -42,6 +63,9 @@ func main() {
 			Lat: *defaultCenterLatArg,
 			Lng: *defaultCenterLngArg,
 		},
+		MaxPhotoX:    *maxPhotoXArg,
+		MaxPhotoY:    *maxPhotoYArg,
+		PhotoQuality: *photoQualityArg,
 	})
 
 	sigs := make(chan os.Signal, 1)

models/models.go

@@ -33,6 +33,11 @@ type Feature struct {
 	PhotoIDs []FeaturePhotoID `json:"photo_ids"`
 }
 
+type BuildInfo struct {
+	VersionHash string     `json:"version_hash"`
+	BuildTime   *time.Time `json:"build_time"`
+}
+
 // transport objects
 
 type Coords struct {
@@ -55,6 +60,7 @@ type Update struct {
 	Update        []Feature              `json:"update"`
 	Delete        []FeatureID            `json:"delete"`
 	DeletePhotos  []FeaturePhotoID       `json:"delete_photos"`
+	Proposals     []Proposal             `json:"proposals"`
 }
 
 type HandshakeChallenge struct {
@@ -69,8 +75,9 @@ type HandshakeResponse struct {
 }
 
 type Data struct {
-	Users    []User    `json:"users"`
-	Features []Feature `json:"features"`
+	Users         []User                   `json:"users"`
+	Features      []Feature                `json:"features"`
+	PhotoMetadata map[string]PhotoMetadata `json:"photo_metadata,omitempty"`
 }
 
 type Photo struct {
@@ -78,3 +85,17 @@ type Photo struct {
 	File        io.ReadCloser
 	Size        int64
 }
+
+type PhotoMetadata struct {
+	ContentType          string         `json:"content_type"`
+	ThumbnailContentType string         `json:"thumbnail_content_type"`
+	Size                 int64          `json:"size"`
+	ID                   FeaturePhotoID `json:"id"`
+	ThumbnailFilename    string         `json:"thumbnail_filename"`
+}
+
+type Proposal struct {
+	OwnerID     UserID `json:"owner_id"`
+	Description string `json:"description"`
+	How         string `json:"how"`
+}

server/constants.go

@@ -2,6 +2,7 @@ package server
 
 const (
 	URIPing              = "/ping"
+	URIBuildInfo         = "/build-info"
 	URIHardFail          = "/hard-fail"
 	URISoftFail          = "/soft-fail"
 	URIReinit            = "/reinit"
@@ -11,6 +12,7 @@ const (
 	URIDataPeople        = "/data/people"
 	URIDataFeatures      = "/data/features"
 	URIDataFeaturesPhoto = "/data/features/:feature/photos/:photo"
+	URIDataProposals     = "/data/proposals"
 	URITileserverRoot    = "/tileserver"
 	URITileserver        = URITileserverRoot + "/*x"
 	URITileTemplate      = URITileserverRoot + "/map/tiles/{z}/{x}/{y}.pbf"

server/handling.go

@@ -4,7 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"math"
 	"net/http"
 	"os"
@@ -81,6 +81,12 @@ func (s *Server) setupRouter() *gin.Engine {
 	router.GET(URIPing, func(gc *gin.Context) {
 		gc.Status(http.StatusNoContent)
 	})
+	router.GET(URIBuildInfo, func(gc *gin.Context) {
+		gc.JSON(http.StatusOK, models.BuildInfo{
+			VersionHash: s.config.VersionHash,
+			BuildTime:   s.config.BuildTime,
+		})
+	})
 	router.GET(URIHardFail, func(gc *gin.Context) {
 		gc.Status(http.StatusNotImplemented)
 	})
@@ -99,6 +105,7 @@ func (s *Server) setupRouter() *gin.Engine {
 	router.GET(URIDataPeople, s.handleGETDataPeople)
 	router.GET(URIDataFeatures, s.handleGETDataFeatures)
 	router.GET(URIDataFeaturesPhoto, s.handleGETDataFeaturesPhoto)
+	router.GET(URIDataProposals, s.handleGETDataProposals)
 
 	// tileserver
 	router.GET(URITileserver, gin.WrapH(s.tileserverSvSet.Handler()))
@@ -107,7 +114,7 @@ func (s *Server) setupRouter() *gin.Engine {
 }
 
 func (s *Server) handlePOSTReset(gc *gin.Context) {
-	err := s.reinitDB()
+	err := s.initDB(true)
 	if err != nil {
 		internalError(gc, err)
 		return
@@ -155,12 +162,40 @@ func (s *Server) handlePOSTHandshake(gc *gin.Context) {
 }
 
 func (s *Server) handleGETData(gc *gin.Context) {
-	data, err := s.getData()
-	if err != nil {
-		internalError(gc, err)
+	accept := gc.GetHeader("Accept")
+	if accept == "application/json" {
+		data, err := s.getDataOnly()
+		if err != nil {
+			internalError(gc, err)
+			return
+		}
+		gc.JSON(http.StatusOK, data)
+		return
+	} else if accept == "application/zip" {
+		file, err := s.getDataWithPhotos()
+		defer func() {
+			file.Close()
+			os.Remove(file.Name())
+		}()
+		if err != nil {
+			internalError(gc, err)
+			return
+		}
+		fi, err := file.Stat()
+		if err != nil {
+			internalError(gc, err)
+			return
+		}
+		size := fi.Size()
+		_, err = file.Seek(0, 0)
+		if err != nil {
+			internalError(gc, err)
+			return
+		}
+		gc.DataFromReader(http.StatusOK, size, "application/zip", file, nil)
 		return
 	}
-	gc.JSON(http.StatusOK, data)
+	gc.String(http.StatusNotAcceptable, "%s is not acceptable", accept)
 }
 
 func (s *Server) handlePOSTData(gc *gin.Context) {
@@ -223,7 +258,7 @@ func (s *Server) handlePOSTDataMultipart(gc *gin.Context) {
 			internalError(gc, fmt.Errorf("failed to open 'data' 'file': %w", err))
 			return
 		}
-		dataBytes, err := ioutil.ReadAll(df)
+		dataBytes, err := io.ReadAll(df)
 		if err != nil {
 			internalError(gc, fmt.Errorf("failed to open 'data' 'file': %w", err))
 			return
@@ -320,3 +355,12 @@ func (s *Server) handleGETDataFeaturesPhoto(gc *gin.Context) {
 
 	gc.Data(http.StatusOK, contentType, photoBytes)
 }
+
+func (s *Server) handleGETDataProposals(gc *gin.Context) {
+	proposals, err := s.getProposals(nil)
+	if err != nil {
+		internalError(gc, err)
+		return
+	}
+	gc.JSON(http.StatusOK, proposals)
+}

server/server.go

@@ -1,6 +1,8 @@
 package server
 
 import (
+	"archive/zip"
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -21,19 +23,25 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-//go:embed initdb.sql
-var initDB string
+//go:embed sql_schema/V1_init.sql
+var sql_v1 string
+
+//go:embed sql_schema/V2_proposals.sql
+var sql_v2 string
 
 type Server struct {
-	config          ServerConfig
-	dbpool          *sqlitex.Pool
-	log             *logrus.Logger
-	ctx             context.Context
-	tileserverSvSet *mbsh.ServiceSet
-	mapPackSize     int64
+	config           ServerConfig
+	dbpool           *sqlitex.Pool
+	checkpointNotice chan struct{}
+	log              *logrus.Logger
+	ctx              context.Context
+	tileserverSvSet  *mbsh.ServiceSet
+	mapPackSize      int64
 }
 
 type ServerConfig struct {
+	VersionHash   string
+	BuildTime     *time.Time
 	Port          int
 	DbPath        string
 	TilepackPath  string
@@ -41,6 +49,9 @@ type ServerConfig struct {
 	ReinitDB      bool
 	MinZoom       int
 	DefaultCenter models.Coords
+	MaxPhotoX     int
+	MaxPhotoY     int
+	PhotoQuality  int
 }
 
 func New(config ServerConfig) *Server {
@@ -55,6 +66,8 @@ func (s *Server) Run(ctx context.Context) {
 
 	s.ctx = ctx
 	s.setupDB()
+	defer s.cleanupDb()
+
 	s.setupTiles()
 
 	router := s.setupRouter()
@@ -72,15 +85,32 @@ func (s *Server) Run(ctx context.Context) {
 
 	<-s.ctx.Done()
 	s.log.Info("Shutting down server...")
+	defer s.log.Info("Server exitting.")
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
 	if err := server.Shutdown(ctx); err != nil {
 		s.log.WithError(err).Fatal("Server forced to shutdown.")
 	}
+}
+
+func (s *Server) cleanupDb() {
+	close(s.checkpointNotice)
+	s.log.Info("Closing db connection pool...")
 	s.dbpool.Close()
 
-	s.log.Info("Server exitting.")
+	// manually force truncate checkpoint
+	conn, err := sqlite.OpenConn(fmt.Sprintf("file:%s", s.config.DbPath), 0)
+	if err != nil {
+		s.log.WithError(err).Error("Failed to open connection for final checkpoint.")
+		return
+	}
+	err = sqlitex.Exec(conn, "vacuum", nil)
+	if err != nil {
+		s.log.WithError(err).Error("Failed to vacuum db.")
+	}
+	s.checkpointDb(conn, true)
+	conn.Close()
 }
 
 func (s *Server) getDbConn() *sqlite.Conn {
@@ -92,19 +122,86 @@ func (s *Server) getDbConn() *sqlite.Conn {
 	return conn
 }
 
+func (s *Server) checkpointDb(conn *sqlite.Conn, truncate bool) {
+	var query string
+	if truncate {
+		query = "PRAGMA wal_checkpoint(TRUNCATE)"
+	} else {
+		query = "PRAGMA wal_checkpoint(RESTART)"
+	}
+	stmt, _, err := conn.PrepareTransient(query)
+	if err != nil {
+		s.log.WithError(err).Error("Failed to prepare checkpoint query.")
+		return
+	}
+	defer stmt.Finalize()
+
+	has, err := stmt.Step()
+	if err != nil {
+		s.log.WithError(err).Error("Failed to step through checkpoint query.")
+		return
+	}
+	if !has {
+		s.log.Error("Checkpoint query returned no rows.")
+		return
+	}
+
+	blocked := stmt.ColumnInt(0)
+	noWalPages := stmt.ColumnInt(1)
+	noReclaimedPages := stmt.ColumnInt(2)
+	if blocked == 1 {
+		s.log.Warn("Checkpoint query was blocked.")
+	}
+	s.log.Debugf("Checkpoint complete. %d pages written to WAL, %d pages written back to DB.", noWalPages, noReclaimedPages)
+}
+
 func (s *Server) setupDB() {
+	sqlitex.PoolCloseTimeout = time.Second * 10
+	s.log.Debugf("Using db %s", s.config.DbPath)
 	dbpool, err := sqlitex.Open(fmt.Sprintf("file:%s", s.config.DbPath), 0, 10)
 	if err != nil {
 		s.log.WithError(err).Fatal("Failed to open/create DB.")
 	}
 	s.dbpool = dbpool
+	s.checkpointNotice = make(chan struct{})
 
-	if s.config.ReinitDB {
-		err = s.reinitDB()
-		if err != nil {
-			s.log.WithError(err).Fatal("init DB transaction failed")
-		}
+	err = s.initDB(s.config.ReinitDB)
+	if err != nil {
+		s.log.WithError(err).Fatal("init DB transaction failed")
 	}
+
+	// aggressively checkpoint the database on idle times
+	go func() {
+		s.log.Debug("Starting manual restart checkpointing.")
+		defer s.log.Debug("Manual restart checkpointing stopped.")
+		delay := time.Minute * 15
+		var (
+			timer <-chan time.Time
+			ok    bool
+		)
+		for {
+			select {
+			case _, ok = <-s.checkpointNotice:
+				if !ok {
+					return
+				}
+				timer = time.After(delay)
+			case <-timer:
+				func() {
+					conn := s.dbpool.Get(s.ctx)
+					defer s.dbpool.Put(conn)
+					s.checkpointDb(conn, false)
+					timer = nil
+				}()
+			}
+		}
+	}()
+}
+
+func (s *Server) requestCheckpoint() {
+	go func() {
+		s.checkpointNotice <- struct{}{}
+	}()
 }
 
 func (s *Server) setupTiles() {
@@ -135,12 +232,62 @@ func (s *Server) setupTiles() {
 	s.mapPackSize = info.Size()
 }
 
-func (s *Server) reinitDB() error {
-	s.log.Info("Reinitializing DB.")
+func (s *Server) initDB(reinit bool) error {
+	s.log.Info("Initializing DB.")
 	conn := s.getDbConn()
 	defer s.dbpool.Put(conn)
 
-	return sqlitex.ExecScript(conn, initDB)
+	defer s.requestCheckpoint()
+
+	if reinit {
+		s.log.Warn("REinitializing DB.")
+		tables := []string{}
+		err := sqlitex.Exec(conn, "select name from sqlite_master where type = 'table'", func(stmt *sqlite.Stmt) error {
+			tables = append(tables, stmt.ColumnText(0))
+			return nil
+		})
+		if err != nil {
+			return fmt.Errorf("failed to get table names: %w", err)
+		}
+
+		for _, table := range tables {
+			err = sqlitex.Exec(conn, "drop table "+table, nil)
+			if err != nil {
+				return fmt.Errorf("failed to drop tables: %w", err)
+			}
+		}
+
+		err = sqlitex.Exec(conn, "PRAGMA user_version = 0", nil)
+		if err != nil {
+			return fmt.Errorf("failed to reset user version: %w", err)
+		}
+	}
+	var version int
+	err := sqlitex.Exec(conn, "PRAGMA user_version", func(stmt *sqlite.Stmt) error {
+		version = stmt.ColumnInt(0)
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to get user version: %w", err)
+	}
+
+	s.log.Debugf("Current db version: %d", version)
+	if version <= 0 {
+		s.log.Debugf("Running db migration V1")
+		err = sqlitex.ExecScript(conn, sql_v1)
+		if err != nil {
+			return fmt.Errorf("failed to run V1 init script")
+		}
+	}
+	if version <= 1 {
+		s.log.Debugf("Running db migration V2")
+		err = sqlitex.ExecScript(conn, sql_v2)
+		if err != nil {
+			return fmt.Errorf("failed to run V2 init script")
+		}
+	}
+
+	return nil
 }
 
 func (s *Server) handshake(hc models.HandshakeChallenge) (models.UserID, error) {
@@ -177,6 +324,7 @@ func (s *Server) handshake(hc models.HandshakeChallenge) (models.UserID, error)
 				return 0, err
 			}
 			id = ptrInt64(conn.LastInsertRowID())
+			s.requestCheckpoint()
 		}
 		return *id, nil
 	}()
@@ -187,35 +335,148 @@ func (s *Server) handshake(hc models.HandshakeChallenge) (models.UserID, error)
 	return models.UserID(userID), nil
 }
 
-func (s *Server) getData() (models.Data, error) {
+func (s *Server) getData(conn *sqlite.Conn) (data models.Data, err error) {
+	err = s.deleteExpiredFeatures(conn)
+	if err != nil {
+		return models.Data{}, fmt.Errorf("failed to delete expired featured: %w", err)
+	}
+	people, err := s.getPeople(conn)
+	if err != nil {
+		return models.Data{}, fmt.Errorf("failed to retreive people: %w", err)
+	}
+	features, err := s.getFeatures(conn)
+	if err != nil {
+		return models.Data{}, fmt.Errorf("failed to retreive features: %w", err)
+	}
+
+	return models.Data{
+		Users:    people,
+		Features: features,
+	}, nil
+}
+
+func (s *Server) getDataOnly() (data models.Data, err error) {
+	conn := s.getDbConn()
+	defer s.dbpool.Put(conn)
+	defer sqlitex.Save(conn)(&err)
+
+	data, err = s.getData(conn)
+	if err != nil {
+		return models.Data{}, fmt.Errorf("failed to get json data: %w", err)
+	}
+	return data, err
+}
+
+func (s *Server) getDataWithPhotos() (file *os.File, err error) {
 	conn := s.getDbConn()
 	defer s.dbpool.Put(conn)
 
-	return func() (data models.Data, err error) {
-		defer sqlitex.Save(conn)(&err)
+	defer sqlitex.Save(conn)(&err)
 
-		err = s.deleteExpiredFeatures(conn)
-		if err != nil {
-			return models.Data{}, fmt.Errorf("failed to delete expired featured: %w", err)
+	makePhotoFilename := func(id models.FeaturePhotoID) string {
+		return fmt.Sprintf("img%d", id)
+	}
+	makeThumbnailFilename := func(id models.FeaturePhotoID) string {
+		return fmt.Sprintf("thumb_%s", makePhotoFilename(id))
+	}
+
+	data, err := s.getData(conn)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get json data: %w", err)
+	}
+
+	data.PhotoMetadata = make(map[string]models.PhotoMetadata, 100)
+	err = sqlitex.Exec(conn, "select id, content_type, length(contents) from feature_photos", func(stmt *sqlite.Stmt) error {
+		id := models.FeaturePhotoID(stmt.ColumnInt64(0))
+		contentType := stmt.ColumnText(1)
+		fileSize := stmt.ColumnInt64(2)
+		data.PhotoMetadata[makePhotoFilename(id)] = models.PhotoMetadata{
+			ContentType:       contentType,
+			Size:              fileSize,
+			ID:                id,
+			ThumbnailFilename: makeThumbnailFilename(id),
 		}
-		people, err := s.getPeople(conn)
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to collect photo metadata: %w", err)
+	}
+
+	dataBytes, err := json.MarshalIndent(&data, "", "  ")
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal json data: %w", err)
+	}
+
+	f, err := os.CreateTemp("", "")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temporary file: %w", err)
+	}
+
+	zw := zip.NewWriter(f)
+	defer zw.Close()
+	w, err := zw.Create("data.json")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create data zip entry: %w", err)
+	}
+	_, err = w.Write(dataBytes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to write data zip entry: %w", err)
+	}
+
+	err = sqlitex.Exec(conn, "select id from feature_photos fp where exists (select 1 from features f where f.id = fp.feature_id)", func(stmt *sqlite.Stmt) error {
+		id := stmt.ColumnInt64(0)
+
+		blob, err := conn.OpenBlob("", "feature_photos", "thumbnail_contents", id, false)
 		if err != nil {
-			return models.Data{}, fmt.Errorf("failed to retreive people: %w", err)
+			return fmt.Errorf("failed to open photo ID %d thumbnail content blob: %w", id, err)
 		}
-		features, err := s.getFeatures(conn)
+		err = func() error {
+			defer blob.Close()
+			w, err := zw.Create(makeThumbnailFilename(models.FeaturePhotoID(id)))
+			if err != nil {
+				return fmt.Errorf("failed to create zip entry: %w", err)
+			}
+			_, err = io.Copy(w, blob)
+			if err != nil {
+				return fmt.Errorf("failed to write zip entry: %w", err)
+			}
+			return nil
+		}()
 		if err != nil {
-			return models.Data{}, fmt.Errorf("failed to retreive features: %w", err)
+			return fmt.Errorf("failed to write photo ID %d thumbnail: %w", id, err)
 		}
 
-		return models.Data{
-			Users:    people,
-			Features: features,
-		}, nil
-	}()
+		blob, err = conn.OpenBlob("", "feature_photos", "contents", id, false)
+		if err != nil {
+			return fmt.Errorf("failed to open photo ID %d photo content blob: %w", id, err)
+		}
+		err = func() error {
+			defer blob.Close()
+			w, err := zw.Create(makePhotoFilename(models.FeaturePhotoID(id)))
+			if err != nil {
+				return fmt.Errorf("failed to create zip entry: %w", err)
+			}
+			_, err = io.Copy(w, blob)
+			if err != nil {
+				return fmt.Errorf("failed to write zip entry: %w", err)
+			}
+			return nil
+		}()
+		if err != nil {
+			return fmt.Errorf("failed to write photo ID %d photo: %w", id, err)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to collect photo files: %w", err)
+	}
+
+	return f, nil
 }
 
 func (s *Server) update(data models.Update, photos map[string]models.Photo) error {
-	s.log.Debugf("Updating data: %d created, %d created photos, %d updated, %d deleted, %d deleted photos, %d photo files", len(data.Create), len(data.CreatedPhotos), len(data.Update), len(data.Delete), len(data.DeletePhotos), len(photos))
+	s.log.Debugf("Updating data: %d created, %d created photos, %d updated, %d deleted, %d deleted photos, %d photo files, %d proposals", len(data.Create), len(data.CreatedPhotos), len(data.Update), len(data.Delete), len(data.DeletePhotos), len(photos), len(data.Proposals))
 	conn := s.getDbConn()
 	defer s.dbpool.Put(conn)
 
@@ -262,6 +523,13 @@ func (s *Server) update(data models.Update, photos map[string]models.Photo) erro
 				return
 			}
 		}
+		if data.Proposals != nil {
+			err = s.addProposals(conn, data.Proposals)
+			if err != nil {
+				err = fmt.Errorf("failed to add proposals: %w", err)
+				return
+			}
+		}
 
 		return nil
 	}()
@@ -279,6 +547,7 @@ func (s *Server) deleteExpiredFeatures(conn *sqlite.Conn) error {
 	if err != nil {
 		return fmt.Errorf("failed to delete expired features: %w", err)
 	}
+	s.requestCheckpoint()
 	return nil
 }
 
@@ -420,11 +689,12 @@ func (s *Server) addFeatures(conn *sqlite.Conn, features []models.Feature) (map[
 
 		localIDMapping[feature.ID] = models.FeatureID(conn.LastInsertRowID())
 	}
+	s.requestCheckpoint()
 	return localIDMapping, nil
 }
 
 func (s *Server) addPhotos(conn *sqlite.Conn, createdFeatureMapping, addedFeatureMapping map[models.FeatureID][]string, createdIDMapping map[models.FeatureID]models.FeatureID, photos map[string]models.Photo) error {
-	stmt, err := conn.Prepare("insert into feature_photos(feature_id, content_type, file_contents) values(?, ?, ?)")
+	stmt, err := conn.Prepare("insert into feature_photos(feature_id, thumbnail_content_type, content_type, thumbnail_contents, contents) values(?, ?, ?, ?, ?)")
 	if err != nil {
 		return fmt.Errorf("failed to prepare statement: %w", err)
 	}
@@ -436,6 +706,10 @@ func (s *Server) addPhotos(conn *sqlite.Conn, createdFeatureMapping, addedFeatur
 			if !ok {
 				return errs.NewErrPhotoNotProvided(photoName)
 			}
+			thumbnail, ok := photos[fmt.Sprintf("thumb_%s", photoName)]
+			if !ok {
+				return errs.NewErrPhotoThumbnailNotProvided(photoName)
+			}
 			if !checkImageContentType(photo.ContentType) {
 				return errs.NewErrUnsupportedContentType(photoName)
 			}
@@ -449,24 +723,51 @@ func (s *Server) addPhotos(conn *sqlite.Conn, createdFeatureMapping, addedFeatur
 				return fmt.Errorf("failed to clear bindings of prepared statement: %w", err)
 			}
 
+			resizedPhoto, err := func() ([]byte, error) {
+				defer photo.File.Close()
+
+				resized, err := resizePhoto(s.config.MaxPhotoX, s.config.MaxPhotoY, s.config.PhotoQuality, photo.File)
+				if err != nil {
+					return nil, err
+				}
+
+				return resized, nil
+			}()
+
 			stmt.BindInt64(1, int64(featureID))
-			stmt.BindText(2, photo.ContentType)
-			stmt.BindZeroBlob(3, photo.Size)
+			stmt.BindText(2, thumbnail.ContentType)
+			stmt.BindText(3, "image/jpeg")
+			stmt.BindZeroBlob(4, thumbnail.Size)
+			stmt.BindZeroBlob(5, int64(len(resizedPhoto)))
 
 			_, err = stmt.Step()
 			if err != nil {
 				return fmt.Errorf("failed to evaluate prepared statement: %w", err)
 			}
 
-			blob, err := conn.OpenBlob("", "feature_photos", "file_contents", conn.LastInsertRowID(), true)
+			blob, err := conn.OpenBlob("", "feature_photos", "thumbnail_contents", conn.LastInsertRowID(), true)
+			if err != nil {
+				return fmt.Errorf("failed to open thumbnail content blob: %w", err)
+			}
+			err = func() error {
+				defer blob.Close()
+				defer thumbnail.File.Close()
+
+				_, err := io.Copy(blob, thumbnail.File)
+				return err
+			}()
+			if err != nil {
+				return fmt.Errorf("failed to write to thumbnail content blob: %w", err)
+			}
+
+			blob, err = conn.OpenBlob("", "feature_photos", "contents", conn.LastInsertRowID(), true)
 			if err != nil {
 				return fmt.Errorf("failed to open photo content blob: %w", err)
 			}
 			err = func() error {
 				defer blob.Close()
-				defer photo.File.Close()
 
-				_, err := io.Copy(blob, photo.File)
+				_, err = io.Copy(blob, bytes.NewBuffer(resizedPhoto))
 				return err
 			}()
 			if err != nil {
@@ -476,18 +777,26 @@ func (s *Server) addPhotos(conn *sqlite.Conn, createdFeatureMapping, addedFeatur
 		return nil
 	}
 
+	changed := false
+	defer func() {
+		if changed {
+			s.requestCheckpoint()
+		}
+	}()
 	for localFeatureID, photoNames := range createdFeatureMapping {
 		featureID, ok := createdIDMapping[localFeatureID]
 		if !ok {
 			return errs.NewErrFeatureForPhotoNotExists(int64(localFeatureID))
 		}
 		err = uploadPhotos(featureID, photoNames)
+		changed = true
 		if err != nil {
 			return fmt.Errorf("failed to upload photos for created features: %w", err)
 		}
 	}
 	for featureID, photoNames := range addedFeatureMapping {
 		err = uploadPhotos(featureID, photoNames)
+		changed = true
 		if err != nil {
 			return fmt.Errorf("failed to upload photos for existing features: %w", err)
 		}
@@ -545,6 +854,7 @@ func (s *Server) updateFeatures(conn *sqlite.Conn, features []models.Feature) er
 			return fmt.Errorf("failed to evaluate prepared statement: %w", err)
 		}
 	}
+	s.requestCheckpoint()
 	return nil
 }
 
@@ -572,6 +882,7 @@ func (s *Server) deleteFeatures(conn *sqlite.Conn, featureIDs []models.FeatureID
 			return fmt.Errorf("failed to evaluate prepared statement: %w", err)
 		}
 	}
+	s.requestCheckpoint()
 	return nil
 }
 
@@ -599,6 +910,7 @@ func (s *Server) deletePhotos(conn *sqlite.Conn, photoIDs []models.FeaturePhotoI
 			return fmt.Errorf("failed to evaluate prepared statement: %w", err)
 		}
 	}
+	s.requestCheckpoint()
 	return nil
 }
 
@@ -609,7 +921,7 @@ func (s *Server) getPhoto(featureID models.FeatureID, photoID models.FeaturePhot
 	var contentType *string = nil
 	var data []byte = nil
 	found := false
-	err := sqlitex.Exec(conn, "select content_type, file_contents from feature_photos where id = ? and feature_id = ?", func(stmt *sqlite.Stmt) error {
+	err := sqlitex.Exec(conn, "select content_type, contents from feature_photos where id = ? and feature_id = ?", func(stmt *sqlite.Stmt) error {
 		if found {
 			return fmt.Errorf("multiple photos returned for feature id %d, photo id %d", featureID, photoID)
 		}
@@ -629,3 +941,54 @@ func (s *Server) getPhoto(featureID models.FeatureID, photoID models.FeaturePhot
 
 	return data, *contentType, nil
 }
+
+func (s *Server) addProposals(conn *sqlite.Conn, proposals []models.Proposal) error {
+	stmt, err := conn.Prepare("insert into proposals(owner_id, description, how) values(?, ?, ?)")
+	if err != nil {
+		return fmt.Errorf("failed to prepare statement: %w", err)
+	}
+	defer stmt.Finalize()
+
+	for _, proposal := range proposals {
+		err := stmt.Reset()
+		if err != nil {
+			return fmt.Errorf("failed to reset prepared statement: %w", err)
+		}
+		err = stmt.ClearBindings()
+		if err != nil {
+			return fmt.Errorf("failed to clear bindings of prepared statement: %w", err)
+		}
+
+		stmt.BindInt64(1, int64(proposal.OwnerID))
+		stmt.BindText(2, proposal.Description)
+		stmt.BindText(3, proposal.How)
+
+		_, err = stmt.Step()
+		if err != nil {
+			return fmt.Errorf("failed to evaluate prepared statement: %w", err)
+		}
+	}
+	s.requestCheckpoint()
+	return nil
+}
+
+func (s *Server) getProposals(conn *sqlite.Conn) ([]models.Proposal, error) {
+	if conn == nil {
+		conn = s.getDbConn()
+		defer s.dbpool.Put(conn)
+	}
+
+	proposals := make([]models.Proposal, 0, 100)
+	err := sqlitex.Exec(conn, "select owner_id, description, how from proposals", func(stmt *sqlite.Stmt) error {
+		proposals = append(proposals, models.Proposal{
+			OwnerID:     models.UserID(stmt.ColumnInt(0)),
+			Description: stmt.ColumnText(1),
+			How:         stmt.ColumnText(2),
+		})
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get proposals from db: %w", err)
+	}
+	return proposals, nil
+}

server/sql_schema/V1_init.sql

@@ -1,12 +1,10 @@
-DROP TABLE IF EXISTS users;
-CREATE TABLE IF NOT EXISTS users (
+CREATE TABLE users (
   id integer PRIMARY KEY AUTOINCREMENT,
   name text NOT NULL UNIQUE
 );
 INSERT INTO users(id, name) VALUES(0, 'system');
 
-DROP TABLE IF EXISTS features;
-CREATE TABLE IF NOT EXISTS features (
+CREATE TABLE features (
   id integer PRIMARY KEY AUTOINCREMENT,
   owner_id integer NOT NULL REFERENCES users(id) ON DELETE CASCADE,
   name text NOT NULL,
@@ -15,10 +13,13 @@ CREATE TABLE IF NOT EXISTS features (
   geom text NOT NULL
 );
 
-DROP TABLE IF EXISTS feature_photos;
-CREATE TABLE IF NOT EXISTS feature_photos (
+CREATE TABLE feature_photos (
   id integer PRIMARY KEY AUTOINCREMENT,
   feature_id integer NOT NULL REFERENCES features(id) ON DELETE CASCADE,
+  thumbnail_content_type text NOT NULL,
   content_type text NOT NULL,
-  file_contents blob NOT NULL
-);
+  thumbnail_contents blob NOT NULL,
+  contents blob NOT NULL
+);
+
+PRAGMA user_version = 1;

server/sql_schema/V2_proposals.sql

@@ -0,0 +1,7 @@
+CREATE TABLE proposals (
+  owner_id integer NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
+  description text NOT NULL,
+  how text NOT NULL
+);
+
+PRAGMA user_version = 2;

server/utils.go

@@ -1,6 +1,15 @@
 package server
 
 import (
+	"bytes"
+	"fmt"
+	"image"
+	"image/jpeg"
+	"io"
+	"math"
+
+	"golang.org/x/image/draw"
+
 	"cernobor.cz/oko-server/models"
 )
 
@@ -32,3 +41,57 @@ func isUniqueFeatureID(features []models.Feature) bool {
 	}
 	return true
 }
+
+func resizePhoto(maxX, maxY, quality int, data io.Reader) ([]byte, error) {
+	src, _, err := image.Decode(data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode image: %w", err)
+	}
+
+	if maxX == 0 && maxY == 0 {
+		output := &bytes.Buffer{}
+		jpeg.Encode(output, src, &jpeg.Options{
+			Quality: 90,
+		})
+
+		return output.Bytes(), nil
+	}
+
+	var dst draw.Image
+	srcX := src.Bounds().Max.X
+	srcY := src.Bounds().Max.Y
+	srcRatio := float64(srcX) / float64(srcY)
+	if maxX == 0 && srcY > maxY {
+		newX := int(math.Round(float64(maxY) * srcRatio))
+		newY := maxY
+		dst = image.NewRGBA(image.Rect(0, 0, newX, newY))
+		draw.CatmullRom.Scale(dst, dst.Bounds(), src, src.Bounds(), draw.Over, nil)
+	} else if maxY == 0 && srcX > maxX {
+		newX := maxX
+		newY := int(math.Round(float64(maxX) / srcRatio))
+		dst = image.NewRGBA(image.Rect(0, 0, newX, newY))
+		draw.CatmullRom.Scale(dst, dst.Bounds(), src, src.Bounds(), draw.Over, nil)
+	} else if srcX > maxX || srcY > maxY {
+		tgtRatio := float64(maxX) / float64(maxY)
+		var newX, newY int
+		if srcRatio > tgtRatio {
+			newX = maxX
+			newY = int(math.Round(float64(maxX) / srcRatio))
+		} else {
+			newX = int(math.Round(float64(maxY) * srcRatio))
+			newY = maxY
+		}
+		dst = image.NewRGBA(image.Rect(0, 0, newX, newY))
+		draw.CatmullRom.Scale(dst, dst.Bounds(), src, src.Bounds(), draw.Over, nil)
+	} else {
+		dst = image.NewRGBA(image.Rect(0, 0, srcX, srcY))
+		draw.Copy(dst, image.Point{X: 0, Y: 0}, src, src.Bounds(), draw.Over, nil)
+	}
+
+	output := &bytes.Buffer{}
+	jpeg.Encode(output, dst, &jpeg.Options{
+		Quality: quality,
+	})
+
+	return output.Bytes(), nil
+}