refactoring, support for storing feature photos

* errors put into separate package
* added table to store feature photos
* added API endpoints for serving photos
* POST /data extended to accept photos using a multipart/form-data body
This commit is contained in:
zegkljan 2022-01-30 01:34:38 +01:00
parent 395b17d1be
commit 5deff38890
9 changed files with 462 additions and 53 deletions

63
errs/errors.go Normal file
View File

@ -0,0 +1,63 @@
package errs
import "fmt"
type ConstError string
func (e ConstError) Error() string {
return string(e)
}
const (
ErrUserNotExists = ConstError("user does not exist")
ErrUserAlreadyExists = ConstError("user already exists")
ErrPhotoNotExists = ConstError("photo does not exist (for the given feature)")
ErrAttemptedSystemUser = ConstError("attempted to associate with system user")
ErrUnsupportedImageFormat = ConstError("unsupported image format")
ErrUnsupportedImageContentType = ConstError("unsupported image content type")
ErrNonUniqueCreatedFeatureIDs = ConstError("created features do not have unique IDs")
)
type EErrPhotoNotProvided *ErrPhotoNotProvided
type ErrPhotoNotProvided struct {
Reference string
}
func NewErrPhotoNotProvided(reference string) *ErrPhotoNotProvided {
return &ErrPhotoNotProvided{
Reference: reference,
}
}
func (e *ErrPhotoNotProvided) Error() string {
return fmt.Sprintf("referenced photo %s which was not provided", e.Reference)
}
type ErrFeatureForPhotoNotExists struct {
PhotoFeatureReference int64
}
func NewErrFeatureForPhotoNotExists(reference int64) *ErrFeatureForPhotoNotExists {
return &ErrFeatureForPhotoNotExists{
PhotoFeatureReference: reference,
}
}
func (e *ErrFeatureForPhotoNotExists) Error() string {
return fmt.Sprintf("photos were referenced for feature with local ID %d which does not exist in posted created features", e.PhotoFeatureReference)
}
type ErrUnsupportedContentType struct {
Reference string
}
func NewErrUnsupportedContentType(reference string) *ErrUnsupportedContentType {
return &ErrUnsupportedContentType{
Reference: reference,
}
}
func (e *ErrUnsupportedContentType) Error() string {
return fmt.Sprintf("photo %s has an unsupported Content-Type", e.Reference)
}

View File

@ -3,6 +3,7 @@ package main
import (
"context"
"flag"
"fmt"
"os"
"os/signal"
"syscall"
@ -18,6 +19,12 @@ func main() {
flag.Parse()
if *tilepackFileArg == "" {
fmt.Fprintln(os.Stderr, "Tile pack not specified.")
flag.Usage()
os.Exit(1)
}
s := server.New(*dbFileArg, *tilepackFileArg, *apkFileArg)
sigs := make(chan os.Signal, 1)

View File

@ -1,11 +1,16 @@
package models
import geojson "github.com/paulmach/go.geojson"
import (
"io"
geojson "github.com/paulmach/go.geojson"
)
// core objects
type UserID int
type FeatureID int
type UserID int64
type FeatureID int64
type FeaturePhotoID int64
type User struct {
ID UserID `json:"id"`
@ -13,20 +18,29 @@ type User struct {
}
type Feature struct {
// ID is an ID of the feature.
// When the feature is submitted by a client for creation (i.e. in Update.Create) it is considered a 'local' ID which must be unique across all submitted features.
ID FeatureID `json:"id"`
OwnerID *UserID `json:"owner_id"`
Name string `json:"name"`
Description *string `json:"description"`
Category *string `json:"category"`
Geometry geojson.Geometry `json:"geometry"`
// PhotoIDs contains a list IDs of photos associated with the feature.
// When the feature is retrieved from the server, the IDs can be used to retrieve the photos.
// When the feature is submitted by a client (created or updated, i.e. in Update.Create or Update.Update), this list is ignored (as photos are managed through Update.CreatePhotos and Update.DeletePhotos fields).
PhotoIDs []FeaturePhotoID `json:"photo_ids"`
}
// transport objects
type Update struct {
Create []Feature `json:"create"`
Update []Feature `json:"update"`
Delete []FeatureID `json:"delete"`
Create []Feature `json:"create"`
CreatedPhotos map[FeatureID][]string `json:"created_photos"`
AddPhotos map[FeatureID][]string `json:"add_photos"`
Update []Feature `json:"update"`
Delete []FeatureID `json:"delete"`
DeletePhotos []FeaturePhotoID `json:"delete_photos"`
}
type HandshakeChallenge struct {
@ -44,3 +58,9 @@ type Data struct {
Users []User `json:"users"`
Features []Feature `json:"features"`
}
type Photo struct {
ContentType string
File io.ReadCloser
Size int64
}

View File

@ -1,13 +0,0 @@
package server
type ConstError string
func (e ConstError) Error() string {
return string(e)
}
const (
ErrUserNotExists = ConstError("user does not exist")
ErrUserAlreadyExists = ConstError("user already exists")
ErrAttemptedSystemUser = ConstError("attepted to associate with system user")
)

View File

@ -1,13 +1,17 @@
package server
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"math"
"net/http"
"os"
"strconv"
"time"
"cernobor.cz/oko-server/errs"
"cernobor.cz/oko-server/models"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
@ -90,6 +94,7 @@ func (s *Server) setupRouter() *gin.Engine {
router.POST(URIData, s.handlePOSTData)
router.GET(URIDataPeople, s.handleGETDataPeople)
router.GET(URIDataFeatures, s.handleGETDataFeatures)
router.GET(URIDataFeaturesPhoto, s.handleGETDataFeaturesPhoto)
// tileserver
router.GET(URITileserver, gin.WrapH(s.tileserverSvSet.Handler()))
@ -111,11 +116,11 @@ func (s *Server) handlePOSTHandshake(gc *gin.Context) {
id, err := s.handshake(hs)
if err != nil {
if errors.Is(err, ErrUserAlreadyExists) {
if errors.Is(err, errs.ErrUserAlreadyExists) {
gc.Status(http.StatusConflict)
} else if errors.Is(err, ErrUserNotExists) {
} else if errors.Is(err, errs.ErrUserNotExists) {
gc.Status(http.StatusNotFound)
} else if errors.Is(err, ErrAttemptedSystemUser) {
} else if errors.Is(err, errs.ErrAttemptedSystemUser) {
gc.Status(http.StatusForbidden)
} else {
internalError(gc, err)
@ -140,6 +145,17 @@ func (s *Server) handleGETData(gc *gin.Context) {
}
func (s *Server) handlePOSTData(gc *gin.Context) {
switch gc.ContentType() {
case "application/json":
s.handlePOSTDataJSON(gc)
case "multipart/form-data":
s.handlePOSTDataMultipart(gc)
default:
gc.String(http.StatusBadRequest, "unsupported Content-Type")
}
}
func (s *Server) handlePOSTDataJSON(gc *gin.Context) {
var data models.Update
err := gc.ShouldBindJSON(&data)
if err != nil {
@ -147,7 +163,17 @@ func (s *Server) handlePOSTData(gc *gin.Context) {
return
}
err = s.update(data)
if !isUniqueFeatureID(data.Create) {
gc.String(http.StatusBadRequest, "created features do not have unique IDs")
return
}
if data.CreatedPhotos != nil || data.AddPhotos != nil {
gc.String(http.StatusBadRequest, "created_photos and/or add_photos present, but Content-Type is application/json")
return
}
err = s.update(data, nil)
if err != nil {
internalError(gc, fmt.Errorf("failed to update data: %w", err))
return
@ -155,6 +181,84 @@ func (s *Server) handlePOSTData(gc *gin.Context) {
gc.Status(http.StatusNoContent)
}
func (s *Server) handlePOSTDataMultipart(gc *gin.Context) {
form, err := gc.MultipartForm()
if err != nil {
gc.String(http.StatusBadRequest, "malformed multipart/form-data content")
return
}
dataStr, ok := form.Value["data"]
if !ok {
dataFile, ok := form.File["data"]
if !ok {
gc.String(http.StatusBadRequest, "value 'data' is missing from the content")
return
}
if len(dataFile) != 1 {
gc.String(http.StatusBadRequest, "value 'data' does not contain exactly 1 item")
return
}
df, err := dataFile[0].Open()
if err != nil {
internalError(gc, fmt.Errorf("failed to open 'data' 'file': %w", err))
return
}
dataBytes, err := ioutil.ReadAll(df)
if err != nil {
internalError(gc, fmt.Errorf("failed to open 'data' 'file': %w", err))
return
}
dataStr = []string{string(dataBytes)}
}
if len(dataStr) != 1 {
gc.String(http.StatusBadRequest, "value 'data' does not contain exactly 1 item")
return
}
var data models.Update
err = json.Unmarshal([]byte(dataStr[0]), &data)
if err != nil {
gc.String(http.StatusBadRequest, "malformed 'data' value: %v", err)
return
}
if !isUniqueFeatureID(data.Create) {
gc.String(http.StatusBadRequest, "created features do not have unique IDs")
return
}
photos := make(map[string]models.Photo, len(form.File))
for name, fh := range form.File {
if len(fh) != 1 {
gc.String(http.StatusBadRequest, "file item %s does not contain exactly 1 file", name)
return
}
var photo models.Photo
f := fh[0]
photo.ContentType = f.Header.Get("Content-Type")
photo.Size = f.Size
photo.File, err = f.Open()
if err != nil {
internalError(gc, fmt.Errorf("failed to open provided photo file: %w", err))
}
defer photo.File.Close()
photos[name] = photo
}
err = s.update(data, photos)
if err != nil {
var e *errs.ErrUnsupportedContentType
if errors.As(err, &e) {
gc.String(http.StatusBadRequest, e.Error())
return
}
internalError(gc, fmt.Errorf("failed to update data: %w", err))
return
}
gc.Status(http.StatusNoContent)
}
func (s *Server) handleGETDataPeople(gc *gin.Context) {
people, err := s.getPeople(nil)
if err != nil {
@ -172,3 +276,28 @@ func (s *Server) handleGETDataFeatures(gc *gin.Context) {
}
gc.JSON(http.StatusOK, pois)
}
func (s *Server) handleGETDataFeaturesPhoto(gc *gin.Context) {
reqFeatureID, err := strconv.Atoi(gc.Param("feature"))
if err != nil {
gc.String(http.StatusBadRequest, "malformed feature ID")
return
}
reqPhotoID, err := strconv.Atoi(gc.Param("photo"))
if err != nil {
gc.String(http.StatusBadRequest, "malformed photo ID")
return
}
photoBytes, contentType, err := s.getPhoto(models.FeatureID(reqFeatureID), models.FeaturePhotoID(reqPhotoID))
if err != nil {
if errors.Is(err, errs.ErrPhotoNotExists) {
gc.String(http.StatusNotFound, "%v", err)
} else {
internalError(gc, fmt.Errorf("failed to retrieve photo: %w", err))
}
return
}
gc.Data(http.StatusOK, contentType, photoBytes)
}

View File

@ -8,10 +8,17 @@ INSERT INTO users(id, name) VALUES(0, 'system');
DROP TABLE IF EXISTS features;
CREATE TABLE IF NOT EXISTS features (
id integer PRIMARY KEY AUTOINCREMENT,
owner_id integer,
owner_id integer REFERENCES users(id) ON DELETE CASCADE,
name text NOT NULL,
description text,
category text,
geom text NOT NULL,
FOREIGN KEY(owner_id) REFERENCES users(id) ON DELETE CASCADE
geom text NOT NULL
);
DROP TABLE IF EXISTS feature_photos;
CREATE TABLE IF NOT EXISTS feature_photos (
id integer PRIMARY KEY AUTOINCREMENT,
feature_id integer NOT NULL REFERENCES features(id) ON DELETE CASCADE,
content_type text NOT NULL,
file_contents blob NOT NULL
);

View File

@ -4,12 +4,14 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"time"
_ "embed"
"cernobor.cz/oko-server/errs"
"cernobor.cz/oko-server/models"
"crawshaw.io/sqlite"
"crawshaw.io/sqlite/sqlitex"
@ -144,10 +146,10 @@ func (s *Server) handshake(hc models.HandshakeChallenge) (models.UserID, error)
return 0, err
}
if id == nil {
return 0, ErrUserNotExists
return 0, errs.ErrUserNotExists
}
if *id == 0 {
return 0, ErrAttemptedSystemUser
return 0, errs.ErrAttemptedSystemUser
}
} else {
err = sqlitex.Exec(conn, "insert into users(name) values(?) returning id", func(stmt *sqlite.Stmt) error {
@ -155,7 +157,7 @@ func (s *Server) handshake(hc models.HandshakeChallenge) (models.UserID, error)
return nil
}, hc.Name)
if sqlite.ErrCode(err) == sqlite.SQLITE_CONSTRAINT_UNIQUE {
return 0, ErrUserAlreadyExists
return 0, errs.ErrUserAlreadyExists
}
if sqlite.ErrCode(err) != sqlite.SQLITE_OK {
return 0, err
@ -193,20 +195,32 @@ func (s *Server) getData() (models.Data, error) {
}()
}
func (s *Server) update(data models.Update) error {
func (s *Server) update(data models.Update, photos map[string]models.Photo) error {
conn := s.getDbConn()
defer s.dbpool.Put(conn)
return func() (err error) {
defer sqlitex.Save(conn)(&err)
var createdIDMapping map[models.FeatureID]models.FeatureID
if data.Create != nil {
err = s.addFeatures(conn, data.Create)
createdIDMapping, err = s.addFeatures(conn, data.Create)
if err != nil {
err = fmt.Errorf("failed to add features: %w", err)
return
}
}
if data.CreatedPhotos != nil || data.AddPhotos != nil {
if photos == nil {
err = fmt.Errorf("photo assignment present but no photos provided")
return
}
err = s.addPhotos(conn, data.CreatedPhotos, data.AddPhotos, createdIDMapping, photos)
if err != nil {
err = fmt.Errorf("failed to add photos: %w", err)
return
}
}
if data.Update != nil {
err = s.updateFeatures(conn, data.Update)
if err != nil {
@ -221,6 +235,13 @@ func (s *Server) update(data models.Update) error {
return
}
}
if data.DeletePhotos != nil {
err = s.deletePhotos(conn, data.DeletePhotos)
if err != nil {
err = fmt.Errorf("failed to delete photos: %w", err)
return
}
}
return nil
}()
@ -253,29 +274,44 @@ func (s *Server) getFeatures(conn *sqlite.Conn) ([]models.Feature, error) {
}
features := make([]models.Feature, 0, 100)
err := sqlitex.Exec(conn, "select id, owner_id, name, description, category, geom from features", func(stmt *sqlite.Stmt) error {
id := stmt.ColumnInt(0)
var ownerID *int
err := sqlitex.Exec(conn, `select f.id, f.owner_id, f.name, f.description, f.category, f.geom, '[' || coalesce(group_concat(p.id, ', '), '') || ']'
from features f
left join feature_photos p on f.id = p.feature_id
group by f.id, f.owner_id, f.name, f.description, f.category, f.geom`, func(stmt *sqlite.Stmt) error {
id := stmt.ColumnInt64(0)
var ownerID *int64
if stmt.ColumnType(1) != sqlite.SQLITE_NULL {
ownerID = ptrInt(stmt.ColumnInt(1))
ownerID = ptrInt64(stmt.ColumnInt64(1))
}
name := stmt.ColumnText(2)
var description *string
if stmt.ColumnType(3) != sqlite.SQLITE_NULL {
description = ptrString(stmt.ColumnText(3))
}
var category *string
if stmt.ColumnType(4) != sqlite.SQLITE_NULL {
category = ptrString(stmt.ColumnText(4))
}
geomRaw := stmt.ColumnText(5)
geomRaw := stmt.ColumnText(5)
var geom geojson.Geometry
err := json.Unmarshal([]byte(geomRaw), &geom)
if err != nil {
return fmt.Errorf("failed to parse geometry for point id=%d: %w", id, err)
}
photosRaw := stmt.ColumnText(6)
var photos []models.FeaturePhotoID
err = json.Unmarshal([]byte(photosRaw), &photos)
if err != nil {
return fmt.Errorf("failed to parse list of photo IDs: %w", err)
}
feature := models.Feature{
ID: models.FeatureID(id),
OwnerID: (*models.UserID)(ownerID),
@ -283,6 +319,7 @@ func (s *Server) getFeatures(conn *sqlite.Conn) ([]models.Feature, error) {
Description: description,
Category: category,
Geometry: geom,
PhotoIDs: photos,
}
features = append(features, feature)
@ -294,21 +331,25 @@ func (s *Server) getFeatures(conn *sqlite.Conn) ([]models.Feature, error) {
return features, nil
}
func (s *Server) addFeatures(conn *sqlite.Conn, features []models.Feature) error {
func (s *Server) addFeatures(conn *sqlite.Conn, features []models.Feature) (map[models.FeatureID]models.FeatureID, error) {
stmt, err := conn.Prepare("insert into features(owner_id, name, description, category, geom) values(?, ?, ?, ?, ?)")
if err != nil {
return fmt.Errorf("failed to prepare statement: %w", err)
return nil, fmt.Errorf("failed to prepare statement: %w", err)
}
defer stmt.Finalize()
localIDMapping := make(map[models.FeatureID]models.FeatureID, len(features))
for _, feature := range features {
if _, ok := localIDMapping[feature.ID]; ok {
return nil, errs.ErrNonUniqueCreatedFeatureIDs
}
err := stmt.Reset()
if err != nil {
return fmt.Errorf("failed to reset prepared statement: %w", err)
return nil, fmt.Errorf("failed to reset prepared statement: %w", err)
}
err = stmt.ClearBindings()
if err != nil {
return fmt.Errorf("failed to clear bindings of prepared statement: %w", err)
return nil, fmt.Errorf("failed to clear bindings of prepared statement: %w", err)
}
if feature.OwnerID == nil {
@ -329,13 +370,87 @@ func (s *Server) addFeatures(conn *sqlite.Conn, features []models.Feature) error
}
geomBytes, err := json.Marshal(feature.Geometry)
if err != nil {
return fmt.Errorf("failed to marshal geometry of feature ID %d: %w", feature.ID, err)
return nil, fmt.Errorf("failed to marshal geometry of feature ID %d: %w", feature.ID, err)
}
stmt.BindText(5, string(geomBytes))
_, err = stmt.Step()
if err != nil {
return fmt.Errorf("failed to evaluate prepared statement: %w", err)
return nil, fmt.Errorf("failed to evaluate prepared statement: %w", err)
}
localIDMapping[feature.ID] = models.FeatureID(conn.LastInsertRowID())
}
return localIDMapping, nil
}
func (s *Server) addPhotos(conn *sqlite.Conn, createdFeatureMapping, addedFeatureMapping map[models.FeatureID][]string, createdIDMapping map[models.FeatureID]models.FeatureID, photos map[string]models.Photo) error {
stmt, err := conn.Prepare("insert into feature_photos(feature_id, content_type, file_contents) values(?, ?, ?)")
if err != nil {
return fmt.Errorf("failed to prepare statement: %w", err)
}
defer stmt.Finalize()
uploadPhotos := func(featureID models.FeatureID, photoNames []string) error {
for _, photoName := range photoNames {
photo, ok := photos[photoName]
if !ok {
return errs.NewErrPhotoNotProvided(photoName)
}
if !checkImageContentType(photo.ContentType) {
return errs.NewErrUnsupportedContentType(photoName)
}
err := stmt.Reset()
if err != nil {
return fmt.Errorf("failed to reset prepared statement: %w", err)
}
err = stmt.ClearBindings()
if err != nil {
return fmt.Errorf("failed to clear bindings of prepared statement: %w", err)
}
stmt.BindInt64(1, int64(featureID))
stmt.BindText(2, photo.ContentType)
stmt.BindZeroBlob(3, photo.Size)
_, err = stmt.Step()
if err != nil {
return fmt.Errorf("failed to evaluate prepared statement: %w", err)
}
blob, err := conn.OpenBlob("", "feature_photos", "file_contents", conn.LastInsertRowID(), true)
if err != nil {
return fmt.Errorf("failed to open photo content blob: %w", err)
}
err = func() error {
defer blob.Close()
defer photo.File.Close()
_, err := io.Copy(blob, photo.File)
return err
}()
if err != nil {
return fmt.Errorf("failed to write to photo content blob: %w", err)
}
}
return nil
}
for localFeatureID, photoNames := range createdFeatureMapping {
featureID, ok := createdIDMapping[localFeatureID]
if !ok {
return errs.NewErrFeatureForPhotoNotExists(int64(localFeatureID))
}
err = uploadPhotos(featureID, photoNames)
if err != nil {
return fmt.Errorf("failed to upload photos for created features: %w", err)
}
}
for featureID, photoNames := range addedFeatureMapping {
err = uploadPhotos(featureID, photoNames)
if err != nil {
return fmt.Errorf("failed to upload photos for existing features: %w", err)
}
}
return nil
@ -415,3 +530,58 @@ func (s *Server) deleteFeatures(conn *sqlite.Conn, featureIDs []models.FeatureID
}
return nil
}
func (s *Server) deletePhotos(conn *sqlite.Conn, photoIDs []models.FeaturePhotoID) error {
stmt, err := conn.Prepare("delete from feature_photos where id = ?")
if err != nil {
return fmt.Errorf("failed to prepare statement: %w", err)
}
defer stmt.Finalize()
for _, photoID := range photoIDs {
err := stmt.Reset()
if err != nil {
return fmt.Errorf("failed to reset prepared statement: %w", err)
}
err = stmt.ClearBindings()
if err != nil {
return fmt.Errorf("failed to clear bindings of prepared statement: %w", err)
}
stmt.BindInt64(1, int64(photoID))
_, err = stmt.Step()
if err != nil {
return fmt.Errorf("failed to evaluate prepared statement: %w", err)
}
}
return nil
}
func (s *Server) getPhoto(featureID models.FeatureID, photoID models.FeaturePhotoID) ([]byte, string, error) {
conn := s.getDbConn()
defer s.dbpool.Put(conn)
var contentType *string = nil
var data []byte = nil
found := false
err := sqlitex.Exec(conn, "select content_type, file_contents from feature_photos where id = ? and feature_id = ?", func(stmt *sqlite.Stmt) error {
if found {
return fmt.Errorf("multiple photos returned for feature id %d, photo id %d", featureID, photoID)
}
contentType = ptrString(stmt.ColumnText(0))
data = make([]byte, stmt.ColumnLen(1))
stmt.ColumnBytes(1, data)
found = true
return nil
}, photoID, featureID)
if err != nil {
return nil, "", fmt.Errorf("photo db query failed: %w", err)
}
if !found {
return nil, "", errs.ErrPhotoNotExists
}
return data, *contentType, nil
}

View File

@ -1,15 +1,16 @@
package server
const (
URIPing = "/ping"
URIHardFail = "/hard-fail"
URISoftFail = "/soft-fail"
URITilepack = "/tilepack"
URIHandshake = "/handshake"
URIData = "/data"
URIDataPeople = "/data/people"
URIDataExtra = "/data/extra"
URIDataFeatures = "/data/features"
URITileserverRoot = "/tileserver"
URITileserver = URITileserverRoot + "/*x"
URIPing = "/ping"
URIHardFail = "/hard-fail"
URISoftFail = "/soft-fail"
URITilepack = "/tilepack"
URIHandshake = "/handshake"
URIData = "/data"
URIDataPeople = "/data/people"
URIDataExtra = "/data/extra"
URIDataFeatures = "/data/features"
URIDataFeaturesPhoto = "/data/features/:feature/photos/:photo"
URITileserverRoot = "/tileserver"
URITileserver = URITileserverRoot + "/*x"
)

View File

@ -1,9 +1,34 @@
package server
import (
"cernobor.cz/oko-server/models"
)
func ptrInt(x int) *int {
return &x
}
func ptrInt64(x int64) *int64 {
return &x
}
func ptrString(x string) *string {
return &x
}
var contentTypes map[string]struct{} = map[string]struct{}{"image/jpeg": {}, "image/png": {}}
func checkImageContentType(contentType string) bool {
_, ok := contentTypes[contentType]
return ok
}
func isUniqueFeatureID(features []models.Feature) bool {
ids := make(map[models.FeatureID]struct{}, len(features))
for _, f := range features {
if _, pres := ids[f.ID]; pres {
return false
}
}
return true
}