From 7deb7e3f392cdf6ddcad306a5bebcbd520616cd5 Mon Sep 17 00:00:00 2001 From: zegkljan Date: Sat, 19 Feb 2022 23:38:58 +0100 Subject: [PATCH] refining photo handling * photo metedata provided with downloaded data * data download recognizes application/json and application/zip accepted types and serves bare json and zip with photos respectively * thumbnail handling and storage --- errs/errors.go | 16 +++++ models/models.go | 12 +++- server/handling.go | 36 ++++++++-- server/initdb.sql | 1 + server/server.go | 170 ++++++++++++++++++++++++++++++++++++++++----- 5 files changed, 211 insertions(+), 24 deletions(-) diff --git a/errs/errors.go b/errs/errors.go index edbae96..e3bafcd 100644 --- a/errs/errors.go +++ b/errs/errors.go @@ -34,6 +34,22 @@ func (e *ErrPhotoNotProvided) Error() string { return fmt.Sprintf("referenced photo %s which was not provided", e.Reference) } +type EErrPhotoThumbnailNotProvided *ErrPhotoThumbnailNotProvided + +type ErrPhotoThumbnailNotProvided struct { + Reference string +} + +func NewErrPhotoThumbnailNotProvided(reference string) *ErrPhotoThumbnailNotProvided { + return &ErrPhotoThumbnailNotProvided{ + Reference: reference, + } +} + +func (e *ErrPhotoThumbnailNotProvided) Error() string { + return fmt.Sprintf("referenced photo %s the thumbnail of which was not provided", e.Reference) +} + type ErrFeatureForPhotoNotExists struct { PhotoFeatureReference int64 } diff --git a/models/models.go b/models/models.go index 3c5b611..7b6955d 100644 --- a/models/models.go +++ b/models/models.go @@ -69,8 +69,9 @@ type HandshakeResponse struct { } type Data struct { - Users []User `json:"users"` - Features []Feature `json:"features"` + Users []User `json:"users"` + Features []Feature `json:"features"` + PhotoMetadata map[string]PhotoMetadata `json:"photo_metadata,omitempty"` } type Photo struct { @@ -78,3 +79,10 @@ type Photo struct { File io.ReadCloser Size int64 } + +type PhotoMetadata struct { + ContentType string `json:"content_type"` + Size int64 `json:"size"` + ID FeaturePhotoID `json:"id"` + ThumbnailFilename string `json:"thumbnail_filename"` +} diff --git a/server/handling.go b/server/handling.go index 47245bf..c8511d2 100644 --- a/server/handling.go +++ b/server/handling.go @@ -155,12 +155,40 @@ func (s *Server) handlePOSTHandshake(gc *gin.Context) { } func (s *Server) handleGETData(gc *gin.Context) { - data, err := s.getData() - if err != nil { - internalError(gc, err) + accept := gc.GetHeader("Accept") + if accept == "application/json" { + data, err := s.getDataOnly() + if err != nil { + internalError(gc, err) + return + } + gc.JSON(http.StatusOK, data) + return + } else if accept == "application/zip" { + file, err := s.getDataWithPhotos() + defer func() { + file.Close() + os.Remove(file.Name()) + }() + if err != nil { + internalError(gc, err) + return + } + fi, err := file.Stat() + if err != nil { + internalError(gc, err) + return + } + size := fi.Size() + _, err = file.Seek(0, 0) + if err != nil { + internalError(gc, err) + return + } + gc.DataFromReader(http.StatusOK, size, "application/zip", file, nil) return } - gc.JSON(http.StatusOK, data) + gc.String(http.StatusNotAcceptable, "%s is not acceptable", accept) } func (s *Server) handlePOSTData(gc *gin.Context) { diff --git a/server/initdb.sql b/server/initdb.sql index 2fff2f3..db28d5d 100644 --- a/server/initdb.sql +++ b/server/initdb.sql @@ -20,5 +20,6 @@ CREATE TABLE IF NOT EXISTS feature_photos ( id integer PRIMARY KEY AUTOINCREMENT, feature_id integer NOT NULL REFERENCES features(id) ON DELETE CASCADE, content_type text NOT NULL, + thumbnail_contents blob NOT NULL, file_contents blob NOT NULL ); \ No newline at end of file diff --git a/server/server.go b/server/server.go index 4e33f31..d9e0bf1 100644 --- a/server/server.go +++ b/server/server.go @@ -1,6 +1,7 @@ package server import ( + "archive/zip" "context" "encoding/json" "fmt" @@ -187,31 +188,144 @@ func (s *Server) handshake(hc models.HandshakeChallenge) (models.UserID, error) return models.UserID(userID), nil } -func (s *Server) getData() (models.Data, error) { +func (s *Server) getData(conn *sqlite.Conn) (data models.Data, err error) { + err = s.deleteExpiredFeatures(conn) + if err != nil { + return models.Data{}, fmt.Errorf("failed to delete expired featured: %w", err) + } + people, err := s.getPeople(conn) + if err != nil { + return models.Data{}, fmt.Errorf("failed to retreive people: %w", err) + } + features, err := s.getFeatures(conn) + if err != nil { + return models.Data{}, fmt.Errorf("failed to retreive features: %w", err) + } + + return models.Data{ + Users: people, + Features: features, + }, nil +} + +func (s *Server) getDataOnly() (data models.Data, err error) { + conn := s.getDbConn() + defer s.dbpool.Put(conn) + defer sqlitex.Save(conn)(&err) + + data, err = s.getData(conn) + if err != nil { + return models.Data{}, fmt.Errorf("failed to get json data: %w", err) + } + return data, err +} + +func (s *Server) getDataWithPhotos() (file *os.File, err error) { conn := s.getDbConn() defer s.dbpool.Put(conn) - return func() (data models.Data, err error) { - defer sqlitex.Save(conn)(&err) + defer sqlitex.Save(conn)(&err) - err = s.deleteExpiredFeatures(conn) - if err != nil { - return models.Data{}, fmt.Errorf("failed to delete expired featured: %w", err) + makePhotoFilename := func(id models.FeaturePhotoID) string { + return fmt.Sprintf("img%d", id) + } + makeThumbnailFilename := func(id models.FeaturePhotoID) string { + return fmt.Sprintf("thumb_%s", makePhotoFilename(id)) + } + + data, err := s.getData(conn) + if err != nil { + return nil, fmt.Errorf("failed to get json data: %w", err) + } + + data.PhotoMetadata = make(map[string]models.PhotoMetadata, 100) + err = sqlitex.Exec(conn, "select id, content_type, length(file_contents) from feature_photos", func(stmt *sqlite.Stmt) error { + id := models.FeaturePhotoID(stmt.ColumnInt64(0)) + contentType := stmt.ColumnText(1) + fileSize := stmt.ColumnInt64(2) + data.PhotoMetadata[makePhotoFilename(id)] = models.PhotoMetadata{ + ContentType: contentType, + Size: fileSize, + ID: id, + ThumbnailFilename: makeThumbnailFilename(id), } - people, err := s.getPeople(conn) + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to collect photo metadata: %w", err) + } + + dataBytes, err := json.MarshalIndent(&data, "", " ") + if err != nil { + return nil, fmt.Errorf("failed to marshal json data: %w", err) + } + + f, err := os.CreateTemp("", "") + if err != nil { + return nil, fmt.Errorf("failed to create temporary file: %w", err) + } + + zw := zip.NewWriter(f) + defer zw.Close() + w, err := zw.Create("data.json") + if err != nil { + return nil, fmt.Errorf("failed to create data zip entry: %w", err) + } + _, err = w.Write(dataBytes) + if err != nil { + return nil, fmt.Errorf("failed to write data zip entry: %w", err) + } + + err = sqlitex.Exec(conn, "select id from feature_photos", func(stmt *sqlite.Stmt) error { + id := stmt.ColumnInt64(0) + + blob, err := conn.OpenBlob("", "feature_photos", "thumbnail_contents", id, false) if err != nil { - return models.Data{}, fmt.Errorf("failed to retreive people: %w", err) + return fmt.Errorf("failed to open photo ID %d thumbnail content blob: %w", id, err) } - features, err := s.getFeatures(conn) + err = func() error { + defer blob.Close() + w, err := zw.Create(makeThumbnailFilename(models.FeaturePhotoID(id))) + if err != nil { + return fmt.Errorf("failed to create zip entry: %w", err) + } + _, err = io.Copy(w, blob) + if err != nil { + return fmt.Errorf("failed to write zip entry: %w", err) + } + return nil + }() if err != nil { - return models.Data{}, fmt.Errorf("failed to retreive features: %w", err) + return fmt.Errorf("failed to write photo ID %d thumbnail: %w", id, err) } - return models.Data{ - Users: people, - Features: features, - }, nil - }() + blob, err = conn.OpenBlob("", "feature_photos", "file_contents", id, false) + if err != nil { + return fmt.Errorf("failed to open photo ID %d photo content blob: %w", id, err) + } + err = func() error { + defer blob.Close() + w, err := zw.Create(makePhotoFilename(models.FeaturePhotoID(id))) + if err != nil { + return fmt.Errorf("failed to create zip entry: %w", err) + } + _, err = io.Copy(w, blob) + if err != nil { + return fmt.Errorf("failed to write zip entry: %w", err) + } + return nil + }() + if err != nil { + return fmt.Errorf("failed to write photo ID %d photo: %w", id, err) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to collect photo files: %w", err) + } + + return f, nil } func (s *Server) update(data models.Update, photos map[string]models.Photo) error { @@ -424,7 +538,7 @@ func (s *Server) addFeatures(conn *sqlite.Conn, features []models.Feature) (map[ } func (s *Server) addPhotos(conn *sqlite.Conn, createdFeatureMapping, addedFeatureMapping map[models.FeatureID][]string, createdIDMapping map[models.FeatureID]models.FeatureID, photos map[string]models.Photo) error { - stmt, err := conn.Prepare("insert into feature_photos(feature_id, content_type, file_contents) values(?, ?, ?)") + stmt, err := conn.Prepare("insert into feature_photos(feature_id, content_type, thumbnail_contents, file_contents) values(?, ?, ?, ?)") if err != nil { return fmt.Errorf("failed to prepare statement: %w", err) } @@ -436,6 +550,10 @@ func (s *Server) addPhotos(conn *sqlite.Conn, createdFeatureMapping, addedFeatur if !ok { return errs.NewErrPhotoNotProvided(photoName) } + thumbnail, ok := photos[fmt.Sprintf("thumb_%s", photoName)] + if !ok { + return errs.NewErrPhotoThumbnailNotProvided(photoName) + } if !checkImageContentType(photo.ContentType) { return errs.NewErrUnsupportedContentType(photoName) } @@ -451,14 +569,30 @@ func (s *Server) addPhotos(conn *sqlite.Conn, createdFeatureMapping, addedFeatur stmt.BindInt64(1, int64(featureID)) stmt.BindText(2, photo.ContentType) - stmt.BindZeroBlob(3, photo.Size) + stmt.BindZeroBlob(3, thumbnail.Size) + stmt.BindZeroBlob(4, photo.Size) _, err = stmt.Step() if err != nil { return fmt.Errorf("failed to evaluate prepared statement: %w", err) } - blob, err := conn.OpenBlob("", "feature_photos", "file_contents", conn.LastInsertRowID(), true) + blob, err := conn.OpenBlob("", "feature_photos", "thumbnail_contents", conn.LastInsertRowID(), true) + if err != nil { + return fmt.Errorf("failed to open thumbnail content blob: %w", err) + } + err = func() error { + defer blob.Close() + defer thumbnail.File.Close() + + _, err := io.Copy(blob, thumbnail.File) + return err + }() + if err != nil { + return fmt.Errorf("failed to write to thumbnail content blob: %w", err) + } + + blob, err = conn.OpenBlob("", "feature_photos", "file_contents", conn.LastInsertRowID(), true) if err != nil { return fmt.Errorf("failed to open photo content blob: %w", err) }