Add support for http loading files from an asset archive

main
Brian Picciano 1 year ago
parent b5e1103caf
commit 68f3215df6
  1. BIN
      src/cmd/load-test-data/archive.tgz
  2. 31
      src/cmd/load-test-data/test-data.yml
  3. 5
      src/go.mod
  4. 7
      src/go.sum
  5. 189
      src/http/assets.go

@ -163,5 +163,36 @@ published_posts:
body: |
This page is almost empty.
- id: markdown-archive-test
title: Markdown Archive Test
description: Test loading assets from an archive (tgz) file
format: md
body: |
This page contains images which are loaded from within an archive file which has been uploaded as a single asset.
There should be 4 images.
{{ Image "archive.tgz/1.jpg" }}
{{ Image "archive.tgz/2.jpg" }}
{{ Image "archive.tgz/foo/3.jpg" }}
{{ Image "archive.tgz/foo/4.jpg" }}
- id: gemtext-archive-test
title: Gemtext Archive Test
description: Test loading assets from an archive (tgz) file
format: gmi
body: |
This page contains images which are loaded from within an archive file which has been uploaded as a single asset.
There should be 4 images.
{{ Image "archive.tgz/1.jpg" "First" }}
{{ Image "archive.tgz/2.jpg" "Second" }}
{{ Image "archive.tgz/foo/3.jpg" "Third" }}
{{ Image "archive.tgz/foo/4.jpg" "Fourth" }}
assets:
galaxy.jpg: ./galaxy.jpg
archive.tgz: ./archive.tgz

@ -12,15 +12,14 @@ require (
github.com/hashicorp/golang-lru v0.5.4
github.com/mattn/go-sqlite3 v1.14.8
github.com/mediocregopher/mediocre-go-lib/v2 v2.0.0-beta.0.0.20220506011745-cbeee71cb1ee
github.com/omeid/go-tarfs v0.0.0-20171018021839-bf0d15c58b89
github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc
github.com/stretchr/testify v1.7.0
github.com/tdemin/gmnhg v0.4.2 // indirect
github.com/tdemin/gmnhg v0.4.2
github.com/tilinna/clock v1.1.0
github.com/ziutek/mymysql v1.5.4 // indirect
golang.org/dl v0.0.0-20190829154251-82a15e2f2ead // indirect
golang.org/x/crypto v0.0.0-20210915214749-c084706c2272
golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
golang.org/x/net v0.0.0-20210917163549-3c21e5b27794 // indirect
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
gopkg.in/yaml.v3 v3.0.1

@ -2,6 +2,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
git.sr.ht/~adnano/go-gemini v0.2.3 h1:oJ+Y0/mheZ4Vg0ABjtf5dlmvq1yoONStiaQvmWWkofc=
git.sr.ht/~adnano/go-gemini v0.2.3/go.mod h1:hQ75Y0i5jSFL+FQ7AzWVAYr5LQsaFC7v3ZviNyj46dY=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
@ -90,6 +91,7 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
@ -144,10 +146,13 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/niklasfasching/go-org v1.5.0 h1:V8IwoSPm/d61bceyWFxxnQLtlvNT+CjiYIhtZLdnMF0=
github.com/niklasfasching/go-org v1.5.0/go.mod h1:sSb8ylwnAG+h8MGFDB3R1D5bxf8wA08REfhjShg3kjA=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/omeid/go-tarfs v0.0.0-20171018021839-bf0d15c58b89 h1:UUYoIjKNJ3XbHT1Nm+0pSs6mjTKgPKcBVa0cA9tIBfQ=
github.com/omeid/go-tarfs v0.0.0-20171018021839-bf0d15c58b89/go.mod h1:OAIjrm1qIDkLYsUJQ5VahN/ZScQYM7jgJCO3zNWYmJ0=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -210,7 +215,6 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/dl v0.0.0-20190829154251-82a15e2f2ead/go.mod h1:IUMfjQLJQd4UTqG1Z90tenwKoCX93Gn3MAQJMOSBsDQ=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@ -294,6 +298,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

@ -2,23 +2,27 @@ package http
import (
"bytes"
"compress/gzip"
"errors"
"fmt"
"image"
"image/jpeg"
"image/png"
"io"
"io/fs"
"net/http"
"path/filepath"
"strings"
"time"
"github.com/mediocregopher/blog.mediocregopher.com/srv/http/apiutil"
"github.com/mediocregopher/blog.mediocregopher.com/srv/post"
"github.com/omeid/go-tarfs"
"golang.org/x/image/draw"
)
func isImgResizable(id string) bool {
switch strings.ToLower(filepath.Ext(id)) {
func isImgResizable(path string) bool {
switch strings.ToLower(filepath.Ext(path)) {
case ".jpg", ".jpeg", ".png":
return true
default:
@ -84,62 +88,173 @@ func (a *api) managePostAssetsHandler() http.Handler {
})
}
func (a *api) getPostAssetHandler() http.Handler {
type postAssetArchiveInfo struct {
path string
id string
subPath string
isGzipped bool
}
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
func extractPostAssetArchiveInfo(path string) (postAssetArchiveInfo, bool) {
id := filepath.Base(r.URL.Path)
var info postAssetArchiveInfo
maxWidth, err := apiutil.StrToInt(r.FormValue("w"), 0)
if err != nil {
apiutil.BadRequest(rw, r, fmt.Errorf("invalid w parameter: %w", err))
return
}
info.path = strings.TrimPrefix(path, "/")
buf := new(bytes.Buffer)
info.id, info.subPath, _ = strings.Cut(info.path, "/")
err = a.params.PostAssetStore.Get(id, buf)
switch {
if errors.Is(err, post.ErrAssetNotFound) {
http.Error(rw, "Asset not found", 404)
return
} else if err != nil {
case strings.HasSuffix(info.id, ".tar.gz"),
strings.HasSuffix(info.id, ".tgz"):
info.isGzipped = true
case strings.HasSuffix(info.id, ".tar"):
// ok
default:
// unsupported
return postAssetArchiveInfo{}, false
}
return info, true
}
func (a *api) writePostAsset(
rw http.ResponseWriter,
r *http.Request,
path string,
from io.ReadSeeker,
) {
maxWidth, err := apiutil.StrToInt(r.FormValue("w"), 0)
if err != nil {
apiutil.BadRequest(rw, r, fmt.Errorf("invalid w parameter: %w", err))
return
}
if maxWidth == 0 {
http.ServeContent(rw, r, path, time.Time{}, from)
return
}
if !isImgResizable(path) {
apiutil.BadRequest(rw, r, fmt.Errorf("cannot resize asset %q", path))
return
}
resizedBuf := new(bytes.Buffer)
if err := resizeImage(resizedBuf, from, float64(maxWidth)); err != nil {
apiutil.InternalServerError(
rw, r,
fmt.Errorf(
"resizing image %q to size %d: %w",
path, maxWidth, err,
),
)
}
http.ServeContent(
rw, r, path, time.Time{}, bytes.NewReader(resizedBuf.Bytes()),
)
}
func (a *api) handleGetPostAssetArchive(
rw http.ResponseWriter,
r *http.Request,
info postAssetArchiveInfo,
) {
buf := new(bytes.Buffer)
err := a.params.PostAssetStore.Get(info.id, buf)
if errors.Is(err, post.ErrAssetNotFound) {
http.Error(rw, "asset not found", 404)
return
} else if err != nil {
apiutil.InternalServerError(
rw, r,
fmt.Errorf("fetching archive asset with id %q: %w", info.id, err),
)
return
}
var from io.Reader = buf
if info.isGzipped {
if from, err = gzip.NewReader(from); err != nil {
apiutil.InternalServerError(
rw, r, fmt.Errorf("fetching asset with id %q: %w", id, err),
rw, r,
fmt.Errorf("decompressing archive asset with id %q: %w", info.id, err),
)
return
}
}
if maxWidth == 0 {
tarFS, err := tarfs.New(from)
if _, err := io.Copy(rw, buf); err != nil {
apiutil.InternalServerError(
rw, r,
fmt.Errorf(
"copying asset with id %q to response writer: %w",
id, err,
),
)
}
if err != nil {
apiutil.InternalServerError(
rw, r,
fmt.Errorf("reading archive asset with id %q as fs: %w", info.id, err),
)
return
}
return
}
f, err := tarFS.Open(info.subPath)
if errors.Is(err, fs.ErrExist) {
http.Error(rw, "Asset not found", 404)
return
if !isImgResizable(id) {
apiutil.BadRequest(rw, r, fmt.Errorf("cannot resize file %q", id))
} else if err != nil {
apiutil.InternalServerError(
rw, r,
fmt.Errorf(
"opening path %q from archive asset with id %q as fs: %w",
info.subPath, info.id, err,
),
)
return
}
defer f.Close()
a.writePostAsset(rw, r, info.path, f)
}
func (a *api) getPostAssetHandler() http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
archiveInfo, ok := extractPostAssetArchiveInfo(r.URL.Path)
if ok {
a.handleGetPostAssetArchive(rw, r, archiveInfo)
return
}
if err := resizeImage(rw, buf, float64(maxWidth)); err != nil {
id := filepath.Base(r.URL.Path)
buf := new(bytes.Buffer)
err := a.params.PostAssetStore.Get(id, buf)
if errors.Is(err, post.ErrAssetNotFound) {
http.Error(rw, "Asset not found", 404)
return
} else if err != nil {
apiutil.InternalServerError(
rw, r,
fmt.Errorf(
"resizing image with id %q to size %d: %w",
id, maxWidth, err,
),
rw, r, fmt.Errorf("fetching asset with id %q: %w", id, err),
)
return
}
a.writePostAsset(rw, r, id, bytes.NewReader(buf.Bytes()))
})
}

Loading…
Cancel
Save