Add support for http loading files from an asset archive

src/cmd/load-test-data/test-data.yml

@@ -163,5 +163,36 @@ published_posts:
     body: |
       This page is almost empty.
 
+  - id: markdown-archive-test
+    title: Markdown Archive Test
+    description: Test loading assets from an archive (tgz) file
+    format: md
+    body: |
+
+      This page contains images which are loaded from within an archive file which has been uploaded as a single asset.
+
+      There should be 4 images.
+
+      {{ Image "archive.tgz/1.jpg" }}
+      {{ Image "archive.tgz/2.jpg" }}
+      {{ Image "archive.tgz/foo/3.jpg" }}
+      {{ Image "archive.tgz/foo/4.jpg" }}
+
+  - id: gemtext-archive-test
+    title: Gemtext Archive Test
+    description: Test loading assets from an archive (tgz) file
+    format: gmi
+    body: |
+
+      This page contains images which are loaded from within an archive file which has been uploaded as a single asset.
+
+      There should be 4 images.
+
+      {{ Image "archive.tgz/1.jpg" "First" }}
+      {{ Image "archive.tgz/2.jpg" "Second" }}
+      {{ Image "archive.tgz/foo/3.jpg" "Third" }}
+      {{ Image "archive.tgz/foo/4.jpg" "Fourth" }}
+
 assets:
   galaxy.jpg: ./galaxy.jpg
+  archive.tgz: ./archive.tgz

src/go.mod

@@ -12,15 +12,14 @@ require (
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/mattn/go-sqlite3 v1.14.8
 	github.com/mediocregopher/mediocre-go-lib/v2 v2.0.0-beta.0.0.20220506011745-cbeee71cb1ee
+	github.com/omeid/go-tarfs v0.0.0-20171018021839-bf0d15c58b89
 	github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc
 	github.com/stretchr/testify v1.7.0
-	github.com/tdemin/gmnhg v0.4.2 // indirect
+	github.com/tdemin/gmnhg v0.4.2
 	github.com/tilinna/clock v1.1.0
 	github.com/ziutek/mymysql v1.5.4 // indirect
-	golang.org/dl v0.0.0-20190829154251-82a15e2f2ead // indirect
 	golang.org/x/crypto v0.0.0-20210915214749-c084706c2272
 	golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9
-	golang.org/x/net v0.0.0-20210917163549-3c21e5b27794 // indirect
 	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	gopkg.in/yaml.v3 v3.0.1

src/go.sum

@@ -2,6 +2,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 git.sr.ht/~adnano/go-gemini v0.2.3 h1:oJ+Y0/mheZ4Vg0ABjtf5dlmvq1yoONStiaQvmWWkofc=
 git.sr.ht/~adnano/go-gemini v0.2.3/go.mod h1:hQ75Y0i5jSFL+FQ7AzWVAYr5LQsaFC7v3ZviNyj46dY=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
 github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
@@ -90,6 +91,7 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
 github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
@@ -144,10 +146,13 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/niklasfasching/go-org v1.5.0 h1:V8IwoSPm/d61bceyWFxxnQLtlvNT+CjiYIhtZLdnMF0=
 github.com/niklasfasching/go-org v1.5.0/go.mod h1:sSb8ylwnAG+h8MGFDB3R1D5bxf8wA08REfhjShg3kjA=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/omeid/go-tarfs v0.0.0-20171018021839-bf0d15c58b89 h1:UUYoIjKNJ3XbHT1Nm+0pSs6mjTKgPKcBVa0cA9tIBfQ=
+github.com/omeid/go-tarfs v0.0.0-20171018021839-bf0d15c58b89/go.mod h1:OAIjrm1qIDkLYsUJQ5VahN/ZScQYM7jgJCO3zNWYmJ0=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -210,7 +215,6 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-golang.org/dl v0.0.0-20190829154251-82a15e2f2ead/go.mod h1:IUMfjQLJQd4UTqG1Z90tenwKoCX93Gn3MAQJMOSBsDQ=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -294,6 +298,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

src/http/assets.go

@@ -2,23 +2,27 @@ package http
 
 import (
 	"bytes"
+	"compress/gzip"
 	"errors"
 	"fmt"
 	"image"
 	"image/jpeg"
 	"image/png"
 	"io"
+	"io/fs"
 	"net/http"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/mediocregopher/blog.mediocregopher.com/srv/http/apiutil"
 	"github.com/mediocregopher/blog.mediocregopher.com/srv/post"
+	"github.com/omeid/go-tarfs"
 	"golang.org/x/image/draw"
 )
 
-func isImgResizable(id string) bool {
+func isImgResizable(path string) bool {
-	switch strings.ToLower(filepath.Ext(id)) {
+	switch strings.ToLower(filepath.Ext(path)) {
 	case ".jpg", ".jpeg", ".png":
 		return true
 	default:
@@ -84,21 +88,161 @@ func (a *api) managePostAssetsHandler() http.Handler {
 	})
 }
 
+type postAssetArchiveInfo struct {
+	path      string
+	id        string
+	subPath   string
+	isGzipped bool
+}
+
+func extractPostAssetArchiveInfo(path string) (postAssetArchiveInfo, bool) {
+
+	var info postAssetArchiveInfo
+
+	info.path = strings.TrimPrefix(path, "/")
+
+	info.id, info.subPath, _ = strings.Cut(info.path, "/")
+
+	switch {
+
+	case strings.HasSuffix(info.id, ".tar.gz"),
+		strings.HasSuffix(info.id, ".tgz"):
+		info.isGzipped = true
+
+	case strings.HasSuffix(info.id, ".tar"):
+		// ok
+
+	default:
+		// unsupported
+		return postAssetArchiveInfo{}, false
+	}
+
+	return info, true
+}
+
+func (a *api) writePostAsset(
+	rw http.ResponseWriter,
+	r *http.Request,
+	path string,
+	from io.ReadSeeker,
+) {
+
+	maxWidth, err := apiutil.StrToInt(r.FormValue("w"), 0)
+	if err != nil {
+		apiutil.BadRequest(rw, r, fmt.Errorf("invalid w parameter: %w", err))
+		return
+	}
+
+	if maxWidth == 0 {
+		http.ServeContent(rw, r, path, time.Time{}, from)
+		return
+	}
+
+	if !isImgResizable(path) {
+		apiutil.BadRequest(rw, r, fmt.Errorf("cannot resize asset %q", path))
+		return
+	}
+
+	resizedBuf := new(bytes.Buffer)
+
+	if err := resizeImage(resizedBuf, from, float64(maxWidth)); err != nil {
+		apiutil.InternalServerError(
+			rw, r,
+			fmt.Errorf(
+				"resizing image %q to size %d: %w",
+				path, maxWidth, err,
+			),
+		)
+	}
+
+	http.ServeContent(
+		rw, r, path, time.Time{}, bytes.NewReader(resizedBuf.Bytes()),
+	)
+}
+
+func (a *api) handleGetPostAssetArchive(
+	rw http.ResponseWriter,
+	r *http.Request,
+	info postAssetArchiveInfo,
+) {
+
+	buf := new(bytes.Buffer)
+
+	err := a.params.PostAssetStore.Get(info.id, buf)
+
+	if errors.Is(err, post.ErrAssetNotFound) {
+		http.Error(rw, "asset not found", 404)
+		return
+	} else if err != nil {
+		apiutil.InternalServerError(
+			rw, r,
+			fmt.Errorf("fetching archive asset with id %q: %w", info.id, err),
+		)
+		return
+	}
+
+	var from io.Reader = buf
+
+	if info.isGzipped {
+
+		if from, err = gzip.NewReader(from); err != nil {
+			apiutil.InternalServerError(
+				rw, r,
+				fmt.Errorf("decompressing archive asset with id %q: %w", info.id, err),
+			)
+			return
+		}
+	}
+
+	tarFS, err := tarfs.New(from)
+
+	if err != nil {
+		apiutil.InternalServerError(
+			rw, r,
+			fmt.Errorf("reading archive asset with id %q as fs: %w", info.id, err),
+		)
+		return
+	}
+
+	f, err := tarFS.Open(info.subPath)
+
+	if errors.Is(err, fs.ErrExist) {
+		http.Error(rw, "Asset not found", 404)
+		return
+
+	} else if err != nil {
+
+		apiutil.InternalServerError(
+			rw, r,
+			fmt.Errorf(
+				"opening path %q from archive asset with id %q as fs: %w",
+				info.subPath, info.id, err,
+			),
+		)
+		return
+	}
+
+	defer f.Close()
+
+	a.writePostAsset(rw, r, info.path, f)
+}
+
 func (a *api) getPostAssetHandler() http.Handler {
 
 	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 
-		id := filepath.Base(r.URL.Path)
+		archiveInfo, ok := extractPostAssetArchiveInfo(r.URL.Path)
 
-		maxWidth, err := apiutil.StrToInt(r.FormValue("w"), 0)
+		if ok {
-		if err != nil {
+			a.handleGetPostAssetArchive(rw, r, archiveInfo)
-			apiutil.BadRequest(rw, r, fmt.Errorf("invalid w parameter: %w", err))
 			return
 		}
 
+		id := filepath.Base(r.URL.Path)
+
 		buf := new(bytes.Buffer)
 
-		err = a.params.PostAssetStore.Get(id, buf)
+		err := a.params.PostAssetStore.Get(id, buf)
 
 		if errors.Is(err, post.ErrAssetNotFound) {
 			http.Error(rw, "Asset not found", 404)
@@ -110,36 +254,7 @@ func (a *api) getPostAssetHandler() http.Handler {
 			return
 		}
 
-		if maxWidth == 0 {
+		a.writePostAsset(rw, r, id, bytes.NewReader(buf.Bytes()))
-
-			if _, err := io.Copy(rw, buf); err != nil {
-				apiutil.InternalServerError(
-					rw, r,
-					fmt.Errorf(
-						"copying asset with id %q to response writer: %w",
-						id, err,
-					),
-				)
-			}
-
-			return
-		}
-
-		if !isImgResizable(id) {
-			apiutil.BadRequest(rw, r, fmt.Errorf("cannot resize file %q", id))
-			return
-		}
-
-		if err := resizeImage(rw, buf, float64(maxWidth)); err != nil {
-			apiutil.InternalServerError(
-				rw, r,
-				fmt.Errorf(
-					"resizing image with id %q to size %d: %w",
-					id, maxWidth, err,
-				),
-			)
-		}
-
 	})
 }