diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 1599cf43..0baf2241 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -31,14 +31,15 @@ func (e DefaultArchiveFactory) Open(ctx context.Context, location url.URL, passw return nil, errors.New("unsupported scheme " + u.Scheme().String()) } - st, err := os.Stat(u.Path()) + path := u.ToFilepath() + st, err := os.Stat(path) if err != nil { return nil, err } if st.IsDir() { - return e.explodedFactory.Open(u.Path(), password) + return e.explodedFactory.Open(path, password) } else { - return e.gozipFactory.Open(u.Path(), password) + return e.gozipFactory.Open(path, password) } } diff --git a/pkg/asset/asset_file.go b/pkg/asset/asset_file.go index 21e68386..e5d7e1aa 100644 --- a/pkg/asset/asset_file.go +++ b/pkg/asset/asset_file.go @@ -49,6 +49,10 @@ func (a *FileAsset) Name() string { } func (a *FileAsset) realPath() string { + // ToFilepath handles Windows drive paths in file URLs, e.g. file:///C:/dir. + if u, ok := a.uri.(url.AbsoluteURL); ok && u.IsFile() { + return filepath.ToSlash(u.ToFilepath()) + } return filepath.ToSlash(a.uri.Path()) } @@ -71,6 +75,33 @@ func (a *FileAsset) MediaType(ctx context.Context) mediatype.MediaType { return *a.mediatype } +// Location implements RelativePublicationAsset +func (a *FileAsset) Location() url.URL { + return a.uri +} + +// CreateRelativeFetcher implements RelativePublicationAsset +func (a *FileAsset) CreateRelativeFetcher(ctx context.Context, root url.URL) (fetcher.Fetcher, error) { + var rootPath string + if u, ok := root.(url.AbsoluteURL); ok { + if !u.IsFile() { + return nil, errors.New("root of a file asset must be a file URL") + } + rootPath = u.ToFilepath() + } else { + rootPath = filepath.FromSlash(root.Path()) + } + + rootPath, err := filepath.Abs(rootPath) + if err != nil { + return nil, err + } + // The fetcher is sandboxed to the publication root: the parser normalizes the + // manifest's HREFs so that none escapes it, and HREFs handed to the publication + // later (e.g. from an incoming server request) must not reach outside it either. + return fetcher.NewFileFetcher("", rootPath), nil +} + // CreateFetcher implements PublicationAsset func (a *FileAsset) CreateFetcher(ctx context.Context, dependencies Dependencies, credentials string) (fetcher.Fetcher, error) { if u, ok := a.uri.(url.AbsoluteURL); ok && !u.IsFile() { diff --git a/pkg/asset/asset_gcs.go b/pkg/asset/asset_gcs.go index 8d1e6879..59ff7794 100644 --- a/pkg/asset/asset_gcs.go +++ b/pkg/asset/asset_gcs.go @@ -86,6 +86,24 @@ func (a *GCSAsset) MediaType(ctx context.Context) mediatype.MediaType { return *a.mediatype } +// Location implements RelativePublicationAsset +func (a *GCSAsset) Location() url.URL { + return a.uri +} + +// CreateRelativeFetcher implements RelativePublicationAsset +func (a *GCSAsset) CreateRelativeFetcher(ctx context.Context, root url.URL) (fetcher.Fetcher, error) { + u, ok := root.(url.AbsoluteURL) + if !ok { + return nil, errors.New("root of a GCS asset must be an absolute GS URL") + } + handle, err := u.ToGSObject(a.client) + if err != nil { + return nil, err + } + return fetcher.NewGCSFetcher("", a.client, handle), nil +} + // CreateFetcher implements PublicationAsset func (a *GCSAsset) CreateFetcher(ctx context.Context, dependencies Dependencies, credentials string) (fetcher.Fetcher, error) { handle, err := a.handle() diff --git a/pkg/asset/asset_http.go b/pkg/asset/asset_http.go index cbf06135..3cfbc16c 100644 --- a/pkg/asset/asset_http.go +++ b/pkg/asset/asset_http.go @@ -111,6 +111,20 @@ func (a *HTTPAsset) MediaType(ctx context.Context) mediatype.MediaType { return *a.mediatype } +// Location implements RelativePublicationAsset +func (a *HTTPAsset) Location() url.URL { + return a.url +} + +// CreateRelativeFetcher implements RelativePublicationAsset +func (a *HTTPAsset) CreateRelativeFetcher(ctx context.Context, root url.URL) (fetcher.Fetcher, error) { + u, ok := root.(url.AbsoluteURL) + if !ok || !u.IsHTTP() { + return nil, errors.New("root of an HTTP asset must be an absolute HTTP(S) URL") + } + return fetcher.NewHTTPFetcher("", a.client, u), nil +} + // CreateFetcher implements PublicationAsset func (a *HTTPAsset) CreateFetcher(ctx context.Context, dependencies Dependencies, credentials string) (fetcher.Fetcher, error) { // We can't determine if the provided path is a directory or not unless it ends in a "/" diff --git a/pkg/asset/asset_publication.go b/pkg/asset/asset_publication.go index 8b1df0af..68164564 100644 --- a/pkg/asset/asset_publication.go +++ b/pkg/asset/asset_publication.go @@ -6,6 +6,7 @@ import ( "github.com/readium/go-toolkit/pkg/archive" "github.com/readium/go-toolkit/pkg/fetcher" "github.com/readium/go-toolkit/pkg/mediatype" + "github.com/readium/go-toolkit/pkg/util/url" ) type Dependencies struct { @@ -18,3 +19,21 @@ type PublicationAsset interface { MediaType(ctx context.Context) mediatype.MediaType // Media type of the asset. If unknown, fallback on `MediaType.Binary`. CreateFetcher(ctx context.Context, dependencies Dependencies, credentials string) (fetcher.Fetcher, error) // Creates a fetcher used to access the asset's content. } + +// A [PublicationAsset] which can also provide access to resources located relative to itself, +// e.g. the resources referenced by a bare (exploded) Readium Web Publication Manifest. +type RelativePublicationAsset interface { + PublicationAsset + + // The URL of the asset within its medium, e.g. a file path URL, or an + // HTTP(S), S3 or GS URL. HREFs relative to the asset resolve against it. + Location() url.URL + + // Creates a fetcher serving the resources surrounding this asset, rooted at [root]: + // a link with HREF `a/file.xhtml` is fetched from `a/file.xhtml` under the root + // directory, whether the asset lives on a local file system or on a remote server. + // + // [root] must be the URL of a directory containing the asset, typically derived + // from [Location] by resolving a `./` or `../` reference against it. + CreateRelativeFetcher(ctx context.Context, root url.URL) (fetcher.Fetcher, error) +} diff --git a/pkg/asset/asset_s3.go b/pkg/asset/asset_s3.go index f449712c..53e98cd2 100644 --- a/pkg/asset/asset_s3.go +++ b/pkg/asset/asset_s3.go @@ -92,6 +92,24 @@ func (a *S3Asset) MediaType(ctx context.Context) mediatype.MediaType { return *a.mediatype } +// Location implements RelativePublicationAsset +func (a *S3Asset) Location() url.URL { + return a.uri +} + +// CreateRelativeFetcher implements RelativePublicationAsset +func (a *S3Asset) CreateRelativeFetcher(ctx context.Context, root url.URL) (fetcher.Fetcher, error) { + u, ok := root.(url.AbsoluteURL) + if !ok { + return nil, errors.New("root of an S3 asset must be an absolute S3 URL") + } + obj, err := u.ToS3Object() + if err != nil { + return nil, err + } + return fetcher.NewS3Fetcher("", a.client, *obj.Bucket, *obj.Key), nil +} + // CreateFetcher implements PublicationAsset func (a *S3Asset) CreateFetcher(ctx context.Context, dependencies Dependencies, credentials string) (fetcher.Fetcher, error) { obj, err := a.object() diff --git a/pkg/fetcher/fetcher.go b/pkg/fetcher/fetcher.go index 0316de7c..133f1671 100644 --- a/pkg/fetcher/fetcher.go +++ b/pkg/fetcher/fetcher.go @@ -2,10 +2,26 @@ package fetcher import ( "context" + "path" + "strings" "github.com/readium/go-toolkit/pkg/manifest" ) +// Reports whether the slash-separated object key [key] stays within [root]: it is +// [root] itself or sits below it. Used by the object-store fetchers to keep a `..` +// in an HREF from addressing objects outside the fetcher's prefix, the way +// [FileFetcher] contains resources within its directory. An empty [root] is the +// bucket root, which only forbids upward (`..`) escapes. +func keyWithinRoot(root, key string) bool { + root = path.Clean(root) + key = path.Clean(key) + if root == "." || root == "/" { + return key != ".." && !strings.HasPrefix(key, "../") + } + return key == root || strings.HasPrefix(key, root+"/") +} + // Fetcher provides access to a Resource from a Link. type Fetcher interface { diff --git a/pkg/fetcher/fetcher_cloud_test.go b/pkg/fetcher/fetcher_cloud_test.go new file mode 100644 index 00000000..89607d56 --- /dev/null +++ b/pkg/fetcher/fetcher_cloud_test.go @@ -0,0 +1,90 @@ +package fetcher + +import ( + "testing" + + "cloud.google.com/go/storage" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/readium/go-toolkit/pkg/manifest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Object keys are raw strings on S3 and GCS: HREFs must be decoded before the lookup, +// and queries/fragments dropped. +func TestS3FetcherGetKey(t *testing.T) { + f := NewS3Fetcher("", &s3.Client{}, "bucket", "pub") + for _, tt := range []struct { + href string + key string + }{ + {"track.mp3", "pub/track.mp3"}, + {"my%20track.mp3", "pub/my track.mp3"}, + {"my track.mp3", "pub/my track.mp3"}, + {"caf%C3%A9.mp3", "pub/café.mp3"}, + {"track.mp3#t=30", "pub/track.mp3"}, + {"track.mp3?token=abc", "pub/track.mp3"}, + {"audio/track.mp3", "pub/audio/track.mp3"}, + } { + res := f.Get(t.Context(), manifest.Link{Href: manifest.MustNewHREFFromString(tt.href, false)}) + s3res, ok := res.(*s3Resource) + require.Truef(t, ok, "expected an s3Resource for %q", tt.href) + assert.Equalf(t, tt.key, s3res.key, "key mismatch for href %q", tt.href) + } +} + +// A `..` in an HREF must not let a request reach objects outside the fetcher's +// key prefix, the way [FileFetcher] contains resources within its directory. +func TestS3FetcherContainsKey(t *testing.T) { + f := NewS3Fetcher("", &s3.Client{}, "bucket", "pub") + for _, href := range []string{ + "../secret.txt", + "../../secret.txt", + "audio/../../secret.txt", + "../pub-other/x.txt", + } { + res := f.Get(t.Context(), manifest.Link{Href: manifest.MustNewHREFFromString(href, false)}) + _, ok := res.(*s3Resource) + assert.Falsef(t, ok, "escaping href %q should not resolve to an object", href) + } + + // Resources within the prefix still resolve, including a `..` that stays inside. + for _, tt := range []struct{ href, key string }{ + {"audio/track.mp3", "pub/audio/track.mp3"}, + {"audio/../cover.jpg", "pub/cover.jpg"}, + } { + res := f.Get(t.Context(), manifest.Link{Href: manifest.MustNewHREFFromString(tt.href, false)}) + s3res, ok := res.(*s3Resource) + require.Truef(t, ok, "href %q should resolve", tt.href) + assert.Equal(t, tt.key, s3res.key) + } +} + +func TestGCSFetcherContainsObjectName(t *testing.T) { + client := &storage.Client{} + f := NewGCSFetcher("", client, client.Bucket("bucket").Object("pub")) + for _, href := range []string{"../secret.txt", "../../secret.txt", "audio/../../secret.txt"} { + res := f.Get(t.Context(), manifest.Link{Href: manifest.MustNewHREFFromString(href, false)}) + _, ok := res.(*gcsResource) + assert.Falsef(t, ok, "escaping href %q should not resolve to an object", href) + } +} + +func TestGCSFetcherGetObjectName(t *testing.T) { + client := &storage.Client{} + f := NewGCSFetcher("", client, client.Bucket("bucket").Object("pub")) + for _, tt := range []struct { + href string + name string + }{ + {"track.mp3", "pub/track.mp3"}, + {"my%20track.mp3", "pub/my track.mp3"}, + {"caf%C3%A9.mp3", "pub/café.mp3"}, + {"track.mp3#t=30", "pub/track.mp3"}, + } { + res := f.Get(t.Context(), manifest.Link{Href: manifest.MustNewHREFFromString(tt.href, false)}) + gcsres, ok := res.(*gcsResource) + require.Truef(t, ok, "expected a gcsResource for %q", tt.href) + assert.Equalf(t, tt.name, gcsres.handle.ObjectName(), "object name mismatch for href %q", tt.href) + } +} diff --git a/pkg/fetcher/fetcher_file.go b/pkg/fetcher/fetcher_file.go index d7c6ec92..a3a67e2e 100644 --- a/pkg/fetcher/fetcher_file.go +++ b/pkg/fetcher/fetcher_file.go @@ -99,7 +99,10 @@ func (f *FileFetcher) Get(ctx context.Context, link manifest.Link) Resource { if err != nil { continue // TODO somehow get this error out? } - if strings.HasPrefix(rapath, iapath) { + // The path must be [itemFile] itself, or sit below it beyond a separator: + // a plain prefix check would let "dir-other" pass as a descendant of "dir". + sep := string(filepath.Separator) + if rapath == iapath || strings.HasPrefix(rapath, strings.TrimSuffix(iapath, sep)+sep) { resource := NewFileResource(link, resourceFile) f.resources = append(f.resources, weak.Make(resource)) return resource diff --git a/pkg/fetcher/fetcher_file_test.go b/pkg/fetcher/fetcher_file_test.go index 32a9fd30..8bcbff9f 100644 --- a/pkg/fetcher/fetcher_file_test.go +++ b/pkg/fetcher/fetcher_file_test.go @@ -2,6 +2,8 @@ package fetcher import ( "bytes" + "os" + "path/filepath" "testing" "github.com/readium/go-toolkit/pkg/manifest" @@ -83,6 +85,27 @@ func TestFileFetcherDirectoryTraversalNotFound(t *testing.T) { assert.Equal(t, NotFound(err.Cause), err, "cannot traverse up a directory using '..'") } +func TestFileFetcherSiblingDirectoryNotFound(t *testing.T) { + // A directory sharing a name prefix with the fetcher's root must not be reachable. + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "pub"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "pub-secret"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "pub", "inside.txt"), []byte("inside"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "pub-secret", "leak.txt"), []byte("leak"), 0o644)) + + fetcher := NewFileFetcher("", filepath.Join(dir, "pub")) + defer fetcher.Close() + + resource := fetcher.Get(t.Context(), manifest.Link{Href: manifest.MustNewHREFFromString("inside.txt", false)}) + bin, err := resource.Read(t.Context(), 0, 0) + require.Nil(t, err) + assert.Equal(t, "inside", string(bin)) + + resource = fetcher.Get(t.Context(), manifest.Link{Href: manifest.MustNewHREFFromString("../pub-secret/leak.txt", false)}) + _, err = resource.Read(t.Context(), 0, 0) + assert.Equal(t, NotFound(err.Cause), err, "cannot escape into a prefix-sharing sibling directory") +} + func TestFileFetcherReadRange(t *testing.T) { resource := testFileFetcher.Get(t.Context(), manifest.Link{Href: manifest.MustNewHREFFromString("file_href", false)}) bin, err := resource.Read(t.Context(), 0, 2) diff --git a/pkg/fetcher/fetcher_gcs.go b/pkg/fetcher/fetcher_gcs.go index 32df770d..5e8cd6e3 100644 --- a/pkg/fetcher/fetcher_gcs.go +++ b/pkg/fetcher/fetcher_gcs.go @@ -122,12 +122,23 @@ func (f *GCSFetcher) Links(ctx context.Context) (manifest.LinkList, error) { // Get implements Fetcher func (f *GCSFetcher) Get(ctx context.Context, link manifest.Link) Resource { - linkHref := link.Href.String() + // Use the decoded path for the object lookup: GCS object names are raw strings, so a + // percent-encoded HREF would never match, and queries/fragments don't belong in names. + var linkHref string + if hrefURL := link.Href.Resolve(nil, nil); hrefURL != nil { + linkHref = hrefURL.Path() + } else { + linkHref = link.Href.String() + } if strings.HasPrefix(linkHref, f.href) { resourceFile := path.Join(f.handle.ObjectName(), strings.TrimPrefix(linkHref, f.href)) - return &gcsResource{ - handle: f.client.Bucket(f.handle.BucketName()).Object(resourceFile), - link: link, + // Keep the resource within the fetcher's object-name prefix: a `..` in the HREF + // must not let an incoming request reach objects elsewhere in the bucket. + if keyWithinRoot(f.handle.ObjectName(), resourceFile) { + return &gcsResource{ + handle: f.client.Bucket(f.handle.BucketName()).Object(resourceFile), + link: link, + } } } diff --git a/pkg/fetcher/fetcher_http.go b/pkg/fetcher/fetcher_http.go index dc3e87f1..a097a44b 100644 --- a/pkg/fetcher/fetcher_http.go +++ b/pkg/fetcher/fetcher_http.go @@ -66,10 +66,16 @@ func (f *HTTPFetcher) Get(ctx context.Context, link manifest.Link) Resource { if strings.HasPrefix(linkHref, f.href) { rurl, err := url.RelativeURLFromString(strings.TrimPrefix(linkHref, f.href)) if err == nil { - return &httpResource{ - link: link, - client: f.client, - url: f.url.Resolve(rurl).(url.AbsoluteURL), + resolved := f.url.Resolve(rurl).(url.AbsoluteURL) + // Keep the resource within the base URL: a `..` or root-absolute HREF must + // not let an incoming request reach a path outside the fetcher's location. + // Relativize returns a relative URL only when [resolved] is under [f.url]. + if _, ok := f.url.Relativize(resolved).(url.RelativeURL); ok { + return &httpResource{ + link: link, + client: f.client, + url: resolved, + } } } } @@ -81,6 +87,18 @@ func (f *HTTPFetcher) Close() { // No-op for HTTP } +// Creates a [Resource] serving the contents of the given HTTP(S) URL. +func NewHTTPResource(link manifest.Link, client *http.Client, url url.AbsoluteURL) Resource { + if client == nil { + panic("NewHTTPResource requires a non-nil client") + } + return &httpResource{ + link: link, + client: client, + url: url, + } +} + // Resource from HTTP type httpResource struct { link manifest.Link diff --git a/pkg/fetcher/fetcher_http_test.go b/pkg/fetcher/fetcher_http_test.go new file mode 100644 index 00000000..6f762e29 --- /dev/null +++ b/pkg/fetcher/fetcher_http_test.go @@ -0,0 +1,38 @@ +package fetcher + +import ( + "net/http" + "testing" + + "github.com/readium/go-toolkit/pkg/manifest" + "github.com/readium/go-toolkit/pkg/util/url" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The HTTP fetcher resolves HREFs against its base URL, but a `..` or root-absolute +// HREF must not let a request reach a path outside the base, matching the containment +// of the file and object-store fetchers. +func TestHTTPFetcherContainsPath(t *testing.T) { + base, err := url.AbsoluteURLFromString("http://example.com/pub/") + require.NoError(t, err) + f := NewHTTPFetcher("", http.DefaultClient, base) + + // Escaping HREFs do not resolve to a network resource. + for _, href := range []string{"../secret.txt", "../../secret.txt", "/secret.txt", "audio/../../secret.txt"} { + res := f.Get(t.Context(), manifest.Link{Href: manifest.MustNewHREFFromString(href, false)}) + _, ok := res.(*httpResource) + assert.Falsef(t, ok, "escaping href %q should not resolve", href) + } + + // HREFs within the base resolve to the expected absolute URL. + for _, tt := range []struct{ href, url string }{ + {"audio/track.mp3", "http://example.com/pub/audio/track.mp3"}, + {"audio/../cover.jpg", "http://example.com/pub/cover.jpg"}, + } { + res := f.Get(t.Context(), manifest.Link{Href: manifest.MustNewHREFFromString(tt.href, false)}) + hres, ok := res.(*httpResource) + require.Truef(t, ok, "href %q should resolve", tt.href) + assert.Equal(t, tt.url, hres.url.String()) + } +} diff --git a/pkg/fetcher/fetcher_s3.go b/pkg/fetcher/fetcher_s3.go index d44b84f2..7954e5e0 100644 --- a/pkg/fetcher/fetcher_s3.go +++ b/pkg/fetcher/fetcher_s3.go @@ -108,14 +108,25 @@ func (f *S3Fetcher) Links(ctx context.Context) (manifest.LinkList, error) { // Get implements Fetcher func (f *S3Fetcher) Get(ctx context.Context, link manifest.Link) Resource { - linkHref := link.Href.String() + // Use the decoded path for the object lookup: S3 keys are raw strings, so a + // percent-encoded HREF would never match, and queries/fragments don't belong in keys. + var linkHref string + if hrefURL := link.Href.Resolve(nil, nil); hrefURL != nil { + linkHref = hrefURL.Path() + } else { + linkHref = link.Href.String() + } if strings.HasPrefix(linkHref, f.href) { resourceFile := path.Join(f.key, strings.TrimPrefix(linkHref, f.href)) - return &s3Resource{ - link: link, - client: f.client, - bucket: f.bucket, - key: resourceFile, + // Keep the resource within the fetcher's key prefix: a `..` in the HREF must + // not let an incoming request reach objects elsewhere in the bucket. + if keyWithinRoot(f.key, resourceFile) { + return &s3Resource{ + link: link, + client: f.client, + bucket: f.bucket, + key: resourceFile, + } } } diff --git a/pkg/manifest/transform.go b/pkg/manifest/transform.go new file mode 100644 index 00000000..aaa17112 --- /dev/null +++ b/pkg/manifest/transform.go @@ -0,0 +1,65 @@ +package manifest + +// Applies [transform] to the HREF of every link of the manifest: the reading order, +// resources, links and table of contents, the metadata's links (subjects, +// contributors and belongsTo collections), including their alternates, children, and +// the links of subcollections. +func (m *Manifest) TransformHREFs(transform func(HREF) HREF) { + m.Links.TransformHREFs(transform) + m.ReadingOrder.TransformHREFs(transform) + m.Resources.TransformHREFs(transform) + m.TableOfContents.TransformHREFs(transform) + m.Metadata.TransformHREFs(transform) + m.Subcollections.TransformHREFs(transform) +} + +// Applies [transform] to the HREF of every link carried by the metadata: the links +// of subjects, of every contributor role, and of the belongsTo collections. +func (m *Metadata) TransformHREFs(transform func(HREF) HREF) { + for i := range m.Subjects { + m.Subjects[i].Links.TransformHREFs(transform) + } + for _, c := range []Contributors{ + m.Authors, m.Translators, m.Editors, m.Artists, m.Illustrators, + m.Letterers, m.Pencilers, m.Colorists, m.Inkers, m.Narrators, + m.Contributors, m.Publishers, m.Imprints, + } { + c.TransformHREFs(transform) + } + for _, collections := range m.BelongsTo { + collections.TransformHREFs(transform) + } +} + +// Applies [transform] to the HREF of every link of every contributor of the list. +func (cs Contributors) TransformHREFs(transform func(HREF) HREF) { + for i := range cs { + cs[i].Links.TransformHREFs(transform) + } +} + +// Applies [transform] to the HREF of every link of the list, including their +// alternates and children. +func (ll LinkList) TransformHREFs(transform func(HREF) HREF) { + for i := range ll { + ll[i].TransformHREFs(transform) + } +} + +// Applies [transform] to the link's HREF, and to those of its alternates and children. +func (l *Link) TransformHREFs(transform func(HREF) HREF) { + l.Href = transform(l.Href) + l.Alternates.TransformHREFs(transform) + l.Children.TransformHREFs(transform) +} + +// Applies [transform] to the HREF of every link of every collection of the map, +// including subcollections. +func (pcm PublicationCollectionMap) TransformHREFs(transform func(HREF) HREF) { + for _, collections := range pcm { + for i := range collections { + collections[i].Links.TransformHREFs(transform) + collections[i].Subcollections.TransformHREFs(transform) + } + } +} diff --git a/pkg/mediatype/mediatype_of.go b/pkg/mediatype/mediatype_of.go index 220ddd2c..956cad67 100644 --- a/pkg/mediatype/mediatype_of.go +++ b/pkg/mediatype/mediatype_of.go @@ -13,16 +13,20 @@ import ( var Sniffers = []Sniffer{ SniffEPUB, SniffLPF, + // SniffOPDS must come before SniffWebpub: an OPDS 2 publication is a RWPM with + // acquisition links, which the WebPub content heuristics would otherwise claim. + SniffOPDS, + SniffLCPLicense, + // SniffWebpub must come before SniffArchive: a package containing a RWPM manifest.json + // could otherwise be claimed as a CBZ or ZAB by the archive content heuristics. + SniffWebpub, SniffArchive, SniffPDF, SniffXHTML, SniffHTML, SniffBitmap, SniffAudio, - SniffOPDS, - SniffLCPLicense, SniffW3CWPUB, - SniffWebpub, // Note SniffKnown and SniffSystem aren't here! } @@ -67,6 +71,11 @@ func of(ctx context.Context, content SnifferContent, mediaTypes []string, fileEx mediaTypes: mediaTypes, fileExtensions: fileExtensions, } + // Open the archive (if the content is one) up front, so the sniffers that + // inspect archive entries share a single handle through the memoized context + // instead of each reopening and leaking it. It's closed once sniffing is done. + context.ContentAsArchive(ctx) + defer context.Close() for _, sniffer := range sniffers { mediaType := sniffer(ctx, context) if mediaType != nil { diff --git a/pkg/mediatype/sniffer.go b/pkg/mediatype/sniffer.go index 5e3d54fa..0d4ea304 100644 --- a/pkg/mediatype/sniffer.go +++ b/pkg/mediatype/sniffer.go @@ -91,7 +91,17 @@ func SniffOPDS(ctx context.Context, context SnifferContext) *MediaType { } // OPDS 2 (Heavy) - // TODO requires context.ContentAsRWPM() + // Only classify JSON that decodes as a RWPM (a `metadata` object with a title), + // like the Kotlin toolkit, so arbitrary JSON carrying an OPDS-vocabulary link + // isn't misread as an OPDS 2 document. + if js := context.ContentAsJSON(); js != nil && rwpmHasMetadataTitle(js) { + if rwpmSelfLinkMatches(js, &OPDS2) { + return &OPDS2 + } + if rwpmHasLinkWithRelPrefix(js, "http://opds-spec.org/acquisition") { + return &OPDS2Publication + } + } // OPDS Authentication Document (Heavy) if context.ContainsJSONKeys("id", "title", "authentication") { @@ -210,9 +220,55 @@ func SniffWebpub(ctx context.Context, context SnifferContext) *MediaType { return &LCPProtectedPDF } - // isManifest := true - // TODO implement heavy sniffing, which requires context.ContentAsRWPM() - // https://github.com/readium/r2-shared-kotlin/blob/develop/r2-shared/src/main/java/org/readium/r2/shared/util/mediatype/Sniffer.kt#L165 + // Heavy sniffing. + // Reads a RWPM, either as a bare manifest or from a `manifest.json` entry in a package. + isManifest := true + rwpm := context.ContentAsJSON() + if !isRWPMJSON(rwpm) { + rwpm = nil + if bin := context.ReadArchiveEntryAt(ctx, "manifest.json"); bin != nil { + var js map[string]interface{} + if json.Unmarshal(bin, &js) == nil && isRWPMJSON(js) { + isManifest = false + rwpm = js + } + } + } + if rwpm == nil { + return nil + } + + isLCPProtected := !isManifest && + (context.ContainsArchiveEntryAt(ctx, "license.lcpl") || rwpmHasLCPScheme(rwpm)) + + if rwpmConformsTo(rwpm, rwpmProfileAudiobook, MediaType.IsAudio) { + if isManifest { + return &ReadiumAudiobookManifest + } + if isLCPProtected { + return &LCPProtectedAudiobook + } + return &ReadiumAudiobook + } + if rwpmConformsTo(rwpm, rwpmProfileDivina, MediaType.IsBitmap) { + if isManifest { + return &ReadiumDivinaManifest + } + return &ReadiumDivina + } + if isLCPProtected && rwpmConformsTo(rwpm, rwpmProfilePDF, func(mt MediaType) bool { return mt.Matches(&PDF) }) { + return &LCPProtectedPDF + } + if rwpmSelfLinkMatches(rwpm, &ReadiumWebpubManifest) { + if isManifest { + return &ReadiumWebpubManifest + } + return &ReadiumWebpub + } + if !isManifest { + // Any package containing a RWPM is a Readium Web Publication. + return &ReadiumWebpub + } return nil } diff --git a/pkg/mediatype/sniffer_content.go b/pkg/mediatype/sniffer_content.go index ce0490bc..9bb4a105 100644 --- a/pkg/mediatype/sniffer_content.go +++ b/pkg/mediatype/sniffer_content.go @@ -20,14 +20,17 @@ type SnifferFileContent struct { buffer []byte } -func NewSnifferFileContent(file fs.File) SnifferFileContent { - return SnifferFileContent{file: file} +// The returned pointer must be shared by every reader of the content: for a file which +// is not an [io.ReadSeeker], the content is buffered on first read, and a copy would +// lose the buffer while the underlying file is already drained. +func NewSnifferFileContent(file fs.File) *SnifferFileContent { + return &SnifferFileContent{file: file} } const MaxReadSize = 5 * 1024 * 1024 // 5MB // Read implements SnifferContent -func (s SnifferFileContent) Read() []byte { +func (s *SnifferFileContent) Read() []byte { info, err := s.file.Stat() if err != nil { return nil @@ -53,7 +56,7 @@ func (s SnifferFileContent) Read() []byte { } // Stream implements SnifferContent -func (s SnifferFileContent) Stream() io.Reader { +func (s *SnifferFileContent) Stream() io.Reader { if of, ok := s.file.(*os.File); ok { of.Seek(0, io.SeekStart) return bufio.NewReader(s.file) diff --git a/pkg/mediatype/sniffer_context.go b/pkg/mediatype/sniffer_context.go index ea0302d0..e2e6eb0d 100644 --- a/pkg/mediatype/sniffer_context.go +++ b/pkg/mediatype/sniffer_context.go @@ -28,6 +28,7 @@ type SnifferContext struct { _contentAsJSON map[string]interface{} _loadedContentAsJSON bool _contentAsArchive archive.Archive + _contentAsArchiveErr error _loadedContentAsArchive bool } @@ -158,41 +159,46 @@ func (s SnifferContext) ContentAsXML() *XMLNode { // Content as an Archive instance. // Warning: Archive is only supported for a local file, for now. +// The archive is opened once and memoized (both the handle and any error), so callers +// must not close it; it's released when the sniffing context is closed with [Close]. func (s *SnifferContext) ContentAsArchive(ctx context.Context) (archive.Archive, error) { if !s._loadedContentAsArchive { s._loadedContentAsArchive = true - switch s.content.(type) { - case SnifferFileContent: - { - fileSniffer := s.content.(SnifferFileContent) - u, err := url.FromFilepath(fileSniffer.Name()) - if err != nil { - return nil, err - } - a, err := archive.NewArchiveFactory().Open(ctx, u, "") - if err != nil { - return nil, err - } - s._contentAsArchive = a - } - case SnifferBytesContent: - { - fileSniffer := s.content.(SnifferBytesContent) - a, err := archive.NewArchiveFactory().OpenBytes(ctx, fileSniffer.bytes, "") - if err != nil { - return nil, err - } - s._contentAsArchive = a - } - default: - { - return nil, errors.New("SnifferContent type does not support opening as an archive") - } + s._contentAsArchive, s._contentAsArchiveErr = s.openArchive(ctx) + } + return s._contentAsArchive, s._contentAsArchiveErr +} + +func (s *SnifferContext) openArchive(ctx context.Context) (archive.Archive, error) { + switch content := s.content.(type) { + case *SnifferFileContent: + u, err := url.FromFilepath(content.Name()) + if err != nil { + return nil, err } + return archive.NewArchiveFactory().Open(ctx, u, "") + case SnifferBytesContent: + return archive.NewArchiveFactory().OpenBytes(ctx, content.bytes, "") + default: + return nil, errors.New("SnifferContent type does not support opening as an archive") + } +} + +// Close releases any resources opened while sniffing the content, such as the archive +// handle memoized by [ContentAsArchive] (which otherwise leaks its underlying file +// until garbage collection). Safe to call more than once. Accessing the archive after +// closing fails gracefully rather than returning a closed handle. +func (s *SnifferContext) Close() { + if s._contentAsArchive != nil { + s._contentAsArchive.Close() + s._contentAsArchive = nil } - return s._contentAsArchive, nil + s._loadedContentAsArchive = true + s._contentAsArchiveErr = errClosedSnifferContext } +var errClosedSnifferContext = errors.New("the sniffing context is closed") + // Content parsed as generic JSON interface. func (s SnifferContext) ContentAsJSON() map[string]interface{} { if !s._loadedContentAsJSON { @@ -211,11 +217,6 @@ func (s SnifferContext) ContentAsJSON() map[string]interface{} { return s._contentAsJSON } -// Content parsed as a Readium Web Publication Manifest. -func (s SnifferContext) ContentAsRWPM() { - panic("Not implemented!") // TODO think out the best go equivalent (without circular imports) -} - // Raw bytes stream of the content. // A byte stream can be useful when sniffers only need to read a few bytes at the beginning of the file. func (s SnifferContext) Stream() io.Reader { @@ -279,7 +280,7 @@ func (s SnifferContext) ContainsJSONKeys(keys ...string) bool { } // Returns whether an Archive entry exists in this file. -func (s SnifferContext) ContainsArchiveEntryAt(ctx context.Context, path string) bool { +func (s *SnifferContext) ContainsArchiveEntryAt(ctx context.Context, path string) bool { a, err := s.ContentAsArchive(ctx) if err != nil { return false @@ -292,7 +293,7 @@ func (s SnifferContext) ContainsArchiveEntryAt(ctx context.Context, path string) } // Returns the Archive entry data at the given [path] in this file. -func (s SnifferContext) ReadArchiveEntryAt(ctx context.Context, path string) []byte { +func (s *SnifferContext) ReadArchiveEntryAt(ctx context.Context, path string) []byte { a, err := s.ContentAsArchive(ctx) if err != nil { return nil diff --git a/pkg/mediatype/sniffer_rwpm.go b/pkg/mediatype/sniffer_rwpm.go new file mode 100644 index 00000000..d404f9fd --- /dev/null +++ b/pkg/mediatype/sniffer_rwpm.go @@ -0,0 +1,218 @@ +package mediatype + +import "strings" + +// Duck-typed inspection of a Readium Web Publication Manifest parsed as generic JSON. +// The manifest package depends on mediatype, so sniffers can't use [manifest.Manifest]; +// instead these helpers mirror the conformance heuristics of Manifest.ConformsTo on the +// raw JSON. + +// Readium Web Publication Profiles, from the registry: +// https://readium.org/webpub-manifest/profiles/ +const ( + rwpmProfileAudiobook = "https://readium.org/webpub-manifest/profiles/audiobook" + rwpmProfileDivina = "https://readium.org/webpub-manifest/profiles/divina" + rwpmProfilePDF = "https://readium.org/webpub-manifest/profiles/pdf" +) + +const lcpSchemeURI = "http://readium.org/2014/01/lcp" + +// Returns whether the JSON has a `metadata` object bearing a valid title: a plain +// string or a localized-string object whose translations are all strings, as +// [manifest.LocalizedStringFromJSON] requires. This is the minimal RWPM signature, +// shared by Readium manifests and OPDS 2 documents. +func rwpmHasMetadataTitle(js map[string]interface{}) bool { + metadata, ok := js["metadata"].(map[string]interface{}) + if !ok { + return false + } + switch title := metadata["title"].(type) { + case string: // Plain title + return true + case map[string]interface{}: // Localized title: every translation must be a string + for _, v := range title { + if _, ok := v.(string); !ok { + return false + } + } + return true + default: + return false + } +} + +// Returns whether the JSON has the shape of a RWPM that [manifest.ManifestFromJSON] +// would accept: a `metadata` object bearing a title, along with a `readingOrder` +// (or legacy `spine`) array of link objects which have an `href` and a valid `type`. +func isRWPMJSON(js map[string]interface{}) bool { + if js == nil { + return false + } + + if !rwpmHasMetadataTitle(js) { + return false + } + + rawLinks, ok := js["readingOrder"].([]interface{}) + if !ok { + if rawLinks, ok = js["spine"].([]interface{}); !ok { + return false + } + } + for _, raw := range rawLinks { + link, ok := raw.(map[string]interface{}) + if !ok { + return false + } + if href, ok := link["href"].(string); !ok || href == "" { + return false + } + // A missing `type` only drops the link from the reading order, + // but an unparsable one fails the whole manifest. + if typ, ok := link["type"].(string); ok && typ != "" { + if _, err := NewOfString(typ); err != nil { + return false + } + } + } + return true +} + +// Returns the link objects of the given collection ("readingOrder", "resources", "links"...). +func rwpmLinks(js map[string]interface{}, key string) []map[string]interface{} { + raw, ok := js[key].([]interface{}) + if !ok { + return nil + } + links := make([]map[string]interface{}, 0, len(raw)) + for _, v := range raw { + if link, ok := v.(map[string]interface{}); ok { + links = append(links, link) + } + } + return links +} + +// Returns the media types of the reading order (or legacy spine) links. +// Links without a valid media type are skipped, like [manifest.ManifestFromJSON] drops them. +func rwpmReadingOrderTypes(js map[string]interface{}) []MediaType { + links := rwpmLinks(js, "readingOrder") + if links == nil { + links = rwpmLinks(js, "spine") + } + types := make([]MediaType, 0, len(links)) + for _, link := range links { + typ, ok := link["type"].(string) + if !ok { + continue + } + mt, err := NewOfString(typ) + if err != nil { + continue + } + types = append(types, mt) + } + return types +} + +// Returns whether the RWPM conforms to the given profile, mirroring +// [manifest.Manifest.ConformsTo]: either the profile is declared in +// `metadata.conformsTo`, or every reading order resource satisfies [matches]. +func rwpmConformsTo(js map[string]interface{}, profile string, matches func(mt MediaType) bool) bool { + types := rwpmReadingOrderTypes(js) + if len(types) == 0 { + return false + } + + metadata, _ := js["metadata"].(map[string]interface{}) + switch conformsTo := metadata["conformsTo"].(type) { + case string: + if conformsTo == profile { + return true + } + case []interface{}: + for _, v := range conformsTo { + if v == profile { + return true + } + } + } + + for _, mt := range types { + if !matches(mt) { + return false + } + } + return true +} + +// Returns whether the `self` link of the RWPM matches one of the given media types. +func rwpmSelfLinkMatches(js map[string]interface{}, mediaTypes ...*MediaType) bool { + for _, link := range rwpmLinks(js, "links") { + rels, ok := link["rel"].([]interface{}) + if !ok { + rels = []interface{}{link["rel"]} + } + isSelf := false + for _, rel := range rels { + if rel == "self" { + isSelf = true + break + } + } + if !isSelf { + continue + } + typ, ok := link["type"].(string) + if !ok { + continue + } + mt, err := NewOfString(typ) + if err != nil { + continue + } + if mt.Matches(mediaTypes...) { + return true + } + } + return false +} + +// Returns whether a link of the RWPM's `links` collection has a relation starting with +// the given prefix. +func rwpmHasLinkWithRelPrefix(js map[string]interface{}, prefix string) bool { + for _, link := range rwpmLinks(js, "links") { + rels, ok := link["rel"].([]interface{}) + if !ok { + rels = []interface{}{link["rel"]} + } + for _, rel := range rels { + if s, ok := rel.(string); ok && strings.HasPrefix(s, prefix) { + return true + } + } + } + return false +} + +// Returns whether a reading order resource of the RWPM is encrypted with the LCP scheme. +func rwpmHasLCPScheme(js map[string]interface{}) bool { + links := rwpmLinks(js, "readingOrder") + if links == nil { + links = rwpmLinks(js, "spine") + } + for _, link := range links { + properties, ok := link["properties"].(map[string]interface{}) + if !ok { + continue + } + encrypted, ok := properties["encrypted"].(map[string]interface{}) + if !ok { + continue + } + if encrypted["scheme"] == lcpSchemeURI { + return true + } + } + return false +} diff --git a/pkg/mediatype/sniffer_test.go b/pkg/mediatype/sniffer_test.go index 990cdc99..391c1349 100644 --- a/pkg/mediatype/sniffer_test.go +++ b/pkg/mediatype/sniffer_test.go @@ -1,6 +1,8 @@ package mediatype import ( + "archive/zip" + "bytes" "io" "mime" "os" @@ -32,15 +34,17 @@ func TestSnifferFromMetadata(t *testing.T) { assert.Equal(t, &ReadiumAudiobook, Of([]string{"application/audiobook+zip"}, []string{"audiobook"}, Sniffers), "\"audiobook\" in a slice + \"application/audiobook+zip\" in a slice should be a Readium audiobook") } -/* -TODO needs webpub heavy parsing. See func SniffWebpub in sniffer.go for details. -CBZ sniffing is implemented below as a temporary alternative. +// Opens a file in testdata and sniffs its media type from the content alone. +func sniffFixture(t *testing.T, filename string) *MediaType { + t.Helper() + f, err := os.Open(filepath.Join("testdata", filename)) + require.NoError(t, err) + defer f.Close() + return OfFileOnly(t.Context(), f) +} func TestSnifferFromFile(t *testing.T) { - testAudiobook, err := os.Open(filepath.Join("testdata", "audiobook.json")) - require.NoError(t, err) - defer testAudiobook.Close() - assert.Equal(t, &ReadiumAudiobookManifest, OfFileOnly(testAudiobook)) + assert.Equal(t, &ReadiumAudiobookManifest, sniffFixture(t, "audiobook.json")) } func TestSnifferFromBytes(t *testing.T) { @@ -49,24 +53,92 @@ func TestSnifferFromBytes(t *testing.T) { testAudiobookBytes, err := io.ReadAll(testAudiobook) testAudiobook.Close() require.NoError(t, err) - assert.Equal(t, &ReadiumAudiobookManifest, MediaTypeOfBytesOnly(testAudiobookBytes)) -} -*/ + assert.Equal(t, &ReadiumAudiobookManifest, OfBytesOnly(t.Context(), testAudiobookBytes)) +} + +// JSON which manifest.ManifestFromJSON would reject must not sniff as a RWPM. +func TestSnifferRejectsInvalidRWPM(t *testing.T) { + // No metadata title. + assert.Nil(t, OfBytesOnly(t.Context(), []byte(`{"metadata":{"whatever":1},"readingOrder":[{"href":"a.mp3","type":"audio/mpeg"}]}`))) + // A reading order link without href. + assert.Nil(t, OfBytesOnly(t.Context(), []byte(`{"metadata":{"title":"T"},"readingOrder":[{"type":"audio/mpeg"}]}`))) + // A reading order link with an unparsable type. + assert.Nil(t, OfBytesOnly(t.Context(), []byte(`{"metadata":{"title":"T"},"readingOrder":[{"href":"a.mp3","type":"!!!"}]}`))) + // A localized title with a non-string translation, which ManifestFromJSON rejects. + assert.Nil(t, OfBytesOnly(t.Context(), []byte(`{"metadata":{"title":{"en":5}},"readingOrder":[{"href":"a.mp3","type":"audio/mpeg"}]}`))) +} + +// The OPDS 2 heavy sniffing only classifies JSON that decodes as a RWPM, so arbitrary +// JSON carrying an OPDS-vocabulary link is not misread as an OPDS 2 document. +func TestSnifferOPDS2RequiresManifestShape(t *testing.T) { + // An acquisition link but no metadata: not an OPDS 2 publication. + assert.Nil(t, OfBytesOnly(t.Context(), []byte(`{"links":[{"rel":"http://opds-spec.org/acquisition/buy","href":"/buy"}]}`))) + // A self link to an OPDS feed but no metadata: not an OPDS 2 feed. + assert.Nil(t, OfBytesOnly(t.Context(), []byte(`{"links":[{"rel":"self","href":"/feed","type":"application/opds+json"}]}`))) +} + +// The archive opened during heavy sniffing is memoized (opened once, not per sniffer +// or per entry read) and released when the sniffing context is closed. +func TestSnifferContextArchiveLifecycle(t *testing.T) { + f, err := os.Open(filepath.Join("testdata", "webpub-package.unknown")) + require.NoError(t, err) + defer f.Close() -func TestSnifferFromFile(t *testing.T) { - testCbz, err := os.Open(filepath.Join("testdata", "cbz.unknown")) + sc := &SnifferContext{content: NewSnifferFileContent(f)} + + a1, err := sc.ContentAsArchive(t.Context()) require.NoError(t, err) - defer testCbz.Close() - assert.Equal(t, &CBZ, OfFileOnly(t.Context(), testCbz), "test CBZ should be identified by heavy Sniffer") + require.NotNil(t, a1) + + // A second call, and archive-entry reads, reuse the same handle instead of reopening. + a2, err := sc.ContentAsArchive(t.Context()) + require.NoError(t, err) + assert.True(t, a1 == a2, "archive should be memoized, not reopened") + assert.True(t, sc.ContainsArchiveEntryAt(t.Context(), "manifest.json")) + + // Closing releases the handle, and accessing the archive afterwards fails + // gracefully rather than handing back a closed handle or panicking. + sc.Close() + assert.Nil(t, sc._contentAsArchive) + assert.False(t, sc.ContainsArchiveEntryAt(t.Context(), "manifest.json")) + assert.Nil(t, sc.ReadArchiveEntryAt(t.Context(), "manifest.json")) + sc.Close() // Idempotent. +} + +// Opening non-archive content as an archive memoizes the error, so the archive-entry +// helpers fail gracefully (no nil-handle panic) even when called repeatedly. +func TestSnifferContextArchiveErrorMemoized(t *testing.T) { + f, err := os.Open(filepath.Join("testdata", "audiobook.json")) // Not an archive. + require.NoError(t, err) + defer f.Close() + + sc := &SnifferContext{content: NewSnifferFileContent(f)} + _, err = sc.ContentAsArchive(t.Context()) + require.Error(t, err) + assert.False(t, sc.ContainsArchiveEntryAt(t.Context(), "manifest.json")) + assert.Nil(t, sc.ReadArchiveEntryAt(t.Context(), "manifest.json")) + sc.Close() // Must not panic when nothing was opened. } -func TestSnifferFromBytes(t *testing.T) { - testCbz, err := os.Open(filepath.Join("testdata", "cbz.unknown")) +// Content sniffing must not consume a non-seekable file: a sniffer running after one +// which read the whole stream (e.g. as JSON) still sees the content. +func TestSnifferNonSeekableFile(t *testing.T) { + pdf, err := os.ReadFile(filepath.Join("testdata", "pdf.unknown")) + require.NoError(t, err) + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.Create("doc") // No extension, so only the content identifies it + require.NoError(t, err) + _, err = w.Write(pdf) + require.NoError(t, err) + require.NoError(t, zw.Close()) + + zr, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) require.NoError(t, err) - testCbzBytes, err := io.ReadAll(testCbz) - testCbz.Close() + f, err := zr.Open("doc") require.NoError(t, err) - assert.Equal(t, &CBZ, OfBytesOnly(t.Context(), testCbzBytes), "test CBZ's bytes should be identified by heavy Sniffer") + defer f.Close() + assert.Equal(t, &PDF, OfFileOnly(t.Context(), f)) } func TestSnifferUnknownFormat(t *testing.T) { @@ -89,15 +161,13 @@ func TestSnifferValidMediaTypeFallback(t *testing.T) { func TestSniffAudiobook(t *testing.T) { assert.Equal(t, &ReadiumAudiobook, OfExtension("audiobook")) assert.Equal(t, &ReadiumAudiobook, OfString("application/audiobook+zip")) - // TODO needs webpub heavy parsing. See func SniffWebpub in sniffer.go for details. - // assert.Equal(t, &ReadiumAudiobook, OfFileOnly("audiobook")) + assert.Equal(t, &ReadiumAudiobook, sniffFixture(t, "audiobook-package.unknown")) } func TestSniffAudiobookManifest(t *testing.T) { assert.Equal(t, &ReadiumAudiobookManifest, OfString("application/audiobook+json")) - // TODO needs webpub heavy parsing. See func SniffWebpub in sniffer.go for details. - // assert.Equal(t, &ReadiumAudiobookManifest, OfFileOnly("audiobook.json")) - // assert.Equal(t, &ReadiumAudiobookManifest, OfFileOnly("audiobook-wrongtype.json")) + assert.Equal(t, &ReadiumAudiobookManifest, sniffFixture(t, "audiobook.json")) + assert.Equal(t, &ReadiumAudiobookManifest, sniffFixture(t, "audiobook-wrongtype.json")) } func TestSniffAVIF(t *testing.T) { @@ -127,14 +197,12 @@ func TestSniffCBZ(t *testing.T) { func TestSniffDiViNa(t *testing.T) { assert.Equal(t, &ReadiumDivina, OfExtension("divina")) assert.Equal(t, &ReadiumDivina, OfString("application/divina+zip")) - // TODO needs webpub heavy parsing. See func SniffWebpub in sniffer.go for details. - // assert.Equal(t, &DIVINA, OfFileOnly("divina-package.unknown")) + assert.Equal(t, &ReadiumDivina, sniffFixture(t, "divina-package.unknown")) } func TestSniffDiViNaManifest(t *testing.T) { assert.Equal(t, &ReadiumDivinaManifest, OfString("application/divina+json")) - // TODO needs webpub heavy parsing. See func SniffWebpub in sniffer.go for details. - // assert.Equal(t, &DIVINA_MANIFEST, OfFileOnly("divina.json")) + assert.Equal(t, &ReadiumDivinaManifest, sniffFixture(t, "divina.json")) } func TestSniffEPUB(t *testing.T) { @@ -209,65 +277,44 @@ func TestSniffOPDS1Entry(t *testing.T) { func TestSniffOPDS2Feed(t *testing.T) { assert.Equal(t, &OPDS2, OfString("application/opds+json")) - - /* - // TODO needs webpub heavy parsing. See func SniffOPDS in sniffer.go for details. - testOPDS2Feed, err := os.Open(filepath.Join("testdata", "opds2-feed.json")) - assert.NoError(t, err) - defer testOPDS2Feed.Close() - assert.Equal(t, &OPDS2, OfFileOnly(testOPDS2Feed)) - */ + assert.Equal(t, &OPDS2, sniffFixture(t, "opds2-feed.json")) } func TestSniffOPDS2Publication(t *testing.T) { assert.Equal(t, &OPDS2Publication, OfString("application/opds-publication+json")) + assert.Equal(t, &OPDS2Publication, sniffFixture(t, "opds2-publication.json")) +} - /* - // TODO needs webpub heavy parsing. See func SniffOPDS in sniffer.go for details. - testOPDS2Feed, err := os.Open(filepath.Join("testdata", "opds2-publication.json")) - assert.NoError(t, err) - defer testOPDS2Feed.Close() - assert.Equal(t, &OPDS2_PUBLICATION, OfFileOnly(testOPDS2Feed)) - */ +// An OPDS 2 publication with a reading order is also a valid RWPM: the acquisition +// links must take precedence over the Readium manifest heuristics. +func TestSniffOPDS2PublicationWithReadingOrder(t *testing.T) { + pub := []byte(`{ + "metadata": {"title": "An audiobook for sale"}, + "links": [ + {"rel": "self", "href": "http://example.org/pub", "type": "application/opds-publication+json"}, + {"rel": "http://opds-spec.org/acquisition/buy", "href": "/buy", "type": "application/audiobook+zip"} + ], + "readingOrder": [{"href": "chapter1.mp3", "type": "audio/mpeg"}] + }`) + assert.Equal(t, &OPDS2Publication, OfBytesOnly(t.Context(), pub)) } func TestSniffOPDSAuthenticationDocument(t *testing.T) { assert.Equal(t, &OPDSAuthentication, OfString("application/opds-authentication+json")) assert.Equal(t, &OPDSAuthentication, OfString("application/vnd.opds.authentication.v1.0+json")) - - /* - // TODO needs webpub heavy parsing. See func SniffOPDS in sniffer.go for details. - testOPDSAuthDoc, err := os.Open(filepath.Join("testdata", "opds-authentication.json")) - assert.NoError(t, err) - defer testOPDSAuthDoc.Close() - assert.Equal(t, &OPDS_AUTHENTICATION, OfFileOnly(testOPDSAuthDoc)) - */ + assert.Equal(t, &OPDSAuthentication, sniffFixture(t, "opds-authentication.json")) } func TestSniffLCPProtectedAudiobook(t *testing.T) { assert.Equal(t, &LCPProtectedAudiobook, OfExtension("lcpa")) assert.Equal(t, &LCPProtectedAudiobook, OfString("application/audiobook+lcp")) - - /* - // TODO needs webpub heavy parsing. See func SniffWebpub in sniffer.go for details. - testLCPAudiobook, err := os.Open(filepath.Join("testdata", "audiobook-lcp.unknown")) - assert.NoError(t, err) - defer testLCPAudiobook.Close() - assert.Equal(t, &LCP_PROTECTED_AUDIOBOOK, OfFileOnly(testLCPAudiobook)) - */ + assert.Equal(t, &LCPProtectedAudiobook, sniffFixture(t, "audiobook-lcp.unknown")) } func TestSniffLCPProtectedPDF(t *testing.T) { assert.Equal(t, &LCPProtectedPDF, OfExtension("lcpdf")) assert.Equal(t, &LCPProtectedPDF, OfString("application/pdf+lcp")) - - /* - // TODO needs webpub heavy parsing. See func SniffWebpub in sniffer.go for details. - testLCPPDF, err := os.Open(filepath.Join("testdata", "pdf-lcp.unknown")) - assert.NoError(t, err) - defer testLCPPDF.Close() - assert.Equal(t, &LCP_PROTECTED_PDF, OfFileOnly(testLCPPDF)) - */ + assert.Equal(t, &LCPProtectedPDF, sniffFixture(t, "pdf-lcp.unknown")) } func TestSniffLCPLicenseDocument(t *testing.T) { @@ -325,16 +372,12 @@ func TestSniffWEBP(t *testing.T) { func TestSniffWebPub(t *testing.T) { assert.Equal(t, &ReadiumWebpub, OfExtension("webpub")) assert.Equal(t, &ReadiumWebpub, OfString("application/webpub+zip")) - - // TODO needs webpub heavy parsing. See func SniffWebpub in sniffer.go for details. - // assert.Equal(t, &ReadiumWebpub, OfFileOnly("webpub-package.unknown")) + assert.Equal(t, &ReadiumWebpub, sniffFixture(t, "webpub-package.unknown")) } func TestSniffWebPubManifest(t *testing.T) { assert.Equal(t, &ReadiumWebpubManifest, OfString("application/webpub+json")) - - // TODO needs webpub heavy parsing. See func SniffWebpub in sniffer.go for details. - // assert.Equal(t, &ReadiumWebpubManifest, OfFileOnly("webpub.json")) + assert.Equal(t, &ReadiumWebpubManifest, sniffFixture(t, "webpub.json")) } func TestSniffW3CWPUBManifest(t *testing.T) { diff --git a/pkg/mediatype/testdata/opds-authentication.json b/pkg/mediatype/testdata/opds-authentication.json index 0b0bdfeb..a86f4990 100644 --- a/pkg/mediatype/testdata/opds-authentication.json +++ b/pkg/mediatype/testdata/opds-authentication.json @@ -21,7 +21,7 @@ "type": "http://opds-spec.org/auth/oauth/implicit", "links": [ {"rel": "authenticate", "href": "http://example.com/oauth", "type": "text/html"}, - {"rel": "refresh", "href": "http://example.com/oauth/refresh", "type": "application/json"}, + {"rel": "refresh", "href": "http://example.com/oauth/refresh", "type": "application/json"} ] } ] diff --git a/pkg/parser/webpub/fetcher.go b/pkg/parser/webpub/fetcher.go new file mode 100644 index 00000000..cdaa9e65 --- /dev/null +++ b/pkg/parser/webpub/fetcher.go @@ -0,0 +1,45 @@ +package webpub + +import ( + "context" + "net/http" + + "github.com/pkg/errors" + "github.com/readium/go-toolkit/pkg/fetcher" + "github.com/readium/go-toolkit/pkg/manifest" + "github.com/readium/go-toolkit/pkg/util/url" +) + +// Serves the resources of a bare (exploded) Readium Web Publication Manifest: +// relative HREFs are resolved against the manifest's location by the [relative] +// fetcher, while absolute HTTP(S) HREFs are fetched with the [client], since a +// manifest is free to reference resources hosted anywhere. +type manifestResourceFetcher struct { + relative fetcher.Fetcher + client *http.Client +} + +// Links implements fetcher.Fetcher +func (f *manifestResourceFetcher) Links(ctx context.Context) (manifest.LinkList, error) { + return f.relative.Links(ctx) +} + +// Get implements fetcher.Fetcher +func (f *manifestResourceFetcher) Get(ctx context.Context, link manifest.Link) fetcher.Resource { + if !link.Href.IsTemplated() { + if u, ok := link.URL(nil, nil).(url.AbsoluteURL); ok { + if u.IsHTTP() && f.client != nil { + return fetcher.NewHTTPResource(link, f.client, u) + } + return fetcher.NewFailureResource(link, fetcher.NotFound( + errors.New("cannot fetch absolute HREF "+u.String()), + )) + } + } + return f.relative.Get(ctx, link) +} + +// Close implements fetcher.Fetcher +func (f *manifestResourceFetcher) Close() { + f.relative.Close() +} diff --git a/pkg/parser/webpub/parser.go b/pkg/parser/webpub/parser.go index 2bef8931..11c80dbd 100644 --- a/pkg/parser/webpub/parser.go +++ b/pkg/parser/webpub/parser.go @@ -3,18 +3,24 @@ package webpub import ( "context" "net/http" + "path" + "strings" "github.com/pkg/errors" "github.com/readium/go-toolkit/pkg/asset" - ftchr "github.com/readium/go-toolkit/pkg/fetcher" + "github.com/readium/go-toolkit/pkg/content/iterator" + "github.com/readium/go-toolkit/pkg/fetcher" "github.com/readium/go-toolkit/pkg/manifest" "github.com/readium/go-toolkit/pkg/mediatype" + "github.com/readium/go-toolkit/pkg/parser/epub" + "github.com/readium/go-toolkit/pkg/parser/pdf" "github.com/readium/go-toolkit/pkg/pub" + "github.com/readium/go-toolkit/pkg/util/url" ) +// Parses any Readium Web Publication package or manifest, e.g. WebPub, Audiobook, DiViNa, LCPDF... type WebPubParser struct { client *http.Client - // pdfFactory may never be needed } func NewParser(client *http.Client) WebPubParser { @@ -24,22 +30,22 @@ func NewParser(client *http.Client) WebPubParser { } // Parse implements PublicationParser -func (p WebPubParser) Parse(ctx context.Context, asset asset.PublicationAsset, fetcher ftchr.Fetcher) (*pub.Builder, error) { - lFetcher := fetcher - mediaType := asset.MediaType(ctx) - +func (p WebPubParser) Parse(ctx context.Context, a asset.PublicationAsset, f fetcher.Fetcher) (*pub.Builder, error) { + mediaType := a.MediaType(ctx) if !isMediatypeReadiumWebPubProfile(mediaType) { return nil, nil } isPackage := !mediaType.IsRwpm() + lFetcher := f var manifestJSON map[string]interface{} if isPackage { res := lFetcher.Get(ctx, manifest.Link{Href: manifest.MustNewHREFFromString("manifest.json", false)}) - mjr, err := ftchr.ReadResourceAsJSON(ctx, res) - if err != nil { - return nil, err + mjr, rerr := fetcher.ReadResourceAsJSON(ctx, res) + res.Close() + if rerr != nil { + return nil, errors.Wrap(rerr, "failed reading manifest.json from package") } manifestJSON = mjr } else { @@ -51,39 +57,169 @@ func (p WebPubParser) Parse(ctx context.Context, asset asset.PublicationAsset, f if len(links) == 0 { return nil, errors.New("links is empty") } - mj, rerr := ftchr.ReadResourceAsJSON(ctx, lFetcher.Get(ctx, links[0])) + res := lFetcher.Get(ctx, links[0]) + mj, rerr := fetcher.ReadResourceAsJSON(ctx, res) + res.Close() if rerr != nil { - return nil, rerr.Cause + return nil, errors.Wrap(rerr, "failed reading manifest") } manifestJSON = mj } - manifest, err := manifest.ManifestFromJSON(manifestJSON, isPackage) + m, err := manifest.ManifestFromJSON(manifestJSON, isPackage) if err != nil { return nil, errors.Wrap(err, "failed parsing RWPM Manifest") } - // For a manifest, we discard the [fetcher] provided by the Streamer, because it was only - // used to read the manifest file. We use an [HttpFetcher] instead to serve the remote resources. + if err := checkProfileRequirements(mediaType, m); err != nil { + return nil, err + } + if !isPackage { - /*baseURL := "" - if link := manifest.LinkWithRel("self"); link != nil { - baseURL = path.Base(link.Href) - }*/ + // For a bare manifest, we discard the [fetcher] provided by the Streamer, because it was + // only used to read the manifest file. Resources are fetched relative to the manifest's + // own location instead, whether it's on a local file system or on a remote server. + relativeAsset, ok := a.(asset.RelativePublicationAsset) + if !ok { + return nil, errors.New("asset does not provide access to resources relative to the manifest") + } + + // The publication is rooted at the topmost directory reached by the manifest's + // HREFs, e.g. one level above the manifest for `../audio/track.mp3`. + maxUp := 0 + m.TransformHREFs(func(h manifest.HREF) manifest.HREF { + if up := parentEscapeDepth(h); up > maxUp { + maxUp = up + } + return h + }) + + location := relativeAsset.Location() + rootRef := "./" + if maxUp > 0 { + rootRef = strings.Repeat("../", maxUp) + } + root := location.Resolve(url.MustURLFromString(rootRef)) - panic("remote HttpFetcher not implemented!") + // HREFs reaching outside the manifest's directory are normalized to the publication + // root, like the resources of a package: `../audio/track.mp3` referenced from + // `manifest/manifest.json` becomes `audio/track.mp3`, and a `cover.jpg` next to the + // manifest becomes `manifest/cover.jpg`. This keeps the manifest and the fetcher + // consistent when the publication is served under a single base URL. + if maxUp > 0 { + dir := location.Resolve(url.MustURLFromString("./")) + m.TransformHREFs(func(h manifest.HREF) manifest.HREF { + if h.IsTemplated() { + return h + } + u := h.Resolve(nil, nil) + if u == nil { + return h + } + // Fragment-only and query-only HREFs (e.g. a `#part1` TOC entry) point + // within the manifest itself and must not become path links. + if u.Path() == "" { + return h + } + rel := root.Relativize(dir.Resolve(u)) + if _, ok := rel.(url.AbsoluteURL); ok { + // The HREF points outside the publication root: an absolute URL + // (e.g. a resource hosted elsewhere) or a root-absolute path. + // Leave it untouched rather than storing a mangled absolute form; + // absolute HTTP(S) HREFs are fetched as-is and other HREFs resolve + // against the publication root by the relative fetcher. + return h + } + return manifest.NewHREF(rel) + }) + } - lFetcher = fetcher // TODO HttpFetcher using p.client + relativeFetcher, err := relativeAsset.CreateRelativeFetcher(ctx, root) + if err != nil { + return nil, errors.Wrap(err, "failed creating a fetcher for the manifest's resources") + } + lFetcher = &manifestResourceFetcher{ + relative: relativeFetcher, + client: p.client, + } + } + + serviceFactories := make(map[pub.ServiceName]pub.ServiceFactory) + + switch { + case m.ConformsTo(manifest.ProfilePDF): + // The positions service only supports a single PDF in the reading order. + if len(m.ReadingOrder) == 1 { + serviceFactories[pub.PositionsService_Name] = pdf.PositionsServiceFactory() + } + case m.ConformsTo(manifest.ProfileDivina): + serviceFactories[pub.PositionsService_Name] = pub.PerResourcePositionsServiceFactory(mediatype.MustNewOfString("image/*")) + case m.ConformsTo(manifest.ProfileEPUB): + serviceFactories[pub.PositionsService_Name] = epub.PositionsServiceFactory(nil) } + // Add content service for WebPubs with HTML contents. + for _, link := range m.ReadingOrder { + if link.MediaType != nil && link.MediaType.IsHTML() { + serviceFactories[pub.ContentService_Name] = pub.DefaultContentServiceFactory([]iterator.ResourceContentIteratorFactory{ + iterator.HTMLFactory(), + }) + break + } + } + + var servicesBuilder *pub.ServicesBuilder + if len(serviceFactories) > 0 { + servicesBuilder = pub.NewServicesBuilder(serviceFactories) + } + return pub.NewBuilder(*m, lFetcher, servicesBuilder), nil +} + +// Checks that the manifest conforms to the profiles implied by the asset's media type. +// The requirements are based on the media type rather than `metadata.conformsTo`, which a bare +// manifest is not required to declare: [manifest.Manifest.ConformsTo] also infers conformance +// from the reading order. +func checkProfileRequirements(mt mediatype.MediaType, m *manifest.Manifest) error { // Checks the requirements from the LCPDF specification. // https://readium.org/lcp-specs/notes/lcp-for-pdf.html - readingOrder := manifest.ReadingOrder - if mediaType.Equal(&mediatype.LCPProtectedPDF) && (len(readingOrder) == 0 || !readingOrder.AllMatchMediaType(&mediatype.PDF)) { - return nil, errors.New("invalid LCP protected PDF") + if mt.Equal(&mediatype.LCPProtectedPDF) && (len(m.ReadingOrder) == 0 || !m.ReadingOrder.AllMatchMediaType(&mediatype.PDF)) { + return errors.New("publication does not conform to the LCPDF specification") + } + + if mt.Matches(&mediatype.ReadiumAudiobook, &mediatype.ReadiumAudiobookManifest, &mediatype.LCPProtectedAudiobook) && + !m.ConformsTo(manifest.ProfileAudiobook) { + return errors.New("publication does not conform to the Audiobook profile") } - return pub.NewBuilder(*manifest, lFetcher, nil), nil // TODO services! + if mt.Matches(&mediatype.ReadiumDivina, &mediatype.ReadiumDivinaManifest) && + !m.ConformsTo(manifest.ProfileDivina) { + return errors.New("publication does not conform to the Divina profile") + } + + return nil +} + +// Returns how many levels above the manifest's directory the HREF reaches, +// e.g. 1 for `../audio/track.mp3` and 0 for `audio/track.mp3`. +func parentEscapeDepth(h manifest.HREF) int { + if h.IsTemplated() { + return 0 + } + u := h.Resolve(nil, nil) + if u == nil { + return 0 + } + if _, ok := u.(url.AbsoluteURL); ok { + return 0 + } + + p := path.Clean(u.Path()) + depth := 0 + for p == ".." || strings.HasPrefix(p, "../") { + depth++ + p = strings.TrimPrefix(strings.TrimPrefix(p, ".."), "/") + } + return depth } func isMediatypeReadiumWebPubProfile(mt mediatype.MediaType) bool { diff --git a/pkg/parser/webpub/parser_test.go b/pkg/parser/webpub/parser_test.go new file mode 100644 index 00000000..829e83ec --- /dev/null +++ b/pkg/parser/webpub/parser_test.go @@ -0,0 +1,482 @@ +package webpub + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/readium/go-toolkit/pkg/archive" + "github.com/readium/go-toolkit/pkg/asset" + "github.com/readium/go-toolkit/pkg/manifest" + "github.com/readium/go-toolkit/pkg/mediatype" + "github.com/readium/go-toolkit/pkg/pub" + "github.com/readium/go-toolkit/pkg/util/url" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// parseFileAsset opens the file at [path] with the given media type and runs it +// through the parser, the same way the Streamer would. +func parseFileAsset(t *testing.T, client *http.Client, path string, mt *mediatype.MediaType) (*pub.Builder, error) { + t.Helper() + uri, err := url.FromFilepath(path) + require.NoError(t, err) + a := asset.FileWithMediaType(uri, mt) + f, err := a.CreateFetcher(t.Context(), asset.Dependencies{ + ArchiveFactory: archive.NewArchiveFactory(), + }, "") + require.NoError(t, err) + if client == nil { + client = http.DefaultClient + } + return NewParser(client).Parse(t.Context(), a, f) +} + +func readPublicationResource(t *testing.T, p *pub.Publication, href string) ([]byte, error) { + t.Helper() + res := p.Get(t.Context(), manifest.Link{Href: manifest.MustNewHREFFromString(href, false)}) + defer res.Close() + data, rerr := res.Read(t.Context(), 0, 0) + if rerr != nil { + return nil, rerr + } + return data, nil +} + +func TestParseBareAudiobookManifest(t *testing.T) { + builder, err := parseFileAsset(t, nil, "testdata/audio/manifest.json", &mediatype.ReadiumAudiobookManifest) + require.NoError(t, err) + require.NotNil(t, builder) + + m := builder.Manifest + assert.Equal(t, "The Art of Letters", m.Metadata.Title()) + assert.True(t, m.ConformsTo(manifest.ProfileAudiobook)) + assert.Len(t, m.ReadingOrder, 3) + + p := builder.Build() + defer p.Close() + + // Resources are fetched relative to the manifest's location: a sibling file + // of the manifest must be reachable through the publication. + data, err := readPublicationResource(t, p, "manifest_noconform1.json") + require.NoError(t, err) + expected, err := os.ReadFile("testdata/audio/manifest_noconform1.json") + require.NoError(t, err) + assert.Equal(t, expected, data) + + // The manifest itself is reachable at its own name. + data, err = readPublicationResource(t, p, "manifest.json") + require.NoError(t, err) + var mjson map[string]interface{} + require.NoError(t, json.Unmarshal(data, &mjson)) + + // A missing sibling fails. + _, err = readPublicationResource(t, p, "missing.json") + assert.Error(t, err) +} + +func TestParseBareManifestConformance(t *testing.T) { + for _, tt := range []struct { + name string + file string + mediaType *mediatype.MediaType + wantErr bool + }{ + {"explicit conformsTo", "manifest.json", &mediatype.ReadiumAudiobookManifest, false}, + {"no conformsTo, schema.org type", "manifest_noconform1.json", &mediatype.ReadiumAudiobookManifest, false}, + {"no conformsTo, no type", "manifest_noconform2.json", &mediatype.ReadiumAudiobookManifest, false}, + {"no conformsTo, cover link only", "manifest_noconform3.json", &mediatype.ReadiumAudiobookManifest, false}, + {"not an audiobook as audiobook", "manifest_notaudiobook.json", &mediatype.ReadiumAudiobookManifest, true}, + {"not an audiobook as generic webpub", "manifest_notaudiobook.json", &mediatype.ReadiumWebpubManifest, false}, + } { + t.Run(tt.name, func(t *testing.T) { + builder, err := parseFileAsset(t, nil, "testdata/audio/"+tt.file, tt.mediaType) + if tt.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + require.NotNil(t, builder) + } + }) + } +} + +func TestParseAudiobookPackage(t *testing.T) { + builder, err := parseFileAsset(t, nil, "testdata/audio/art_letters.audiobook", &mediatype.ReadiumAudiobook) + require.NoError(t, err) + require.NotNil(t, builder) + + m := builder.Manifest + assert.Equal(t, "The Art of Letters", m.Metadata.Title()) + assert.True(t, m.ConformsTo(manifest.ProfileAudiobook)) + require.Len(t, m.ReadingOrder, 3) + + // Resources come from inside the package. + p := builder.Build() + defer p.Close() + res := p.Get(t.Context(), m.ReadingOrder[0]) + defer res.Close() + length, rerr := res.Length(t.Context()) + require.Nil(t, rerr) + assert.Greater(t, length, int64(0)) +} + +func TestParsePDFPackage(t *testing.T) { + for _, tt := range []struct { + name string + mediaType *mediatype.MediaType + }{ + {"as webpub", &mediatype.ReadiumWebpub}, + {"as LCP protected PDF", &mediatype.LCPProtectedPDF}, + } { + t.Run(tt.name, func(t *testing.T) { + builder, err := parseFileAsset(t, nil, "testdata/pdf/pdf.webpub", tt.mediaType) + require.NoError(t, err) + require.NotNil(t, builder) + + m := builder.Manifest + assert.True(t, m.ConformsTo(manifest.ProfilePDF)) + require.Len(t, m.ReadingOrder, 1) + assert.NotNil(t, builder.ServicesBuilder.Get(pub.PositionsService_Name)) + }) + } +} + +func TestParseLCPDFRequirements(t *testing.T) { + // An audiobook package parsed as LCPDF must be rejected: the reading order + // does not contain PDF documents. + _, err := parseFileAsset(t, nil, "testdata/audio/art_letters.audiobook", &mediatype.LCPProtectedPDF) + assert.Error(t, err) +} + +func TestParseBareManifestOverHTTP(t *testing.T) { + ts := httptest.NewServer(http.FileServer(http.Dir("testdata/audio"))) + defer ts.Close() + + manifestURL, err := url.AbsoluteURLFromString(ts.URL + "/manifest.json") + require.NoError(t, err) + a := asset.HTTPWithMediaType(ts.Client(), manifestURL, &mediatype.ReadiumAudiobookManifest) + f, err := a.CreateFetcher(t.Context(), asset.Dependencies{}, "") + require.NoError(t, err) + + builder, err := NewParser(ts.Client()).Parse(t.Context(), a, f) + require.NoError(t, err) + require.NotNil(t, builder) + assert.Equal(t, "The Art of Letters", builder.Manifest.Metadata.Title()) + + p := builder.Build() + defer p.Close() + + // Relative HREFs resolve against the manifest's URL. + data, err := readPublicationResource(t, p, "manifest_noconform1.json") + require.NoError(t, err) + expected, err := os.ReadFile("testdata/audio/manifest_noconform1.json") + require.NoError(t, err) + assert.Equal(t, expected, data) +} + +func TestParseBareManifestAbsoluteHrefs(t *testing.T) { + // A local manifest referencing a remote resource with an absolute HTTP HREF: + // relative HREFs are served from the manifest's directory, absolute HTTP(S) + // HREFs with the parser's HTTP client. + remote := []byte("remote audio bytes") + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/track.opus" { + http.NotFound(w, r) + return + } + w.Write(remote) + })) + defer ts.Close() + + local := []byte("local cover bytes") + dir := t.TempDir() + manifestJSON := `{ + "@context": "https://readium.org/webpub-manifest/context.jsonld", + "metadata": { + "@type": "http://schema.org/Audiobook", + "title": "Absolute HREF test" + }, + "links": [], + "readingOrder": [ + {"href": "` + ts.URL + `/track.opus", "type": "audio/opus"} + ], + "resources": [ + {"href": "images/cover.jpg", "type": "image/jpeg", "rel": "cover"} + ] + }` + require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest.json"), []byte(manifestJSON), 0o644)) + require.NoError(t, os.Mkdir(filepath.Join(dir, "images"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "images", "cover.jpg"), local, 0o644)) + + builder, err := parseFileAsset(t, ts.Client(), filepath.ToSlash(filepath.Join(dir, "manifest.json")), &mediatype.ReadiumAudiobookManifest) + require.NoError(t, err) + require.NotNil(t, builder) + + p := builder.Build() + defer p.Close() + + data, err := readPublicationResource(t, p, ts.URL+"/track.opus") + require.NoError(t, err) + assert.Equal(t, remote, data) + + data, err = readPublicationResource(t, p, "images/cover.jpg") + require.NoError(t, err) + assert.Equal(t, local, data) +} + +// A bare manifest in a subdirectory referencing resources in sibling directories +// through `../` HREFs, a common layout for exploded publications: +// +// pub/manifest/manifest.json -> ../audiobook/track.opus, ../cover/cover.jpg +// +// The parser normalizes the HREFs to the publication root (`pub/`), like the +// resources of a package, so that the publication can be served under a single +// base URL: `audiobook/track.opus`, `cover/cover.jpg`, `manifest/manifest.json`. +func TestParseExplodedPublicationLayout(t *testing.T) { + audio := []byte("audio bytes") + cover := []byte("cover bytes") + dir := t.TempDir() + manifestJSON := `{ + "@context": "https://readium.org/webpub-manifest/context.jsonld", + "metadata": { + "@type": "http://schema.org/Audiobook", + "title": "Exploded layout test" + }, + "links": [ + {"href": "../cover/cover.jpg", "type": "image/jpeg", "rel": "cover"}, + {"href": "manifest.json", "type": "application/audiobook+json", "rel": "self"} + ], + "readingOrder": [ + {"href": "../audiobook/track.opus", "type": "audio/opus", "duration": 60} + ], + "toc": [ + {"href": "#part1", "title": "Part 1"}, + {"href": "?page=2", "title": "Page 2"} + ] + }` + for sub, content := range map[string][]byte{ + filepath.Join("manifest", "manifest.json"): []byte(manifestJSON), + filepath.Join("audiobook", "track.opus"): audio, + filepath.Join("cover", "cover.jpg"): cover, + } { + require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, sub)), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, sub), content, 0o644)) + } + manifestPath := filepath.ToSlash(filepath.Join(dir, "manifest", "manifest.json")) + + for _, tt := range []struct { + name string + mediaType *mediatype.MediaType + }{ + {"explicit media type", &mediatype.ReadiumAudiobookManifest}, + {"sniffed media type", nil}, // relies on SniffWebpub's heavy sniffing + } { + t.Run(tt.name, func(t *testing.T) { + builder, err := parseFileAsset(t, nil, manifestPath, tt.mediaType) + require.NoError(t, err) + require.NotNil(t, builder) + + m := builder.Manifest + assert.True(t, m.ConformsTo(manifest.ProfileAudiobook)) + + // HREFs are normalized to the publication root. + require.Len(t, m.ReadingOrder, 1) + assert.Equal(t, "audiobook/track.opus", m.ReadingOrder[0].Href.String()) + require.NotNil(t, m.Links.FirstWithRel("cover")) + assert.Equal(t, "cover/cover.jpg", m.Links.FirstWithRel("cover").Href.String()) + require.NotNil(t, m.Links.FirstWithRel("self")) + assert.Equal(t, "manifest/manifest.json", m.Links.FirstWithRel("self").Href.String()) + + // Fragment-only and query-only HREFs point within the manifest itself + // and must not be rewritten into path links. + require.Len(t, m.TableOfContents, 2) + assert.Equal(t, "#part1", m.TableOfContents[0].Href.String()) + assert.Equal(t, "?page=2", m.TableOfContents[1].Href.String()) + + p := builder.Build() + defer p.Close() + + // Resources are fetched through the manifest's own links... + res := p.Get(t.Context(), m.ReadingOrder[0]) + data, rerr := res.Read(t.Context(), 0, 0) + res.Close() + require.Nil(t, rerr) + assert.Equal(t, audio, data) + + // ...and through root-relative HREFs, as a web server would request them. + data, err = readPublicationResource(t, p, "cover/cover.jpg") + require.NoError(t, err) + assert.Equal(t, cover, data) + + data, err = readPublicationResource(t, p, "manifest/manifest.json") + require.NoError(t, err) + assert.Equal(t, []byte(manifestJSON), data) + }) + } + + // The same layout served over HTTP: the manifest URL is in a subdirectory, and the + // normalized HREFs must resolve against the publication root on the server. + t.Run("over HTTP", func(t *testing.T) { + ts := httptest.NewServer(http.FileServer(http.Dir(dir))) + defer ts.Close() + + manifestURL, err := url.AbsoluteURLFromString(ts.URL + "/manifest/manifest.json") + require.NoError(t, err) + a := asset.HTTPWithMediaType(ts.Client(), manifestURL, &mediatype.ReadiumAudiobookManifest) + f, err := a.CreateFetcher(t.Context(), asset.Dependencies{}, "") + require.NoError(t, err) + + builder, err := NewParser(ts.Client()).Parse(t.Context(), a, f) + require.NoError(t, err) + require.NotNil(t, builder) + require.Len(t, builder.Manifest.ReadingOrder, 1) + assert.Equal(t, "audiobook/track.opus", builder.Manifest.ReadingOrder[0].Href.String()) + + p := builder.Build() + defer p.Close() + + data, err := readPublicationResource(t, p, "audiobook/track.opus") + require.NoError(t, err) + assert.Equal(t, audio, data) + + data, err = readPublicationResource(t, p, "cover/cover.jpg") + require.NoError(t, err) + assert.Equal(t, cover, data) + }) +} + +// HREF normalization for an exploded manifest with maxUp>0 must handle the HREFs +// that don't relativize to the publication root, and reach metadata links too. +func TestParseExplodedManifestHrefNormalization(t *testing.T) { + dir := t.TempDir() + // `../audiobook/track.opus` forces the publication root one level above the + // manifest, triggering normalization of every other HREF. + manifestJSON := `{ + "@context": "https://readium.org/webpub-manifest/context.jsonld", + "metadata": { + "@type": "http://schema.org/Audiobook", + "title": "HREF normalization test", + "author": [ + {"name": "Author", "links": [{"href": "bio.json", "type": "application/json"}]} + ] + }, + "links": [ + {"href": "http://other-host.invalid/pub/audio/remote.opus", "type": "audio/opus", "rel": "alternate"}, + {"href": "/site-cover.jpg", "type": "image/jpeg", "rel": "cover"} + ], + "readingOrder": [ + {"href": "../audiobook/track.opus", "type": "audio/opus", "duration": 60} + ] + }` + require.NoError(t, os.MkdirAll(filepath.Join(dir, "manifest"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest", "manifest.json"), []byte(manifestJSON), 0o644)) + + builder, err := parseFileAsset(t, nil, filepath.ToSlash(filepath.Join(dir, "manifest", "manifest.json")), &mediatype.ReadiumAudiobookManifest) + require.NoError(t, err) + require.NotNil(t, builder) + m := builder.Manifest + + // A `../` HREF is rebased to the publication root. + assert.Equal(t, "audiobook/track.opus", m.ReadingOrder[0].Href.String()) + + // A cross-origin absolute HREF is left untouched, not stripped to a path under + // the root (which would make the publication serve the wrong resource). + require.NotNil(t, m.Links.FirstWithRel("alternate")) + assert.Equal(t, "http://other-host.invalid/pub/audio/remote.opus", m.Links.FirstWithRel("alternate").Href.String()) + + // A root-absolute HREF is left as-is, not mangled into a `file://` URL. + require.NotNil(t, m.Links.FirstWithRel("cover")) + assert.Equal(t, "/site-cover.jpg", m.Links.FirstWithRel("cover").Href.String()) + + // Metadata links are normalized like the rest: a link sitting next to the + // manifest is rebased under the manifest's own directory. + require.Len(t, m.Metadata.Authors, 1) + require.Len(t, m.Metadata.Authors[0].Links, 1) + assert.Equal(t, "manifest/bio.json", m.Metadata.Authors[0].Links[0].Href.String()) +} + +// The fetcher created for a bare manifest is sandboxed to the publication root: +// HREFs handed to the publication later (e.g. from an incoming server request) +// must not reach files outside of it, even with `..` or root-absolute paths. +func TestParseExplodedPublicationSandboxed(t *testing.T) { + base := t.TempDir() + dir := filepath.Join(base, "pub") + audio := []byte("audio bytes") + require.NoError(t, os.WriteFile(filepath.Join(base, "secret.txt"), []byte("secret"), 0o644)) + manifestJSON := `{ + "@context": "https://readium.org/webpub-manifest/context.jsonld", + "metadata": { + "@type": "http://schema.org/Audiobook", + "title": "Sandbox test" + }, + "links": [], + "readingOrder": [ + {"href": "../audiobook/track.opus", "type": "audio/opus", "duration": 60} + ] + }` + for sub, content := range map[string][]byte{ + filepath.Join("manifest", "manifest.json"): []byte(manifestJSON), + filepath.Join("audiobook", "track.opus"): audio, + } { + require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, sub)), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, sub), content, 0o644)) + } + + builder, err := parseFileAsset(t, nil, filepath.ToSlash(filepath.Join(dir, "manifest", "manifest.json")), &mediatype.ReadiumAudiobookManifest) + require.NoError(t, err) + require.NotNil(t, builder) + p := builder.Build() + defer p.Close() + + // Resources inside the root are served, with queries and fragments dropped. + data, err := readPublicationResource(t, p, "audiobook/track.opus?token=abc#t=30") + require.NoError(t, err) + assert.Equal(t, audio, data) + + // Escape attempts fail, even though the target file exists. + for _, href := range []string{ + "../secret.txt", + "../../secret.txt", + "audiobook/../../secret.txt", + "/secret.txt", + } { + _, err := readPublicationResource(t, p, href) + assert.Error(t, err, href) + } +} + +func TestParseServiceFactories(t *testing.T) { + // An EPUB-profile WebPub gets a positions service and, having HTML contents, + // a content service. + dir := t.TempDir() + manifestJSON := `{ + "@context": "https://readium.org/webpub-manifest/context.jsonld", + "metadata": { + "conformsTo": "https://readium.org/webpub-manifest/profiles/epub", + "title": "EPUB profile test" + }, + "links": [], + "readingOrder": [ + {"href": "chapter1.xhtml", "type": "application/xhtml+xml"} + ] + }` + require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest.json"), []byte(manifestJSON), 0o644)) + + builder, err := parseFileAsset(t, nil, filepath.ToSlash(filepath.Join(dir, "manifest.json")), &mediatype.ReadiumWebpubManifest) + require.NoError(t, err) + require.NotNil(t, builder) + + assert.NotNil(t, builder.ServicesBuilder.Get(pub.PositionsService_Name)) + assert.NotNil(t, builder.ServicesBuilder.Get(pub.ContentService_Name)) +} + +// The parser must not swallow assets which aren't WebPub flavored. +func TestParseSkipsOtherMediaTypes(t *testing.T) { + builder, err := parseFileAsset(t, nil, "testdata/audio/manifest.json", &mediatype.JSON) + assert.NoError(t, err) + assert.Nil(t, builder) +} diff --git a/pkg/parser/webpub/testdata/audio/art_letters.audiobook b/pkg/parser/webpub/testdata/audio/art_letters.audiobook new file mode 100644 index 00000000..1c184447 --- /dev/null +++ b/pkg/parser/webpub/testdata/audio/art_letters.audiobook @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d07c9d142a912827d221b169aac707a4465daa9b7ff5ada9118df07e6eba70dd +size 24871490 diff --git a/pkg/parser/webpub/testdata/audio/manifest.json b/pkg/parser/webpub/testdata/audio/manifest.json new file mode 100644 index 00000000..9bc3c22b --- /dev/null +++ b/pkg/parser/webpub/testdata/audio/manifest.json @@ -0,0 +1,50 @@ +{ + "@context": "http://readium.org/webpub-manifest/context.jsonld", + "metadata": { + "@type": "http://schema.org/Audiobook", + "conformsTo": "https://readium.org/webpub-manifest/profiles/audiobook", + "identifier": "ArtLettersAudiobookTest", + "author": "Robert Lynd", + "title": "The Art of Letters", + "duration": 68512, + "language": "en", + "tracks": 82 + }, + "links": [ + { + "type": "image/jpeg", + "rel": "cover", + "height": 180, + "width": 180, + "href": "img/art_letters_1809_librivox.jpeg" + }, + { + "type": "application/audiobook+json", + "rel": "alternate", + "href": "https://mickael.menu/pub-samples/art_letters.json" + } + ], + "readingOrder": [ + { + "type": "audio/opus", + "duration": 62, + "title": "00 - Dedication", + "href": "audio/artofletters_00_lynd.opus", + "bitrate": 32 + }, + { + "type": "audio/opus", + "duration": 1050, + "title": "01 - Mr. Pepys", + "href": "audio/artofletters_01_lynd.opus", + "bitrate": 32 + }, + { + "type": "audio/opus", + "duration": 789, + "title": "02 - John Bunyan", + "href": "audio/artofletters_02_lynd.opus", + "bitrate": 32 + } + ] +} diff --git a/pkg/parser/webpub/testdata/audio/manifest_noconform1.json b/pkg/parser/webpub/testdata/audio/manifest_noconform1.json new file mode 100644 index 00000000..302d4358 --- /dev/null +++ b/pkg/parser/webpub/testdata/audio/manifest_noconform1.json @@ -0,0 +1,49 @@ +{ + "@context": "http://readium.org/webpub-manifest/context.jsonld", + "metadata": { + "@type": "http://schema.org/Audiobook", + "identifier": "ArtLettersAudiobookTest", + "author": "Robert Lynd", + "title": "The Art of Letters", + "duration": 68512, + "language": "en", + "tracks": 82 + }, + "links": [ + { + "type": "image/jpeg", + "rel": "cover", + "height": 180, + "width": 180, + "href": "img/art_letters_1809_librivox.jpeg" + }, + { + "type": "application/audiobook+json", + "rel": "alternate", + "href": "https://mickael.menu/pub-samples/art_letters.json" + } + ], + "readingOrder": [ + { + "type": "audio/opus", + "duration": 62, + "title": "00 - Dedication", + "href": "audio/artofletters_00_lynd.opus", + "bitrate": 32 + }, + { + "type": "audio/opus", + "duration": 1050, + "title": "01 - Mr. Pepys", + "href": "audio/artofletters_01_lynd.opus", + "bitrate": 32 + }, + { + "type": "audio/opus", + "duration": 789, + "title": "02 - John Bunyan", + "href": "audio/artofletters_02_lynd.opus", + "bitrate": 32 + } + ] +} diff --git a/pkg/parser/webpub/testdata/audio/manifest_noconform2.json b/pkg/parser/webpub/testdata/audio/manifest_noconform2.json new file mode 100644 index 00000000..2ed27ef0 --- /dev/null +++ b/pkg/parser/webpub/testdata/audio/manifest_noconform2.json @@ -0,0 +1,48 @@ +{ + "@context": "http://readium.org/webpub-manifest/context.jsonld", + "metadata": { + "identifier": "ArtLettersAudiobookTest", + "author": "Robert Lynd", + "title": "The Art of Letters", + "duration": 68512, + "language": "en", + "tracks": 82 + }, + "links": [ + { + "type": "image/jpeg", + "rel": "cover", + "height": 180, + "width": 180, + "href": "img/art_letters_1809_librivox.jpeg" + }, + { + "type": "application/audiobook+json", + "rel": "alternate", + "href": "https://mickael.menu/pub-samples/art_letters.json" + } + ], + "readingOrder": [ + { + "type": "audio/opus", + "duration": 62, + "title": "00 - Dedication", + "href": "audio/artofletters_00_lynd.opus", + "bitrate": 32 + }, + { + "type": "audio/opus", + "duration": 1050, + "title": "01 - Mr. Pepys", + "href": "audio/artofletters_01_lynd.opus", + "bitrate": 32 + }, + { + "type": "audio/opus", + "duration": 789, + "title": "02 - John Bunyan", + "href": "audio/artofletters_02_lynd.opus", + "bitrate": 32 + } + ] +} diff --git a/pkg/parser/webpub/testdata/audio/manifest_noconform3.json b/pkg/parser/webpub/testdata/audio/manifest_noconform3.json new file mode 100644 index 00000000..b96bd9b2 --- /dev/null +++ b/pkg/parser/webpub/testdata/audio/manifest_noconform3.json @@ -0,0 +1,43 @@ +{ + "@context": "http://readium.org/webpub-manifest/context.jsonld", + "metadata": { + "identifier": "ArtLettersAudiobookTest", + "author": "Robert Lynd", + "title": "The Art of Letters", + "duration": 68512, + "language": "en", + "tracks": 82 + }, + "links": [ + { + "type": "image/jpeg", + "rel": "cover", + "height": 180, + "width": 180, + "href": "img/art_letters_1809_librivox.jpeg" + } + ], + "readingOrder": [ + { + "type": "audio/opus", + "duration": 62, + "title": "00 - Dedication", + "href": "audio/artofletters_00_lynd.opus", + "bitrate": 32 + }, + { + "type": "audio/opus", + "duration": 1050, + "title": "01 - Mr. Pepys", + "href": "audio/artofletters_01_lynd.opus", + "bitrate": 32 + }, + { + "type": "audio/opus", + "duration": 789, + "title": "02 - John Bunyan", + "href": "audio/artofletters_02_lynd.opus", + "bitrate": 32 + } + ] +} diff --git a/pkg/parser/webpub/testdata/audio/manifest_notaudiobook.json b/pkg/parser/webpub/testdata/audio/manifest_notaudiobook.json new file mode 100644 index 00000000..f207aaa9 --- /dev/null +++ b/pkg/parser/webpub/testdata/audio/manifest_notaudiobook.json @@ -0,0 +1,50 @@ +{ + "@context": "http://readium.org/webpub-manifest/context.jsonld", + "metadata": { + "identifier": "ArtLettersAudiobookTest", + "author": "Robert Lynd", + "title": "The Art of Letters", + "duration": 68512, + "language": "en", + "tracks": 82 + }, + "links": [ + { + "type": "image/jpeg", + "rel": "cover", + "height": 180, + "width": 180, + "href": "img/art_letters_1809_librivox.jpeg" + } + ], + "readingOrder": [ + { + "type": "image/jpeg", + "rel": "cover", + "height": 180, + "width": 180, + "href": "img/art_letters_1809_librivox.jpeg" + }, + { + "type": "audio/opus", + "duration": 62, + "title": "00 - Dedication", + "href": "audio/artofletters_00_lynd.opus", + "bitrate": 32 + }, + { + "type": "audio/opus", + "duration": 1050, + "title": "01 - Mr. Pepys", + "href": "audio/artofletters_01_lynd.opus", + "bitrate": 32 + }, + { + "type": "audio/opus", + "duration": 789, + "title": "02 - John Bunyan", + "href": "audio/artofletters_02_lynd.opus", + "bitrate": 32 + } + ] +} diff --git a/pkg/parser/webpub/testdata/pdf/pdf.webpub b/pkg/parser/webpub/testdata/pdf/pdf.webpub new file mode 100644 index 00000000..2422030f --- /dev/null +++ b/pkg/parser/webpub/testdata/pdf/pdf.webpub @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d088a83f7bd84a8d99cfb465cd14d4d4e13d25d9e8ae0ac5365f140d0a6feea5 +size 238059 diff --git a/pkg/util/url/url.go b/pkg/util/url/url.go index 208aedbc..280404a2 100644 --- a/pkg/util/url/url.go +++ b/pkg/util/url/url.go @@ -134,7 +134,7 @@ func (u RelativeURL) Relativize(url URL) URL { if len(u.url.Opaque) > 0 || len(url.url.Opaque) > 0 { return url } - if u.url.Scheme != url.url.Scheme && u.url.Host != url.url.Host { + if u.url.Scheme != url.url.Scheme || u.url.Host != url.url.Host { return url } @@ -273,7 +273,7 @@ func (u AbsoluteURL) Relativize(url URL) URL { if len(u.url.Opaque) > 0 || len(url.url.Opaque) > 0 { return url } - if u.url.Scheme != url.url.Scheme && u.url.Host != url.url.Host { + if u.url.Scheme != url.url.Scheme || u.url.Host != url.url.Host { return url } @@ -365,7 +365,13 @@ func (u AbsoluteURL) ToFilepath() string { if !u.IsFile() { return "" } - return filepath.FromSlash(u.url.Path) + p := u.url.Path + // Strip the root slash of a Windows drive path: file:///C:/dir is the path C:/dir. + if len(p) >= 3 && p[0] == '/' && p[2] == ':' && + (('a' <= p[1] && p[1] <= 'z') || ('A' <= p[1] && p[1] <= 'Z')) { + p = p[1:] + } + return filepath.FromSlash(p) } // Creates a [AbsoluteURL] from its encoded string representation. @@ -410,9 +416,28 @@ func FromEPUBHref(href string) (URL, error) { return u, nil } +// Creates a file URL from a filepath, made absolute against the current working +// directory when relative: a file URL path must be rooted, since resolving relative +// references against it prepends a root slash anyway (RFC 3986 merge), silently +// turning a working-directory-relative path into a file system-absolute one. func FromFilepath(path string) (URL, error) { + apath, err := filepath.Abs(path) + if err != nil { + return nil, err + } + p := filepath.ToSlash(apath) + // filepath.Abs drops any trailing separator, but in a URL it distinguishes a + // directory, keeping relative references inside it when resolved against the URL. + if !strings.HasSuffix(p, "/") && + (strings.HasSuffix(path, "/") || strings.HasSuffix(path, string(filepath.Separator))) { + p += "/" + } + // A Windows drive path like C:\dir becomes file:///C:/dir. + if !strings.HasPrefix(p, "/") { + p = "/" + p + } return AbsoluteURLFromGo(&gurl.URL{ - Path: filepath.ToSlash(path), + Path: p, Scheme: SchemeFile.String(), }) } diff --git a/pkg/util/url/url_test.go b/pkg/util/url/url_test.go index 0e0c2b49..1c22fc24 100644 --- a/pkg/util/url/url_test.go +++ b/pkg/util/url/url_test.go @@ -2,6 +2,7 @@ package url import ( gurl "net/url" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -337,6 +338,20 @@ func TestRelativizeHttpURL(t *testing.T) { assert.Equal(t, ur, base.Relativize(u)) } +// A URL that differs from the base only by host (or only by scheme) must not be +// relativized, even when its path shares the base's path prefix: dropping the host +// would silently repoint the URL at the base's origin. +func TestRelativizeDifferentOriginSamePath(t *testing.T) { + base, _ := URLFromString("http://example.com/foo/") + for _, k := range []string{ + "http://other-host.invalid/foo/quz/baz", // Same scheme, different host + "https://example.com/foo/quz/baz", // Different scheme, same host + } { + u, _ := URLFromString(k) + assert.Equal(t, u, base.Relativize(u), k) + } +} + func TestRelativizeFileURL(t *testing.T) { base, _ := URLFromString("file:///root/foo") for k, v := range map[string]string{ @@ -425,3 +440,45 @@ func TestNormalize(t *testing.T) { u, _ = URLFromString("http://user:password@example.com:443/foo?b=b&a=a#fragment") assert.Equal(t, "http://user:password@example.com:443/foo?b=b&a=a#fragment", u.Normalize().String()) } + +// Windows drive paths round-trip through file URLs in the file:///C:/dir form, +// so that resolving relative references does not corrupt the drive letter. +func TestFilepathWindowsDrive(t *testing.T) { + base, err := AbsoluteURLFromString("file:///C:/pub/manifest.json") + if !assert.NoError(t, err) { + return + } + rel, err := RelativeURLFromString("../audio/track.mp3") + if !assert.NoError(t, err) { + return + } + resolved := base.Resolve(rel).(AbsoluteURL) + assert.Equal(t, "file:///C:/audio/track.mp3", resolved.String()) + assert.Equal(t, filepath.FromSlash("C:/audio/track.mp3"), resolved.ToFilepath()) +} + +func TestFilepathUnix(t *testing.T) { + if filepath.Separator == '\\' { + t.Skip("unix-style absolute path expectations are not stable on Windows") + } + u, err := FromFilepath("/pub/manifest.json") + if !assert.NoError(t, err) { + return + } + assert.Equal(t, "file:///pub/manifest.json", u.String()) + assert.Equal(t, filepath.FromSlash("/pub/manifest.json"), u.(AbsoluteURL).ToFilepath()) +} + +// A relative filepath is made absolute, so that resolving relative references against +// the resulting URL cannot silently turn it into a file system-absolute path. +func TestFilepathRelative(t *testing.T) { + u, err := FromFilepath(filepath.Join("some", "relative", "file.txt")) + if !assert.NoError(t, err) { + return + } + abs, err := filepath.Abs(filepath.Join("some", "relative", "file.txt")) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, abs, u.(AbsoluteURL).ToFilepath()) +}