From 2bc74a6f2763bf09ab78e4aedf74c15fb4ae3e61 Mon Sep 17 00:00:00 2001 From: chrysn Date: Sat, 24 Oct 2020 00:43:24 +0200 Subject: [PATCH 01/13] coapfileserver: New module to serve VFS via CoAP This is starting point for a file server. It works but does not do block-wise (truncating large files or directory listings), ETag or write support, and fails to handle read errors (after a successful open) properly. --- sys/Makefile | 3 + sys/Makefile.dep | 4 + sys/include/net/coapfileserver.h | 71 ++++++ .../application_layer/coapfileserver/Makefile | 5 + .../coapfileserver/coapfileserver.c | 217 ++++++++++++++++++ .../application_layer/coapfileserver/doc.txt | 56 +++++ 6 files changed, 356 insertions(+) create mode 100644 sys/include/net/coapfileserver.h create mode 100644 sys/net/application_layer/coapfileserver/Makefile create mode 100644 sys/net/application_layer/coapfileserver/coapfileserver.c create mode 100644 sys/net/application_layer/coapfileserver/doc.txt diff --git a/sys/Makefile b/sys/Makefile index 9abc50975fcf..52d8a0faf6e7 100644 --- a/sys/Makefile +++ b/sys/Makefile @@ -179,6 +179,9 @@ endif ifneq (,$(filter ztimer_xtimer_compat,$(USEMODULE))) FILTER += xtimer endif +ifneq (,$(filter coapfileserver,$(USEMODULE))) + DIRS += net/application_layer/coapfileserver +endif DIRS += $(dir $(wildcard $(addsuffix /Makefile, $(filter-out $(FILTER), $(USEMODULE))))) diff --git a/sys/Makefile.dep b/sys/Makefile.dep index e96a15b89b58..c2fbeb4cc5d3 100644 --- a/sys/Makefile.dep +++ b/sys/Makefile.dep @@ -760,6 +760,10 @@ ifneq (,$(filter devfs,$(USEMODULE))) USEMODULE += vfs endif +ifneq (,$(filter coapfileserver,$(USEMODULE))) + USEMODULE += vfs +endif + ifneq (,$(filter vfs,$(USEMODULE))) USEMODULE += posix_headers ifeq (native, $(BOARD)) diff --git a/sys/include/net/coapfileserver.h b/sys/include/net/coapfileserver.h new file mode 100644 index 000000000000..57b5142bf8c7 --- /dev/null +++ b/sys/include/net/coapfileserver.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2020 chrysn + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +/** + * @ingroup net_coapfileserver + * @{ + * + * @file + * @brief Resource handler for the CoAP file system server + * + * @author chrysn + * + * @} + */ + +#ifndef COAPFILESERVER_H +#define COAPFILESERVER_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/** File server starting point + * + * This struct needs to be present at the ctx of a coapfileserver_handler entry + * in a resource list. + * + */ +struct coapfileserver_entry { + /** Path in the VFS that should be served. + * + * This does not need a trailing slash. */ + char *nameprefix; + /** Number of leading path components to ignore as they are the common prefix. + * + * If the file server is bound to the "/" resource, make this 0; if there + * is an entry + * + * ``{ "/files/sd", COAP_GET | COAP_MATCH_SUBTREE, coapfileserver_handler, files_sd }`` + * + * then ``files_sd.strip_path`` needs to be 2. + * + * */ + uint8_t strip_path; +}; + +/** + * File server handler + * + * Serve a directory from the VFS as a CoAP resource tree. + * + * @p ctx pointer to a @ref coapfileserver_entry + * + * @see net_coapfileserver + */ +ssize_t coapfileserver_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void *ctx); + +#ifdef __cplusplus +} +#endif + +#endif + +/** @} */ diff --git a/sys/net/application_layer/coapfileserver/Makefile b/sys/net/application_layer/coapfileserver/Makefile new file mode 100644 index 000000000000..6202039760cd --- /dev/null +++ b/sys/net/application_layer/coapfileserver/Makefile @@ -0,0 +1,5 @@ +MODULE = coapfileserver + +SRC = coapfileserver.c + +include $(RIOTBASE)/Makefile.base diff --git a/sys/net/application_layer/coapfileserver/coapfileserver.c b/sys/net/application_layer/coapfileserver/coapfileserver.c new file mode 100644 index 000000000000..a0d3ad5771e0 --- /dev/null +++ b/sys/net/application_layer/coapfileserver/coapfileserver.c @@ -0,0 +1,217 @@ +/* + * Copyright (C) 2020 chrysn + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define ENABLE_DEBUG 0 +#include "debug.h" + +/** Maximum length of an expressible path, including the trailing 0 character. */ +#define COAPFILESERVER_PATH_MAX (64) + +static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void *ctx); +static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void *ctx); +/** Create a CoAP response for a given errno (eg. EACCESS -> 4.03 Forbidden + * etc., defaulting to 5.03 Internal Server Error) */ +static size_t coapfileserver_errno_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, int err); + +ssize_t coapfileserver_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void *ctx) { + struct coapfileserver_entry *entry = (struct coapfileserver_entry *)ctx; + char namebuf[COAPFILESERVER_PATH_MAX]; + /** Index in namebuf. Must not point at the last entry as that will be + * zeroed to get a 0-terminated string. */ + size_t namelength = 0; + bool trailing_slash = false; + /** If no path component comes along at all, it'll count as a trailing + * slash no matter the trailing_slash value */ + bool any_component = false; + uint8_t errorcode = COAP_CODE_INTERNAL_SERVER_ERROR; + + uint8_t strip_remaining = entry->strip_path; + namelength += namebuf - strncpy(namebuf, entry->nameprefix, sizeof(namebuf) - 1); + + coap_optpos_t opt; + bool is_first = true; + uint8_t *value; + ssize_t optlen; + while ((optlen = coap_opt_get_next(pdu, &opt, &value, is_first)) != + -ENOENT) + { + if (optlen < 0) { + errorcode = COAP_CODE_BAD_REQUEST; + goto error; + } + + is_first = false; + switch (opt.opt_num) { + case COAP_OPT_URI_PATH: + if (strip_remaining != 0) { + strip_remaining -= 1; + continue; + } + if (trailing_slash) { + errorcode = COAP_CODE_BAD_REQUEST; + goto error; + } + any_component = true; + if (optlen == 0) { + trailing_slash = true; + continue; + } + if (memchr(value, '0', optlen) != NULL || + memchr(value, '/', optlen) != NULL) { + /* Path can not be expressed in the file system */ + errorcode = COAP_CODE_PATH_NOT_FOUND; + goto error; + } + size_t newlength = namelength + 1 + optlen; + if (newlength > sizeof(namebuf) - 1) { + /* Path too long, therefore can't exist in this mapping */ + errorcode = COAP_CODE_PATH_NOT_FOUND; + goto error; + } + namebuf[namelength] = '/'; + memcpy(&namebuf[namelength] + 1, value, optlen); + namelength = newlength; + break; + default: + if (opt.opt_num & 1) { + errorcode = COAP_CODE_BAD_REQUEST; + goto error; + } else { + /* Ignoring elective option */ + } + } + } + + namebuf[namelength] = '\0'; + bool is_directory = trailing_slash | !any_component; + + /* Note to self: As we parse more options than just Uri-Path, we'll likely + * pass a struct pointer later. So far, those could even be hooked into the + * resource list, but that'll go away once we parse more options */ + if (is_directory) { + return coapfileserver_directory_handler(pdu, buf, len, namebuf); + } else { + return coapfileserver_file_handler(pdu, buf, len, namebuf); + } + +error: + return gcoap_response(pdu, buf, len, errorcode); +} + +static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void *ctx) { + /** + * ToDo: + * + * * Blockwise + * + * * ETag + * + * * Error handling on late read errors + */ + const char *name = (char *)ctx; + + int fd = vfs_open(name, O_RDONLY, 0); + if (fd < 0) + return coapfileserver_errno_handler(pdu, buf, len, fd); + + gcoap_resp_init(pdu, buf, len, COAP_CODE_CONTENT); + size_t resp_len = coap_opt_finish(pdu, COAP_OPT_FINISH_PAYLOAD); + + int read = vfs_read(fd, pdu->payload, pdu->payload_len); + assert(read >= 0); /* Can't rewind PDU yet */ + + vfs_close(fd); + + if (read == 0) { + /* Rewind to clear payload marker */ + read -= 1; + } + + return resp_len + read; +} + +static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void *ctx) { + /** + * ToDo: + * + * * Produce actual link format (considering the 'origin' part, ie. always + * giving full paths including path-so-far from the request PDU), or + * serve CoRAL right away + * + * * Blockwise + * + * * Directories don't have their trailing slashes yet + */ + const char *name = (char *)ctx; + vfs_DIR dir; + + int err = vfs_opendir(&dir, name); + if (err != 0) { + return coapfileserver_errno_handler(pdu, buf, len, err); + } + DEBUG("coapfileserver: Serving directory listing\n"); + + gcoap_resp_init(pdu, buf, len, COAP_CODE_CONTENT); + coap_opt_add_format(pdu, COAP_FORMAT_LINK); + size_t resp_len = coap_opt_finish(pdu, COAP_OPT_FINISH_PAYLOAD); + + vfs_dirent_t entry; + ssize_t payload_cursor = 0; + while (vfs_readdir(&dir, &entry) > 0) { + size_t entry_len = strlen((char*)&entry.d_name); + /* maybe ",", "<>", and the length without leading slash */ + ssize_t need_bytes = (payload_cursor == 0 ? 0 : 1) + 2 + entry_len - 1; + if (payload_cursor + need_bytes > pdu->payload_len) { + /* Without blockwise, this is the best approximation we can do */ + DEBUG("coapfileserver: Directory listing truncated\n"); + break; + } + if (payload_cursor != 0) { + pdu->payload[payload_cursor++] = ','; + } + pdu->payload[payload_cursor++] = '<'; + memcpy(&pdu->payload[payload_cursor], &entry.d_name[1], entry_len - 1); + payload_cursor += entry_len - 1; + pdu->payload[payload_cursor++] = '>'; + } + vfs_closedir(&dir); + + if (payload_cursor == 0) { + /* Rewind to clear payload marker */ + payload_cursor -= 1; + } + + return resp_len + payload_cursor; +} + +static size_t coapfileserver_errno_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, int err) +{ + uint8_t code; + switch (err) { + case -EACCES: + code = COAP_CODE_FORBIDDEN; break; + case -ENOENT: + code = COAP_CODE_PATH_NOT_FOUND; break; + default: + code = COAP_CODE_INTERNAL_SERVER_ERROR; + }; + DEBUG("coapfileserver: Rejecting error %d (%s) as %d.%02d\n", err, strerror(err), code >> 5, code & 0x1f); + return gcoap_response(pdu, buf, len, code); +} diff --git a/sys/net/application_layer/coapfileserver/doc.txt b/sys/net/application_layer/coapfileserver/doc.txt new file mode 100644 index 000000000000..1716728ec28d --- /dev/null +++ b/sys/net/application_layer/coapfileserver/doc.txt @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2020 chrysn + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +/** + * @defgroup net_coapfileserver CoAP file server + * @ingroup net + * @brief Library for serving files from the VFS to CoAP clients + * + * # About + * + * This maps files in the local file system onto a resources in CoAP. In that, + * is is what is called a static web server in the unconstrained web. + * + * As usual, GET operations are used to read files. + * + * Directories are expressed to URIs with trailing slashes. + * + * + * # Usage + * + * * ``USEMODULE += coapfileserver`` + * + * * Have a @ref coapfileserver_entry populated with the path you want to serve, + * and the number of path components to strip from incoming requests: + * + * ``` + * static const struct coapfileserver_entry files_sd = { + * "/sd", + * 2 + * }; + * ``` + * + * * Enter a @ref coapfileserver_handler handler into your CoAP server's + * resource list like this: + * + * ``` + * static const coap_resource_t _resources[] = { + * ..., + * { "/files/sd", COAP_GET | COAP_MATCH_SUBTREE, coapfileserver_handler, files_sd }, + * ... + * } + * ``` + * + * The allowed methods dictate whether it's read-only (``COAP_GET``) or (in the + * future) read-write (``1COAP_GET | COAP_PUT | COAP_DELETE``). + * + */ From abe369f98532deb1ec145f6b94c9920b889c1ade Mon Sep 17 00:00:00 2001 From: chrysn Date: Tue, 30 Jun 2020 17:17:02 +0200 Subject: [PATCH 02/13] coap: Add ETag code from RFC7252 --- sys/include/net/coap.h | 1 + 1 file changed, 1 insertion(+) diff --git a/sys/include/net/coap.h b/sys/include/net/coap.h index 168211d6bdfb..e82668ac6a7d 100644 --- a/sys/include/net/coap.h +++ b/sys/include/net/coap.h @@ -36,6 +36,7 @@ extern "C" { * @{ */ #define COAP_OPT_URI_HOST (3) +#define COAP_OPT_ETAG (4) #define COAP_OPT_OBSERVE (6) #define COAP_OPT_LOCATION_PATH (8) #define COAP_OPT_URI_PATH (11) From 4ae2c60a8878101ef4fef6439a050bfa3d94054b Mon Sep 17 00:00:00 2001 From: chrysn Date: Tue, 30 Jun 2020 18:02:19 +0200 Subject: [PATCH 03/13] coapfileserver: Add support for ETag The note on partially cached versions is not applicable yet (for lack of block-wise support), but will become relevant. --- .../coapfileserver/coapfileserver.c | 117 +++++++++++++++--- .../application_layer/coapfileserver/doc.txt | 7 ++ 2 files changed, 105 insertions(+), 19 deletions(-) diff --git a/sys/net/application_layer/coapfileserver/coapfileserver.c b/sys/net/application_layer/coapfileserver/coapfileserver.c index a0d3ad5771e0..fa7e15995a80 100644 --- a/sys/net/application_layer/coapfileserver/coapfileserver.c +++ b/sys/net/application_layer/coapfileserver/coapfileserver.c @@ -24,16 +24,47 @@ /** Maximum length of an expressible path, including the trailing 0 character. */ #define COAPFILESERVER_PATH_MAX (64) -static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void *ctx); -static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void *ctx); +/** Constant ETag length */ +#define ETAG_LENGTH 8 + +/** Data extracted from a request on a file */ +struct requestdata { + /** 0-terminated expanded file name in the VFS */ + char namebuf[COAPFILESERVER_PATH_MAX]; + bool etag_sent; + uint8_t etag[ETAG_LENGTH]; +}; + +/* See stat_etag */ +union stattag { + struct stat stat; + struct { + uint8_t etag[ETAG_LENGTH]; + /* This will be as many repetitions of ETAG_LENGTH that together + * with the original etag field id's larger than the stat */ + uint8_t etag_padding[(sizeof(struct stat) - 1) / ETAG_LENGTH * ETAG_LENGTH]; + }; +}; + +/** Build an ETag based on the given file's VFS stat. If the stat fails, + * returns the error and leaves stattag in any state; otherwise there's an etag + * in the stattag's field */ +static int stat_etag(char *filename, union stattag *stattag); + +/* These are almost but but not quite coap_handler_t -- they require the + * request to be already fully read and digested into the last argument. */ + +static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, struct requestdata *request); +static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, struct requestdata *request); /** Create a CoAP response for a given errno (eg. EACCESS -> 4.03 Forbidden * etc., defaulting to 5.03 Internal Server Error) */ static size_t coapfileserver_errno_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, int err); ssize_t coapfileserver_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void *ctx) { struct coapfileserver_entry *entry = (struct coapfileserver_entry *)ctx; - char namebuf[COAPFILESERVER_PATH_MAX]; - /** Index in namebuf. Must not point at the last entry as that will be + struct requestdata request; + request.etag_sent = false; + /** Index in request.namebuf. Must not point at the last entry as that will be * zeroed to get a 0-terminated string. */ size_t namelength = 0; bool trailing_slash = false; @@ -43,7 +74,7 @@ ssize_t coapfileserver_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void * uint8_t errorcode = COAP_CODE_INTERNAL_SERVER_ERROR; uint8_t strip_remaining = entry->strip_path; - namelength += namebuf - strncpy(namebuf, entry->nameprefix, sizeof(namebuf) - 1); + namelength += request.namebuf - strncpy(request.namebuf, entry->nameprefix, sizeof(request.namebuf) - 1); coap_optpos_t opt; bool is_first = true; @@ -80,15 +111,29 @@ ssize_t coapfileserver_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void * goto error; } size_t newlength = namelength + 1 + optlen; - if (newlength > sizeof(namebuf) - 1) { + if (newlength > sizeof(request.namebuf) - 1) { /* Path too long, therefore can't exist in this mapping */ errorcode = COAP_CODE_PATH_NOT_FOUND; goto error; } - namebuf[namelength] = '/'; - memcpy(&namebuf[namelength] + 1, value, optlen); + request.namebuf[namelength] = '/'; + memcpy(&request.namebuf[namelength] + 1, value, optlen); namelength = newlength; break; + case COAP_OPT_ETAG: + if (optlen != sizeof(request.etag)) { + /* Can't be a matching tag, no use in carrying that */ + continue; + } + if (request.etag_sent) { + /* We can reasonably only check for a limited sized set, + * and it size is 1 here (sending multiple ETags is + * possible but rare) */ + continue; + } + request.etag_sent = true; + memcpy(request.etag, value, sizeof(request.etag)); + break; default: if (opt.opt_num & 1) { errorcode = COAP_CODE_BAD_REQUEST; @@ -99,39 +144,50 @@ ssize_t coapfileserver_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void * } } - namebuf[namelength] = '\0'; + request.namebuf[namelength] = '\0'; bool is_directory = trailing_slash | !any_component; /* Note to self: As we parse more options than just Uri-Path, we'll likely * pass a struct pointer later. So far, those could even be hooked into the * resource list, but that'll go away once we parse more options */ if (is_directory) { - return coapfileserver_directory_handler(pdu, buf, len, namebuf); + return coapfileserver_directory_handler(pdu, buf, len, &request); } else { - return coapfileserver_file_handler(pdu, buf, len, namebuf); + return coapfileserver_file_handler(pdu, buf, len, &request); } error: return gcoap_response(pdu, buf, len, errorcode); } -static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void *ctx) { +static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, struct requestdata *request) +{ /** * ToDo: * * * Blockwise * - * * ETag - * * * Error handling on late read errors */ - const char *name = (char *)ctx; + int err; - int fd = vfs_open(name, O_RDONLY, 0); + union stattag stattag; + err = stat_etag(request->namebuf, &stattag); + if (err < 0) + return coapfileserver_errno_handler(pdu, buf, len, err); + + if (request->etag_sent && memcmp(stattag.etag, request->etag, ETAG_LENGTH) == 0) { + gcoap_resp_init(pdu, buf, len, COAP_CODE_VALID); + coap_opt_add_opaque(pdu, COAP_OPT_ETAG, stattag.etag, ETAG_LENGTH); + return coap_opt_finish(pdu, COAP_OPT_FINISH_NONE); + } + + int fd = vfs_open(request->namebuf, O_RDONLY, 0); if (fd < 0) return coapfileserver_errno_handler(pdu, buf, len, fd); gcoap_resp_init(pdu, buf, len, COAP_CODE_CONTENT); + coap_opt_add_opaque(pdu, COAP_OPT_ETAG, stattag.etag, ETAG_LENGTH); size_t resp_len = coap_opt_finish(pdu, COAP_OPT_FINISH_PAYLOAD); int read = vfs_read(fd, pdu->payload, pdu->payload_len); @@ -147,7 +203,8 @@ static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t return resp_len + read; } -static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void *ctx) { +static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, struct requestdata *request) +{ /** * ToDo: * @@ -159,10 +216,9 @@ static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, s * * * Directories don't have their trailing slashes yet */ - const char *name = (char *)ctx; vfs_DIR dir; - int err = vfs_opendir(&dir, name); + int err = vfs_opendir(&dir, request->namebuf); if (err != 0) { return coapfileserver_errno_handler(pdu, buf, len, err); } @@ -215,3 +271,26 @@ static size_t coapfileserver_errno_handler(coap_pkt_t *pdu, uint8_t *buf, size_t DEBUG("coapfileserver: Rejecting error %d (%s) as %d.%02d\n", err, strerror(err), code >> 5, code & 0x1f); return gcoap_response(pdu, buf, len, code); } + +static int stat_etag(char *filename, union stattag *stattag) +{ + int err; + + memset(stattag, 0, sizeof(union stattag)); + err = vfs_stat(filename, &stattag->stat); + if (err < 0) + return err; + + /* Normalizing fields whose value can change without affecting the ETag */ + stattag->stat.st_nlink = 0; + memset(&stattag->stat.st_atim, 0, sizeof(stattag->stat.st_atim)); + + /* Build a compact ETag by XORing the stat onto itself */ + for (unsigned int i = 0; i < sizeof(stattag->etag_padding) / sizeof(stattag->etag); ++i) { + for (int j = 0; j < ETAG_LENGTH; ++j) { + stattag->etag[j] ^= stattag->etag_padding[i * ETAG_LENGTH + j]; + } + } + + return 0; +} diff --git a/sys/net/application_layer/coapfileserver/doc.txt b/sys/net/application_layer/coapfileserver/doc.txt index 1716728ec28d..4633b0b42645 100644 --- a/sys/net/application_layer/coapfileserver/doc.txt +++ b/sys/net/application_layer/coapfileserver/doc.txt @@ -24,6 +24,13 @@ * created implicitly when files are PUT under them; they can be DELETEd when * empty -->. * + * @note The file server uses ETag for cache validation. The ETags are built + * from the file system stat values. As clients rely on the ETag to differ when + * the file changes, it is important that file modification times are set. The + * precise time values do not matter, but if a file is changed in place and + * neither its length nor its modification time is varied, then clients will + * not become aware of the change or may even mix up the versions half way + * through if they have a part of the old version cached. * * # Usage * From 5ac989856610c2a667a571b3f102eb6803b910ac Mon Sep 17 00:00:00 2001 From: chrysn Date: Tue, 30 Jun 2020 20:06:10 +0200 Subject: [PATCH 04/13] coapfileserver: Serve files blockwise --- .../coapfileserver/coapfileserver.c | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/sys/net/application_layer/coapfileserver/coapfileserver.c b/sys/net/application_layer/coapfileserver/coapfileserver.c index fa7e15995a80..1c3c356ae450 100644 --- a/sys/net/application_layer/coapfileserver/coapfileserver.c +++ b/sys/net/application_layer/coapfileserver/coapfileserver.c @@ -31,6 +31,8 @@ struct requestdata { /** 0-terminated expanded file name in the VFS */ char namebuf[COAPFILESERVER_PATH_MAX]; + uint32_t blocknum2; + unsigned int szx2; /* would prefer uint8_t but that's what coap_get_blockopt gives */ bool etag_sent; uint8_t etag[ETAG_LENGTH]; }; @@ -64,6 +66,8 @@ ssize_t coapfileserver_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void * struct coapfileserver_entry *entry = (struct coapfileserver_entry *)ctx; struct requestdata request; request.etag_sent = false; + request.blocknum2 = 0; + request.szx2 = CONFIG_NANOCOAP_BLOCK_SIZE_EXP_MAX; /** Index in request.namebuf. Must not point at the last entry as that will be * zeroed to get a 0-terminated string. */ size_t namelength = 0; @@ -134,6 +138,11 @@ ssize_t coapfileserver_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void * request.etag_sent = true; memcpy(request.etag, value, sizeof(request.etag)); break; + case COAP_OPT_BLOCK2: + /* Could be more efficient now that we already know where it + * is, but meh */ + coap_get_blockopt(pdu, COAP_OPT_BLOCK2, &request.blocknum2, &request.szx2); + break; default: if (opt.opt_num & 1) { errorcode = COAP_CODE_BAD_REQUEST; @@ -165,8 +174,6 @@ static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t /** * ToDo: * - * * Blockwise - * * * Error handling on late read errors */ int err; @@ -188,13 +195,47 @@ static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t gcoap_resp_init(pdu, buf, len, COAP_CODE_CONTENT); coap_opt_add_opaque(pdu, COAP_OPT_ETAG, stattag.etag, ETAG_LENGTH); + /* To best see how this works set CONFIG_GCAOP_PDU_BUF_SIZE to 532 or 533. + * If we did a sharper estimation (factoring in the block2 size option with + * the current blockum), we'd even pack 512 bytes into 530 until block + * numbers get large enough to eat another byte, which is when the block + * size would decrease in-flight. */ + size_t remaining_length = len - (pdu->payload - buf); + remaining_length -= 5; /* maximum block2 option usable in nanocoap */ + remaining_length -= 1; /* payload marker */ + /* > 0: To not wrap around; if that still won't fit that's later caught in + * an assertion */ + while (coap_szx2size(request->szx2) > remaining_length && request->szx2 > 0) { + request->szx2 --; + request->blocknum2 <<= 1; + } + coap_block_slicer_t slicer; + coap_block_slicer_init(&slicer, request->blocknum2, coap_szx2size(request->szx2)); + coap_opt_add_block2(pdu, &slicer, true); size_t resp_len = coap_opt_finish(pdu, COAP_OPT_FINISH_PAYLOAD); - int read = vfs_read(fd, pdu->payload, pdu->payload_len); + err = vfs_lseek(fd, slicer.start, SEEK_SET); + assert(err >= 0); /* Can't rewind PDU yet */ + + /* That'd only happen if the buffer is too small for even a 16-byte block, + * or if the above calculations were wrong. + * + * Not using payload_len here as that's needlessly underestimating the + * space by CONFIG_GCOAP_RESP_OPTIONS_BUF + * */ + assert(pdu->payload + slicer.end - slicer.start <= buf + len); + int read = vfs_read(fd, pdu->payload, slicer.end - slicer.start); assert(read >= 0); /* Can't rewind PDU yet */ + uint8_t morebuf; + int more = vfs_read(fd, &morebuf, 1); + assert(more >= 0); /* Can't rewind PDU yet */ + vfs_close(fd); + slicer.cur = slicer.end + more; + coap_block2_finish(&slicer); + if (read == 0) { /* Rewind to clear payload marker */ read -= 1; From 7d02c94f894206bae3d8b731dec36020f3a85413 Mon Sep 17 00:00:00 2001 From: chrysn Date: Tue, 30 Jun 2020 20:32:34 +0200 Subject: [PATCH 05/13] coapfileserver: Error handling --- .../coapfileserver/coapfileserver.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/sys/net/application_layer/coapfileserver/coapfileserver.c b/sys/net/application_layer/coapfileserver/coapfileserver.c index 1c3c356ae450..fa13c41b4634 100644 --- a/sys/net/application_layer/coapfileserver/coapfileserver.c +++ b/sys/net/application_layer/coapfileserver/coapfileserver.c @@ -171,11 +171,6 @@ ssize_t coapfileserver_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void * static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, struct requestdata *request) { - /** - * ToDo: - * - * * Error handling on late read errors - */ int err; union stattag stattag; @@ -215,7 +210,8 @@ static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t size_t resp_len = coap_opt_finish(pdu, COAP_OPT_FINISH_PAYLOAD); err = vfs_lseek(fd, slicer.start, SEEK_SET); - assert(err >= 0); /* Can't rewind PDU yet */ + if (err < 0) + goto late_err; /* That'd only happen if the buffer is too small for even a 16-byte block, * or if the above calculations were wrong. @@ -225,11 +221,13 @@ static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t * */ assert(pdu->payload + slicer.end - slicer.start <= buf + len); int read = vfs_read(fd, pdu->payload, slicer.end - slicer.start); - assert(read >= 0); /* Can't rewind PDU yet */ + if (err < 0) + goto late_err; uint8_t morebuf; int more = vfs_read(fd, &morebuf, 1); - assert(more >= 0); /* Can't rewind PDU yet */ + if (err < 0) + goto late_err; vfs_close(fd); @@ -242,6 +240,11 @@ static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t } return resp_len + read; + +late_err: + vfs_close(fd); + coap_hdr_set_code(pdu->hdr, COAP_CODE_INTERNAL_SERVER_ERROR); + return coap_get_total_hdr_len(pdu); } static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, struct requestdata *request) From ab858b418a6e0dd92d5701e7f655fc39aeaea86f Mon Sep 17 00:00:00 2001 From: chrysn Date: Tue, 30 Jun 2020 20:57:05 +0200 Subject: [PATCH 06/13] coapfileserver: Cling to filesystem example The filesystem example is ideal to demonstrate the fileserver as it has a usable default set of demo files, and different file systems can be mounted for experimentation. The impact on the filesystem example outside the coapfileserver context is minimal: Only a few lines of includes (2), static configuration (17 including comments and whitespace) and startup code (2) are added under ifdef guards that clearly mark them as optional and Gcoap related. They are only enabled if NETWORK=1 is set. --- examples/filesystem/Makefile | 8 ++++++++ examples/filesystem/README.md | 18 ++++++++++++++++++ examples/filesystem/main.c | 30 ++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/examples/filesystem/Makefile b/examples/filesystem/Makefile index c215dc37968d..be0508edc02e 100644 --- a/examples/filesystem/Makefile +++ b/examples/filesystem/Makefile @@ -24,6 +24,14 @@ USEMODULE += shell USEMODULE += shell_commands USEMODULE += ps +ifeq (${NETWORK},1) +USEMODULE += gcoap +USEMODULE += coapfileserver +USEMODULE += gnrc_ipv6_default +USEMODULE += gnrc_netdev_default +USEMODULE += auto_init_gnrc_netif +endif + # Use MTD (flash layer) USEMODULE += mtd # USEMODULE += mtd_sdcard diff --git a/examples/filesystem/README.md b/examples/filesystem/README.md index 31dd8c642cbd..85ce99e60a30 100644 --- a/examples/filesystem/README.md +++ b/examples/filesystem/README.md @@ -76,3 +76,21 @@ Hello World! cat /const/hello-riot Hello RIOT! ``` + +## Network file server + +If ``NETWORK=1`` is passed to make, the board's default network setup is +enabled, and a CoAP server is started that exports the file system at the path +``/vfs``. + +You can access the file system from another board that runs the gcoap example: + +``` +> coap get fe80::3c63:beff:fe85:ca96%6 5683 /vfs/const/ +, +> coap get fe80::3c63:beff:fe85:ca96%6 5683 /vfs/const/hello-riot +--- blockwise start --- +gcoap: response Success, code 2.05, 13 bytes +00000000 48 65 6C 6C 6F 20 52 49 4F 54 21 0A 00 +--- blockwise complete --- +``` diff --git a/examples/filesystem/main.c b/examples/filesystem/main.c index 759d61c4ab9c..34d15dfb4dc6 100644 --- a/examples/filesystem/main.c +++ b/examples/filesystem/main.c @@ -26,6 +26,11 @@ #include "shell.h" #include "board.h" /* MTD_0 is defined in board.h */ +#ifdef MODULE_COAPFILESERVER +#include +#include +#endif + /* Configure MTD device for SD card if none is provided */ #if !defined(MTD_0) && MODULE_MTD_SDCARD #include "mtd_sdcard.h" @@ -309,6 +314,26 @@ static const shell_command_t shell_commands[] = { { NULL, NULL, NULL } }; +#ifdef MODULE_COAPFILESERVER +/* Export all the file system */ +static const struct coapfileserver_entry vfs = { "", 1 }; + +/* CoAP resources. Must be sorted by path (ASCII order). */ +static const coap_resource_t _resources[] = { + { "/vfs", COAP_GET | COAP_MATCH_SUBTREE, coapfileserver_handler, (void*)&vfs }, +}; + +static gcoap_listener_t _listener = { + &_resources[0], + ARRAY_SIZE(_resources), + NULL, + NULL +}; + +#define MAIN_QUEUE_SIZE (4) +static msg_t _main_msg_queue[MAIN_QUEUE_SIZE]; +#endif + int main(void) { #if defined(MTD_0) && (defined(MODULE_SPIFFS) || defined(MODULE_LITTLEFS) || defined(MODULE_LITTLEFS2)) @@ -326,6 +351,11 @@ int main(void) puts("constfs mounted successfully"); } +#ifdef MODULE_COAPFILESERVER + msg_init_queue(_main_msg_queue, MAIN_QUEUE_SIZE); + gcoap_register_listener(&_listener); +#endif + char line_buf[SHELL_DEFAULT_BUFSIZE]; shell_run(shell_commands, line_buf, SHELL_DEFAULT_BUFSIZE); From 73b2ccf2864f6582ea0a02ec4424d396081373d2 Mon Sep 17 00:00:00 2001 From: chrysn Date: Sat, 24 Oct 2020 01:04:21 +0200 Subject: [PATCH 07/13] coapfileserver: Work around different slash conventions Workaround-For: https://github.com/RIOT-OS/RIOT/issues/14635 --- .../coapfileserver/coapfileserver.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/sys/net/application_layer/coapfileserver/coapfileserver.c b/sys/net/application_layer/coapfileserver/coapfileserver.c index fa13c41b4634..16556d88cc97 100644 --- a/sys/net/application_layer/coapfileserver/coapfileserver.c +++ b/sys/net/application_layer/coapfileserver/coapfileserver.c @@ -275,9 +275,15 @@ static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, s vfs_dirent_t entry; ssize_t payload_cursor = 0; while (vfs_readdir(&dir, &entry) > 0) { - size_t entry_len = strlen((char*)&entry.d_name); - /* maybe ",", "<>", and the length without leading slash */ - ssize_t need_bytes = (payload_cursor == 0 ? 0 : 1) + 2 + entry_len - 1; + char *entry_name = (char*)entry.d_name; + /* Ignore leading slashes presented by some file systems; see + * https://github.com/RIOT-OS/RIOT/issues/14635 */ + while (entry_name[0] == '/') { + entry_name += 1; + } + size_t entry_len = strlen(entry_name); + /* maybe ",", "<>", and the length */ + ssize_t need_bytes = (payload_cursor == 0 ? 0 : 1) + 2 + entry_len; if (payload_cursor + need_bytes > pdu->payload_len) { /* Without blockwise, this is the best approximation we can do */ DEBUG("coapfileserver: Directory listing truncated\n"); @@ -287,8 +293,8 @@ static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, s pdu->payload[payload_cursor++] = ','; } pdu->payload[payload_cursor++] = '<'; - memcpy(&pdu->payload[payload_cursor], &entry.d_name[1], entry_len - 1); - payload_cursor += entry_len - 1; + memcpy(&pdu->payload[payload_cursor], entry_name, entry_len); + payload_cursor += entry_len; pdu->payload[payload_cursor++] = '>'; } vfs_closedir(&dir); From 1c3f63f1237740276ac7e73f3542345f83fc8170 Mon Sep 17 00:00:00 2001 From: chrysn Date: Sat, 24 Oct 2020 01:08:41 +0200 Subject: [PATCH 08/13] coapfileserver: Hide . and .. --- sys/net/application_layer/coapfileserver/coapfileserver.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sys/net/application_layer/coapfileserver/coapfileserver.c b/sys/net/application_layer/coapfileserver/coapfileserver.c index 16556d88cc97..23465d057d78 100644 --- a/sys/net/application_layer/coapfileserver/coapfileserver.c +++ b/sys/net/application_layer/coapfileserver/coapfileserver.c @@ -282,6 +282,10 @@ static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, s entry_name += 1; } size_t entry_len = strlen(entry_name); + if (entry_len <= 2 && memcmp(entry_name, "..", entry_len) == 0) { + /* Up pointers don't work the same way in URI semantics */ + continue; + } /* maybe ",", "<>", and the length */ ssize_t need_bytes = (payload_cursor == 0 ? 0 : 1) + 2 + entry_len; if (payload_cursor + need_bytes > pdu->payload_len) { From 604dc911fe285b2d9ae8695ec07203a6213d853c Mon Sep 17 00:00:00 2001 From: chrysn Date: Sat, 24 Oct 2020 01:21:22 +0200 Subject: [PATCH 09/13] coapfileserver: Work around missing stat in fatfs --- .../coapfileserver/coapfileserver.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/sys/net/application_layer/coapfileserver/coapfileserver.c b/sys/net/application_layer/coapfileserver/coapfileserver.c index 23465d057d78..602b7eb4fda4 100644 --- a/sys/net/application_layer/coapfileserver/coapfileserver.c +++ b/sys/net/application_layer/coapfileserver/coapfileserver.c @@ -332,8 +332,20 @@ static int stat_etag(char *filename, union stattag *stattag) memset(stattag, 0, sizeof(union stattag)); err = vfs_stat(filename, &stattag->stat); - if (err < 0) + if (err < 0) { + /* Some file systems, like only implement fstat. FIXME: run on file + * handles all the time and only use fstat. (Not trivial because it + * needs going through all filesystems to check whether there isn't one + * that only does stat and not fstat) */ + int fd = vfs_open(filename, O_RDONLY, 0); + if (fd < 0) { + return fd; + } + err = vfs_fstat(fd, &stattag->stat); + vfs_close(fd); + return err; + } /* Normalizing fields whose value can change without affecting the ETag */ stattag->stat.st_nlink = 0; From 527d37a6670f0f576710c9f2ed398cd5589c5349 Mon Sep 17 00:00:00 2001 From: chrysn Date: Sun, 25 Oct 2020 12:56:09 +0100 Subject: [PATCH 10/13] coapfileserver: Server links with correct trailing slashes --- .../coapfileserver/coapfileserver.c | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/sys/net/application_layer/coapfileserver/coapfileserver.c b/sys/net/application_layer/coapfileserver/coapfileserver.c index 602b7eb4fda4..4cf0ffc16345 100644 --- a/sys/net/application_layer/coapfileserver/coapfileserver.c +++ b/sys/net/application_layer/coapfileserver/coapfileserver.c @@ -61,6 +61,12 @@ static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, s /** Create a CoAP response for a given errno (eg. EACCESS -> 4.03 Forbidden * etc., defaulting to 5.03 Internal Server Error) */ static size_t coapfileserver_errno_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, int err); +/** Return true if path/name is a directory. + * + * Currently assembling the values in a buffer, this could be done without a + * buffer if an openat / vfs_openat was present. + * */ +static bool entry_is_dir(char *path, char *name); ssize_t coapfileserver_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void *ctx) { struct coapfileserver_entry *entry = (struct coapfileserver_entry *)ctx; @@ -258,7 +264,6 @@ static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, s * * * Blockwise * - * * Directories don't have their trailing slashes yet */ vfs_DIR dir; @@ -286,8 +291,9 @@ static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, s /* Up pointers don't work the same way in URI semantics */ continue; } + bool is_dir = entry_is_dir(request->namebuf, entry_name); /* maybe ",", "<>", and the length */ - ssize_t need_bytes = (payload_cursor == 0 ? 0 : 1) + 2 + entry_len; + ssize_t need_bytes = (payload_cursor == 0 ? 0 : 1) + 2 + entry_len + (is_dir ? 1 : 0); if (payload_cursor + need_bytes > pdu->payload_len) { /* Without blockwise, this is the best approximation we can do */ DEBUG("coapfileserver: Directory listing truncated\n"); @@ -299,6 +305,9 @@ static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, s pdu->payload[payload_cursor++] = '<'; memcpy(&pdu->payload[payload_cursor], entry_name, entry_len); payload_cursor += entry_len; + if (is_dir) { + pdu->payload[payload_cursor++] = '/'; + } pdu->payload[payload_cursor++] = '>'; } vfs_closedir(&dir); @@ -360,3 +369,37 @@ static int stat_etag(char *filename, union stattag *stattag) return 0; } + +static bool entry_is_dir(char *path, char *name) +{ + /* FIXME have an openat or fstatat + * + * or just build the long name outside because it's printed later, or use + * it from the printed buffer? */ + char buf[COAPFILESERVER_PATH_MAX]; + /* snprintf may be expensive, but without a strlcpy it's a pain to do + * manually */ + int printed = snprintf(buf, sizeof(buf), "%s/%s", path, name); + if (printed <= 0) { + /* Returning true on error as a slash in a file name is more likely to + * cause suspicion than the lack of one at a directory */ + return true; + } + + struct stat stat; + memset(&stat, 0, sizeof(stat)); + int err = vfs_stat(buf, &stat); + if (err != 0) { + int fd = vfs_open(buf, O_RDONLY, 0); + if (fd < 0) { + return true; + } + err = vfs_fstat(fd, &stat); + vfs_close(fd); + + if (err != 0) { + return true; + } + } + return (stat.st_mode & S_IFMT) == S_IFDIR; +} From 893874ce80a3824d75d75a8c3d203c2fb2fea1bc Mon Sep 17 00:00:00 2001 From: chrysn Date: Sun, 25 Oct 2020 12:57:50 +0100 Subject: [PATCH 11/13] coapfileserver: Adopt different reading of RFC6690 With the "resolved agains the anchor" reading obsolete, link targets can easily be expressed relatively. --- sys/net/application_layer/coapfileserver/coapfileserver.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sys/net/application_layer/coapfileserver/coapfileserver.c b/sys/net/application_layer/coapfileserver/coapfileserver.c index 4cf0ffc16345..57549d5e07d3 100644 --- a/sys/net/application_layer/coapfileserver/coapfileserver.c +++ b/sys/net/application_layer/coapfileserver/coapfileserver.c @@ -18,7 +18,7 @@ #include #include -#define ENABLE_DEBUG 0 +#define ENABLE_DEBUG 1 #include "debug.h" /** Maximum length of an expressible path, including the trailing 0 character. */ @@ -258,10 +258,6 @@ static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, s /** * ToDo: * - * * Produce actual link format (considering the 'origin' part, ie. always - * giving full paths including path-so-far from the request PDU), or - * serve CoRAL right away - * * * Blockwise * */ From 87072a24a78543249a2742b526bd05af10d83bc9 Mon Sep 17 00:00:00 2001 From: chrysn Date: Sun, 25 Oct 2020 22:46:15 +0100 Subject: [PATCH 12/13] fixup! coapfileserver: New module to serve VFS via CoAP headerguards check found I'm not using the right names for the guards --- sys/include/net/coapfileserver.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sys/include/net/coapfileserver.h b/sys/include/net/coapfileserver.h index 57b5142bf8c7..70e132bde830 100644 --- a/sys/include/net/coapfileserver.h +++ b/sys/include/net/coapfileserver.h @@ -18,8 +18,8 @@ * @} */ -#ifndef COAPFILESERVER_H -#define COAPFILESERVER_H +#ifndef NET_COAPFILESERVER_H +#define NET_COAPFILESERVER_H #ifdef __cplusplus extern "C" { @@ -66,6 +66,6 @@ ssize_t coapfileserver_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void * } #endif -#endif +#endif /* NET_COAPFILESERVER_H */ /** @} */ From 92ea550a97c8583e3dc5f03bf07e931cc7ed6d7e Mon Sep 17 00:00:00 2001 From: chrysn Date: Mon, 26 Oct 2020 13:51:55 +0100 Subject: [PATCH 13/13] fixup! coapfileserver: New module to serve VFS via CoAP Whitespace clean-up --- sys/net/application_layer/coapfileserver/doc.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/net/application_layer/coapfileserver/doc.txt b/sys/net/application_layer/coapfileserver/doc.txt index 4633b0b42645..e4ec679a9e75 100644 --- a/sys/net/application_layer/coapfileserver/doc.txt +++ b/sys/net/application_layer/coapfileserver/doc.txt @@ -12,7 +12,7 @@ * @brief Library for serving files from the VFS to CoAP clients * * # About - * + * * This maps files in the local file system onto a resources in CoAP. In that, * is is what is called a static web server in the unconstrained web. *