diff --git a/examples/filesystem/Makefile b/examples/filesystem/Makefile index c215dc37968d..be0508edc02e 100644 --- a/examples/filesystem/Makefile +++ b/examples/filesystem/Makefile @@ -24,6 +24,14 @@ USEMODULE += shell USEMODULE += shell_commands USEMODULE += ps +ifeq (${NETWORK},1) +USEMODULE += gcoap +USEMODULE += coapfileserver +USEMODULE += gnrc_ipv6_default +USEMODULE += gnrc_netdev_default +USEMODULE += auto_init_gnrc_netif +endif + # Use MTD (flash layer) USEMODULE += mtd # USEMODULE += mtd_sdcard diff --git a/examples/filesystem/README.md b/examples/filesystem/README.md index 31dd8c642cbd..85ce99e60a30 100644 --- a/examples/filesystem/README.md +++ b/examples/filesystem/README.md @@ -76,3 +76,21 @@ Hello World! cat /const/hello-riot Hello RIOT! ``` + +## Network file server + +If ``NETWORK=1`` is passed to make, the board's default network setup is +enabled, and a CoAP server is started that exports the file system at the path +``/vfs``. + +You can access the file system from another board that runs the gcoap example: + +``` +> coap get fe80::3c63:beff:fe85:ca96%6 5683 /vfs/const/ +, +> coap get fe80::3c63:beff:fe85:ca96%6 5683 /vfs/const/hello-riot +--- blockwise start --- +gcoap: response Success, code 2.05, 13 bytes +00000000 48 65 6C 6C 6F 20 52 49 4F 54 21 0A 00 +--- blockwise complete --- +``` diff --git a/examples/filesystem/main.c b/examples/filesystem/main.c index 759d61c4ab9c..34d15dfb4dc6 100644 --- a/examples/filesystem/main.c +++ b/examples/filesystem/main.c @@ -26,6 +26,11 @@ #include "shell.h" #include "board.h" /* MTD_0 is defined in board.h */ +#ifdef MODULE_COAPFILESERVER +#include +#include +#endif + /* Configure MTD device for SD card if none is provided */ #if !defined(MTD_0) && MODULE_MTD_SDCARD #include "mtd_sdcard.h" @@ -309,6 +314,26 @@ static const shell_command_t shell_commands[] = { { NULL, NULL, NULL } }; +#ifdef MODULE_COAPFILESERVER +/* Export all the file system */ +static const struct coapfileserver_entry vfs = { "", 1 }; + +/* CoAP resources. Must be sorted by path (ASCII order). */ +static const coap_resource_t _resources[] = { + { "/vfs", COAP_GET | COAP_MATCH_SUBTREE, coapfileserver_handler, (void*)&vfs }, +}; + +static gcoap_listener_t _listener = { + &_resources[0], + ARRAY_SIZE(_resources), + NULL, + NULL +}; + +#define MAIN_QUEUE_SIZE (4) +static msg_t _main_msg_queue[MAIN_QUEUE_SIZE]; +#endif + int main(void) { #if defined(MTD_0) && (defined(MODULE_SPIFFS) || defined(MODULE_LITTLEFS) || defined(MODULE_LITTLEFS2)) @@ -326,6 +351,11 @@ int main(void) puts("constfs mounted successfully"); } +#ifdef MODULE_COAPFILESERVER + msg_init_queue(_main_msg_queue, MAIN_QUEUE_SIZE); + gcoap_register_listener(&_listener); +#endif + char line_buf[SHELL_DEFAULT_BUFSIZE]; shell_run(shell_commands, line_buf, SHELL_DEFAULT_BUFSIZE); diff --git a/sys/Makefile b/sys/Makefile index 9abc50975fcf..52d8a0faf6e7 100644 --- a/sys/Makefile +++ b/sys/Makefile @@ -179,6 +179,9 @@ endif ifneq (,$(filter ztimer_xtimer_compat,$(USEMODULE))) FILTER += xtimer endif +ifneq (,$(filter coapfileserver,$(USEMODULE))) + DIRS += net/application_layer/coapfileserver +endif DIRS += $(dir $(wildcard $(addsuffix /Makefile, $(filter-out $(FILTER), $(USEMODULE))))) diff --git a/sys/Makefile.dep b/sys/Makefile.dep index e96a15b89b58..c2fbeb4cc5d3 100644 --- a/sys/Makefile.dep +++ b/sys/Makefile.dep @@ -760,6 +760,10 @@ ifneq (,$(filter devfs,$(USEMODULE))) USEMODULE += vfs endif +ifneq (,$(filter coapfileserver,$(USEMODULE))) + USEMODULE += vfs +endif + ifneq (,$(filter vfs,$(USEMODULE))) USEMODULE += posix_headers ifeq (native, $(BOARD)) diff --git a/sys/include/net/coap.h b/sys/include/net/coap.h index 168211d6bdfb..e82668ac6a7d 100644 --- a/sys/include/net/coap.h +++ b/sys/include/net/coap.h @@ -36,6 +36,7 @@ extern "C" { * @{ */ #define COAP_OPT_URI_HOST (3) +#define COAP_OPT_ETAG (4) #define COAP_OPT_OBSERVE (6) #define COAP_OPT_LOCATION_PATH (8) #define COAP_OPT_URI_PATH (11) diff --git a/sys/include/net/coapfileserver.h b/sys/include/net/coapfileserver.h new file mode 100644 index 000000000000..70e132bde830 --- /dev/null +++ b/sys/include/net/coapfileserver.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2020 chrysn + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +/** + * @ingroup net_coapfileserver + * @{ + * + * @file + * @brief Resource handler for the CoAP file system server + * + * @author chrysn + * + * @} + */ + +#ifndef NET_COAPFILESERVER_H +#define NET_COAPFILESERVER_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/** File server starting point + * + * This struct needs to be present at the ctx of a coapfileserver_handler entry + * in a resource list. + * + */ +struct coapfileserver_entry { + /** Path in the VFS that should be served. + * + * This does not need a trailing slash. */ + char *nameprefix; + /** Number of leading path components to ignore as they are the common prefix. + * + * If the file server is bound to the "/" resource, make this 0; if there + * is an entry + * + * ``{ "/files/sd", COAP_GET | COAP_MATCH_SUBTREE, coapfileserver_handler, files_sd }`` + * + * then ``files_sd.strip_path`` needs to be 2. + * + * */ + uint8_t strip_path; +}; + +/** + * File server handler + * + * Serve a directory from the VFS as a CoAP resource tree. + * + * @p ctx pointer to a @ref coapfileserver_entry + * + * @see net_coapfileserver + */ +ssize_t coapfileserver_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void *ctx); + +#ifdef __cplusplus +} +#endif + +#endif /* NET_COAPFILESERVER_H */ + +/** @} */ diff --git a/sys/net/application_layer/coapfileserver/Makefile b/sys/net/application_layer/coapfileserver/Makefile new file mode 100644 index 000000000000..6202039760cd --- /dev/null +++ b/sys/net/application_layer/coapfileserver/Makefile @@ -0,0 +1,5 @@ +MODULE = coapfileserver + +SRC = coapfileserver.c + +include $(RIOTBASE)/Makefile.base diff --git a/sys/net/application_layer/coapfileserver/coapfileserver.c b/sys/net/application_layer/coapfileserver/coapfileserver.c new file mode 100644 index 000000000000..57549d5e07d3 --- /dev/null +++ b/sys/net/application_layer/coapfileserver/coapfileserver.c @@ -0,0 +1,401 @@ +/* + * Copyright (C) 2020 chrysn + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define ENABLE_DEBUG 1 +#include "debug.h" + +/** Maximum length of an expressible path, including the trailing 0 character. */ +#define COAPFILESERVER_PATH_MAX (64) + +/** Constant ETag length */ +#define ETAG_LENGTH 8 + +/** Data extracted from a request on a file */ +struct requestdata { + /** 0-terminated expanded file name in the VFS */ + char namebuf[COAPFILESERVER_PATH_MAX]; + uint32_t blocknum2; + unsigned int szx2; /* would prefer uint8_t but that's what coap_get_blockopt gives */ + bool etag_sent; + uint8_t etag[ETAG_LENGTH]; +}; + +/* See stat_etag */ +union stattag { + struct stat stat; + struct { + uint8_t etag[ETAG_LENGTH]; + /* This will be as many repetitions of ETAG_LENGTH that together + * with the original etag field id's larger than the stat */ + uint8_t etag_padding[(sizeof(struct stat) - 1) / ETAG_LENGTH * ETAG_LENGTH]; + }; +}; + +/** Build an ETag based on the given file's VFS stat. If the stat fails, + * returns the error and leaves stattag in any state; otherwise there's an etag + * in the stattag's field */ +static int stat_etag(char *filename, union stattag *stattag); + +/* These are almost but but not quite coap_handler_t -- they require the + * request to be already fully read and digested into the last argument. */ + +static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, struct requestdata *request); +static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, struct requestdata *request); +/** Create a CoAP response for a given errno (eg. EACCESS -> 4.03 Forbidden + * etc., defaulting to 5.03 Internal Server Error) */ +static size_t coapfileserver_errno_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, int err); +/** Return true if path/name is a directory. + * + * Currently assembling the values in a buffer, this could be done without a + * buffer if an openat / vfs_openat was present. + * */ +static bool entry_is_dir(char *path, char *name); + +ssize_t coapfileserver_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, void *ctx) { + struct coapfileserver_entry *entry = (struct coapfileserver_entry *)ctx; + struct requestdata request; + request.etag_sent = false; + request.blocknum2 = 0; + request.szx2 = CONFIG_NANOCOAP_BLOCK_SIZE_EXP_MAX; + /** Index in request.namebuf. Must not point at the last entry as that will be + * zeroed to get a 0-terminated string. */ + size_t namelength = 0; + bool trailing_slash = false; + /** If no path component comes along at all, it'll count as a trailing + * slash no matter the trailing_slash value */ + bool any_component = false; + uint8_t errorcode = COAP_CODE_INTERNAL_SERVER_ERROR; + + uint8_t strip_remaining = entry->strip_path; + namelength += request.namebuf - strncpy(request.namebuf, entry->nameprefix, sizeof(request.namebuf) - 1); + + coap_optpos_t opt; + bool is_first = true; + uint8_t *value; + ssize_t optlen; + while ((optlen = coap_opt_get_next(pdu, &opt, &value, is_first)) != + -ENOENT) + { + if (optlen < 0) { + errorcode = COAP_CODE_BAD_REQUEST; + goto error; + } + + is_first = false; + switch (opt.opt_num) { + case COAP_OPT_URI_PATH: + if (strip_remaining != 0) { + strip_remaining -= 1; + continue; + } + if (trailing_slash) { + errorcode = COAP_CODE_BAD_REQUEST; + goto error; + } + any_component = true; + if (optlen == 0) { + trailing_slash = true; + continue; + } + if (memchr(value, '0', optlen) != NULL || + memchr(value, '/', optlen) != NULL) { + /* Path can not be expressed in the file system */ + errorcode = COAP_CODE_PATH_NOT_FOUND; + goto error; + } + size_t newlength = namelength + 1 + optlen; + if (newlength > sizeof(request.namebuf) - 1) { + /* Path too long, therefore can't exist in this mapping */ + errorcode = COAP_CODE_PATH_NOT_FOUND; + goto error; + } + request.namebuf[namelength] = '/'; + memcpy(&request.namebuf[namelength] + 1, value, optlen); + namelength = newlength; + break; + case COAP_OPT_ETAG: + if (optlen != sizeof(request.etag)) { + /* Can't be a matching tag, no use in carrying that */ + continue; + } + if (request.etag_sent) { + /* We can reasonably only check for a limited sized set, + * and it size is 1 here (sending multiple ETags is + * possible but rare) */ + continue; + } + request.etag_sent = true; + memcpy(request.etag, value, sizeof(request.etag)); + break; + case COAP_OPT_BLOCK2: + /* Could be more efficient now that we already know where it + * is, but meh */ + coap_get_blockopt(pdu, COAP_OPT_BLOCK2, &request.blocknum2, &request.szx2); + break; + default: + if (opt.opt_num & 1) { + errorcode = COAP_CODE_BAD_REQUEST; + goto error; + } else { + /* Ignoring elective option */ + } + } + } + + request.namebuf[namelength] = '\0'; + bool is_directory = trailing_slash | !any_component; + + /* Note to self: As we parse more options than just Uri-Path, we'll likely + * pass a struct pointer later. So far, those could even be hooked into the + * resource list, but that'll go away once we parse more options */ + if (is_directory) { + return coapfileserver_directory_handler(pdu, buf, len, &request); + } else { + return coapfileserver_file_handler(pdu, buf, len, &request); + } + +error: + return gcoap_response(pdu, buf, len, errorcode); +} + +static ssize_t coapfileserver_file_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, struct requestdata *request) +{ + int err; + + union stattag stattag; + err = stat_etag(request->namebuf, &stattag); + if (err < 0) + return coapfileserver_errno_handler(pdu, buf, len, err); + + if (request->etag_sent && memcmp(stattag.etag, request->etag, ETAG_LENGTH) == 0) { + gcoap_resp_init(pdu, buf, len, COAP_CODE_VALID); + coap_opt_add_opaque(pdu, COAP_OPT_ETAG, stattag.etag, ETAG_LENGTH); + return coap_opt_finish(pdu, COAP_OPT_FINISH_NONE); + } + + int fd = vfs_open(request->namebuf, O_RDONLY, 0); + if (fd < 0) + return coapfileserver_errno_handler(pdu, buf, len, fd); + + gcoap_resp_init(pdu, buf, len, COAP_CODE_CONTENT); + coap_opt_add_opaque(pdu, COAP_OPT_ETAG, stattag.etag, ETAG_LENGTH); + /* To best see how this works set CONFIG_GCAOP_PDU_BUF_SIZE to 532 or 533. + * If we did a sharper estimation (factoring in the block2 size option with + * the current blockum), we'd even pack 512 bytes into 530 until block + * numbers get large enough to eat another byte, which is when the block + * size would decrease in-flight. */ + size_t remaining_length = len - (pdu->payload - buf); + remaining_length -= 5; /* maximum block2 option usable in nanocoap */ + remaining_length -= 1; /* payload marker */ + /* > 0: To not wrap around; if that still won't fit that's later caught in + * an assertion */ + while (coap_szx2size(request->szx2) > remaining_length && request->szx2 > 0) { + request->szx2 --; + request->blocknum2 <<= 1; + } + coap_block_slicer_t slicer; + coap_block_slicer_init(&slicer, request->blocknum2, coap_szx2size(request->szx2)); + coap_opt_add_block2(pdu, &slicer, true); + size_t resp_len = coap_opt_finish(pdu, COAP_OPT_FINISH_PAYLOAD); + + err = vfs_lseek(fd, slicer.start, SEEK_SET); + if (err < 0) + goto late_err; + + /* That'd only happen if the buffer is too small for even a 16-byte block, + * or if the above calculations were wrong. + * + * Not using payload_len here as that's needlessly underestimating the + * space by CONFIG_GCOAP_RESP_OPTIONS_BUF + * */ + assert(pdu->payload + slicer.end - slicer.start <= buf + len); + int read = vfs_read(fd, pdu->payload, slicer.end - slicer.start); + if (err < 0) + goto late_err; + + uint8_t morebuf; + int more = vfs_read(fd, &morebuf, 1); + if (err < 0) + goto late_err; + + vfs_close(fd); + + slicer.cur = slicer.end + more; + coap_block2_finish(&slicer); + + if (read == 0) { + /* Rewind to clear payload marker */ + read -= 1; + } + + return resp_len + read; + +late_err: + vfs_close(fd); + coap_hdr_set_code(pdu->hdr, COAP_CODE_INTERNAL_SERVER_ERROR); + return coap_get_total_hdr_len(pdu); +} + +static ssize_t coapfileserver_directory_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, struct requestdata *request) +{ + /** + * ToDo: + * + * * Blockwise + * + */ + vfs_DIR dir; + + int err = vfs_opendir(&dir, request->namebuf); + if (err != 0) { + return coapfileserver_errno_handler(pdu, buf, len, err); + } + DEBUG("coapfileserver: Serving directory listing\n"); + + gcoap_resp_init(pdu, buf, len, COAP_CODE_CONTENT); + coap_opt_add_format(pdu, COAP_FORMAT_LINK); + size_t resp_len = coap_opt_finish(pdu, COAP_OPT_FINISH_PAYLOAD); + + vfs_dirent_t entry; + ssize_t payload_cursor = 0; + while (vfs_readdir(&dir, &entry) > 0) { + char *entry_name = (char*)entry.d_name; + /* Ignore leading slashes presented by some file systems; see + * https://github.com/RIOT-OS/RIOT/issues/14635 */ + while (entry_name[0] == '/') { + entry_name += 1; + } + size_t entry_len = strlen(entry_name); + if (entry_len <= 2 && memcmp(entry_name, "..", entry_len) == 0) { + /* Up pointers don't work the same way in URI semantics */ + continue; + } + bool is_dir = entry_is_dir(request->namebuf, entry_name); + /* maybe ",", "<>", and the length */ + ssize_t need_bytes = (payload_cursor == 0 ? 0 : 1) + 2 + entry_len + (is_dir ? 1 : 0); + if (payload_cursor + need_bytes > pdu->payload_len) { + /* Without blockwise, this is the best approximation we can do */ + DEBUG("coapfileserver: Directory listing truncated\n"); + break; + } + if (payload_cursor != 0) { + pdu->payload[payload_cursor++] = ','; + } + pdu->payload[payload_cursor++] = '<'; + memcpy(&pdu->payload[payload_cursor], entry_name, entry_len); + payload_cursor += entry_len; + if (is_dir) { + pdu->payload[payload_cursor++] = '/'; + } + pdu->payload[payload_cursor++] = '>'; + } + vfs_closedir(&dir); + + if (payload_cursor == 0) { + /* Rewind to clear payload marker */ + payload_cursor -= 1; + } + + return resp_len + payload_cursor; +} + +static size_t coapfileserver_errno_handler(coap_pkt_t *pdu, uint8_t *buf, size_t len, int err) +{ + uint8_t code; + switch (err) { + case -EACCES: + code = COAP_CODE_FORBIDDEN; break; + case -ENOENT: + code = COAP_CODE_PATH_NOT_FOUND; break; + default: + code = COAP_CODE_INTERNAL_SERVER_ERROR; + }; + DEBUG("coapfileserver: Rejecting error %d (%s) as %d.%02d\n", err, strerror(err), code >> 5, code & 0x1f); + return gcoap_response(pdu, buf, len, code); +} + +static int stat_etag(char *filename, union stattag *stattag) +{ + int err; + + memset(stattag, 0, sizeof(union stattag)); + err = vfs_stat(filename, &stattag->stat); + if (err < 0) { + /* Some file systems, like only implement fstat. FIXME: run on file + * handles all the time and only use fstat. (Not trivial because it + * needs going through all filesystems to check whether there isn't one + * that only does stat and not fstat) */ + int fd = vfs_open(filename, O_RDONLY, 0); + if (fd < 0) { + return fd; + } + err = vfs_fstat(fd, &stattag->stat); + vfs_close(fd); + + return err; + } + + /* Normalizing fields whose value can change without affecting the ETag */ + stattag->stat.st_nlink = 0; + memset(&stattag->stat.st_atim, 0, sizeof(stattag->stat.st_atim)); + + /* Build a compact ETag by XORing the stat onto itself */ + for (unsigned int i = 0; i < sizeof(stattag->etag_padding) / sizeof(stattag->etag); ++i) { + for (int j = 0; j < ETAG_LENGTH; ++j) { + stattag->etag[j] ^= stattag->etag_padding[i * ETAG_LENGTH + j]; + } + } + + return 0; +} + +static bool entry_is_dir(char *path, char *name) +{ + /* FIXME have an openat or fstatat + * + * or just build the long name outside because it's printed later, or use + * it from the printed buffer? */ + char buf[COAPFILESERVER_PATH_MAX]; + /* snprintf may be expensive, but without a strlcpy it's a pain to do + * manually */ + int printed = snprintf(buf, sizeof(buf), "%s/%s", path, name); + if (printed <= 0) { + /* Returning true on error as a slash in a file name is more likely to + * cause suspicion than the lack of one at a directory */ + return true; + } + + struct stat stat; + memset(&stat, 0, sizeof(stat)); + int err = vfs_stat(buf, &stat); + if (err != 0) { + int fd = vfs_open(buf, O_RDONLY, 0); + if (fd < 0) { + return true; + } + err = vfs_fstat(fd, &stat); + vfs_close(fd); + + if (err != 0) { + return true; + } + } + return (stat.st_mode & S_IFMT) == S_IFDIR; +} diff --git a/sys/net/application_layer/coapfileserver/doc.txt b/sys/net/application_layer/coapfileserver/doc.txt new file mode 100644 index 000000000000..e4ec679a9e75 --- /dev/null +++ b/sys/net/application_layer/coapfileserver/doc.txt @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2020 chrysn + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +/** + * @defgroup net_coapfileserver CoAP file server + * @ingroup net + * @brief Library for serving files from the VFS to CoAP clients + * + * # About + * + * This maps files in the local file system onto a resources in CoAP. In that, + * is is what is called a static web server in the unconstrained web. + * + * As usual, GET operations are used to read files. + * + * Directories are expressed to URIs with trailing slashes. + * + * @note The file server uses ETag for cache validation. The ETags are built + * from the file system stat values. As clients rely on the ETag to differ when + * the file changes, it is important that file modification times are set. The + * precise time values do not matter, but if a file is changed in place and + * neither its length nor its modification time is varied, then clients will + * not become aware of the change or may even mix up the versions half way + * through if they have a part of the old version cached. + * + * # Usage + * + * * ``USEMODULE += coapfileserver`` + * + * * Have a @ref coapfileserver_entry populated with the path you want to serve, + * and the number of path components to strip from incoming requests: + * + * ``` + * static const struct coapfileserver_entry files_sd = { + * "/sd", + * 2 + * }; + * ``` + * + * * Enter a @ref coapfileserver_handler handler into your CoAP server's + * resource list like this: + * + * ``` + * static const coap_resource_t _resources[] = { + * ..., + * { "/files/sd", COAP_GET | COAP_MATCH_SUBTREE, coapfileserver_handler, files_sd }, + * ... + * } + * ``` + * + * The allowed methods dictate whether it's read-only (``COAP_GET``) or (in the + * future) read-write (``1COAP_GET | COAP_PUT | COAP_DELETE``). + * + */