From 97b6043435883ea8fb12d94d809ff16edf5b294f Mon Sep 17 00:00:00 2001 From: "hrushikesh.bhosale" Date: Thu, 30 Jan 2025 17:18:15 +0530 Subject: [PATCH] feat(esp_http_server): Dynamically allocate http server's scratch buffer In this commit, esp_http_server's http_parser scratch is made dynamic. User is asked to give limit size for header and URI, according to which scratch buufer allocates memory upto limits --- components/esp_http_server/Kconfig | 5 +- .../esp_http_server/include/esp_http_server.h | 10 +++- .../esp_http_server/src/esp_httpd_priv.h | 11 +++-- components/esp_http_server/src/httpd_parse.c | 42 ++++++++++++----- components/esp_http_server/src/httpd_txrx.c | 47 ++++++++++++++----- .../esp_https_server/src/https_server.c | 2 +- .../protocols/https_server/simple/main/main.c | 1 - 7 files changed, 83 insertions(+), 35 deletions(-) diff --git a/components/esp_http_server/Kconfig b/components/esp_http_server/Kconfig index 5ce708b3ac..682a0d6fd6 100644 --- a/components/esp_http_server/Kconfig +++ b/components/esp_http_server/Kconfig @@ -1,15 +1,16 @@ menu "HTTP Server" + config HTTPD_MAX_REQ_HDR_LEN int "Max HTTP Request Header Length" - default 512 + default 2000 help This sets the maximum supported size of headers section in HTTP request packet to be processed by the server config HTTPD_MAX_URI_LEN int "Max HTTP URI Length" - default 512 + default 2000 help This sets the maximum supported size of HTTP request URI to be processed by the server diff --git a/components/esp_http_server/include/esp_http_server.h b/components/esp_http_server/include/esp_http_server.h index 273d7581a8..725a2ed8b9 100644 --- a/components/esp_http_server/include/esp_http_server.h +++ b/components/esp_http_server/include/esp_http_server.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -53,6 +53,8 @@ initializer that should be kept in sync #define HTTPD_DEFAULT_CONFIG() { \ .task_priority = tskIDLE_PRIORITY+5, \ .stack_size = 4096, \ + .hdr_buf_size_limit = HTTPD_MAX_REQ_HDR_LEN, \ + .uri_buf_size_limit = HTTPD_MAX_URI_LEN, \ .core_id = tskNO_AFFINITY, \ .task_caps = (MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT), \ .server_port = 80, \ @@ -173,6 +175,12 @@ typedef struct httpd_config { BaseType_t core_id; /*!< The core the HTTP server task will run on */ uint32_t task_caps; /*!< The memory capabilities to use when allocating the HTTP server task's stack */ + /** + * Size limits for the header and URI buffers respectively. These are just limits, actually allocation would be depend upon actual size of URI/header. + * These are set to the values defined in the Kconfig file. + */ + uint16_t hdr_buf_size_limit; /*!< Size limit for the header buffer */ + uint16_t uri_buf_size_limit; /*!< Size limit for the URI buffer */ /** * TCP Port number for receiving and transmitting HTTP traffic */ diff --git a/components/esp_http_server/src/esp_httpd_priv.h b/components/esp_http_server/src/esp_httpd_priv.h index e56418236e..ace6fe84fe 100644 --- a/components/esp_http_server/src/esp_httpd_priv.h +++ b/components/esp_http_server/src/esp_httpd_priv.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -35,9 +35,6 @@ extern "C" { * exceed the scratch buffer size and should at least be 8 bytes */ #define PARSER_BLOCK_SIZE 128 -/* Calculate the maximum size needed for the scratch buffer */ -#define HTTPD_SCRATCH_BUF MAX(HTTPD_MAX_REQ_HDR_LEN, HTTPD_MAX_URI_LEN) - /* Formats a log string to prepend context function name */ #define LOG_FMT(x) "%s: " x, __func__ @@ -88,7 +85,11 @@ struct sock_db { */ struct httpd_req_aux { struct sock_db *sd; /*!< Pointer to socket database */ - char scratch[HTTPD_SCRATCH_BUF + 1]; /*!< Temporary buffer for our operations (1 byte extra for null termination) */ + char *scratch; /*!< Temporary buffer for our operations (1 byte extra for null termination) */ + uint16_t scratch_size_limit; /*!< Scratch buffer size limit */ + uint16_t scratch_cur_size; /*!< Scratch buffer cur size */ + uint16_t hdr_buf_size_limit; /*!< Header buffer size limit */ + uint16_t uri_buf_size_limit; /*!< URI buffer size limit */ size_t remaining_len; /*!< Amount of data remaining to be fetched */ char *status; /*!< HTTP response's status code */ char *content_type; /*!< HTTP response's content type */ diff --git a/components/esp_http_server/src/httpd_parse.c b/components/esp_http_server/src/httpd_parse.c index b520c871b1..d969a417d4 100644 --- a/components/esp_http_server/src/httpd_parse.c +++ b/components/esp_http_server/src/httpd_parse.c @@ -112,7 +112,8 @@ static esp_err_t cb_url(http_parser *parser, const char *at, size_t length) { parser_data_t *parser_data = (parser_data_t *) parser->data; - + httpd_req_t *req = parser_data->req; + struct httpd_req_aux *raux = req->aux; if (parser_data->status == PARSING_IDLE) { ESP_LOGD(TAG, LOG_FMT("message begin")); @@ -130,9 +131,9 @@ static esp_err_t cb_url(http_parser *parser, ESP_LOGD(TAG, LOG_FMT("processing url = %.*s"), (int)length, at); /* Update length of URL string */ - if ((parser_data->last.length += length) > HTTPD_MAX_URI_LEN) { + if ((parser_data->last.length += length) > raux->uri_buf_size_limit) { ESP_LOGW(TAG, LOG_FMT("URI length (%"NEWLIB_NANO_COMPAT_FORMAT") greater than supported (%d)"), - NEWLIB_NANO_COMPAT_CAST(parser_data->last.length), HTTPD_MAX_URI_LEN); + NEWLIB_NANO_COMPAT_CAST(parser_data->last.length), raux->uri_buf_size_limit); parser_data->error = HTTPD_414_URI_TOO_LONG; parser_data->status = PARSING_FAILED; return ESP_FAIL; @@ -215,7 +216,7 @@ static esp_err_t cb_header_field(http_parser *parser, const char *at, size_t len parser_data->last.at = ra->scratch; parser_data->last.length = 0; parser_data->status = PARSING_HDR_FIELD; - + ra->scratch_size_limit = ra->hdr_buf_size_limit; /* Stop parsing for now and give control to process */ if (pause_parsing(parser, at) != ESP_OK) { parser_data->error = HTTPD_500_INTERNAL_SERVER_ERROR; @@ -232,6 +233,7 @@ static esp_err_t cb_header_field(http_parser *parser, const char *at, size_t len parser_data->last.at = at; parser_data->last.length = 0; parser_data->status = PARSING_HDR_FIELD; + ra->scratch_size_limit = ra->hdr_buf_size_limit; /* Increment header count */ ra->req_hdrs_count++; @@ -413,7 +415,6 @@ static esp_err_t cb_headers_complete(http_parser *parser) static esp_err_t cb_on_body(http_parser *parser, const char *at, size_t length) { parser_data_t *parser_data = (parser_data_t *) parser->data; - /* Check previous status */ if (parser_data->status != PARSING_BODY) { ESP_LOGE(TAG, LOG_FMT("unexpected state transition")); @@ -445,7 +446,6 @@ static esp_err_t cb_on_body(http_parser *parser, const char *at, size_t length) static esp_err_t cb_no_body(http_parser *parser) { parser_data_t *parser_data = (parser_data_t *) parser->data; - /* Check previous status */ if (parser_data->status == PARSING_URL) { ESP_LOGD(TAG, LOG_FMT("no headers")); @@ -485,11 +485,21 @@ static int read_block(httpd_req_t *req, size_t offset, size_t length) struct httpd_req_aux *raux = req->aux; /* Limits the read to scratch buffer size */ - ssize_t buf_len = MIN(length, (sizeof(raux->scratch) - offset)); + ssize_t buf_len = MIN(length, (raux->scratch_size_limit - offset)); if (buf_len <= 0) { return 0; } - + if (raux->scratch == NULL && buf_len < raux->scratch_size_limit) { + raux->scratch = (char*) malloc(buf_len); + } + else if (raux->scratch != NULL && buf_len < raux->scratch_size_limit) { + raux->scratch = (char*) realloc(raux->scratch, raux->scratch_cur_size + buf_len); + } + if (raux->scratch == NULL) { + return 0; + } + raux->scratch_cur_size += buf_len; + ESP_LOGD(TAG, "scratch size = %d", raux->scratch_cur_size); /* Receive data into buffer. If data is pending (from unrecv) then return * immediately after receiving pending data, as pending data may just complete * this request packet. */ @@ -527,19 +537,20 @@ static int parse_block(http_parser *parser, size_t offset, size_t length) httpd_req_t *req = data->req; struct httpd_req_aux *raux = req->aux; size_t nparsed = 0; - + data->last.at = raux->scratch; if (!length) { /* Parsing is still happening but nothing to * parse means no more space left on buffer, * therefore it can be inferred that the * request URI/header must be too long */ - ESP_LOGW(TAG, LOG_FMT("request URI/header too long")); switch (data->status) { case PARSING_URL: + ESP_LOGW(TAG, LOG_FMT("request URI too long")); data->error = HTTPD_414_URI_TOO_LONG; break; case PARSING_HDR_FIELD: case PARSING_HDR_VALUE: + ESP_LOGW(TAG, LOG_FMT("request header too long")); data->error = HTTPD_431_REQ_HDR_FIELDS_TOO_LARGE; break; default: @@ -649,7 +660,6 @@ static esp_err_t httpd_parse_req(struct httpd_data *hd) /* This is used by the callbacks to track * data usage of the buffer */ parser_data.raw_datalen = blk_len + offset; - /* Parse data block from buffer */ if ((offset = parse_block(&parser, offset, blk_len)) < 0) { /* HTTP error occurred. @@ -679,13 +689,17 @@ static void init_req(httpd_req_t *r, httpd_config_t *config) static void init_req_aux(struct httpd_req_aux *ra, httpd_config_t *config) { ra->sd = 0; - memset(ra->scratch, 0, sizeof(ra->scratch)); ra->remaining_len = 0; ra->status = 0; ra->content_type = 0; ra->first_chunk_sent = 0; ra->req_hdrs_count = 0; ra->resp_hdrs_count = 0; + ra->scratch = NULL; + ra->scratch_cur_size = 0; + ra->hdr_buf_size_limit = config->hdr_buf_size_limit; + ra->uri_buf_size_limit = config->uri_buf_size_limit; + ra->scratch_size_limit = ra->uri_buf_size_limit; #if CONFIG_HTTPD_WS_SUPPORT ra->ws_handshake_detect = false; #endif @@ -716,6 +730,10 @@ static void httpd_req_cleanup(httpd_req_t *r) /* Clear out the request and request_aux structures */ ra->sd = NULL; + free(ra->scratch); + ra->scratch = NULL; + ra->scratch_cur_size = 0; + ra->scratch_size_limit = 0; r->handle = NULL; r->aux = NULL; r->user_ctx = NULL; diff --git a/components/esp_http_server/src/httpd_txrx.c b/components/esp_http_server/src/httpd_txrx.c index fa50378f69..35bb4b73c8 100644 --- a/components/esp_http_server/src/httpd_txrx.c +++ b/components/esp_http_server/src/httpd_txrx.c @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -132,7 +132,6 @@ int httpd_recv_with_opt(httpd_req_t *r, char *buf, size_t buf_len, bool halt_aft } return ret; } - ESP_LOGD(TAG, LOG_FMT("received length = %"NEWLIB_NANO_COMPAT_FORMAT), NEWLIB_NANO_COMPAT_CAST((ret + pending_len))); return ret + pending_len; } @@ -243,16 +242,27 @@ esp_err_t httpd_resp_send(httpd_req_t *r, const char *buf, ssize_t buf_len) if (buf_len == HTTPD_RESP_USE_STRLEN) { buf_len = strlen(buf); } - + size_t required_size = snprintf(NULL, 0, httpd_hdr_str, ra->status, ra->content_type, buf_len) + 1; + if (required_size >= HTTPD_MAX_REQ_HDR_LEN) { + return ESP_ERR_HTTPD_RESP_HDR; + } + if (ra->scratch == NULL) + ra->scratch = malloc(required_size); + else { + ra->scratch = realloc(ra->scratch, required_size); + } + if (ra->scratch == NULL) { + ESP_LOGE(TAG, "Unable to allocate scratch buffer"); + return ESP_ERR_HTTPD_ALLOC_MEM; + } + ra->scratch_cur_size = required_size; + ESP_LOGD(TAG, "scratch size = %d", ra->scratch_cur_size); /* Request headers are no longer available */ ra->req_hdrs_count = 0; /* Size of essential headers is limited by scratch buffer size */ - if (snprintf(ra->scratch, sizeof(ra->scratch), httpd_hdr_str, - ra->status, ra->content_type, buf_len) >= sizeof(ra->scratch)) { - return ESP_ERR_HTTPD_RESP_HDR; - } - + snprintf(ra->scratch, ra->scratch_cur_size, httpd_hdr_str, ra->status, ra->content_type, buf_len); + ESP_LOG_BUFFER_HEXDUMP(TAG, ra->scratch, strlen(ra->scratch), ESP_LOG_INFO); /* Sending essential headers */ if (httpd_send_all(r, ra->scratch, strlen(ra->scratch)) != ESP_OK) { return ESP_ERR_HTTPD_RESP_SEND; @@ -319,13 +329,24 @@ esp_err_t httpd_resp_send_chunk(httpd_req_t *r, const char *buf, ssize_t buf_len /* Request headers are no longer available */ ra->req_hdrs_count = 0; - + size_t required_size = snprintf(NULL, 0, httpd_chunked_hdr_str, ra->status, ra->content_type) + 1; + if (required_size >= HTTPD_MAX_REQ_HDR_LEN) { + return ESP_ERR_HTTPD_RESP_HDR; + } + if (ra->scratch == NULL) + ra->scratch = malloc(required_size); + else { + ra->scratch = realloc(ra->scratch, required_size); + } + if (ra->scratch == NULL) { + ESP_LOGE(TAG, "Unable to allocate scratch buffer"); + return ESP_ERR_HTTPD_ALLOC_MEM; + } + ra->scratch_cur_size = required_size; + ESP_LOGD(TAG, "scratch size = %d", ra->scratch_cur_size); if (!ra->first_chunk_sent) { /* Size of essential headers is limited by scratch buffer size */ - if (snprintf(ra->scratch, sizeof(ra->scratch), httpd_chunked_hdr_str, - ra->status, ra->content_type) >= sizeof(ra->scratch)) { - return ESP_ERR_HTTPD_RESP_HDR; - } + snprintf(ra->scratch, ra->scratch_cur_size, httpd_chunked_hdr_str, ra->status, ra->content_type); /* Sending essential headers */ if (httpd_send_all(r, ra->scratch, strlen(ra->scratch)) != ESP_OK) { diff --git a/components/esp_https_server/src/https_server.c b/components/esp_https_server/src/https_server.c index ed3d432a94..0ef519c2e4 100644 --- a/components/esp_https_server/src/https_server.c +++ b/components/esp_https_server/src/https_server.c @@ -377,7 +377,7 @@ exit: } /** Start the server */ -esp_err_t httpd_ssl_start(httpd_handle_t *pHandle, struct httpd_ssl_config *config) +esp_err_t httpd_ssl_start(httpd_handle_t *pHandle, struct httpd_ssl_config *config) { assert(config != NULL); assert(pHandle != NULL); diff --git a/examples/protocols/https_server/simple/main/main.c b/examples/protocols/https_server/simple/main/main.c index 7c58891e9a..d6300c5af5 100644 --- a/examples/protocols/https_server/simple/main/main.c +++ b/examples/protocols/https_server/simple/main/main.c @@ -44,7 +44,6 @@ static esp_err_t root_get_handler(httpd_req_t *req) { httpd_resp_set_type(req, "text/html"); httpd_resp_send(req, "

Hello Secure World!

", HTTPD_RESP_USE_STRLEN); - return ESP_OK; }