Bug#989851: pre-approval unblock: uwsgi/2.0.19.1-8
Package: release.debian.org
Severity: normal
User: release.debian.org@packages.debian.org
Usertags: unblock
Dear release team,
The Glance and the Swift packages both use, in some cases, the HTTP
transfer mode:
Transfer-Encoding: chunked
In this mode, no Content-Lenght: header is set, and the client is sending
the data using chunked.
As a result, both Swift and Glance are only partially working. For swift,
the issue happens mostly when someone sends large documents. Upload is
interrupted and breaks. For Glance, it happens when doing a command like
this one:
openstack server image create --name my-name <ID-OF-VIRTUAL-MACHINE>
Unfortunately, the Uwsgi version 2.0.19, as in Bullseye and Sid, does not
have the feature. The patch was written 4 years ago, but upstream didn't
release a new upstream release containing the patch.
Therefore, I have backported the patch to the version 2.0.19.1 as in
Bullseye (it applied cleanly with only a few fuzz hunks). Please find the
attached debdiff containing the patch I'd like to add.
As this only adds the specific feature, and that it must be activated
using the directive:
wsgi-manage-chunked-input = true
(or with --wsgi-manage-chunked-input on the command line), then I don't
think there's any regression risk for Bullseye. However, this repairs
Glance and Swift, as soon as we add the directive, which Glance and
Swift already have (there will be no need to re-upload Glance or Swift).
Note that I already tested all of this in production, and in fact, I
just would like to apply what I've done in production to what's in Debian
official (rather than having to provide a modified uwsgi in an unofficial
repository).
[ Checklist ]
[x] all changes are documented in the d/changelog
[x] I reviewed all changes and I approve them
[x] attach debdiff against the package in testing
Please allow me to unblock uwsgi/2.0.19.1-8 with the attached patch,
targetting Bullseye,
Cheers,
Thomas Goirand (zigo)
>From 722db2ea22eb454ed678bd6ff8b1c2f287df4802 Mon Sep 17 00:00:00 2001
From: Thomas Goirand <zigo@debian.org>
Date: Fri, 11 Jun 2021 11:09:19 +0200
Subject: [PATCH] * Add upstream patch to support Transfer-Encoding:
chuncked, necessary for OpenStack Glance and Swift over uwsgi.
---
debian/changelog | 9 +
.../Add_support_for_chunked_encoding.patch | 241 ++++++++++++++++++
debian/patches/series | 1 +
3 files changed, 251 insertions(+)
create mode 100644 debian/patches/Add_support_for_chunked_encoding.patch
diff --git a/debian/changelog b/debian/changelog
index 70389417..b0372124 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,12 @@
+uwsgi (2.0.19.1-7.1) UNRELEASED; urgency=medium
+
+ [ Thomas Goirand ]
+ * Non-maintainer upload.
+ * Add upstream patch to support Transfer-Encoding: chuncked, necessary for
+ OpenStack Glance and Swift over uwsgi.
+
+ -- Thomas Goirand <zigo@debian.org> Fri, 11 Jun 2021 11:08:33 +0200
+
uwsgi (2.0.19.1-7) unstable; urgency=medium
* add patch cherry-picked upstream
diff --git a/debian/patches/Add_support_for_chunked_encoding.patch b/debian/patches/Add_support_for_chunked_encoding.patch
new file mode 100644
index 00000000..496a1e75
--- /dev/null
+++ b/debian/patches/Add_support_for_chunked_encoding.patch
@@ -0,0 +1,241 @@
+Subject: preliminary implementation of #1428
+ This implements support for transfer-encoding: chuncked
+Author: Unbit <info@unbit.it>
+Date: Thu, 9 Nov 2017 16:40:44 +0100
+Last-Update: 2021-06-11
+
+Index: uwsgi/core/chunked.c
+===================================================================
+--- uwsgi.orig/core/chunked.c
++++ uwsgi/core/chunked.c
+@@ -83,6 +83,83 @@ static ssize_t uwsgi_chunked_readline(st
+
+ */
+
++struct uwsgi_buffer *uwsgi_chunked_read_smart(struct wsgi_request *wsgi_req, size_t len, int timeout) {
++ // check for buffer
++ if (!wsgi_req->body_chunked_buf)
++ wsgi_req->body_chunked_buf = uwsgi_buffer_new(uwsgi.page_size);
++ // first case: asking for all
++ if (!len) {
++ for(;;) {
++ size_t chunked_len = 0;
++ char *buf = uwsgi_chunked_read(wsgi_req, &chunked_len, timeout, 0);
++ if (chunked_len == 0) {
++ struct uwsgi_buffer *ret = uwsgi_buffer_new(wsgi_req->body_chunked_buf->pos);
++ if (uwsgi_buffer_append(ret, wsgi_req->body_chunked_buf->buf, wsgi_req->body_chunked_buf->pos)) {
++ uwsgi_buffer_destroy(ret);
++ return NULL;
++ }
++ uwsgi_buffer_decapitate(wsgi_req->body_chunked_buf, wsgi_req->body_chunked_buf->pos);
++ return ret;
++ }
++ if (uwsgi_buffer_append(wsgi_req->body_chunked_buf, buf, chunked_len)) {
++ return NULL;
++ }
++ }
++ }
++
++ // asking for littler part
++ if (len <= wsgi_req->body_chunked_buf->pos) {
++ struct uwsgi_buffer *ret = uwsgi_buffer_new(len);
++ if (uwsgi_buffer_append(ret, wsgi_req->body_chunked_buf->buf, len)) {
++ uwsgi_buffer_destroy(ret);
++ return NULL;
++ }
++ uwsgi_buffer_decapitate(wsgi_req->body_chunked_buf, len);
++ return ret;
++ }
++
++ // more data required
++ size_t remains = len;
++ struct uwsgi_buffer *ret = uwsgi_buffer_new(remains);
++ if (wsgi_req->body_chunked_buf->pos > 0) {
++ if (uwsgi_buffer_append(ret, wsgi_req->body_chunked_buf->buf, wsgi_req->body_chunked_buf->pos)) {
++ uwsgi_buffer_destroy(ret);
++ return NULL;
++ }
++ remains -= wsgi_req->body_chunked_buf->pos;
++ uwsgi_buffer_decapitate(wsgi_req->body_chunked_buf, wsgi_req->body_chunked_buf->pos);
++ }
++
++ while(remains) {
++ size_t chunked_len = 0;
++ char *buf = uwsgi_chunked_read(wsgi_req, &chunked_len, timeout, 0);
++ if (chunked_len == 0) {
++ break;
++ }
++ if (uwsgi_buffer_append(wsgi_req->body_chunked_buf, buf, chunked_len)) {
++ uwsgi_buffer_destroy(ret);
++ return NULL;
++ }
++
++ if (chunked_len > remains) {
++ if (uwsgi_buffer_append(ret, wsgi_req->body_chunked_buf->buf, wsgi_req->body_chunked_buf->pos - (chunked_len - remains))) {
++ uwsgi_buffer_destroy(ret);
++ return NULL;
++ }
++ uwsgi_buffer_decapitate(wsgi_req->body_chunked_buf, wsgi_req->body_chunked_buf->pos - (chunked_len - remains));
++ return ret;
++ }
++ remains -= chunked_len;
++ }
++
++ if (uwsgi_buffer_append(ret, wsgi_req->body_chunked_buf->buf, wsgi_req->body_chunked_buf->pos)) {
++ uwsgi_buffer_destroy(ret);
++ return NULL;
++ }
++ uwsgi_buffer_decapitate(wsgi_req->body_chunked_buf, wsgi_req->body_chunked_buf->pos);
++ return ret;
++}
++
+ char *uwsgi_chunked_read(struct wsgi_request *wsgi_req, size_t *len, int timeout, int nb) {
+
+ if (!wsgi_req->chunked_input_buf) {
+Index: uwsgi/core/protocol.c
+===================================================================
+--- uwsgi.orig/core/protocol.c
++++ uwsgi/core/protocol.c
+@@ -560,6 +560,12 @@ static int uwsgi_proto_check_22(struct w
+ wsgi_req->scheme_len = len;
+ }
+
++ if (!uwsgi_proto_key("HTTP_TRANSFER_ENCODING", 22)) {
++ if (!uwsgi_strnicmp(buf, len, "chunked", 7)) {
++ wsgi_req->body_is_chunked = 1;
++ }
++ }
++
+ return 0;
+ }
+
+Index: uwsgi/core/utils.c
+===================================================================
+--- uwsgi.orig/core/utils.c
++++ uwsgi/core/utils.c
+@@ -1182,6 +1182,10 @@ void uwsgi_close_request(struct wsgi_req
+ uwsgi_buffer_destroy(wsgi_req->chunked_input_buf);
+ }
+
++ if (wsgi_req->body_chunked_buf) {
++ uwsgi_buffer_destroy(wsgi_req->body_chunked_buf);
++ }
++
+ // free websocket engine
+ if (wsgi_req->websocket_buf) {
+ uwsgi_buffer_destroy(wsgi_req->websocket_buf);
+Index: uwsgi/plugins/python/python_plugin.c
+===================================================================
+--- uwsgi.orig/plugins/python/python_plugin.c
++++ uwsgi/plugins/python/python_plugin.c
+@@ -174,6 +174,8 @@ struct uwsgi_option uwsgi_python_options
+
+ {"python-worker-override", required_argument, 0, "override worker with the specified python script", uwsgi_opt_set_str, &up.worker_override, 0},
+
++ {"wsgi-manage-chunked-input", no_argument, 0, "manage chunked input via the wsgi.input_terminated extension", uwsgi_opt_true, &up.wsgi_manage_chunked_input, 0},
++
+ {0, 0, 0, 0, 0, 0, 0},
+ };
+
+Index: uwsgi/plugins/python/uwsgi_python.h
+===================================================================
+--- uwsgi.orig/plugins/python/uwsgi_python.h
++++ uwsgi/plugins/python/uwsgi_python.h
+@@ -214,6 +214,8 @@ struct uwsgi_python {
+ int wsgi_disable_file_wrapper;
+
+ char *worker_override;
++
++ int wsgi_manage_chunked_input;
+ };
+
+
+Index: uwsgi/plugins/python/wsgi_handlers.c
+===================================================================
+--- uwsgi.orig/plugins/python/wsgi_handlers.c
++++ uwsgi/plugins/python/wsgi_handlers.c
+@@ -62,7 +62,20 @@ static PyObject *uwsgi_Input_read(uwsgi_
+ ssize_t rlen = 0;
+
+ UWSGI_RELEASE_GIL
+- char *buf = uwsgi_request_body_read(wsgi_req, arg_len, &rlen);
++ char *buf = NULL;
++ if (wsgi_req->body_is_chunked && up.wsgi_manage_chunked_input) {
++ struct uwsgi_buffer *ubuf = uwsgi_chunked_read_smart(wsgi_req, arg_len, uwsgi.socket_timeout);
++ UWSGI_GET_GIL
++ if (!ubuf) {
++ return PyErr_Format(PyExc_IOError, "error during chunked read(%ld) on wsgi.input", arg_len);
++ }
++ PyObject *ret = PyString_FromStringAndSize(ubuf->buf, ubuf->pos);
++ uwsgi_buffer_destroy(ubuf);
++ return ret;
++ }
++ else {
++ buf = uwsgi_request_body_read(wsgi_req, arg_len, &rlen);
++ }
+ UWSGI_GET_GIL
+ if (buf == uwsgi.empty) {
+ return PyString_FromString("");
+Index: uwsgi/plugins/python/wsgi_subhandler.c
+===================================================================
+--- uwsgi.orig/plugins/python/wsgi_subhandler.c
++++ uwsgi/plugins/python/wsgi_subhandler.c
+@@ -179,6 +179,15 @@ void *uwsgi_request_subhandler_wsgi(stru
+
+ PyDict_SetItemString(wsgi_req->async_environ, "wsgi.input", wsgi_req->async_input);
+
++ if (up.wsgi_manage_chunked_input) {
++ if (wsgi_req->body_is_chunked) {
++ PyDict_SetItemString(wsgi_req->async_environ, "wsgi.input_terminated", Py_True);
++ }
++ else {
++ PyDict_SetItemString(wsgi_req->async_environ, "wsgi.input_terminated", Py_False);
++ }
++ }
++
+ if (!up.wsgi_disable_file_wrapper)
+ PyDict_SetItemString(wsgi_req->async_environ, "wsgi.file_wrapper", wi->sendfile);
+
+Index: uwsgi/t/python/wsgi_chunked.py
+===================================================================
+--- /dev/null
++++ uwsgi/t/python/wsgi_chunked.py
+@@ -0,0 +1,16 @@
++def application(environ, start_response):
++ print(environ)
++ start_response('200 OK', [])
++ if not environ['wsgi.input_terminated']:
++ return []
++# print(environ['wsgi.input'].read())
++ data = environ['wsgi.input'].read(2)
++ print(data)
++ data = environ['wsgi.input'].read(2)
++ print(data)
++ data = environ['wsgi.input'].read(2)
++ print(data)
++ data = environ['wsgi.input'].read(6)
++ print(data)
++ print(environ['wsgi.input'].read())
++ return [data]
+Index: uwsgi/uwsgi.h
+===================================================================
+--- uwsgi.orig/uwsgi.h
++++ uwsgi/uwsgi.h
+@@ -1617,6 +1617,9 @@ struct wsgi_request {
+ // used for protocol parsers requiring EOF signaling
+ int proto_parser_eof;
+
++ int body_is_chunked;
++ struct uwsgi_buffer *body_chunked_buf;
++
+ // 64bit range, deprecates size_t __range_from, __range_to
+ enum uwsgi_range range_parsed;
+ int64_t range_from;
+@@ -4506,6 +4509,7 @@ struct uwsgi_buffer *uwsgi_websocket_rec
+ struct uwsgi_buffer *uwsgi_websocket_recv_nb(struct wsgi_request *);
+
+ char *uwsgi_chunked_read(struct wsgi_request *, size_t *, int, int);
++struct uwsgi_buffer *uwsgi_chunked_read_smart(struct wsgi_request *, size_t, int);
+
+ uint16_t uwsgi_be16(char *);
+ uint32_t uwsgi_be32(char *);
diff --git a/debian/patches/series b/debian/patches/series
index b8c2938a..82ecc1f0 100644
--- a/debian/patches/series
+++ b/debian/patches/series
@@ -20,3 +20,4 @@
1016_readline.patch
1017_python3-compat.patch
2001_ensure_verbose_build.patch
+Add_support_for_chunked_encoding.patch
--
2.30.2
Reply to: