blackbeard420/kore
1
0
Fork 0
mirror of https://github.com/jorisvink/kore synced 2025-03-08 19:19:02 -05:00
Code Issues Projects Releases Wiki Activity

Python: Several fixes for our async curl support.

Browse Source 
- Fix the curl-extract-opt.sh generation script to work on newer
  curl releases as the header changed slightly.
- Use the correct handles when calling curl_easy_setopt() inside
  of our setopt functions exported via Python.
- Add a curl.setbody() method, allowing a body to be sent to be set.
  (eg when sending mail via SMTP).
- Regen of our python_curlopt.h from 7.71.1
This commit is contained in:
Joris Vink 2020-07-02 08:41:17 +02:00
parent 122a86013b
commit e38c6e5d30
5 changed files with 172 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -57,7 +57,7 @@ Requirements
(note: libressl 3.0.0+ works as a replacement)
Requirement for asynchronous curl (optional)
* libcurl
* libcurl (7.64.0 or higher)
Requirements for background tasks (optional)
* pthreads

15
include/kore/python_methods.h
View File

@ -795,10 +795,17 @@ static PyTypeObject pyhttp_file_type = {
#define CURL_CLIENT_OP_RUN 1
#define CURL_CLIENT_OP_RESULT 2
struct pycurl_slist {
struct curl_slist *slist;
LIST_ENTRY(pycurl_slist) list;
};
struct pycurl_handle {
PyObject_HEAD
char *url;
struct kore_curl curl;
struct kore_curl curl;
char *url;
struct kore_buf *body;
LIST_HEAD(, pycurl_slist) slists;
};
struct pycurl_handle_op {
@ -816,15 +823,19 @@ static void pycurl_handle_op_dealloc(struct pycurl_handle_op *);
static PyObject *pycurl_handle_run(struct pycurl_handle *, PyObject *);
static PyObject *pycurl_handle_setopt(struct pycurl_handle *, PyObject *);
static PyObject *pycurl_handle_setbody(struct pycurl_handle *, PyObject *);
static PyObject *pycurl_handle_setopt_string(struct pycurl_handle *,
int, PyObject *);
static PyObject *pycurl_handle_setopt_long(struct pycurl_handle *,
int, PyObject *);
static PyObject *pycurl_handle_setopt_slist(struct pycurl_handle *,
int, PyObject *);
static PyMethodDef pycurl_handle_methods[] = {
METHOD("run", pycurl_handle_run, METH_VARARGS),
METHOD("setopt", pycurl_handle_setopt, METH_VARARGS),
METHOD("setbody", pycurl_handle_setbody, METH_VARARGS),
METHOD(NULL, NULL, -1)
};

17
misc/curl-extract-opt.sh
View File

@ -32,15 +32,16 @@ struct {
} py_curlopt[] = {
__EOF
egrep "^.*CINIT\(.*\),$" "$1/curl.h" | \
egrep "^.*CURLOPT\(.*\),$" "$1/curl.h" | \
cut -d'(' -f 2 | cut -d ')' -f 1 | sed 's/,/ /g' | \
sed 's/OBJECTPOINT/NULL/g' | \
sed 's/STRINGPOINT/pycurl_handle_setopt_string/g' | \
sed 's/LONG/pycurl_handle_setopt_long/g' | \
sed 's/SLISTPOINT/NULL/g' | \
sed 's/FUNCTIONPOINT/NULL/g' | \
sed 's/OFF_T/NULL/g' | \
awk '{ printf "\t{ \"CURLOPT_%s\", %s, %s },\n", $1, $3, $2 } '
sed 's/CURLOPTTYPE_OBJECTPOINT/NULL/g' | \
sed 's/CURLOPTTYPE_STRINGPOINT/pycurl_handle_setopt_string/g' | \
sed 's/CURLOPTTYPE_LONG/pycurl_handle_setopt_long/g' | \
sed 's/CURLOPTTYPE_SLISTPOINT/pycurl_handle_setopt_slist/g' | \
sed 's/CURLOPTTYPE_FUNCTIONPOINT/NULL/g' | \
sed 's/CURLOPTTYPE_OFF_T/NULL/g' | \
sed 's/CURLOPTTYPE_BLOB/NULL/g' | \
awk '{ printf "\t{ \"%s\", %s, %s },\n", $1, $3, $2 } '
echo "\t{ NULL, 0, 0 }"
echo "};"

48
misc/curl/python_curlopt.h
View File

@ -1,4 +1,4 @@
/* Auto generated on Sat Jan 18 19:42:48 CET 2020 from 7.64.1 */
/* Auto generated on Wed Jul 1 14:38:36 CEST 2020 from 7.71.1 */
struct {
const char *name;
@ -26,12 +26,12 @@ struct {
{ "CURLOPT_LOW_SPEED_TIME", 20, pycurl_handle_setopt_long },
{ "CURLOPT_RESUME_FROM", 21, pycurl_handle_setopt_long },
{ "CURLOPT_COOKIE", 22, pycurl_handle_setopt_string },
{ "CURLOPT_HTTPHEADER", 23, NULL },
{ "CURLOPT_HTTPHEADER", 23, pycurl_handle_setopt_slist },
{ "CURLOPT_HTTPPOST", 24, NULL },
{ "CURLOPT_SSLCERT", 25, pycurl_handle_setopt_string },
{ "CURLOPT_KEYPASSWD", 26, pycurl_handle_setopt_string },
{ "CURLOPT_CRLF", 27, pycurl_handle_setopt_long },
{ "CURLOPT_QUOTE", 28, NULL },
{ "CURLOPT_QUOTE", 28, pycurl_handle_setopt_slist },
{ "CURLOPT_HEADERDATA", 29, NULL },
{ "CURLOPT_COOKIEFILE", 31, pycurl_handle_setopt_string },
{ "CURLOPT_SSLVERSION", 32, pycurl_handle_setopt_long },
@ -39,8 +39,21 @@ struct {
{ "CURLOPT_TIMEVALUE", 34, pycurl_handle_setopt_long },
{ "CURLOPT_CUSTOMREQUEST", 36, pycurl_handle_setopt_string },
{ "CURLOPT_STDERR", 37, NULL },
{ "CURLOPT_POSTQUOTE", 39, NULL },
{ "CURLOPT_POSTQUOTE", 39, pycurl_handle_setopt_slist },
{ "CURLOPT_OBSOLETE40", 40, NULL },
{ "CURLOPT_VERBOSE", 41, pycurl_handle_setopt_long },
{ "CURLOPT_HEADER", 42, pycurl_handle_setopt_long },
{ "CURLOPT_NOPROGRESS", 43, pycurl_handle_setopt_long },
{ "CURLOPT_NOBODY", 44, pycurl_handle_setopt_long },
{ "CURLOPT_FAILONERROR", 45, pycurl_handle_setopt_long },
{ "CURLOPT_UPLOAD", 46, pycurl_handle_setopt_long },
{ "CURLOPT_POST", 47, pycurl_handle_setopt_long },
{ "CURLOPT_DIRLISTONLY", 48, pycurl_handle_setopt_long },
{ "CURLOPT_APPEND", 50, pycurl_handle_setopt_long },
{ "CURLOPT_NETRC", 51, pycurl_handle_setopt_long },
{ "CURLOPT_FOLLOWLOCATION", 52, pycurl_handle_setopt_long },
{ "CURLOPT_TRANSFERTEXT", 53, pycurl_handle_setopt_long },
{ "CURLOPT_PUT", 54, pycurl_handle_setopt_long },
{ "CURLOPT_PROGRESSFUNCTION", 56, NULL },
{ "CURLOPT_PROGRESSDATA", 57, NULL },
{ "CURLOPT_AUTOREFERER", 58, pycurl_handle_setopt_long },
@ -53,8 +66,9 @@ struct {
{ "CURLOPT_CAINFO", 65, pycurl_handle_setopt_string },
{ "CURLOPT_MAXREDIRS", 68, pycurl_handle_setopt_long },
{ "CURLOPT_FILETIME", 69, pycurl_handle_setopt_long },
{ "CURLOPT_TELNETOPTIONS", 70, NULL },
{ "CURLOPT_TELNETOPTIONS", 70, pycurl_handle_setopt_slist },
{ "CURLOPT_MAXCONNECTS", 71, pycurl_handle_setopt_long },
{ "CURLOPT_OBSOLETE72", 72, pycurl_handle_setopt_long },
{ "CURLOPT_FRESH_CONNECT", 74, pycurl_handle_setopt_long },
{ "CURLOPT_FORBID_REUSE", 75, pycurl_handle_setopt_long },
{ "CURLOPT_RANDOM_FILE", 76, pycurl_handle_setopt_string },
@ -72,8 +86,9 @@ struct {
{ "CURLOPT_SSLKEYTYPE", 88, pycurl_handle_setopt_string },
{ "CURLOPT_SSLENGINE", 89, pycurl_handle_setopt_string },
{ "CURLOPT_SSLENGINE_DEFAULT", 90, pycurl_handle_setopt_long },
{ "CURLOPT_DNS_USE_GLOBAL_CACHE", 91, pycurl_handle_setopt_long },
{ "CURLOPT_DNS_CACHE_TIMEOUT", 92, pycurl_handle_setopt_long },
{ "CURLOPT_PREQUOTE", 93, NULL },
{ "CURLOPT_PREQUOTE", 93, pycurl_handle_setopt_slist },
{ "CURLOPT_DEBUGFUNCTION", 94, NULL },
{ "CURLOPT_DEBUGDATA", 95, NULL },
{ "CURLOPT_COOKIESESSION", 96, pycurl_handle_setopt_long },
@ -84,7 +99,7 @@ struct {
{ "CURLOPT_PROXYTYPE", 101, pycurl_handle_setopt_long },
{ "CURLOPT_ACCEPT_ENCODING", 102, pycurl_handle_setopt_string },
{ "CURLOPT_PRIVATE", 103, NULL },
{ "CURLOPT_HTTP200ALIASES", 104, NULL },
{ "CURLOPT_HTTP200ALIASES", 104, pycurl_handle_setopt_slist },
{ "CURLOPT_UNRESTRICTED_AUTH", 105, pycurl_handle_setopt_long },
{ "CURLOPT_FTP_USE_EPRT", 106, pycurl_handle_setopt_long },
{ "CURLOPT_HTTPAUTH", 107, pycurl_handle_setopt_long },
@ -150,6 +165,7 @@ struct {
{ "CURLOPT_PROXYPASSWORD", 176, pycurl_handle_setopt_string },
{ "CURLOPT_NOPROXY", 177, pycurl_handle_setopt_string },
{ "CURLOPT_TFTP_BLKSIZE", 178, pycurl_handle_setopt_long },
{ "CURLOPT_SOCKS5_GSSAPI_SERVICE", 179, pycurl_handle_setopt_string },
{ "CURLOPT_SOCKS5_GSSAPI_NEC", 180, pycurl_handle_setopt_long },
{ "CURLOPT_PROTOCOLS", 181, pycurl_handle_setopt_long },
{ "CURLOPT_REDIR_PROTOCOLS", 182, pycurl_handle_setopt_long },
@ -157,7 +173,7 @@ struct {
{ "CURLOPT_SSH_KEYFUNCTION", 184, NULL },
{ "CURLOPT_SSH_KEYDATA", 185, NULL },
{ "CURLOPT_MAIL_FROM", 186, pycurl_handle_setopt_string },
{ "CURLOPT_MAIL_RCPT", 187, NULL },
{ "CURLOPT_MAIL_RCPT", 187, pycurl_handle_setopt_slist },
{ "CURLOPT_FTP_USE_PRET", 188, pycurl_handle_setopt_long },
{ "CURLOPT_RTSP_REQUEST", 189, pycurl_handle_setopt_long },
{ "CURLOPT_RTSP_SESSION_ID", 190, pycurl_handle_setopt_string },
@ -173,7 +189,7 @@ struct {
{ "CURLOPT_FNMATCH_FUNCTION", 200, NULL },
{ "CURLOPT_CHUNK_DATA", 201, NULL },
{ "CURLOPT_FNMATCH_DATA", 202, NULL },
{ "CURLOPT_RESOLVE", 203, NULL },
{ "CURLOPT_RESOLVE", 203, pycurl_handle_setopt_slist },
{ "CURLOPT_TLSAUTH_USERNAME", 204, pycurl_handle_setopt_string },
{ "CURLOPT_TLSAUTH_PASSWORD", 205, pycurl_handle_setopt_string },
{ "CURLOPT_TLSAUTH_TYPE", 206, pycurl_handle_setopt_string },
@ -198,7 +214,7 @@ struct {
{ "CURLOPT_SSL_ENABLE_NPN", 225, pycurl_handle_setopt_long },
{ "CURLOPT_SSL_ENABLE_ALPN", 226, pycurl_handle_setopt_long },
{ "CURLOPT_EXPECT_100_TIMEOUT_MS", 227, pycurl_handle_setopt_long },
{ "CURLOPT_PROXYHEADER", 228, NULL },
{ "CURLOPT_PROXYHEADER", 228, pycurl_handle_setopt_slist },
{ "CURLOPT_HEADEROPT", 229, pycurl_handle_setopt_long },
{ "CURLOPT_PINNEDPUBLICKEY", 230, pycurl_handle_setopt_string },
{ "CURLOPT_UNIX_SOCKET_PATH", 231, pycurl_handle_setopt_string },
@ -213,7 +229,7 @@ struct {
{ "CURLOPT_STREAM_DEPENDS", 240, NULL },
{ "CURLOPT_STREAM_DEPENDS_E", 241, NULL },
{ "CURLOPT_TFTP_NO_OPTIONS", 242, pycurl_handle_setopt_long },
{ "CURLOPT_CONNECT_TO", 243, NULL },
{ "CURLOPT_CONNECT_TO", 243, pycurl_handle_setopt_slist },
{ "CURLOPT_TCP_FASTOPEN", 244, pycurl_handle_setopt_long },
{ "CURLOPT_KEEP_SENDING_ON_ERROR", 245, pycurl_handle_setopt_long },
{ "CURLOPT_PROXY_CAINFO", 246, pycurl_handle_setopt_string },
@ -258,5 +274,15 @@ struct {
{ "CURLOPT_HTTP09_ALLOWED", 285, pycurl_handle_setopt_long },
{ "CURLOPT_ALTSVC_CTRL", 286, pycurl_handle_setopt_long },
{ "CURLOPT_ALTSVC", 287, pycurl_handle_setopt_string },
{ "CURLOPT_MAXAGE_CONN", 288, pycurl_handle_setopt_long },
{ "CURLOPT_SASL_AUTHZID", 289, pycurl_handle_setopt_string },
{ "CURLOPT_MAIL_RCPT_ALLLOWFAILS", 290, pycurl_handle_setopt_long },
{ "CURLOPT_SSLCERT_BLOB", 291, NULL },
{ "CURLOPT_SSLKEY_BLOB", 292, NULL },
{ "CURLOPT_PROXY_SSLCERT_BLOB", 293, NULL },
{ "CURLOPT_PROXY_SSLKEY_BLOB", 294, NULL },
{ "CURLOPT_ISSUERCERT_BLOB", 295, NULL },
{ "CURLOPT_PROXY_ISSUERCERT", 296, pycurl_handle_setopt_string },
{ "CURLOPT_PROXY_ISSUERCERT_BLOB", 297, NULL },
{ NULL, 0, 0 }
};

117
src/python.c
View File

@ -5404,6 +5404,9 @@ python_kore_curl_handle(PyObject *self, PyObject *args)
handle->url = kore_strdup(url);
memset(&handle->curl, 0, sizeof(handle->curl));
handle->body = NULL;
LIST_INIT(&handle->slists);
if (!kore_curl_init(&handle->curl, handle->url, KORE_CURL_ASYNC)) {
Py_DECREF((PyObject *)handle);
PyErr_SetString(PyExc_RuntimeError, "failed to setup call");
@ -5416,12 +5419,65 @@ python_kore_curl_handle(PyObject *self, PyObject *args)
static void
pycurl_handle_dealloc(struct pycurl_handle *handle)
{
struct pycurl_slist *psl;
while ((psl = LIST_FIRST(&handle->slists))) {
LIST_REMOVE(psl, list);
curl_slist_free_all(psl->slist);
kore_free(psl);
}
if (handle->body != NULL)
kore_buf_free(handle->body);
kore_free(handle->url);
kore_curl_cleanup(&handle->curl);
PyObject_Del((PyObject *)handle);
}
static PyObject *
pycurl_handle_setbody(struct pycurl_handle *handle, PyObject *args)
{
PyObject *obj;
char *ptr;
Py_ssize_t length;
if (!PyArg_ParseTuple(args, "O", &obj))
return (NULL);
if (handle->body != NULL) {
PyErr_SetString(PyExc_RuntimeError,
"curl handle already has body attached");
return (NULL);
}
if (!PyBytes_CheckExact(obj)) {
PyErr_SetString(PyExc_RuntimeError,
"curl.setbody expects bytes");
return (NULL);
}
if (PyBytes_AsStringAndSize(obj, &ptr, &length) == -1)
return (NULL);
if (length < 0) {
PyErr_SetString(PyExc_TypeError, "invalid length");
return (NULL);
}
handle->body = kore_buf_alloc(length);
kore_buf_append(handle->body, ptr, length);
kore_buf_reset(handle->body);
curl_easy_setopt(handle->curl.handle,
CURLOPT_READFUNCTION, kore_curl_frombuf);
curl_easy_setopt(handle->curl.handle, CURLOPT_READDATA, handle->body);
curl_easy_setopt(handle->curl.handle, CURLOPT_UPLOAD, 1);
Py_RETURN_TRUE;
}
static PyObject *
pycurl_handle_setopt(struct pycurl_handle *handle, PyObject *args)
{
@ -5459,14 +5515,15 @@ pycurl_handle_setopt_string(struct pycurl_handle *handle, int idx,
if (!PyUnicode_Check(obj)) {
PyErr_Format(PyExc_RuntimeError,
"option '%s' requires a string as argument",
py_curlopt[idx]);
py_curlopt[idx].name);
return (NULL);
}
if ((str = PyUnicode_AsUTF8(obj)) == NULL)
return (NULL);
curl_easy_setopt(&handle->curl, py_curlopt[idx].value, str);
curl_easy_setopt(handle->curl.handle,
CURLOPTTYPE_OBJECTPOINT + py_curlopt[idx].value, str);
Py_RETURN_TRUE;
}
@ -5478,7 +5535,8 @@ pycurl_handle_setopt_long(struct pycurl_handle *handle, int idx, PyObject *obj)
if (!PyLong_CheckExact(obj)) {
PyErr_Format(PyExc_RuntimeError,
"option '%s' requires a long as argument", py_curlopt[idx]);
"option '%s' requires a long as argument",
py_curlopt[idx].name);
return (NULL);
}
@ -5487,7 +5545,51 @@ pycurl_handle_setopt_long(struct pycurl_handle *handle, int idx, PyObject *obj)
if (val == -1 && PyErr_Occurred())
return (NULL);
curl_easy_setopt(&handle->curl, py_curlopt[idx].value, val);
curl_easy_setopt(handle->curl.handle,
CURLOPTTYPE_LONG + py_curlopt[idx].value, val);
Py_RETURN_TRUE;
}
static PyObject *
pycurl_handle_setopt_slist(struct pycurl_handle *handle, int idx, PyObject *obj)
{
struct pycurl_slist *psl;
PyObject *item;
const char *sval;
struct curl_slist *slist;
Py_ssize_t list_len, i;
if (!PyList_CheckExact(obj)) {
PyErr_Format(PyExc_RuntimeError,
"option '%s' requires a list as argument",
py_curlopt[idx].name);
return (NULL);
}
slist = NULL;
list_len = PyList_Size(obj);
for (i = 0; i < list_len; i++) {
if ((item = PyList_GetItem(obj, i)) == NULL)
return (NULL);
if (!PyUnicode_Check(item))
return (NULL);
if ((sval = PyUnicode_AsUTF8AndSize(item, NULL)) == NULL)
return (NULL);
if ((slist = curl_slist_append(slist, sval)) == NULL)
fatal("%s: curl_slist_append failed", __func__);
}
psl = kore_calloc(1, sizeof(*psl));
psl->slist = slist;
LIST_INSERT_HEAD(&handle->slists, psl, list);
curl_easy_setopt(handle->curl.handle,
CURLOPTTYPE_OBJECTPOINT + py_curlopt[idx].value, slist);
Py_RETURN_TRUE;
}
@ -5539,6 +5641,11 @@ pycurl_handle_op_iternext(struct pycurl_handle_op *op)
Py_RETURN_NONE;
}
if (op->handle->body != NULL) {
kore_buf_free(op->handle->body);
op->handle->body = NULL;
}
if (!kore_curl_success(&op->handle->curl)) {
/* Do not log the url here, may contain some sensitive data. */
PyErr_Format(PyExc_RuntimeError, "request failed: %s",
@ -5684,7 +5791,7 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs)
((headers = PyDict_GetItemString(kwargs, "headers")) != NULL)) {
if (!PyDict_CheckExact(headers)) {
PyErr_SetString(PyExc_RuntimeError,
"header keyword must be a dict");
"headers keyword must be a dict");
return (NULL);
}
}