vendor OpenSSL.crypto, and pylsqpack (v1.0.3), plus making the packag…

…e abi3 (#2)

* vendor OpenSSL.crypto instead of requiring whole package and backport aiortc#373 by kdashg

* vendor pylsqpack and upgrade it to v1.0.3 (major v2 available but breaking) and (todo) not abi3 ready

* fix manifest include vendor (lsqpack)

* make pylsqpack abi3 ready

* bump version to 0.10.0

* reformat connection.py

* fix tests.yml workflow

* fix xxhash include dir in setup.py

* fix docs job and tests on windows

* fix test on windows (choco install openssl issue, headers non-existent)

* fix test on windows (choco install openssl issue, headers non-existent)*2

.github/workflows/tests.yml

@@ -1,12 +1,16 @@
 name: tests
 
-on: [push, pull_request]
+on:
+  pull_request:
+  workflow_dispatch:
 
 jobs:
   docs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: 'true'
       - uses: actions/setup-python@v4
         with:
           python-version: 3.7
@@ -50,13 +54,16 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         python:
+          - "3.12-dev"
           - "3.11"
           - "3.10"
           - "3.9"
           - "3.8"
           - "3.7"
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: 'true'
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
@@ -67,12 +74,19 @@ jobs:
           echo "AIOQUIC_SKIP_TESTS=chacha20" >> $GITHUB_ENV
           echo "CFLAGS=-I/usr/local/opt/openssl/include" >> $GITHUB_ENV
           echo "LDFLAGS=-L/usr/local/opt/openssl/lib" >> $GITHUB_ENV
+#      - name: Install OpenSSL
+#        if: matrix.os == 'windows-latest'
+#        run: |
+#          choco install openssl --no-progress --verbose
+#          echo "INCLUDE=C:\\Progra~1\\OpenSSL-Win64\\include" >> $GITHUB_ENV
+#          echo "LIB=C:\\Progra~1\\OpenSSL-Win64\\lib" >> $GITHUB_ENV
+#        shell: bash
       - name: Install OpenSSL
         if: matrix.os == 'windows-latest'
         run: |
-          choco install openssl --no-progress
-          echo "INCLUDE=C:\Progra~1\OpenSSL-Win64\include" >> $GITHUB_ENV
-          echo "LIB=C:\Progra~1\OpenSSL-Win64\lib" >> $GITHUB_ENV
+          python scripts\\fetch-vendor.py C:\\cibw\\vendor
+          echo "INCLUDE=C:\\cibw\\vendor\\include" >> $GITHUB_ENV
+          echo "LIB=C:\\cibw\\vendor\\lib" >> $GITHUB_ENV
         shell: bash
       - name: Run tests
         run: |
@@ -89,6 +103,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: 'true'
       - uses: actions/setup-python@v4
         with:
           python-version: 3.7
@@ -122,6 +138,8 @@ jobs:
             arch: x86
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: 'true'
       - uses: actions/setup-python@v4
         with:
           python-version: 3.7
@@ -130,12 +148,14 @@ jobs:
         uses: docker/setup-qemu-action@v2
       - name: Build wheels
         env:
+          CIBW_BUILD_FRONTEND: build
           CIBW_ARCHS: ${{ matrix.arch }}
           CIBW_BEFORE_BUILD: python scripts/fetch-vendor.py /tmp/vendor
           CIBW_BEFORE_BUILD_WINDOWS: python scripts\fetch-vendor.py C:\cibw\vendor
           CIBW_ENVIRONMENT: AIOQUIC_SKIP_TESTS=ipv6,loss CFLAGS=-I/tmp/vendor/include LDFLAGS=-L/tmp/vendor/lib
           CIBW_ENVIRONMENT_WINDOWS: AIOQUIC_SKIP_TESTS=ipv6,loss INCLUDE=C:\\cibw\\vendor\\include LIB=C:\\cibw\\vendor\\lib
-          CIBW_SKIP: cp36-* pp36-* *-musllinux*
+          CIBW_SKIP: "*-musllinux*"
+          CIBW_BUILD: cp39-*
           CIBW_TEST_COMMAND: python -m unittest discover -t {project} -s {project}/tests
           # there are no wheels for cryptography on these platforms
           CIBW_TEST_SKIP: "*-{manylinux_i686,win32} pp*"
@@ -158,7 +178,7 @@ jobs:
           name: dist
           path: dist/
       - name: Publish to PyPI
-        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')
+        if: startsWith(github.event.ref, 'refs/tags/')
         uses: pypa/gh-action-pypi-publish@master
         with:
           user: __token__

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vendor/ls-qpack"]
+	path = vendor/ls-qpack
+	url = https://github.com/litespeedtech/ls-qpack.git

MANIFEST.in

@@ -2,3 +2,4 @@ include LICENSE
 recursive-include docs *.py *.rst Makefile
 recursive-include examples *.css *.html *.py *.txt
 recursive-include tests *.bin *.pem *.py
+recursive-include vendor *

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools"]
+requires = ["setuptools", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -24,11 +24,11 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: Internet :: WWW/HTTP",
 ]
 dependencies = [
-    "pylsqpack>=0.3.3,<0.4.0",
-    "pyopenssl>=22",
+    "cryptography>=41.0.0,<42.0.0",
 ]
 dynamic = ["version"]
 

setup.py

@@ -1,26 +1,71 @@
 import sys
+import os
+from wheel.bdist_wheel import bdist_wheel
 
 import setuptools
 
+include_dirs = [
+    os.path.join("vendor", "ls-qpack"),
+    os.path.join("vendor", "ls-qpack", "deps", "xxhash"),
+]
+
 if sys.platform == "win32":
     extra_compile_args = []
-    libraries = ["libcrypto", "advapi32", "crypt32", "gdi32", "user32", "ws2_32"]
+    libraries = [
+        "libcrypto",
+        "advapi32",
+        "crypt32",
+        "gdi32",
+        "user32",
+        "ws2_32",
+    ]
+    include_dirs.append(
+        os.path.join("vendor", "ls-qpack", "wincompat"),
+    )
 else:
     extra_compile_args = ["-std=c99"]
     libraries = ["crypto"]
 
+
+class bdist_wheel_abi3(bdist_wheel):
+    def get_tag(self):
+        python, abi, plat = super().get_tag()
+
+        if python.startswith("cp"):
+            return "cp37", "abi3", plat
+
+        return python, abi, plat
+
+
 setuptools.setup(
     ext_modules=[
         setuptools.Extension(
             "aioquic._buffer",
             extra_compile_args=extra_compile_args,
             sources=["src/aioquic/_buffer.c"],
+            define_macros=[("Py_LIMITED_API", "0x03070000")],
+            py_limited_api=True,
         ),
         setuptools.Extension(
             "aioquic._crypto",
             extra_compile_args=extra_compile_args,
             libraries=libraries,
             sources=["src/aioquic/_crypto.c"],
+            define_macros=[("Py_LIMITED_API", "0x03070000")],
+            py_limited_api=True,
+        ),
+        setuptools.Extension(
+            "aioquic._vendor.pylsqpack._binding",
+            extra_compile_args=extra_compile_args,
+            include_dirs=include_dirs,
+            sources=[
+                "src/aioquic/_vendor/pylsqpack/binding.c",
+                "vendor/ls-qpack/lsqpack.c",
+                "vendor/ls-qpack/deps/xxhash/xxhash.c",
+            ],
+            define_macros=[("Py_LIMITED_API", "0x03070000")],
+            py_limited_api=True,
         ),
     ],
+    cmdclass={"bdist_wheel": bdist_wheel_abi3},
 )

src/aioquic/__init__.py

@@ -1 +1 @@
-__version__ = "0.9.21"
+__version__ = "0.10.0"

src/aioquic/_buffer.c

@@ -5,6 +5,11 @@
 
 #define MODULE_NAME "aioquic._buffer"
 
+// https://foss.heptapod.net/pypy/pypy/-/issues/3770
+#ifndef Py_None
+#define Py_None (&_Py_NoneStruct)
+#endif
+
 static PyObject *BufferReadError;
 static PyObject *BufferWriteError;
 
@@ -15,6 +20,8 @@ typedef struct {
     uint8_t *pos;
 } BufferObject;
 
+static PyObject *BufferType;
+
 #define CHECK_READ_BOUNDS(self, len) \
     if (len < 0 || self->pos + len > self->end) { \
         PyErr_SetString(BufferReadError, "Read out of bounds"); \
@@ -54,7 +61,10 @@ static void
 Buffer_dealloc(BufferObject *self)
 {
     free(self->base);
-    Py_TYPE(self)->tp_free((PyObject *) self);
+    PyTypeObject *tp = Py_TYPE(self);
+    freefunc free = PyType_GetSlot(tp, Py_tp_free);
+    free(self);
+    Py_DECREF(tp);
 }
 
 static PyObject *
@@ -360,44 +370,21 @@ static PyGetSetDef Buffer_getset[] = {
     {NULL}
 };
 
-static PyTypeObject BufferType = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    MODULE_NAME ".Buffer",              /* tp_name */
-    sizeof(BufferObject),               /* tp_basicsize */
-    0,                                  /* tp_itemsize */
-    (destructor)Buffer_dealloc,         /* tp_dealloc */
-    0,                                  /* tp_print */
-    0,                                  /* tp_getattr */
-    0,                                  /* tp_setattr */
-    0,                                  /* tp_reserved */
-    0,                                  /* tp_repr */
-    0,                                  /* tp_as_number */
-    0,                                  /* tp_as_sequence */
-    0,                                  /* tp_as_mapping */
-    0,                                  /* tp_hash  */
-    0,                                  /* tp_call */
-    0,                                  /* tp_str */
-    0,                                  /* tp_getattro */
-    0,                                  /* tp_setattro */
-    0,                                  /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,                 /* tp_flags */
-    "Buffer objects",                   /* tp_doc */
-    0,                                  /* tp_traverse */
-    0,                                  /* tp_clear */
-    0,                                  /* tp_richcompare */
-    0,                                  /* tp_weaklistoffset */
-    0,                                  /* tp_iter */
-    0,                                  /* tp_iternext */
-    Buffer_methods,                     /* tp_methods */
-    0,                                  /* tp_members */
-    Buffer_getset,                      /* tp_getset */
-    0,                                  /* tp_base */
-    0,                                  /* tp_dict */
-    0,                                  /* tp_descr_get */
-    0,                                  /* tp_descr_set */
-    0,                                  /* tp_dictoffset */
-    (initproc)Buffer_init,              /* tp_init */
-    0,                                  /* tp_alloc */
+static PyType_Slot BufferType_slots[] = {
+    {Py_tp_dealloc, Buffer_dealloc},
+    {Py_tp_methods, Buffer_methods},
+    {Py_tp_doc, "Buffer objects"},
+    {Py_tp_getset, Buffer_getset},
+    {Py_tp_init, Buffer_init},
+    {0, 0},
+};
+
+static PyType_Spec BufferType_spec = {
+    MODULE_NAME ".Buffer",
+    sizeof(BufferObject),
+    0,
+    Py_TPFLAGS_DEFAULT,
+    BufferType_slots
 };
 
 
@@ -431,11 +418,15 @@ PyInit__buffer(void)
     Py_INCREF(BufferWriteError);
     PyModule_AddObject(m, "BufferWriteError", BufferWriteError);
 
-    BufferType.tp_new = PyType_GenericNew;
-    if (PyType_Ready(&BufferType) < 0)
+    BufferType = PyType_FromSpec(&BufferType_spec);
+    if (BufferType == NULL)
         return NULL;
-    Py_INCREF(&BufferType);
-    PyModule_AddObject(m, "Buffer", (PyObject *)&BufferType);
+
+    PyObject *o = PyType_FromSpec(&BufferType_spec);
+    if (o == NULL)
+        return NULL;
+
+    PyModule_AddObject(m, "Buffer", o);
 
     return m;
 }