Merge pull request #118 from scw007/msgpack

add support for msgpack

mitmproxy2swagger/mitmproxy2swagger.py

@@ -10,6 +10,7 @@ import traceback
 import urllib
 from typing import Any, Optional, Sequence, Union
 
+import msgpack
 import ruamel.yaml
 from mitmproxy.exceptions import FlowReadException
 
@@ -240,6 +241,15 @@ def main(override_args: Optional[Sequence[str]] = None):
                         pass
                     except json.decoder.JSONDecodeError:
                         pass
+
+                    # try to parse the body as msgpack, if it's not json
+                    if body_val is None:
+                        try:
+                            body_val = msgpack.loads(req.get_request_body())
+                            content_type = "application/msgpack"
+                        except Exception:
+                            pass
+
                     if content_type is None:
                         # try to parse the body as form data
                         try:
@@ -278,28 +288,38 @@ def main(override_args: Optional[Sequence[str]] = None):
                             content_to_set,
                         )
 
-            # try parsing the response as json
             response_body = req.get_response_body()
             if response_body is not None:
+                # try parsing the response as json
                 try:
-                    response_json = json.loads(response_body)
+                    response_parsed = json.loads(response_body)
+                    response_content_type = "application/json"
                 except UnicodeDecodeError:
-                    response_json = None
+                    response_parsed = None
                 except json.decoder.JSONDecodeError:
-                    response_json = None
+                    response_parsed = None
-                if response_json is not None:
+
+                if response_parsed is None:
+                    # try parsing the response as msgpack, if it's not json
+                    try:
+                        response_parsed = msgpack.loads(response_body)
+                        response_content_type = "application/msgpack"
+                    except Exception:
+                        response_parsed = None
+
+                if response_parsed is not None:
                     resp_data_to_set = {
                         "description": req.get_response_reason(),
                         "content": {
-                            "application/json": {
+                            response_content_type: {
-                                "schema": swagger_util.value_to_schema(response_json)
+                                "schema": swagger_util.value_to_schema(response_parsed)
                             }
                         },
                     }
                     if args.examples:
-                        resp_data_to_set["content"]["application/json"][
+                        resp_data_to_set["content"][response_content_type][
                             "example"
-                        ] = swagger_util.limit_example_size(response_json)
+                        ] = swagger_util.limit_example_size(response_parsed)
                     if args.headers:
                         resp_data_to_set["headers"] = swagger_util.response_to_headers(
                             req.get_response_headers()
@@ -310,6 +330,7 @@ def main(override_args: Optional[Sequence[str]] = None):
                         str(status),
                         resp_data_to_set,
                     )
+
             if (
                 "responses" in swagger["paths"][path_template_to_set][method]
                 and len(swagger["paths"][path_template_to_set][method]["responses"])

mitmproxy2swagger/test_mitmproxy2swagger.py

@@ -68,3 +68,38 @@ def test_mitmproxy2swagger_generates_headers_for_flow_files():
         get_nested_key(data, "paths./post.post.responses.200.headers.content-type")
         is not None
     )
+
+
+def test_mitmproxy2swagger_parses_msgpack_requests_and_responses():
+    data = mitmproxy2swagger_e2e_test(
+        "testdata/msgpack_flows",
+        "http://localhost:8082/",
+        [
+            "--format",
+            "flow",
+        ],
+    )
+    assert data is not None
+
+    assert (
+        get_nested_key(data, "paths./.post.responses.200.content.application/msgpack")
+        is not None
+    )
+    assert (
+        get_nested_key(
+            data,
+            "paths./.post.responses.200.content.application/msgpack.schema.properties.new_field.type",
+        )
+        == "string"
+    )
+    assert (
+        get_nested_key(data, "paths./.post.requestBody.content.application/msgpack")
+        is not None
+    )
+    assert (
+        get_nested_key(
+            data,
+            "paths./.post.requestBody.content.application/msgpack.schema.properties.field1.type",
+        )
+        == "string"
+    )

mitmproxy2swagger/test_openapi_compliance.py

@@ -57,3 +57,17 @@ def test_mitmproxy2swagger_compliance_from_form_data_file_with_headers():
     )
     assert data is not None
     validate_spec(data)
+
+
+def test_mitmproxy2swagger_compliance_from_msgpack_file_with_headers():
+    data = mitmproxy2swagger_e2e_test(
+        "testdata/msgpack_flows",
+        "http://localhost:8082/",
+        [
+            "--format",
+            "flow",
+            "--headers",
+        ],
+    )
+    assert data is not None
+    validate_spec(data)

poetry.lock

@@ -1,10 +1,9 @@
-# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
 
 [[package]]
 name = "aioquic-mitmproxy"
 version = "0.9.21.1"
 description = "Fork of aioquic (https://github.com/aiortc/aioquic) with adjustments for mitmproxy."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -25,7 +24,6 @@ dev = ["coverage[toml] (>=7.2.2)"]
 name = "asgiref"
 version = "3.7.2"
 description = "ASGI specs, helper code, and adapters"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -43,7 +41,6 @@ tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"]
 name = "attrs"
 version = "23.1.0"
 description = "Classes Without Boilerplate"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -62,7 +59,6 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte
 name = "black"
 version = "23.12.1"
 description = "The uncompromising code formatter."
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -109,7 +105,6 @@ uvloop = ["uvloop (>=0.15.2)"]
 name = "blinker"
 version = "1.7.0"
 description = "Fast, simple object-to-object and broadcast signaling"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -121,7 +116,6 @@ files = [
 name = "brotli"
 version = "1.1.0"
 description = "Python bindings for the Brotli compression library"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -214,7 +208,6 @@ files = [
 name = "certifi"
 version = "2023.11.17"
 description = "Python package for providing Mozilla's CA Bundle."
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -226,7 +219,6 @@ files = [
 name = "cffi"
 version = "1.16.0"
 description = "Foreign Function Interface for Python calling C code."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -291,7 +283,6 @@ pycparser = "*"
 name = "cfgv"
 version = "3.4.0"
 description = "Validate configuration and produce human readable error messages."
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -303,7 +294,6 @@ files = [
 name = "charset-normalizer"
 version = "3.3.2"
 description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
-category = "dev"
 optional = false
 python-versions = ">=3.7.0"
 files = [
@@ -403,7 +393,6 @@ files = [
 name = "click"
 version = "8.1.7"
 description = "Composable command line interface toolkit"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -418,7 +407,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""}
 name = "colorama"
 version = "0.4.6"
 description = "Cross-platform colored terminal text."
-category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
 files = [
@@ -430,7 +418,6 @@ files = [
 name = "coverage"
 version = "7.4.0"
 description = "Code coverage measurement for Python"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -498,7 +485,6 @@ toml = ["tomli"]
 name = "cryptography"
 version = "41.0.7"
 description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -544,7 +530,6 @@ test-randomorder = ["pytest-randomly"]
 name = "distlib"
 version = "0.3.8"
 description = "Distribution utilities"
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -556,7 +541,6 @@ files = [
 name = "docformatter"
 version = "1.7.5"
 description = "Formats docstrings to follow PEP 257"
-category = "dev"
 optional = false
 python-versions = ">=3.7,<4.0"
 files = [
@@ -576,7 +560,6 @@ tomli = ["tomli (>=2.0.0,<3.0.0)"]
 name = "exceptiongroup"
 version = "1.2.0"
 description = "Backport of PEP 654 (exception groups)"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -591,7 +574,6 @@ test = ["pytest (>=6)"]
 name = "filelock"
 version = "3.13.1"
 description = "A platform independent file lock."
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -608,7 +590,6 @@ typing = ["typing-extensions (>=4.8)"]
 name = "flake8"
 version = "6.1.0"
 description = "the modular source code checker: pep8 pyflakes and co"
-category = "dev"
 optional = false
 python-versions = ">=3.8.1"
 files = [
@@ -625,7 +606,6 @@ pyflakes = ">=3.1.0,<3.2.0"
 name = "flake8-pyproject"
 version = "1.2.3"
 description = "Flake8 plug-in loading the configuration from pyproject.toml"
-category = "dev"
 optional = false
 python-versions = ">= 3.6"
 files = [
@@ -643,7 +623,6 @@ dev = ["pyTest", "pyTest-cov"]
 name = "flask"
 version = "3.0.0"
 description = "A simple framework for building complex web applications."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -666,7 +645,6 @@ dotenv = ["python-dotenv"]
 name = "h11"
 version = "0.14.0"
 description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -678,7 +656,6 @@ files = [
 name = "h2"
 version = "4.1.0"
 description = "HTTP/2 State-Machine based protocol implementation"
-category = "main"
 optional = false
 python-versions = ">=3.6.1"
 files = [
@@ -694,7 +671,6 @@ hyperframe = ">=6.0,<7"
 name = "hpack"
 version = "4.0.0"
 description = "Pure-Python HPACK header compression"
-category = "main"
 optional = false
 python-versions = ">=3.6.1"
 files = [
@@ -706,7 +682,6 @@ files = [
 name = "hyperframe"
 version = "6.0.1"
 description = "HTTP/2 framing layer for Python"
-category = "main"
 optional = false
 python-versions = ">=3.6.1"
 files = [
@@ -718,7 +693,6 @@ files = [
 name = "identify"
 version = "2.5.33"
 description = "File identification library for Python"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -733,7 +707,6 @@ license = ["ukkonen"]
 name = "idna"
 version = "3.6"
 description = "Internationalized Domain Names in Applications (IDNA)"
-category = "dev"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -745,7 +718,6 @@ files = [
 name = "iniconfig"
 version = "2.0.0"
 description = "brain-dead simple config-ini parsing"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -757,7 +729,6 @@ files = [
 name = "isort"
 version = "5.13.2"
 description = "A Python utility / library to sort Python imports."
-category = "dev"
 optional = false
 python-versions = ">=3.8.0"
 files = [
@@ -772,7 +743,6 @@ colors = ["colorama (>=0.4.6)"]
 name = "itsdangerous"
 version = "2.1.2"
 description = "Safely pass data to untrusted environments and back."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -784,7 +754,6 @@ files = [
 name = "jinja2"
 version = "3.1.2"
 description = "A very fast and expressive template engine."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -802,7 +771,6 @@ i18n = ["Babel (>=2.7)"]
 name = "json-stream"
 version = "2.3.2"
 description = "Streaming JSON encoder and decoder"
-category = "main"
 optional = false
 python-versions = "<4,>=3.5"
 files = [
@@ -821,7 +789,6 @@ requests = ["requests"]
 name = "json-stream-rs-tokenizer"
 version = "0.4.25"
 description = "A faster tokenizer for the json-stream Python library"
-category = "main"
 optional = false
 python-versions = ">=3.7,<4"
 files = [
@@ -890,7 +857,6 @@ test = ["json-stream (==2.3.2)", "json-stream-rs-tokenizer[benchmark]", "pytest
 name = "jsonschema"
 version = "4.20.0"
 description = "An implementation of JSON Schema validation for Python"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -912,7 +878,6 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-
 name = "jsonschema-path"
 version = "0.3.2"
 description = "JSONSchema Spec with object-oriented paths"
-category = "dev"
 optional = false
 python-versions = ">=3.8.0,<4.0.0"
 files = [
@@ -930,7 +895,6 @@ requests = ">=2.31.0,<3.0.0"
 name = "jsonschema-specifications"
 version = "2023.12.1"
 description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -945,7 +909,6 @@ referencing = ">=0.31.0"
 name = "kaitaistruct"
 version = "0.10"
 description = "Kaitai Struct declarative parser generator for binary data: runtime library for Python"
-category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"
 files = [
@@ -957,7 +920,6 @@ files = [
 name = "lazy-object-proxy"
 version = "1.10.0"
 description = "A fast and thorough lazy object proxy."
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1004,7 +966,6 @@ files = [
 name = "ldap3"
 version = "2.9.1"
 description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -1019,7 +980,6 @@ pyasn1 = ">=0.4.6"
 name = "markupsafe"
 version = "2.1.3"
 description = "Safely add untrusted strings to HTML/XML markup."
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1043,6 +1003,16 @@ files = [
     {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"},
     {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"},
     {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"},
@@ -1079,7 +1049,6 @@ files = [
 name = "mccabe"
 version = "0.7.0"
 description = "McCabe checker, plugin for flake8"
-category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1091,7 +1060,6 @@ files = [
 name = "mitmproxy"
 version = "10.1.6"
 description = "An interactive, SSL/TLS-capable intercepting proxy for HTTP/1, HTTP/2, and WebSockets."
-category = "main"
 optional = false
 python-versions = ">=3.10"
 files = [
@@ -1134,7 +1102,6 @@ dev = ["build (>=0.10.0)", "click (>=7.0,<8.2)", "hypothesis (>=5.8,<7)", "pdoc
 name = "mitmproxy-macos"
 version = "0.4.1"
 description = ""
-category = "main"
 optional = false
 python-versions = ">=3.10"
 files = [
@@ -1145,7 +1112,6 @@ files = [
 name = "mitmproxy-rs"
 version = "0.4.1"
 description = ""
-category = "main"
 optional = false
 python-versions = ">=3.10"
 files = [
@@ -1164,7 +1130,6 @@ mitmproxy_windows = {version = "0.4.1", markers = "os_name == \"nt\""}
 name = "mitmproxy-windows"
 version = "0.4.1"
 description = ""
-category = "main"
 optional = false
 python-versions = ">=3.10"
 files = [
@@ -1175,7 +1140,6 @@ files = [
 name = "msgpack"
 version = "1.0.7"
 description = "MessagePack serializer"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1241,7 +1205,6 @@ files = [
 name = "mypy"
 version = "1.8.0"
 description = "Optional static typing for Python"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1289,7 +1252,6 @@ reports = ["lxml"]
 name = "mypy-extensions"
 version = "1.0.0"
 description = "Type system extensions for programs checked with the mypy type checker."
-category = "dev"
 optional = false
 python-versions = ">=3.5"
 files = [
@@ -1301,7 +1263,6 @@ files = [
 name = "nodeenv"
 version = "1.8.0"
 description = "Node.js virtual environment builder"
-category = "dev"
 optional = false
 python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*"
 files = [
@@ -1316,7 +1277,6 @@ setuptools = "*"
 name = "openapi-schema-validator"
 version = "0.6.2"
 description = "OpenAPI schema validation for Python"
-category = "dev"
 optional = false
 python-versions = ">=3.8.0,<4.0.0"
 files = [
@@ -1333,7 +1293,6 @@ rfc3339-validator = "*"
 name = "openapi-spec-validator"
 version = "0.7.1"
 description = "OpenAPI 2.0 (aka Swagger) and OpenAPI 3 spec validator"
-category = "dev"
 optional = false
 python-versions = ">=3.8.0,<4.0.0"
 files = [
@@ -1351,7 +1310,6 @@ openapi-schema-validator = ">=0.6.0,<0.7.0"
 name = "packaging"
 version = "23.2"
 description = "Core utilities for Python packages"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1363,7 +1321,6 @@ files = [
 name = "passlib"
 version = "1.7.4"
 description = "comprehensive password hashing framework supporting over 30 schemes"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -1381,7 +1338,6 @@ totp = ["cryptography"]
 name = "pathable"
 version = "0.4.3"
 description = "Object-oriented paths"
-category = "dev"
 optional = false
 python-versions = ">=3.7.0,<4.0.0"
 files = [
@@ -1393,7 +1349,6 @@ files = [
 name = "pathspec"
 version = "0.12.1"
 description = "Utility library for gitignore style pattern matching of file paths."
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1405,7 +1360,6 @@ files = [
 name = "platformdirs"
 version = "4.1.0"
 description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1421,7 +1375,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co
 name = "pluggy"
 version = "1.3.0"
 description = "plugin and hook calling mechanisms for python"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1437,7 +1390,6 @@ testing = ["pytest", "pytest-benchmark"]
 name = "pre-commit"
 version = "3.6.0"
 description = "A framework for managing and maintaining multi-language pre-commit hooks."
-category = "dev"
 optional = false
 python-versions = ">=3.9"
 files = [
@@ -1456,7 +1408,6 @@ virtualenv = ">=20.10.0"
 name = "protobuf"
 version = "4.25.1"
 description = ""
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1477,7 +1428,6 @@ files = [
 name = "publicsuffix2"
 version = "2.20191221"
 description = "Get a public suffix for a domain name using the Public Suffix List. Forked from and using the same API as the publicsuffix package."
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -1489,7 +1439,6 @@ files = [
 name = "pyasn1"
 version = "0.5.1"
 description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)"
-category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
 files = [
@@ -1501,7 +1450,6 @@ files = [
 name = "pyasn1-modules"
 version = "0.3.0"
 description = "A collection of ASN.1-based protocols modules"
-category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
 files = [
@@ -1516,7 +1464,6 @@ pyasn1 = ">=0.4.6,<0.6.0"
 name = "pycodestyle"
 version = "2.11.1"
 description = "Python style guide checker"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1528,7 +1475,6 @@ files = [
 name = "pycparser"
 version = "2.21"
 description = "C parser in Python"
-category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
@@ -1540,7 +1486,6 @@ files = [
 name = "pydivert"
 version = "2.1.0"
 description = "Python binding to windivert driver"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -1556,7 +1501,6 @@ test = ["codecov (>=2.0.5)", "hypothesis (>=3.5.3)", "mock (>=1.0.1)", "pytest (
 name = "pyflakes"
 version = "3.1.0"
 description = "passive checker of Python programs"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1568,7 +1512,6 @@ files = [
 name = "pylsqpack"
 version = "0.3.18"
 description = "Python wrapper for the ls-qpack QPACK library"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1596,7 +1539,6 @@ files = [
 name = "pyopenssl"
 version = "23.3.0"
 description = "Python wrapper module around the OpenSSL library"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1615,7 +1557,6 @@ test = ["flaky", "pretend", "pytest (>=3.0.1)"]
 name = "pyparsing"
 version = "3.1.1"
 description = "pyparsing module - Classes and methods to define and execute parsing grammars"
-category = "main"
 optional = false
 python-versions = ">=3.6.8"
 files = [
@@ -1630,7 +1571,6 @@ diagrams = ["jinja2", "railroad-diagrams"]
 name = "pyperclip"
 version = "1.8.2"
 description = "A cross-platform clipboard module for Python. (Only handles plain text for now.)"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -1641,7 +1581,6 @@ files = [
 name = "pytest"
 version = "7.4.3"
 description = "pytest: simple powerful testing with Python"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1664,7 +1603,6 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no
 name = "pytest-asyncio"
 version = "0.23.2"
 description = "Pytest support for asyncio"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1683,7 +1621,6 @@ testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"]
 name = "pytest-cov"
 version = "4.1.0"
 description = "Pytest plugin for measuring coverage."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1702,7 +1639,6 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale
 name = "pyyaml"
 version = "6.0.1"
 description = "YAML parser and emitter for Python"
-category = "dev"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -1762,7 +1698,6 @@ files = [
 name = "referencing"
 version = "0.31.1"
 description = "JSON Referencing + Python"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1778,7 +1713,6 @@ rpds-py = ">=0.7.0"
 name = "requests"
 version = "2.31.0"
 description = "Python HTTP for Humans."
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1800,7 +1734,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 name = "rfc3339-validator"
 version = "0.1.4"
 description = "A pure python RFC3339 validator"
-category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 files = [
@@ -1815,7 +1748,6 @@ six = "*"
 name = "rpds-py"
 version = "0.15.2"
 description = "Python bindings to Rust's persistent data structures (rpds)"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -1924,7 +1856,6 @@ files = [
 name = "ruamel-yaml"
 version = "0.18.5"
 description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order"
-category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -1943,7 +1874,6 @@ jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"]
 name = "ruamel-yaml-clib"
 version = "0.2.8"
 description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml"
-category = "main"
 optional = false
 python-versions = ">=3.6"
 files = [
@@ -2003,7 +1933,6 @@ files = [
 name = "service-identity"
 version = "23.1.0"
 description = "Service identity verification for pyOpenSSL & cryptography."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -2028,7 +1957,6 @@ tests = ["coverage[toml] (>=5.0.2)", "pytest"]
 name = "setuptools"
 version = "69.0.3"
 description = "Easily download, build, install, upgrade, and uninstall Python packages"
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -2045,7 +1973,6 @@ testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jar
 name = "six"
 version = "1.16.0"
 description = "Python 2 and 3 compatibility utilities"
-category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
 files = [
@@ -2057,7 +1984,6 @@ files = [
 name = "sortedcontainers"
 version = "2.4.0"
 description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -2069,7 +1995,6 @@ files = [
 name = "tomli"
 version = "2.0.1"
 description = "A lil' TOML parser"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2081,7 +2006,6 @@ files = [
 name = "tornado"
 version = "6.4"
 description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed."
-category = "main"
 optional = false
 python-versions = ">= 3.8"
 files = [
@@ -2102,7 +2026,6 @@ files = [
 name = "typing-extensions"
 version = "4.9.0"
 description = "Backported and Experimental Type Hints for Python 3.8+"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -2114,7 +2037,6 @@ files = [
 name = "untokenize"
 version = "0.1.1"
 description = "Transforms tokens into original source code (while preserving whitespace)."
-category = "dev"
 optional = false
 python-versions = "*"
 files = [
@@ -2125,7 +2047,6 @@ files = [
 name = "urllib3"
 version = "2.1.0"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
-category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -2142,7 +2063,6 @@ zstd = ["zstandard (>=0.18.0)"]
 name = "urwid-mitmproxy"
 version = "2.1.2.1"
 description = "A full-featured console (xterm et al.) user interface library"
-category = "main"
 optional = false
 python-versions = "*"
 files = [
@@ -2159,7 +2079,6 @@ files = [
 name = "vermin"
 version = "1.6.0"
 description = "Concurrently detect the minimum Python versions needed to run code"
-category = "dev"
 optional = false
 python-versions = ">=3.0"
 files = [
@@ -2171,7 +2090,6 @@ files = [
 name = "virtualenv"
 version = "20.25.0"
 description = "Virtual Python Environment builder"
-category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
@@ -2192,7 +2110,6 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess
 name = "werkzeug"
 version = "3.0.1"
 description = "The comprehensive WSGI web application library."
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -2210,7 +2127,6 @@ watchdog = ["watchdog (>=2.3)"]
 name = "wsproto"
 version = "1.2.0"
 description = "WebSockets state-machine based protocol implementation"
-category = "main"
 optional = false
 python-versions = ">=3.7.0"
 files = [
@@ -2225,7 +2141,6 @@ h11 = ">=0.9.0,<1"
 name = "zstandard"
 version = "0.22.0"
 description = "Zstandard bindings for Python"
-category = "main"
 optional = false
 python-versions = ">=3.8"
 files = [
@@ -2286,4 +2201,4 @@ cffi = ["cffi (>=1.11)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "381a4a9ac8eb52f16dfc3c19b7c51d80bf47dbaf0b0e1c4afbc4b4f6a1eafacb"
+content-hash = "951142a28f6ad1feaf4b5e62e0cd7699fbe59b039178b6955d44df21f9426754"

pyproject.toml

@@ -11,6 +11,7 @@ python = "^3.10"
 mitmproxy = "^10.1.1"
 "ruamel.yaml" = ">=0.17.32,<0.19.0"
 json-stream = "^2.3.2"
+msgpack = "^1.0.7"
 
 
 [tool.poetry.group.dev.dependencies]

testdata/msgpack_flows

@@ -0,0 +1 @@
+2099:9:websocket;0:~8:response;393:6:reason;2:OK,11:status_code;3:200#13:timestamp_end;18:1704284722.2074344^15:timestamp_start;18:1704284722.2070074^8:trailers;0:~7:content;51:ƒ¦field1¦value1¦field2¦value2©new_field«Added Field,7:headers;157:39:6:Server,26:BaseHTTP/0.6 Python/3.11.6,]40:4:Date,29:Wed, 03 Jan 2024 12:25:22 GMT,]39:12:Content-type,19:application/msgpack,]23:14:Content-length,2:51,]]12:http_version;8:HTTP/1.0,}7:request;514:4:path;1:/,9:authority;0:,6:scheme;4:http,6:method;4:POST,4:port;4:8082#4:host;9:localhost;13:timestamp_end;18:1704284722.2042184^15:timestamp_start;18:1704284722.2037127^8:trailers;0:~7:content;29:‚¦field1¦value1¦field2¦value2,7:headers;244:25:4:Host,14:localhost:8082,]40:10:User-Agent,22:python-requests/2.31.0,]46:15:Accept-Encoding,23:gzip, deflate, br, zstd,]15:6:Accept,3:*/*,]28:10:Connection,10:keep-alive,]39:12:Content-Type,19:application/msgpack,]23:14:Content-Length,2:29,]]12:http_version;8:HTTP/1.1,}6:backup;0:~17:timestamp_created;17:1704284722.203833^7:comment;0:;8:metadata;0:}6:marked;0:;9:is_replay;0:~11:intercepted;5:false!11:server_conn;467:3:via;0:~19:timestamp_tcp_setup;18:1704284722.2062707^7:address;19:9:localhost;4:8082#]19:timestamp_tls_setup;0:~13:timestamp_end;17:1704284722.208372^15:timestamp_start;18:1704284722.2047434^3:sni;0:~11:tls_version;0:~11:cipher_list;0:]6:cipher;0:~11:alpn_offers;0:]4:alpn;0:~16:certificate_list;0:]3:tls;5:false!5:error;0:~18:transport_protocol;3:tcp;2:id;36:0d4b5407-f87c-4a56-b1be-0eb43eef8855;8:sockname;20:9:127.0.0.1;5:47522#]8:peername;19:9:127.0.0.1;4:8082#]}11:client_conn;423:10:proxy_mode;7:regular;8:mitmcert;0:~19:timestamp_tls_setup;0:~13:timestamp_end;18:1704284722.2085643^15:timestamp_start;18:1704284722.2027926^3:sni;0:~11:tls_version;0:~11:cipher_list;0:]6:cipher;0:~11:alpn_offers;0:]4:alpn;0:~16:certificate_list;0:]3:tls;5:false!5:error;0:~18:transport_protocol;3:tcp;2:id;36:d298f071-2025-4766-b64d-6a868949b78c;8:sockname;21:3:::1;4:8080#1:0#1:0#]8:peername;22:3:::1;5:60670#1:0#1:0#]}5:error;0:~2:id;36:acc4fde4-062c-4fbd-b7a8-5a46d3a0bcfc;4:type;4:http;7:version;2:20#}

testdata/msgpack_testclient.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+
+import msgpack
+import requests  # type: ignore
+
+# Sample MessagePack data
+msgpack_data = {"field1": "value1", "field2": "value2"}
+
+
+url = "http://localhost:8082"
+headers = {"Content-Type": "application/msgpack"}
+response = requests.post(
+    url,
+    data=msgpack.packb(msgpack_data, use_bin_type=True),
+    headers=headers,
+    proxies={"http": "http://localhost:8080", "https": "http://localhost:8080"},
+)
+
+# Print the response
+print(response.status_code)
+print(response.headers)
+
+# convert the response data from MessagePack to JSON
+data = msgpack.unpackb(response.content, raw=False)
+print(data)

testdata/msgpack_testserver.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+import http.server
+import socketserver
+
+import msgpack
+
+
+class MessagePackHandler(http.server.BaseHTTPRequestHandler):
+    def do_POST(self):
+        content_length = int(self.headers["Content-Length"])
+        raw_data = self.rfile.read(content_length)
+
+        try:
+            # Decode received MessagePack data
+
+            print(raw_data)
+            data = msgpack.unpackb(raw_data, raw=False)
+
+            # Add a new field to the data
+            data["new_field"] = "Added Field"
+
+            # Encode the modified data as MessagePack
+            modified_data = msgpack.packb(data, use_bin_type=True)
+
+            # Send the response
+            self.send_response(200)
+            self.send_header("Content-type", "application/msgpack")
+            self.send_header("Content-length", len(modified_data))
+            self.end_headers()
+            self.wfile.write(modified_data)
+
+        except Exception as e:
+            print(f"Error processing request: {str(e)}")
+            self.send_response(500)
+            self.send_header("Content-type", "text/plain")
+            self.end_headers()
+            self.wfile.write(f"Error processing request: {str(e)}".encode())
+
+
+if __name__ == "__main__":
+    PORT = 8082
+
+    with socketserver.TCPServer(("", PORT), MessagePackHandler) as httpd:
+        print(f"Serving on port {PORT}")
+        httpd.serve_forever()