@ -12,30 +12,34 @@ import ruamel.yaml
import re import re
from . import swagger_util from . import swagger_util
from . har_capture_reader import HarCaptureReader , har_archive_heuristic from . har_capture_reader import HarCaptureReader , har_archive_heuristic
from . mitmproxy_capture_reader import MitmproxyCaptureReader , mitmproxy_dump_file_huristic from . mitmproxy_capture_reader import (
MitmproxyCaptureReader ,
mitmproxy_dump_file_huristic ,
)
from . import console_util from . import console_util
import urllib import urllib
def path_to_regex ( path ) : def path_to_regex ( path ) :
# replace the path template with a regex
# replace the path template with a regex
path = path . replace ( ' \\ ' , ' \\ \\ ' )
path = path . replace ( " \\ " , " \\ \\ " )
path = path . replace ( ' { ' , ' (?P< ' )
path = path . replace ( " { " , " (?P< " )
path = path . replace ( ' } ' , ' >[^/]+) ' )
path = path . replace ( " } " , " >[^/]+) " )
path = path . replace ( ' * ' , ' .* ' )
path = path . replace ( " * " , " .* " )
path = path . replace ( ' / ' , ' \\ / ' )
path = path . replace ( " / " , " \\ / " )
path = path . replace ( ' ? ' , ' \\ ? ' )
path = path . replace ( " ? " , " \\ ? " )
path = path . replace ( ' ( ' , ' \\ ( ' )
path = path . replace ( " ( " , " \\ ( " )
path = path . replace ( ' ) ' , ' \\ ) ' )
path = path . replace ( " ) " , " \\ ) " )
path = path . replace ( ' . ' , ' \\ . ' )
path = path . replace ( " . " , " \\ . " )
path = path . replace ( ' + ' , ' \\ + ' )
path = path . replace ( " + " , " \\ + " )
path = path . replace ( ' [ ' , ' \\ [ ' )
path = path . replace ( " [ " , " \\ [ " )
path = path . replace ( ' ] ' , ' \\ ] ' )
path = path . replace ( " ] " , " \\ ] " )
return " ^ " + path + " $ "
return " ^ " + path + " $ "
def strip_query_string ( path ) : def strip_query_string ( path ) :
# remove the query string from the path
# remove the query string from the path
return path . split ( ' ? ' ) [ 0 ]
return path . split ( " ? " ) [ 0 ]
def set_key_if_not_exists ( dict , key , value ) : def set_key_if_not_exists ( dict , key , value ) :
@ -49,10 +53,10 @@ def progress_callback(progress):
def detect_input_format ( file_path ) : def detect_input_format ( file_path ) :
har_score = har_archive_heuristic ( file_path )
har_score = har_archive_heuristic ( file_path )
mitmproxy_score = mitmproxy_dump_file_huristic ( file_path )
mitmproxy_score = mitmproxy_dump_file_huristic ( file_path )
if ' MITMPROXY2SWAGGER_DEBUG ' in os . environ :
if " MITMPROXY2SWAGGER_DEBUG " in os . environ :
print ( ' har score: ' + str ( har_score ) )
print ( " har score: " + str ( har_score ) )
print ( ' mitmproxy score: ' + str ( mitmproxy_score ) )
print ( " mitmproxy score: " + str ( mitmproxy_score ) )
if har_score > mitmproxy_score :
if har_score > mitmproxy_score :
return HarCaptureReader ( file_path , progress_callback )
return HarCaptureReader ( file_path , progress_callback )
return MitmproxyCaptureReader ( file_path , progress_callback )
return MitmproxyCaptureReader ( file_path , progress_callback )
@ -60,24 +64,41 @@ def detect_input_format(file_path):
def main ( ) : def main ( ) :
parser = argparse . ArgumentParser (
parser = argparse . ArgumentParser (
description = ' Converts a mitmproxy dump file or HAR to a swagger schema. ' )
description = " Converts a mitmproxy dump file or HAR to a swagger schema. "
)
parser . add_argument (
" -i " ,
" --input " ,
help = " The input mitmproxy dump file or HAR dump file (from DevTools) " ,
required = True ,
)
parser . add_argument (
parser . add_argument (
' -i ' , ' --input ' , help = ' The input mitmproxy dump file or HAR dump file (from DevTools) ' , required = True )
" -o " ,
" --output " ,
help = " The output swagger schema file (yaml). If it exists, new endpoints will be added " ,
required = True ,
)
parser . add_argument ( " -p " , " --api-prefix " , help = " The api prefix " , required = True )
parser . add_argument (
parser . add_argument (
' -o ' , ' --output ' , help = ' The output swagger schema file (yaml). If it exists, new endpoints will be added ' , required = True )
" -e " ,
parser . add_argument ( ' -p ' , ' --api-prefix ' , help = ' The api prefix ' , required = True )
" --examples " ,
parser . add_argument ( ' -e ' , ' --examples ' , action = ' store_true ' ,
action = " store_true " ,
help = ' Include examples in the schema. This might expose sensitive information. ' )
help = " Include examples in the schema. This might expose sensitive information. " ,
parser . add_argument ( ' -f ' , ' --format ' , choices = [ ' flow ' , ' har ' ] ,
)
help = ' Override the input file format auto-detection. ' )
parser . add_argument (
" -f " ,
" --format " ,
choices = [ " flow " , " har " ] ,
help = " Override the input file format auto-detection. " ,
)
args = parser . parse_args ( )
args = parser . parse_args ( )
yaml = ruamel . yaml . YAML ( )
yaml = ruamel . yaml . YAML ( )
capture_reader = None
capture_reader = None
if args . format == ' flow ' or args . format == ' mitmproxy ' :
if args . format == " flow " or args . format == " mitmproxy " :
capture_reader = MitmproxyCaptureReader ( args . input , progress_callback )
capture_reader = MitmproxyCaptureReader ( args . input , progress_callback )
elif args . format == ' har ' :
elif args . format == " har " :
capture_reader = HarCaptureReader ( args . input , progress_callback )
capture_reader = HarCaptureReader ( args . input , progress_callback )
else :
else :
capture_reader = detect_input_format ( args . input )
capture_reader = detect_input_format ( args . input )
@ -86,45 +107,46 @@ def main():
# try loading the existing swagger file
# try loading the existing swagger file
try :
try :
with open ( args . output , ' r ' ) as f :
with open ( args . output , " r " ) as f :
swagger = yaml . load ( f )
swagger = yaml . load ( f )
except FileNotFoundError :
except FileNotFoundError :
print ( " No existing swagger file found. Creating new one. " )
print ( " No existing swagger file found. Creating new one. " )
pass
pass
if swagger is None :
if swagger is None :
swagger = ruamel . yaml . comments . CommentedMap ( {
swagger = ruamel . yaml . comments . CommentedMap (
" openapi " : " 3.0.0 " ,
{
" info " : {
" openapi " : " 3.0.0 " ,
" title " : args . input + " Mitmproxy2Swagger " ,
" info " : {
" version " : " 1.0.0 "
" title " : args . input + " Mitmproxy2Swagger " ,
} ,
" version " : " 1.0.0 " ,
} )
} ,
}
)
# strip the trailing slash from the api prefix
# strip the trailing slash from the api prefix
args . api_prefix = args . api_prefix . rstrip ( ' / ' )
args . api_prefix = args . api_prefix . rstrip ( " / " )
if ' servers ' not in swagger or swagger [ ' servers ' ] is None :
if " servers " not in swagger or swagger [ " servers " ] is None :
swagger [ ' servers ' ] = [ ]
swagger [ " servers " ] = [ ]
# add the server if it doesn't exist
# add the server if it doesn't exist
if not any ( server [ ' url ' ] == args . api_prefix for server in swagger [ ' servers ' ] ) :
if not any ( server [ " url " ] == args . api_prefix for server in swagger [ " servers " ] ) :
swagger [ ' servers ' ] . append ( {
swagger [ " servers " ] . append (
" url " : args . api_prefix ,
{ " url " : args . api_prefix , " description " : " The default server " }
" description " : " The default server "
)
} )
if ' paths ' not in swagger or swagger [ ' paths ' ] is None :
if " paths " not in swagger or swagger [ " paths " ] is None :
swagger [ ' paths ' ] = { }
swagger [ " paths " ] = { }
if ' x-path-templates ' not in swagger or swagger [ ' x-path-templates ' ] is None :
if " x-path-templates " not in swagger or swagger [ " x-path-templates " ] is None :
swagger [ ' x-path-templates ' ] = [ ]
swagger [ " x-path-templates " ] = [ ]
path_templates = [ ]
path_templates = [ ]
for path in swagger [ ' paths ' ] :
for path in swagger [ " paths " ] :
path_templates . append ( path )
path_templates . append ( path )
# also add paths from the the x-path-templates array
# also add paths from the the x-path-templates array
if ' x-path-templates ' in swagger and swagger [ ' x-path-templates ' ] is not None :
if " x-path-templates " in swagger and swagger [ " x-path-templates " ] is not None :
for path in swagger [ ' x-path-templates ' ] :
for path in swagger [ " x-path-templates " ] :
path_templates . append ( path )
path_templates . append ( path )
# new endpoints will be added here so that they can be added as comments in the swagger file
# new endpoints will be added here so that they can be added as comments in the swagger file
@ -133,8 +155,7 @@ def main():
print ( " Compiling path " + path )
print ( " Compiling path " + path )
print ( " Compiled to regex: " + path_to_regex ( path ) )
print ( " Compiled to regex: " + path_to_regex ( path ) )
re . compile ( path_to_regex ( path ) )
re . compile ( path_to_regex ( path ) )
path_template_regexes = [ re . compile ( path_to_regex ( path ) )
path_template_regexes = [ re . compile ( path_to_regex ( path ) ) for path in path_templates ]
for path in path_templates ]
try :
try :
for f in capture_reader . captured_requests ( ) :
for f in capture_reader . captured_requests ( ) :
@ -159,20 +180,27 @@ def main():
continue
continue
path_template_to_set = path_templates [ path_template_index ]
path_template_to_set = path_templates [ path_template_index ]
set_key_if_not_exists (
set_key_if_not_exists ( swagger [ " paths " ] , path_template_to_set , { } )
swagger [ ' paths ' ] , path_template_to_set , { } )
set_key_if_not_exists ( swagger [ ' paths ' ] [ path_template_to_set ] , method , {
set_key_if_not_exists (
' summary ' : swagger_util . path_template_to_endpoint_name ( method , path_template_to_set ) ,
swagger [ " paths " ] [ path_template_to_set ] ,
' responses ' : { }
method ,
} )
{
" summary " : swagger_util . path_template_to_endpoint_name (
method , path_template_to_set
) ,
" responses " : { } ,
} ,
)
params = swagger_util . url_to_params ( url , path_template_to_set )
params = swagger_util . url_to_params ( url , path_template_to_set )
if params is not None and len ( params ) > 0 :
if params is not None and len ( params ) > 0 :
set_key_if_not_exists ( swagger [ ' paths ' ] [ path_template_to_set ] [ method ] , ' parameters ' , params )
set_key_if_not_exists (
swagger [ " paths " ] [ path_template_to_set ] [ method ] , " parameters " , params
)
if method not in [ ' get ' , ' head ' ] :
if method not in [ " get " , " head " ] :
body = f . get_request_body ( )
body = f . get_request_body ( )
if body is not None :
if body is not None :
body_val = None
body_val = None
@ -180,7 +208,7 @@ def main():
# try to parse the body as json
# try to parse the body as json
try :
try :
body_val = json . loads ( f . get_request_body ( ) )
body_val = json . loads ( f . get_request_body ( ) )
content_type = ' application/json '
content_type = " application/json "
except UnicodeDecodeError :
except UnicodeDecodeError :
pass
pass
except json . decoder . JSONDecodeError :
except json . decoder . JSONDecodeError :
@ -188,32 +216,40 @@ def main():
if content_type is None :
if content_type is None :
# try to parse the body as form data
# try to parse the body as form data
try :
try :
body_val_bytes = dict ( urllib . parse . parse_qsl ( body , encoding = ' utf-8 ' , keep_blank_values = True ) )
body_val_bytes = dict (
urllib . parse . parse_qsl (
body , encoding = " utf-8 " , keep_blank_values = True
)
)
body_val = { }
body_val = { }
did_find_anything = False
did_find_anything = False
for key , value in body_val_bytes . items ( ) :
for key , value in body_val_bytes . items ( ) :
did_find_anything = True
did_find_anything = True
body_val [ key . decode ( ' utf-8 ' ) ] = value . decode ( ' utf-8 ' )
body_val [ key . decode ( " utf-8 " ) ] = value . decode ( " utf-8 " )
if did_find_anything :
if did_find_anything :
content_type = ' application/x-www-form-urlencoded '
content_type = " application/x-www-form-urlencoded "
else :
else :
body_val = None
body_val = None
except UnicodeDecodeError :
except UnicodeDecodeError :
pass
pass
if body_val is not None :
if body_val is not None :
content_to_set = {
content_to_set = {
' content ' : {
" content " : {
content_type : {
content_type : {
' schema ' : swagger_util . value_to_schema ( body_val )
" schema " : swagger_util . value_to_schema ( body_val )
}
}
}
}
}
}
if args . examples :
if args . examples :
content_to_set [ ' content ' ] [ content_type ] [ ' example ' ] = swagger_util . limit_example_size (
content_to_set [ " content " ] [ content_type ] [
body_val )
" example "
] = swagger_util . limit_example_size ( body_val )
set_key_if_not_exists (
set_key_if_not_exists (
swagger [ ' paths ' ] [ path_template_to_set ] [ method ] , ' requestBody ' , content_to_set )
swagger [ " paths " ] [ path_template_to_set ] [ method ] ,
" requestBody " ,
content_to_set ,
)
# try parsing the response as json
# try parsing the response as json
response_body = f . get_response_body ( )
response_body = f . get_response_body ( )
@ -226,33 +262,45 @@ def main():
response_json = None
response_json = None
if response_json is not None :
if response_json is not None :
resp_data_to_set = {
resp_data_to_set = {
' description ' : f . get_response_reason ( ) ,
" description " : f . get_response_reason ( ) ,
' content ' : {
" content " : {
' application/json ' : {
" application/json " : {
' schema ' : swagger_util . value_to_schema ( response_json )
" schema " : swagger_util . value_to_schema ( response_json )
}
}
}
} ,
}
}
if args . examples :
if args . examples :
resp_data_to_set [ ' content ' ] [ ' application/json ' ] [ ' example ' ] = swagger_util . limit_example_size (
resp_data_to_set [ " content " ] [ " application/json " ] [
response_json )
" example "
set_key_if_not_exists ( swagger [ ' paths ' ] [ path_template_to_set ] [ method ] [ ' responses ' ] , str (
] = swagger_util . limit_example_size ( response_json )
status ) , resp_data_to_set )
set_key_if_not_exists (
swagger [ " paths " ] [ path_template_to_set ] [ method ] [ " responses " ] ,
str ( status ) ,
resp_data_to_set ,
)
except FlowReadException as e :
except FlowReadException as e :
print ( f " Flow file corrupted: { e } " )
print ( f " Flow file corrupted: { e } " )
traceback . print_exception ( * sys . exc_info ( ) )
traceback . print_exception ( * sys . exc_info ( ) )
print ( f " { console_util . ANSI_RED } Failed to parse the input file as ' { capture_reader . name ( ) } ' . " )
print (
f " { console_util . ANSI_RED } Failed to parse the input file as ' { capture_reader . name ( ) } ' . "
)
if not args . format :
if not args . format :
print ( f " It might happen that the input format as incorrectly detected. Please try using ' --format flow ' or ' --format har ' to specify the input format. { console_util . ANSI_RESET } " )
print (
f " It might happen that the input format as incorrectly detected. Please try using ' --format flow ' or ' --format har ' to specify the input format. { console_util . ANSI_RESET } "
)
sys . exit ( 1 )
sys . exit ( 1 )
except ValueError as e :
except ValueError as e :
print ( f " ValueError: { e } " )
print ( f " ValueError: { e } " )
# print stack trace
# print stack trace
traceback . print_exception ( * sys . exc_info ( ) )
traceback . print_exception ( * sys . exc_info ( ) )
print ( f " { console_util . ANSI_RED } Failed to parse the input file as ' { capture_reader . name ( ) } ' . " )
print (
f " { console_util . ANSI_RED } Failed to parse the input file as ' { capture_reader . name ( ) } ' . "
)
if not args . format :
if not args . format :
print ( f " It might happen that the input format as incorrectly detected. Please try using ' --format flow ' or ' --format har ' to specify the input format. { console_util . ANSI_RESET } " )
print (
f " It might happen that the input format as incorrectly detected. Please try using ' --format flow ' or ' --format har ' to specify the input format. { console_util . ANSI_RESET } "
)
sys . exit ( 1 )
sys . exit ( 1 )
new_path_templates . sort ( )
new_path_templates . sort ( )
@ -262,50 +310,53 @@ def main():
new_path_templates_with_suggestions = [ ]
new_path_templates_with_suggestions = [ ]
for idx , path in enumerate ( new_path_templates ) :
for idx , path in enumerate ( new_path_templates ) :
# check if path contains number-only segments
# check if path contains number-only segments
segments = path . split ( ' / ' )
segments = path . split ( " / " )
if any ( segment . isdigit ( ) for segment in segments ) :
if any ( segment . isdigit ( ) for segment in segments ) :
# replace digit segments with {id}, {id1}, {id2} etc
# replace digit segments with {id}, {id1}, {id2} etc
new_segments = [ ]
new_segments = [ ]
param_id = 0
param_id = 0
for segment in segments :
for segment in segments :
if segment . isdigit ( ) :
if segment . isdigit ( ) :
param_name = ' id ' + str ( param_id )
param_name = " id " + str ( param_id )
if param_id == 0 :
if param_id == 0 :
param_name = ' id '
param_name = " id "
new_segments . append ( ' { ' + param_name + ' } ' )
new_segments . append ( " { " + param_name + " } " )
param_id + = 1
param_id + = 1
else :
else :
new_segments . append ( segment )
new_segments . append ( segment )
suggested_path = ' / ' . join ( new_segments )
suggested_path = " / " . join ( new_segments )
# prepend the suggested path to the new_path_templates list
# prepend the suggested path to the new_path_templates list
if suggested_path not in new_path_templates_with_suggestions :
if suggested_path not in new_path_templates_with_suggestions :
new_path_templates_with_suggestions . append (
new_path_templates_with_suggestions . append ( " ignore: " + suggested_path )
" ignore: " + suggested_path )
new_path_templates_with_suggestions . append ( " ignore: " + path )
new_path_templates_with_suggestions . append ( " ignore: " + path )
# remove the ending comments not to add them twice
# remove the ending comments not to add them twice
# append the contents of new_path_templates_with_suggestions to swagger['x-path-templates']
# append the contents of new_path_templates_with_suggestions to swagger['x-path-templates']
for path in new_path_templates_with_suggestions :
for path in new_path_templates_with_suggestions :
swagger [ ' x-path-templates ' ] . append ( path )
swagger [ " x-path-templates " ] . append ( path )
# remove elements already generated
# remove elements already generated
swagger [ ' x-path-templates ' ] = [
swagger [ " x-path-templates " ] = [
path for path in swagger [ ' x-path-templates ' ] if path not in swagger [ ' paths ' ] ]
path for path in swagger [ " x-path-templates " ] if path not in swagger [ " paths " ]
]
# remove duplicates while preserving order
# remove duplicates while preserving order
def f7 ( seq ) :
def f7 ( seq ) :
seen = set ( )
seen = set ( )
seen_add = seen . add
seen_add = seen . add
return [ x for x in seq if not ( x in seen or seen_add ( x ) ) ]
return [ x for x in seq if not ( x in seen or seen_add ( x ) ) ]
swagger [ ' x-path-templates ' ] = f7 ( swagger [ ' x-path-templates ' ] )
swagger [ ' x-path-templates ' ] = ruamel . yaml . comments . CommentedSeq (
swagger [ " x-path-templates " ] = f7 ( swagger [ " x-path-templates " ] )
swagger [ ' x-path-templates ' ] )
swagger [ ' x-path-templates ' ] . yaml_set_start_comment (
swagger [ " x-path-templates " ] = ruamel . yaml . comments . CommentedSeq (
' Remove the ignore: prefix to generate an endpoint with its URL \n Lines that are closer to the top take precedence, the matching is greedy ' )
swagger [ " x-path-templates " ]
)
swagger [ " x-path-templates " ] . yaml_set_start_comment (
" Remove the ignore: prefix to generate an endpoint with its URL \n Lines that are closer to the top take precedence, the matching is greedy "
)
# save the swagger file
# save the swagger file
with open ( args . output , ' w ' ) as f :
with open ( args . output , " w " ) as f :
yaml . dump ( swagger , f )
yaml . dump ( swagger , f )
print ( " Done! " )
print ( " Done! " )
alufers
3 years ago