tpython++compiler.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-
import os, sys, subprocess, random, json
from xml.sax.saxutils import escape
import unrealgen, ast

THREEJS_MISSING = '''

ERROR: because you used `import THREE` in your script the three.js source code is required.
Could not find the three.js source folder in your home directory,
try running the commands below, and then rebuild.

	cd
	git clone https://github.com/mrdoob/three.js.git

'''

UNREAL_VER = '4.24.1'   ## first tested with 4.2.0, current version is 4.23.1

UIFACE_TEMPLATE = '''
#pragma('once')
#include "ModuleManager.h"

class I%s : public IModuleInterface {
	public:
		static I%s& Get() {
			return FModuleManager::LoadModuleChecked<I%s>(FName("%s"));
		}
		static bool IsAvailable(){
			return FModuleManager::Get().IsModuleLoaded(FName("%s"));
		}
};
'''

UPLUGIN_TEMPLATE = '''
{
	"FileVersion" : 3,
	"FriendlyName" : "%s",
	"Version" : 1,
	"VersionName" : "1.0",
	"CreatedBy" : "%s",
	"CreatedByURL" : "%s",
	"EngineVersion" : "%s",
	"Description" : "%s",
	"Category" : "%s",
	"EnabledByDefault" : %s,
	"Modules" :
	[
		{
			"Name" : "%s",
			"Type" : "%s",
			"LoadingPhase" : "%s"
		}
	]
}
'''

UNREAL_BUILD_TEMPLATE = '''
using UnrealBuildTool.Rules;
using System.IO;
 
public class %s : ModuleRules {
	public %s(ReadOnlyTargetRules Target) : base(Target) {
		PrivateIncludePaths.AddRange(new string[] { "%s/Private" });
		PublicIncludePaths.AddRange(new string[] { "%s/Public" });
		PublicDependencyModuleNames.AddRange(new string[] {"%s"});
		var base_path = Path.GetFullPath(
			Path.Combine(
				Path.GetDirectoryName(ModuleDirectory), "../../../3rdparty")
		);
		if (!Directory.Exists(base_path)) {
			Log.TraceError("can not find 3rdparty build folder");
			Log.TraceError(base_path);
		}
		PublicIncludePaths.Add( base_path );
		PublicIncludePaths.AddRange( new string[] {"%s"} );
		var path = Path.Combine(base_path, "libtpython++.so");
		PublicAdditionalLibraries.Add(path);
		PublicAdditionalLibraries.AddRange( new string[] {"%s"});
		PublicDependencyModuleNames.AddRange(
			new string[] {
				"Engine", "Core"
				//"CoreUObject", "Engine", "InputCore", "RHI",
				//"RenderCore", "HTTP", "UMG", "Slate", "SlateCore",
				//"ImageWrapper", "PhysX", "HeadMountedDisplay", "AIModule"
			});
	}
}
'''


def bin_scramble(fname, finfo, mangle_map):
	scram = finfo['scramble']
	ok = False
	for mangled in mangle_map:
		if scram in mangled:
			scram = mangled
			ok = True
	if not ok:
		print('WARN: can not find mangled version of: ' + scram)
		return scram

	xorkey = []
	xscram = []
	for i in range(len(scram)):
		x = int( random.uniform(1,255) )
		xorkey.append(x)
		c = ord(scram[i]) ^ x
		xscram.append( c )

	lambda_scram = [
		'char _[%s];' %len(scram),
		'int __[%s]{%s};' %(len(scram), str(xscram)[1:-1] ),
		'int ___[%s]{%s};' %(len(scram), str(xorkey)[1:-1] ),
		'for (int _i=0; _i<%s; _i++) _[_i]=__[_i]^___[_i];' %len(scram),
		##'std::cout<< std::string(_, %s) <<std::endl;' %len(scram),
		'return std::string(_, %s);' %len(scram)
	]
	lambda_scram = ' '.join(lambda_scram)
	bscram = '( (%s (*)(%s))(dlsym(__libself__,[](){%s}().c_str() )) )' %(finfo['returns'], ','.join(finfo['arg_types']), lambda_scram)
	#bscram = '( (%s (*)(%s))(dlsym(dlopen(NULL, 1),[](){%s}().c_str() )) )' %(finfo['returns'], ','.join(finfo['arg_types']), lambda_scram)
	return bscram

def auto_semicolon(ln):
	s = ln.strip()
	if not s.endswith( ('{', '}', '(', ',', ':') ) and not s.startswith('#'):
		if not s=='else' and not s.startswith( ('if ', 'if(') ):
			if not s.endswith(';') and s:
				if not s.startswith( ("TP_LOOP(", "GENERATED_UCLASS_BODY") ):
					ln += ';'
	return ln

def is_untyped_global_var(s):
	if s.startswith('//'):
		return False
	elif s.startswith('define'):
		return False
	elif s.startswith('('):
		return False
	elif s.count('=') == 1:
		decl, value = s.split('=')
		decl = decl.strip()
		value = value.strip()
		if ' ' not in decl:
			return True
	return False

__guesses = {}
__global_string_vecs = {}
__global_vecs = set()

def guess_type_of_var(s, classes=None, global_auto_unwrap={}):
	assert s.count('=') == 1
	ctype = None
	decl, val = s.split('=')
	if val.endswith(';'):
		val = val[:-1]
	decl = decl.strip()
	val = val.strip()
	assert ' ' not in decl
	if val.isdigit():
		ctype = 'long'
	elif val.count('"')==2 and decl.count('[')==1 and decl.count(']')==1:
		ctype = 'char'
	elif val == "''":
		ctype = 'char'
	elif val == 'False' or val == 'True':
		ctype = 'tp_obj' #'bool'
	elif val == 'None':
		ctype = 'tp_obj'
	elif val.startswith('['):
		is_all_strings = True
		ctype = 'std::vector<tp_obj>'
		#not safe yet#ctype = 'tp_obj' - nested iteration is not yet supported
		assert val.endswith(']')
		if decl not in __global_vecs:
			__global_vecs.add( decl )
		val = val[1:-1].strip()
		vecargs = []
		if ',' not in val and ' for ' in val and ' in ' in val:
			forval = val.split()[-1]
			## special case for reversed strings ##
			if '.reverse()' in val and forval in __global_string_vecs:
				for sv in __global_string_vecs[ forval ]:
					vecargs.append( sv[::-1] )
			else:
				raise SyntaxError("invalid list comp at global level: " + s)
		elif ',' not in val:
			is_all_strings = False
		else:
			for part in val.split(','):  ## this breaks with strings that have `,` in them
				part = part.strip()
				if not part:
					continue
				if part.startswith("'"):
					assert part.endswith("'")
					part = part.replace("'", '"')
					#part = 'tp_obj(' + part + ')'
				elif part.startswith('"'):
					assert part.endswith('"')
				else:
					is_all_strings = False
				vecargs.append( part )
			if is_all_strings:
				__global_string_vecs[ decl ] = vecargs
		if is_all_strings:
			ctype = 'std::vector<std::string>'
			val = 'std::vector<std::string>({\n%s})' % ',\n'.join(vecargs)
		else:
			val = 'std::vector<tp_obj>({%s})' % ','.join(vecargs)
	elif val.startswith('{') and val.endswith('}'):
		#ctype = 'tp_obj'
		is_all_chars = True
		ctype = 'std::unordered_map<std::string,tp_obj>'
		mapargs = []
		for part in val[1:-1].split():
			#print(part)
			assert ':' in part
			if part.endswith(','):
				part = part[:-1]
			key,val = part.split(':')
			if key.startswith("'"):
				assert key.endswith("'")
				if len(key) > 3:
					key = key.replace("'", '"')
					is_all_chars = False
			else:
				is_all_chars = False
			mapargs.append( '{%s,%s}' % (key,val))
		if is_all_chars:
			ctype = 'std::unordered_map<char,tp_obj>'
			val = 'std::unordered_map<char,tp_obj>{%s}' % ','.join(mapargs)
		else:
			val = 'std::unordered_map<std::string,tp_obj>{%s}' % ','.join(mapargs)
		
	elif classes:
		for classname in classes:
			if classname in val:
				#ctype = classname
				ctype = 'tp_obj'
				global_auto_unwrap[ decl ] = classname
				break

	if ctype is None:
		if val.startswith('ord(') and val.endswith(')'):
			ctype = 'int'
		elif val.endswith(')') and '(' in val:  ## some function call
			ctype = 'tp_obj'
		elif val.endswith('*'):  ## some pointer
			ctype = val

	if ctype:
		if decl not in __guesses:
			__guesses[decl] = ctype
		return (ctype, decl, val)
	elif val in __guesses:
		return (__guesses[val], decl, val)

	print('--------classes----------')
	print(classes)
	print('--------prev guesses-----')
	print(__guesses)
	raise RuntimeError('can not guess type for: ' + s)

def get_base_members( classes, class_name, base_members={} ):
	#for m in classes[class_name]:
	base_members.update( classes[class_name]['members'] )
	for base in classes[class_name]['bases']:
		if base in ('tp_obj', 'tpy_subclass', 'Object3D'):
			continue
		get_base_members( classes, base, base_members )

tpy_modules_aot = {}

def pythonicpp( source, header='', file_name='', info={}, swap_self_to_this=False, binary_scramble=False, mangle_map=None, fodg=None, unreal_plugin_name=None, vis_cursor=None, mode='c++' ):
	if not type(source) is list:
		source = source.splitlines()
	print(file_name)
	if file_name.endswith('.aot.pyh'):
		if '--aot-modules' in sys.argv:
			tpy_modules_aot[file_name] = source
		return None

	if type(fodg) is list:
		fodg.append('<?xml version="1.0" encoding="UTF-8"?>')
		fodg.append('<office:document xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:style="urn:oasis:names:tc:opendocument:xmlns:style:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:fo="urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:meta="urn:oasis:names:tc:opendocument:xmlns:meta:1.0" xmlns:number="urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0" xmlns:presentation="urn:oasis:names:tc:opendocument:xmlns:presentation:1.0" xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0" xmlns:chart="urn:oasis:names:tc:opendocument:xmlns:chart:1.0" xmlns:dr3d="urn:oasis:names:tc:opendocument:xmlns:dr3d:1.0" xmlns:math="http://www.w3.org/1998/Math/MathML" xmlns:form="urn:oasis:names:tc:opendocument:xmlns:form:1.0" xmlns:script="urn:oasis:names:tc:opendocument:xmlns:script:1.0" xmlns:config="urn:oasis:names:tc:opendocument:xmlns:config:1.0" xmlns:ooo="http://openoffice.org/2004/office" xmlns:ooow="http://openoffice.org/2004/writer" xmlns:oooc="http://openoffice.org/2004/calc" xmlns:dom="http://www.w3.org/2001/xml-events" xmlns:xforms="http://www.w3.org/2002/xforms" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:smil="urn:oasis:names:tc:opendocument:xmlns:smil-compatible:1.0" xmlns:anim="urn:oasis:names:tc:opendocument:xmlns:animation:1.0" xmlns:rpt="http://openoffice.org/2005/report" xmlns:of="urn:oasis:names:tc:opendocument:xmlns:of:1.2" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:grddl="http://www.w3.org/2003/g/data-view#" xmlns:officeooo="http://openoffice.org/2009/office" xmlns:tableooo="http://openoffice.org/2009/table" xmlns:drawooo="http://openoffice.org/2010/draw" xmlns:calcext="urn:org:documentfoundation:names:experimental:calc:xmlns:calcext:1.0" xmlns:loext="urn:org:documentfoundation:names:experimental:office:xmlns:loext:1.0" xmlns:field="urn:openoffice:names:experimental:ooo-ms-interop:xmlns:field:1.0" xmlns:formx="urn:openoffice:names:experimental:ooxml-odf-interop:xmlns:form:1.0" xmlns:css3t="http://www.w3.org/TR/css3-text/" office:version="1.2" office:mimetype="application/vnd.oasis.opendocument.graphics">')
		fodg.extend([
			'<office:settings>',
			'<config:config-item-set config:name="ooo:view-settings">',
			'<config:config-item config:name="VisibleAreaTop" config:type="int">560</config:config-item>',
			'<config:config-item config:name="VisibleAreaLeft" config:type="int">156</config:config-item>',
			'<config:config-item config:name="VisibleAreaWidth" config:type="int">29131</config:config-item>',
			'<config:config-item config:name="VisibleAreaHeight" config:type="int">14711</config:config-item>',
			'</config:config-item-set>',
			'</office:settings>',

			'<office:automatic-styles>',
			'<style:page-layout style:name="PAGELAYOUT">',
			'<style:page-layout-properties fo:margin-top="1cm" fo:margin-bottom="1cm" fo:margin-left="1cm" fo:margin-right="1cm" fo:page-width="21.59cm" fo:page-height="27.94cm" style:print-orientation="portrait"/>',
			'</style:page-layout>',

			'<style:style style:name="LIGHTBLUE" style:family="paragraph">',
			'<loext:graphic-properties draw:fill-color="#e8f2a1"/>',
			'<style:paragraph-properties fo:text-align="center"/>',
			'</style:style>'
			'<style:style style:name="RED" style:family="paragraph">',
			'<loext:graphic-properties draw:fill-color="#ff0000"/>',
			'<style:paragraph-properties fo:text-align="center"/>',
			'</style:style>'
			'<style:style style:name="GREEN" style:family="paragraph">',
			'<loext:graphic-properties draw:fill-color="#00ff00"/>',
			'<style:paragraph-properties fo:text-align="center"/>',
			'</style:style>'
			'<style:style style:name="BLUE" style:family="paragraph">',
			'<loext:graphic-properties draw:fill-color="#0000ff"/>',
			'<style:paragraph-properties fo:text-align="center"/>',
			'</style:style>'
			'<style:style style:name="YELLOW" style:family="paragraph">',
			'<loext:graphic-properties draw:fill-color="#ffff00"/>',
			'<style:paragraph-properties fo:text-align="center"/>',
			'</style:style>'
			'<style:style style:name="PURPLE" style:family="paragraph">',
			'<loext:graphic-properties draw:fill-color="#ff00ff"/>',
			'<style:paragraph-properties fo:text-align="center"/>',
			'</style:style>'
			'<style:style style:name="BLACK" style:family="paragraph">',
			'<loext:graphic-properties draw:fill-color="#000000"/>',
			'<style:paragraph-properties fo:text-align="center"/>',
			'</style:style>'
			'<style:style style:name="WHITE" style:family="paragraph">',
			'<loext:graphic-properties draw:fill-color="#ffffff"/>',
			'<style:paragraph-properties fo:text-align="center"/>',
			'</style:style>'


			'<style:style style:name="PAGESTYLE" style:family="drawing-page">',
			'<style:drawing-page-properties draw:background-size="border" draw:fill="none"/>',
			'</style:style>',

			'<style:style style:name="DEF" style:family="text">',
			'<style:text-properties fo:font-size="14pt" style:font-size-asian="14pt" style:font-size-complex="14pt"/>',
			'</style:style>',

			'<style:style style:name="FUNC" style:family="text">',
			'<style:text-properties fo:font-size="18pt" style:font-size-asian="18pt" style:font-size-complex="18pt" fo:background-color="#000000" fo:color="#00ff00"/>',
			'</style:style>',
			'</office:automatic-styles>',
			'<office:master-styles>',
			'<style:master-page style:name="Default" style:page-layout-name="PAGELAYOUT" draw:style-name="PAGESTYLE"/>',
			'</office:master-styles>',

			'<office:body>',
			'	<office:drawing>',
			'		<draw:page draw:name="page1" draw:master-page-name="Default">',
		])
	out = []
	if header:
		out.append(header)

	prev = ''
	prevs = ''
	previ = -1
	autobrace = 0
	autofunc = 0
	mods = {'__aot_builtin_module__':[]}
	modname = None
	init_list = []
	in_init_list = False
	init_list_indent = 0
	in_class = False
	class_members = {}
	class_con = []
	class_new = []
	in_struct = False
	struct_stack = []
	struct_name = None
	struct_indent = 0
	in_enum = False
	enum_indent = 0
	class_indent = 0
	class_name = None
	class_has_init = False
	classes = {}
	tp_obj_subclass = False
	nsbrace = -1
	lambdabrace = []
	define = []
	define_ident = -1
	func_name = None
	in_func = False
	is_virt = False
	auto_unwrap = {}
	global_auto_unwrap = {}
	func_indent = -1
	func_locals = {}
	func_globals = []
	func_info = {'name':None, 'args':[], 'returns':None}
	virt_func_dispatch = {}
	fodgx = -16
	fodgy = 0
	fid = 0
	in_vis = True
	macro_indent = []
	in_unreal_plugin = False
	unreal_plugin_cpp = []
	unreal_plugin_iface = []
	unreal_blueprints = {}
	in_blueprint = False
	unreal_blueprint = []
	extern_funcs = []
	em_js = False
	next_line = ''
	user_pythonic = file_name.endswith('__user_pythonic__.pyh')

	if user_pythonic:
		#out.append('#define unwrap(T,o) ((T*)o.pointer.val)')
		out.append('#define unwrap(T,o) static_cast<class T*>(o.pointer.val)')
		# why do these macros sometimes fail with this error?
		#error: expected ‘>’ before ‘*’ token #define unwrap(T,o) static_cast<T*>(o.pointer.val)
		#out.append('#define in >>__is_in__()<')
		out.append('#define in >>')

		if tpy_modules_aot:
			src = []
			for aotmod in tpy_modules_aot:
				#src.extend( tpy_modules_aot[aotmod] )
				for ln in tpy_modules_aot[aotmod]:
					src.append('\t' + ln)

			src.extend( source )
			source = src
			tpy_modules_aot.clear()

	if 'functions' in info:
		functions = info['functions']
	else:
		functions = {}

	for line_num, ln in enumerate(source):
		if line_num+1 < len(source):
			next_line = source[line_num+1]
		oline = ln
		draw_type = 'rectangle'
		color = 'LIGHTBLUE'
		if '(' in ln and ')' in ln:
			draw_type= 'round-rectangle'
		if vis_cursor:
			if line_num + 40 > vis_cursor[0]:
				in_vis = False
			elif line_num < vis_cursor[0]:
				in_vis = False
			else:
				in_vis = True
		indent = 0
		for c in ln:
			if c == '\t':
				indent += 1
			else:
				break

		if in_blueprint:
			unreal_blueprint.append(ln)
			if not ln.strip():
				in_blueprint = False
			continue

		## curved upwards arrow is a reference
		if u'⤴' in ln:
			ln = ln.replace(u'⤴', '&')
		## black rightwards arrow head is a pointer
		if u'⮞' in ln:
			ln = ln.replace(u'⮞', '*')
		if u'×' in ln:
			ln = ln.replace(u'×', '*')
		if u'÷' in ln:
			ln = ln.replace(u'÷', '/')
		if u'≼' in ln:
			ln = ln.replace(u'≼', '<')
		if u'≽' in ln:
			ln = ln.replace(u'≽', '>')
		if u'🔒' in ln:
			ln = ln.replace(u'🔒', ' const ')
		if u'🠊' in ln:
			ln = ln.replace(u'🠊', '->')
		if u'⧟' in ln:
			ln = ln.replace(u'⧟', ',')

		s = ln.strip()

		## check for function calls, or forward defs
		if not s.startswith('def '):
			for fname in functions:
				if s.count(fname+'(')==1 or s.count(fname+')')==1 or s.count(fname+'}')==1:
					prevchar = ln[ ln.index(fname)-1 ]
					if prevchar in '\t +=-*/[]();,?':
						sig = '%s:%s call: `%s`' %(file_name, fname, s)
						if sig not in functions[fname]['calls']:
							functions[fname]['calls'].append(sig)
						if 'scramble' in functions[fname]:
							finfo = functions[fname]
							scram = finfo['scramble']

							ok = False
							if mangle_map and binary_scramble:
								if fname != '__init_libself__' and not 'static' in finfo and 'auto' not in finfo['arg_types'] and 'std::function<tp_obj(tp_vm*)>' not in finfo['arg_types'] and len(finfo['defs'])==1 and '...' not in finfo['arg_types']:
									for mangled in mangle_map:
										if scram in mangled:
											scram = mangled
											ok = True
									if not ok:
										print('WARN: can not find mangled version of: ' + scram)

							if ok and binary_scramble:

								if '--debug-obfuscate' in sys.argv:
									bscram = '( (%s (*)(%s)) ( [](){std::cout<<__libself__<<std::endl<<"%s"<<std::endl; auto fptr=dlsym(__libself__,"%s"); std::cout<<fptr<<std::endl; return fptr;}() ) )' %(finfo['returns'], ','.join(finfo['arg_types']), scram, scram)
								else:
									#bscram = '( (%s (*)(%s))(dlsym(__libself__,"%s")) )' %(finfo['returns'], ','.join(finfo['arg_types']), scram)
									bscram = bin_scramble(fname, finfo, mangle_map)

								ln = ln.replace(fname, bscram)

							else:
								ln = ln.replace(fname, scram)

							s = ln.strip()


		if s.endswith('\\'):
			out.append(ln)
			continue

		if macro_indent and indent <= macro_indent[-1]:
			if 'else:' in s:
				if indent < macro_indent[-1]:  ## TODO support more nested levels
					out.append(('\t'*macro_indent.pop())+'#endif')
				out.append(('\t'*indent)+'#else')
			else:
				out.append(('\t'*macro_indent.pop())+'#endif')

		elif not len(define):
			if in_init_list and indent <= init_list_indent:
				if not out[-1][-1] == '{':
					for outln in out:
						print(outln)
					raise SyntaxError("invalid init_list: syntax")
				out[-1]  = out[-1][:-1]
				out[-1] += ' : ' + ', '.join(init_list) + ' {'
				in_init_list = False
				init_list_indent = 0
				init_list = []
			######################################################
			if lambdabrace and indent <= lambdabrace[-1]:
				brace = lambdabrace.pop()
				b = '\t' * brace
				b += '};'
				out.append(b)

			elif indent <= nsbrace:
				b = '\t' * nsbrace
				b += '}  // end of namespace'
				out.append(b)
				nsbrace = 0
			elif macro_indent and indent <= macro_indent[-1]:
				out.append( ('\t'*macro_indent.pop())+"#endif" )
			elif indent < previ and autobrace:
				braces = previ - indent
				b = '\t' * indent

				if in_func and indent <= func_indent and user_pythonic:
					#if indent == func_indent:
					#	b += 'return None;}'
					#else:
					b += '}' * ((previ - func_indent)-1)
					if func_info['returns'] == 'tp_obj':
						b += 'return None;}'
					else:
						b += '}'
					if em_js:
						em_js = False
						b += '); // end of JS function: %s ' %func_name
					else:
						b += '// end of function: %s ' %func_name
					in_func = False
					func_indent = -1
				else:
					b += '}'*braces
				out.append(b)
				if indent == 0 and in_unreal_plugin:
					out[-1] += ';'
				if indent <= 1 and in_func and em_js:
					out[-1] += ');'
					em_js = False

			############################################
			if in_class and indent <= class_indent:
				in_class = False
				class_indent = 0
				if not out[-1][-1] == '}':
					if class_con:
						out.extend(class_con)
					out.append('}')
				elif class_con:
					out[-1] = out[-1][:-1]  ## trim ending of class }
					out.extend(class_con)
					out.append('}')         ## put back ending of class }

				#out.append('	;// end of class: ' + class_name)
				out[-1] += ';	// end of class: `%s` - members: %s' %(class_name, ', '.join(class_members.keys()))

				if user_pythonic and class_new:
					## generated myclass_new
					out.extend(class_new)

				if class_members and user_pythonic:
					cindex = 0
					found = False
					for oln in out:
						if oln.startswith('class %s:' %class_name):
							found = True
							break
						cindex += 1
					if found:
						base_members = {}
						get_base_members( classes, class_name, base_members )
						members = ['public:']
						for mname in class_members:
							if mname not in base_members:
								members.append(
									'%s %s;' %(class_members[mname], mname)
								)
						out.insert(cindex+1, '\n'.join(members))
					else:
						raise RuntimeError('could not find class def for: ' + class_name)
					classes[class_name]['members'].update(class_members)
					class_members = {}
				class_name = None
				class_con = []
				class_new = []

			elif in_struct and indent <= struct_indent:
				if not out[-1].endswith('}'):
					out[-1] += '}'
				if '.' in struct_name:
					## inline struct def, with variable name at the end
					struct_name, struct_var_name = struct_name.split('.')
					out[-1] += '%s;	// end of struct: %s' %(struct_var_name, struct_name)
				else:
					out[-1] += ';	// end of struct: ' + struct_name
				struct_stack.pop()
				if len(struct_stack):
					struct_name   = struct_stack[-1][0]
					struct_indent = struct_stack[-1][1]
				else:
					in_struct = False
					struct_name = None
					struct_indent = 0
			elif in_enum and indent <= enum_indent:
				if enum_name == 'enum':
					enum_name = 'unnamed'
				if out[-1][-1] == '}':
					out[-1] += ';	// end of enum: ' + enum_name
				else:
					out.append('')  ## just incase there is an #endif at the end of the enum
					out[-1] += '};	// end of enum: ' + enum_name
				in_enum = False


		if not s:
			in_class = False
			if in_func and fodg and in_vis:
				#fodg.append('<draw:enhanced-geometry svg:viewBox="0 0 21600 21600" draw:type="rectangle" draw:enhanced-path="M 0 0 L 21600 0 21600 21600 0 21600 0 0 Z N"/>')
				#fodg.append('</draw:custom-shape>')
				fodgx += 10

			in_func = False
			em_js = False
			func_locals = {}
			func_globals = []
			is_virt = False
			auto_unwrap = {}
			auto_unwrap.update(global_auto_unwrap)

		## early stage preprocessing for user pythonic (AOT) ##
		if user_pythonic and not s.startswith('#') and not s.endswith(";"):
			newln = []
			for part in s.split():
				if part.startswith('set(['):
					if part.endswith('])'):
						sitems = []
						for sitem in part[ len('set([') : -2 ].split(','):
							sitems.append(sitem)
						part = 'tp_uset(std::unordered_set<float>{%s})' % ','.join(sitems)
					elif part.endswith(']):'):
						sitems = []
						for sitem in part[ len('set([') : -3 ].split(','):
							sitems.append(sitem)
						part = 'tp_uset(std::unordered_set<float>{%s})' % ','.join(sitems)
				if part.count('.')==1 and not part.startswith( ('self.', '(self.') ):
					a,b = part.split('.')
					aorig = a


					if a in auto_unwrap:
						if auto_unwrap[a] != "__EXTERNAL_OBJECT__":
							##error: expected unqualified-id before ‘static_cast’
							##a = 'unwrap(%s,%s)' %(auto_unwrap[a], a)
							a = a + ('.template unwrap<class %s>()' %auto_unwrap[a])

					#elif is_virt:
					elif in_func and not em_js:
						hits = []
						for clsname in classes:
							if b.split('[')[0] in classes[clsname]['members']:
								hits.append(clsname)
								## https://stackoverflow.com/questions/3786360/confusing-template-error
								a = a + ('.template unwrap<class %s>()' %clsname)
								## above template workaround is ugly, and sometimes still fails with error: use of ‘this’ in a constant expression,
								## this happens when the class contains a member with the same name as the class being casted to,
								## by prefixing `class ` the problem is solved.
								##a = 'unwrap(%s,%s)' %(clsname, a)
							elif '(' in b:
								if b.split('(')[0] in classes[clsname]['methods']:
									hits.append(clsname)
									a = a + ('.template unwrap<class %s>()' %clsname)
									#a = 'unwrap(%s,%s)' %(clsname, a)
						if not hits and in_class:
							## checks own class members
							if b.split('[')[0] in class_members:
								hits.append(class_name)
								a = a + ('.template unwrap<%s>()' %class_name)
							elif '(' in b:
								if b.split('(')[0] in class_members:
									hits.append(class_name)
									a = a + ('.template unwrap<%s>()' %class_name)

						if not hits and '(' not in a and a not in classes and '->' not in a:
							if b.startswith( ('set(','append(') ):
								## tp_obj base class contains a set and append methods
								if a in __global_vecs:
									part = part.replace('append(', 'push_back(')
								else:
									pass
							elif b.startswith( ('size(', 'clear(', 'push_back(') ):
								## std::vector, std::string, and other c++11 containers
								pass
							elif b.isdigit():
								pass
							elif a.isdigit():
								pass
							elif a.startswith('-') and len(a) >= 2 and a[1:].isdigit():
								pass
							elif a.startswith('&'):
								## lower level pythonic++ must sometimes be used in AOT code
								pass
							elif s.startswith('switch '):
								pass
							else:
								print('WARNING: can not determine types of:')
								print(b)
								print(s)
								print(part)
								if '--fully-static-types' in sys.argv:
									print('=======this class members:')
									print(class_members)
									for cls in classes:
										print(cls)
										print(classes[cls])
									raise RuntimeError("unable to find the class type from member or method use on variable: `%s`" %a)		
						elif len(hits) > 1:
							raise SyntaxError('can not auto unwrap a pointer because multiple classes have the same named members or methods: %s line: `%s`' %(str(hits), s))
						elif len(hits) == 1:
							## not very safe or required to remove *most* of the need for pythonic users to manually use @unwrap
							if aorig.startswith('('):
								aorig = aorig[1:]
							if aorig not in auto_unwrap:
								if '(' in aorig or ')' in aorig:
									#raise SyntaxError(part)
									print('WARN: parser choked on `%s` from line: `%s` ' % (aorig, s) )
								else:
									# the problem with this magic is that a base class with the member could have been picked first
									# then later when the variable is used again, its auto_unwrapp'ed as the base class, 
									# and not the required subclass type.
									#auto_unwrap[aorig] = hits[0]
									pass

					if a != part.split('.')[0]:
						newln.append('%s->%s' %(a,b))
					else:
						newln.append(part)
				else:
					clsnames = list( classes )
					clsnames.sort()
					clsnames.reverse()
					for clsname in clsnames:
						if part.startswith(clsname+'('):
							part = part.replace(clsname+'(', clsname+'_new(')
							if s.count('=') == 1:
								a,b = s.split('=')
								a = a.strip()
								if '.' not in a:
									if a in auto_unwrap and auto_unwrap[a] != clsname:
										raise RuntimeError("auto unwrap error - type redefined : `%s`" %s)
									auto_unwrap[a] = clsname
							break
					newln.append(part)
			s = ' '.join(newln)
			ln = ('\t'*indent) + s

		if user_pythonic and in_class and not in_func:
			lns = ln.strip()
			if lns.endswith(';'):
				lns = lns[:-1]
			if lns and ' ' in lns:
				mname = lns.split()[-1]
				class_members_exp[mname] = lns

		if s.startswith('##'):
			ln = ln.replace('##', '//')
			out.append(ln)
		elif s.startswith('#') and mode=='js':
			ln = ln.replace('#', '//')
			out.append(ln)
		elif user_pythonic and s.startswith("#"):
			## note user can not directly use macro syntax
			pass
		elif s.startswith("global "):
			## using the global keyword in low level pythonic++ (used to implement the VM) is just for clarity, local vars are not auto prefixed, it is done manually ##
			func_globals = [gbl.strip() for gbl in s.split('global ')[-1].split(',')]
			pass
		elif s.startswith("raise "):
			if user_pythonic:
				## currently not used in low level pythonic++ (used to implement the VM)
				if 'NotImplementedError' in s:
					## forced to be forward declared and defined later, using virt_func_dispatch
					indent = previ
				else:
					err = s.split('raise ')[-1]
					err = err.replace('"', "`")
					out.append('print("%s"); throw "%s";' %(err, err))
			else:
				## low level pythonic++ (VM level)
				err = s.split('raise ')[-1]
				err = err.replace('"', "`")
				out.append('std::cout << "%s" << std::endl; throw "%s";' %(err, err))
		#elif user_pythonic and indent==1 and not ln.startswith(' ') and not s.startswith('@') and is_untyped_global_var(s):
		elif user_pythonic and indent <= 1 and not ln.startswith(' ') and not s.startswith('@') and is_untyped_global_var(s):
			ctype, cname, cval = guess_type_of_var(s, classes=classes, global_auto_unwrap=global_auto_unwrap)
			## check if ctype needs to be a const or a define, if used in a switch/case or an array decl size
			requires_const = False
			requires_define = False
			if ctype != 'tp_obj':
				for line in source:
					if cname in line and 'case ' in line and ':' in line:
						#requires_const = True  ## a const is vaild for use in a switch/case, but a define is better
						requires_define = True
					if cname in line and '[' in line and ']' in line:
						if cname in line.split('[')[-1].split(']')[0]:
							## just because its in brackes it is not for sure that's its an array decl size
							if '=' in line:
								pass
							elif '&'+cname in line.split():
								pass
							else:
								requires_define = True
								break
			if requires_define:
				out.append('#define ' + cname + ' ' + cval)
			else:
				if requires_const:
					ctype = 'const ' + ctype
				#out.append(ctype + ' ' + s + ';')
				if '=' in ln:
					out.append('%s %s=%s;' %(ctype, cname, cval))
				else:
					raise RuntimeError(ln)
					out.append('%s %s %s;' %(ctype, cname, cval))
		elif s.startswith('unreal.blueprint') and s.endswith(':'):
			in_blueprint = True
			blueprint_name = s.split('(')[-1].split(')')[0].strip()
			unreal_blueprint = unreal_blueprints[blueprint_name] = []

		elif s == 'unreal.plugin:':
			if not unreal_plugin_name:
				if '.unreal/' in file_name:
					unreal_plugin_name = os.path.split(file_name.split('.unreal/')[0])[-1]
				else:
					raise RuntimeError(file_name)
			in_unreal_plugin = True
			unreal_plugin_lib = list(out)
			out = []
			unreal_plugin_cpp.append('#include "%sPrivatePCH.h"' %unreal_plugin_name)
			unreal_plugin_cpp.append('#include "I%s.h"' %unreal_plugin_name)
			for efunc in extern_funcs:
				unreal_plugin_cpp.append(efunc)
			unreal_plugin_cpp.append('class F%s: I%s {' %(unreal_plugin_name, unreal_plugin_name))

			unreal_plugin_iface.append(UIFACE_TEMPLATE % tuple([unreal_plugin_name]*5) )


		elif s.startswith('import '):
			inc = s.split()[-1]
			if mode=='js':
				if inc == 'THREE':
					threepath = os.path.expanduser('~/three.js')
					if not os.path.isdir(threepath):
						raise RuntimeError(THREEJS_MISSING)
					dat = open(os.path.join(threepath, 'build/three.min.js')).read()
					info['js_header'].append(dat)
				elif inc.endswith('.js'):
					ipath = None
					for arg in sys.argv:
						if arg.startswith('--import-path'):
							assert '=' in arg
							ipath = arg.split('=')[-1]
							break
					if not ipath:
						raise RuntimeError('you must provide the option `--import-path=` to import external js files')
					else:
						jspath = os.path.join(ipath, inc)
						if os.path.isfile(jspath):
							dat = open(jspath).read()
							info['js_header'].append(dat)
						else:
							raise RuntimeError('could not import js file: ' + jspath )

			else:
				if inc.startswith("<"):
					assert inc.endswith(">")
				elif not inc.startswith('"'):
					inc = '"' + inc + '"'
				out.append(('\t'*indent)+'#include ' + inc)
		elif s.startswith('define('):
			assert s.endswith(')')
			if '=' in s:
				defname = s[len('define(') : s.index('=') ]
				defval  = s[ s.index('=')+1 : -1]
				out.append('#define %s %s' %(defname, defval))
			else:
				defname = s[len('define(') : -1]
				out.append('#define %s' %defname)
		elif s.startswith('define ') and s.endswith(':'):
			define_ident = indent
			defname = s[len('define ') : -1 ]
			define.append('#define %s \\' %defname)
			continue
		elif s == 'init_list:':
			init_list_indent = indent
			init_list = []
			in_init_list = True
		elif in_init_list:
			if s.endswith(','):
				s = s[:-1]
			init_list.append(s)
			continue
		elif len(define):
			if indent <= define_ident:
				assert define[-1].endswith('\\')
				define[-1] = define[-1][:-1]
				out.extend(define)
				define = []
				define_ident = -1
				out.append( auto_semicolon(ln) )
			else:
				define.append(ln + '\\')
		elif s.startswith('undef('):
			assert s.endswith(')')
			defname = s.split('(')[-1].split(')')[0]
			out.append('#undef %s' %defname)

		elif s.strip() == 'pass':
			out.append('/*pass*/')

		elif s.startswith('@module'):
			assert s.count('(')==1
			assert s.count(')')==1
			modname = s.split('(')[-1].split(')')[0].strip()
			assert modname
			if modname not in mods:
				mods[modname] = []
			out.append('// module: ' + modname)
		elif s == '@const':
			pass
		elif s == '@static':
			pass
		elif s.startswith('@unwrap('):
			pass
		elif s.startswith('@typedef('):
			pass
		elif s == '@constexpr':
			out.append('constexpr')
		elif s in ('@javascript', '@js'):
			em_js = True
		elif s.startswith('@template('):
			out.append( 'template<%s>' % s[len('@template(') : -1] )
		elif s.startswith('@virtual'):
			out.append( 'virtual' )
		elif s.startswith('@UCLASS'):
			out.append( 'UCLASS' )
		elif s.startswith('@UFUNCTION'):