assorted scanner/parser bug fixes by Pieter (tested by Ronny)

(bug_incomplete_instance_def, bug_layout_rule, bug_nested_guard_in_otherwise,
 parse-bug-18, parse_bug_Real_as_class_name, parse_bug_case,
 parse_bug_constructor_with_name_of_basic_type, parse_bug_lost_brackets_in_pattern,
 parse_bug_no_layout_rule)


git-svn-id: https://svn.cs.ru.nl/repos/clean-compiler/trunk@550 1f8540f1-abd5-4d5b-9d24-4c5ce8603e2d

1 files changed, 178 insertions, 424 deletions

diff --git a/frontend/scanner.icl b/frontend/scanner.icl
index 78c8c32..25ebe79 100644
--- a/frontend/scanner.icl
+++ b/frontend/scanner.icl
@@ -57,7 +57,7 @@ instance getPosition ScanState
 where
 	getPosition (ScanState scan_state)
 		# (position,scan_state) = getPosition scan_state
-		= (position,ScanState scan_state) 
+		= (position,ScanState scan_state)
 
 ::	* RScanState =
 	{	ss_input		::	ScanInput
@@ -67,7 +67,7 @@ where
 	}
 
 ::	* ScanInput
-	=	Input	Input
+	=	Input			Input
 	|	PushedToken		LongToken ScanInput
 
 ::	* Input =
@@ -79,9 +79,7 @@ where
 
 ::	* InputStream
 	=	InFile			* File
-	|	OldLine !Int !{#Char} !InputStream
-//	|	OldChar			! Char ! FilePosition ! InputStream
-//	|	OldChars		! *[Char] ! InputStream
+	|	OldLine 		!Int !{#Char} !InputStream
 
 ::	FilePosition =
 	{	fp_line			::	! Int
@@ -90,9 +88,9 @@ where
 
 ::	LongToken =
 	{	lt_position		::	! FilePosition	// Start position of this token
+	,	lt_index		::	! Int			// The index in the current line
 	,	lt_token		::	! Token			// The token itself
-//	,	lt_chars		::	! [Char]		// The chars in this token
-//	,	lt_context		::	! Context		// The context of the scanning of this token
+	,	lt_context		::	! Context		// The context of the token
 	}
 
 ::	Buffer x
@@ -189,6 +187,9 @@ where
 	|	GenericOpenToken		//		{|
 	|	GenericCloseToken		//		|}
 
+	|	ExistsToken				//		E.
+	|	ForAllToken				//		A.
+
 
 ::	Context
 	=	GeneralContext
@@ -215,7 +216,6 @@ where
 ScanErrIllegal	:== "illegal char in input"
 ScanErrCharErr	:== "wrong character denotation"
 ScanErrNLString	:== "new line in string denotation"
-ScanErrWild		:== "ident should not start with _"
 
 class getFilename state :: !*state -> (!String,!*state)
 
@@ -261,11 +261,52 @@ where
 instance getCharPosition Input
 where getCharPosition input=:{inp_pos} = (inp_pos, input)
 
+class getIndex input :: !*input -> (!Int, !*input)
+
+instance getIndex InputStream
+where
+	getIndex input=:(OldLine index _ _) = (index-1,input)
+	getIndex input = (0,input)
+
+instance getIndex Input
+where
+	getIndex input=:{inp_stream=stream}
+		# (index,stream) = getIndex stream
+		= (index,{input & inp_stream=stream})
+
 class nextToken state :: !Context !*state -> (!Token, !*state)
 
 instance nextToken RScanState
 where
-//	nextToken newContext {ss_input=PushedToken token=:{lt_position,lt_token} rest,ss_tokenBuffer,ss_offsides,ss_useLayout}
+/* RWS ... rolled back from Pieter's version
+
+    this fixes the bug funny_id_after_type, but failes on
+        g = let x = 1 in x
+
+	nextToken newContext (scanState=:{ss_input=inp=:PushedToken token=:{lt_position,lt_token,lt_context,lt_index} rest_inp,ss_tokenBuffer,ss_offsides,ss_useLayout})
+		| lt_context == newContext || notContextDependent lt_token
+		=	(	lt_token
+			,	{ scanState & ss_input = rest_inp , ss_tokenBuffer	= store token ss_tokenBuffer }
+			)  //-->> ("nextToken: pushed token", lt_token)
+		= token_back rest_inp
+		where
+			token_back input=:(Input {inp_pos,inp_stream=OldLine _ string stream,inp_filename,inp_tabsize}) // one old token in wrong context.
+				|	inp_pos.fp_line == lt_position.fp_line
+				#	old_input
+					=	{ inp_stream	= OldLine lt_index string stream
+						, inp_filename	= inp_filename
+						, inp_pos		= lt_position
+						, inp_tabsize	= inp_tabsize
+						} -->> ("token_back in input", lt_token)
+				=	nextToken newContext {ss_input = Input old_input, ss_offsides=ss_offsides, ss_useLayout=ss_useLayout, ss_tokenBuffer=ss_tokenBuffer}
+				=	(	lt_token
+					,	{ss_input = input , ss_tokenBuffer	= store token ss_tokenBuffer, ss_offsides=ss_offsides, ss_useLayout=ss_useLayout}
+					) -->> ("unable to push token_back in input; line is lost",(inp_pos.fp_line,lt_position.fp_line), lt_token)
+			token_back input
+				=	(	lt_token
+					,	{ss_input = input , ss_tokenBuffer	= store token ss_tokenBuffer, ss_offsides=ss_offsides, ss_useLayout=ss_useLayout}
+					) -->> ("unable to push token_back in input; generated token", lt_token)
+*/
 	nextToken newContext scanState=:{ss_input=input=:PushedToken token=:{lt_position,lt_token/*,lt_context*/} rest,ss_tokenBuffer}
 //		| lt_context == newContext || ~ (contextDependent lt_token) || isGeneratedToken lt_token
 
@@ -284,17 +325,20 @@ where
 				& inp_stream = OldToken token inp_stream
 				}  //-->> ("pushTokensBack",token)
 */
+// ... RWS
+
 	nextToken context {ss_input=Input inp,ss_tokenBuffer,ss_offsides,ss_useLayout}
 		# (error, c, inp) 	= SkipWhites inp
+		  (pos, inp)		= inp!inp_pos
+		  (index,inp)		= getIndex inp
 		= case error of
 			Yes string
-				#! (pos, inp)		= inp!inp_pos
 				->	( ErrorToken string
 							,	{	ss_tokenBuffer	= store
 												{	lt_position 	= pos
+												,	lt_index		= index
 												,	lt_token		= ErrorToken string
-											//	,	lt_chars		= []
-											//	,	lt_context		= context
+												,	lt_context		= context
 												}
 												ss_tokenBuffer,
 									ss_input=Input inp,
@@ -303,86 +347,36 @@ where
 							) -->> ("Error token generated",string)
 			no
 				#	(eof, inp)	= EndOfInput inp
-				#!	(pos, inp)	= inp!inp_pos
 				|	eof && c == NewLineChar
 					#	newToken	= EndOfFileToken
-					->	checkOffside pos newToken
+					->	checkOffside pos index newToken
 							{ ss_tokenBuffer	= store
 													{	lt_position 	= pos
+													,	lt_index		= index
 													,	lt_token		= newToken
-												//	,	lt_chars		= []
-												//	,	lt_context		= context
+													,	lt_context		= context
 													}
 													ss_tokenBuffer
 							, ss_input = Input inp,
 							ss_offsides=ss_offsides,	ss_useLayout=ss_useLayout
 							} // -->> ("Token", EndOfFileToken,pos)
 				// otherwise // ~ (eof && c == NewLineChar)
-					#	(token, inp)	= Scan c inp /* {inp & inp_curToken = [c]}*/ context
-				//	#	(chars, inp)	= inp!inp_curToken
-					-> checkOffside pos token
+					#	(token, inp)	= Scan c inp context
+					-> checkOffside pos index token
 						{ ss_input 			= Input inp
 						, ss_tokenBuffer	= store
 												{	lt_position 	= pos
+												,	lt_index		= index
 												,	lt_token		= token
-											//	,	lt_chars		= reverse chars
-											//	,	lt_context		= context
+												,	lt_context		= context
 												}
 												ss_tokenBuffer,
 							ss_offsides=ss_offsides,	ss_useLayout=ss_useLayout
 						}	 //-->> (token,pos)
-
-/*
-		#! (pos, inp)		= inp!inp_pos
-		#!  scanState			= {scanState & ss_input = Input inp }
-//		#! scanState = {ss_input=Input inp, ss_offsides=ss_offsides,	ss_useLayout=ss_useLayout, ss_tokenBuffer=ss_tokenBuffer }
-		= case error of
-			Yes string
-				->	( ErrorToken string
-							,	{	scanState
-								&	ss_tokenBuffer	= store
-												{	lt_position 	= pos
-												,	lt_token		= ErrorToken string
-											//	,	lt_chars		= []
-											//	,	lt_context		= context
-												}
-												scanState.ss_tokenBuffer
-								}
-							) -->> ("Error token generated",string)
-			no
-				->	determineToken c pos scanState
 	where
-			determineToken c pos scanState=:{ss_input=Input inp}
-				#	(eof, inp)	= EndOfInput inp
-	//			#!	(pos, inp)	= inp!inp_pos
-				|	eof && c == NewLineChar
-					#	newToken	= EndOfFileToken
-					=	checkOffside pos newToken
-							{ scanState
-							& ss_tokenBuffer	= store
-													{	lt_position 	= pos
-													,	lt_token		= newToken
-												//	,	lt_chars		= []
-												//	,	lt_context		= context
-													}
-													scanState.ss_tokenBuffer
-							, ss_input = Input inp
-							} // -->> ("Token", EndOfFileToken,pos)
-				// otherwise // ~ (eof && c == NewLineChar)
-					#	(token, inp)	= Scan c inp /* {inp & inp_curToken = [c]}*/ context
-				//	#	(chars, inp)	= inp!inp_curToken
-					= checkOffside pos token
-						{ scanState
-						& ss_input 			= Input inp
-						, ss_tokenBuffer	= store
-												{	lt_position 	= pos
-												,	lt_token		= token
-											//	,	lt_chars		= reverse chars
-											//	,	lt_context		= context
-												}
-												ss_tokenBuffer
-						}	 //-->> (token,pos)
-*/
+		mark_position {inp_stream=input=:(OldLine i _ _),inp_filename,inp_pos,inp_tabsize}
+			= {inp_stream=input, inp_filename=inp_filename, inp_pos={inp_pos &fp_col=1}, inp_tabsize=inp_tabsize}
+		mark_poistion input = input
 	nextToken _ _ = abort "Scanner: Error in nextToken"
 
 class tokenBack state :: !*state -> !*state
@@ -410,26 +404,24 @@ class insertToken state :: !Token !Context !*state -> *state
 instance insertToken RScanState
 where
 	insertToken t c scanState
-/*		#	chars = if (isGeneratedToken t)
-						[]
-						(fromString (toString t))
-*/		#	(pos, scanState=:{ss_input}) = getPosition scanState
+		#	(pos, scanState=:{ss_input}) = getPosition scanState
 		=	{ scanState
 			& ss_input = PushedToken
 							{ lt_position	= pos
+							, lt_index		= pos.fp_col
 							, lt_token		= t
-						//	, lt_chars		= chars
-						//	, lt_context	= c
+							, lt_context	= c
 							}
 							ss_input
 			}
-/*
-isGeneratedToken :: !Token -> Bool
-isGeneratedToken NewDefinitionToken = True
-isGeneratedToken EndGroupToken      = True
-isGeneratedToken (CodeBlockToken _) = True
-isGeneratedToken _					= False
-*/
+
+notContextDependent :: !Token -> Bool
+notContextDependent NewDefinitionToken	= True
+notContextDependent EndGroupToken		= True
+notContextDependent EndOfFileToken		= True
+notContextDependent (ErrorToken _)		= True
+notContextDependent (CodeBlockToken _)	= True
+notContextDependent _					= False
 
 class replaceToken state :: !Token !*state -> *state
 
@@ -447,7 +439,7 @@ SkipWhites {inp_stream=OldLine i line stream,inp_pos={fp_line,fp_col},inp_tabsiz
 		= skip_whites_in_line i fp_col fp_line line inp_tabsize stream inp_filename
 SkipWhites input
 	# (eof, c, input)		= ReadChar input
-	| eof					= (No, NewLineChar, input) // -->> "EOF in SkipWhites"
+	| eof					= (No, NewLineChar, input)
 	| IsWhiteSpace c		= SkipWhites input
 							= TryScanComment c input
 
@@ -490,7 +482,6 @@ TryScanComment c1=:'/' input
 				(No,input)	-> SkipWhites input
 				(er,input)	-> (er, c1, input)
 		_   -> (No, c1, charBack input)
-			
 TryScanComment c input
 	= (No, c, input)
 
@@ -566,7 +557,6 @@ SkipToEndOfLine input
 			= SkipToEndOfLine input
 
 Scan :: !Char !Input !Context -> (!Token, !Input)
-
 Scan '(' input co			= (OpenToken, input)
 Scan ')' input co			= (CloseToken, input)
 Scan '{' input CodeContext	= ScanCodeBlock input
@@ -607,7 +597,7 @@ Scan c0=:'&' input co
 	| eof					= (AndToken, input)
 	| isSpecialChar c1		= ScanOperator 1 input [c1, c0] co
 							= (AndToken, charBack input)
-Scan c0=:'.' input co
+Scan c0=:'.' input co	// PK incorrect ?
 	= case co of
 		TypeContext
 						-> (DotToken, input)
@@ -628,6 +618,23 @@ Scan '\\' input co
 	| eof					= (BackSlashToken, input)
 	| c == '\\'				= (DoubleBackSlashToken, input)
 							= (BackSlashToken, charBack input)
+Scan c0=:'_' input=:{inp_stream=OldLine i line stream,inp_pos} co //PK ..
+	# size	= size line
+	# end_i	= scan_underscores i size line
+		with
+			scan_underscores :: !Int !Int !{#Char} -> Int
+			scan_underscores i size line
+				| i<size && line.[i] == '_'
+					= scan_underscores (i+1) size line
+					= i
+	| end_i<size && IsIdentChar line.[end_i] co
+		= ScanIdentFast (end_i-i+1) {input & inp_stream=OldLine end_i line stream} co
+	| end_i==i
+		= (WildCardToken, input)
+		# pos = {inp_pos & fp_col = inp_pos.fp_col + (end_i-i)}
+		# input =  {input & inp_stream=OldLine end_i line stream,inp_pos=pos}
+		= (ErrorToken (line % (i-1,end_i-1)+++" is an illegal token"),input)
+/* PK
 Scan c0=:'_' input co
 	# (eof, c1, input)		= ReadNormalChar input
 	| eof					= (WildCardToken, input)
@@ -636,6 +643,7 @@ Scan c0=:'_' input co
 		= ScanIdentFast 2 input co
 //	| isSpecialChar c1		= ScanOperator 1 input [c1, c0] co
 							= (WildCardToken, charBack input)
+*/
 Scan c0=:'<' input TypeContext
 	# (eof, c1, input)		= ReadNormalChar input
 	| eof					= (ErrorToken "< just before end of file in TypeContext", input)
@@ -677,12 +685,8 @@ Scan c0=:'+' input co
 
 	# (eof, c1, input)		= ReadNormalChar input
 	| eof					= (IdentToken "+", input)
-//	# new					= newExp input.inp_charBuffer
-//	| IsDigit c1 && new		= ScanNumeral 1 input [c1,c0]
-
 	| IsDigit c1 && new_exp_char previous_char
 		= ScanNumeral 1 input [c1,c0]
-
 							= ScanOperator 0 (charBack input) [c0] co
 Scan c0=:'=' input co
 	# (eof, c, input)		= ReadNormalChar input
@@ -703,8 +707,21 @@ Scan c0=:':' input co
 	| c2 == '='				= (ColonDefinesToken, input)
 							= ScanOperator 1 (charBack input) [c1, c0] co
 Scan c0=:'\'' input co		= ScanChar input [c0]
-Scan c0=:'\"' input co		= ScanString 0 input [c0]
-
+Scan c0=:'\"' input co		= ScanString 0 [c0] input
+// PK ..
+Scan 'E' input TypeContext
+	# (eof,c1,input)		= ReadNormalChar input
+	| eof					= (IdentToken "E", input)
+	| c1 == '.'				= (ExistsToken, input)
+//							= ScanIdent 1 (charBack input) TypeContext
+							= ScanIdentFast 1 (charBack input) TypeContext
+Scan 'A' input TypeContext
+	# (eof,c1,input)		= ReadNormalChar input
+	| eof					= (IdentToken "A", input)
+	| c1 == '.'				= (ForAllToken, input)
+//							= ScanIdent 1 (charBack input) TypeContext
+							= ScanIdentFast 1 (charBack input) TypeContext
+// .. PK
 Scan c    input co
 	| IsDigit c				= ScanNumeral 0 input [c]
 	| IsIdentChar c	co	
@@ -712,18 +729,11 @@ Scan c    input co
 //		= ScanIdent 0 input [c] co
 	| isSpecialChar c		= ScanOperator 0 input [c] co
 							= (ErrorToken ScanErrIllegal, input)
-/*
-newExp :: !(Buffer (Char,FilePosition)) -> Bool
-newExp buffer
-	# (c, _)	= case buffer of
-					Buffer3 _ _ cp	-> cp
-					_				-> (' ',{fp_line=0,fp_col=0})
-	= new_exp_char c
-where
-*/
+
 new_exp_char ',' = True
 new_exp_char '[' = True
 new_exp_char '(' = True
+new_exp_char '{' = True
 new_exp_char '/' = True // to handle end of comment symbol: */
 new_exp_char c	 = isSpace c
 
@@ -739,18 +749,11 @@ ScanIdentFast n input=:{inp_stream=OldLine i line stream,inp_pos} co
 	# pos = {inp_pos & fp_col = inp_pos.fp_col + (end_i-i)}
 	# input =  {input & inp_stream=OldLine end_i line stream,inp_pos=pos}
 	= CheckReserved co (line % (i-n,end_i-1)) input
-/*
-ScanIdent :: !Int !Input ![Char] !Context -> (!Token, !Input)
-ScanIdent n input token co
-	# (eof, c, input)		= ReadNormalChar input
-	| eof					= CheckReserved co (revCharListToString n token) input
-	| IsIdentChar c co		= ScanIdent (n + 1) input [c:token] co
-							= CheckReserved co (revCharListToString n token) (charBack input)
-*/
+
 ScanOperator :: !Int !Input ![Char] !Context -> (!Token, !Input)
 ScanOperator n input token co
 	#  (eof, c, input)		= ReadNormalChar input
-	| eof					= (IdentToken (revCharListToString n token), input)
+	| eof					= CheckReserved co (revCharListToString n token) input
 	| isSpecialChar c		= ScanOperator (n + 1) input [c:token] co
 							= CheckReserved co (revCharListToString n token) (charBack input)
 
@@ -930,8 +933,8 @@ ScanFraction n input chars
 	# (eof, c, input)		= ReadNormalChar input
 	| eof					= (RealToken (revCharListToString n chars), input)
 	| c == 'E'				= case chars of
-								[c:_] | IsDigit c	-> ScanExponentSign (n + 1) input ['E':chars] /* Sjaak, was [c:chars] */
-								_					-> ScanExponentSign (n + 2) input ['E','0':chars] /* Sjaak, idem */
+								[c:_] | IsDigit c	-> ScanExponentSign (n + 1) input ['E':chars]
+								_					-> ScanExponentSign (n + 2) input ['E','0':chars]
 	| IsDigit c				= ScanFraction (n + 1) input [c:chars]
 							= case chars of
 								[c:_] | IsDigit c	-> (RealToken (revCharListToString n chars), charBack input)
@@ -973,65 +976,57 @@ ScanChar input chars
 	# (eof, c, input)		= ReadNormalChar input
 	| eof					= (ErrorToken "End of file inside Char denotation", input)
 	| '\\' <> c				= ScanEndOfChar 1 [c: chars] input
-	# (chars, n, input)		= ScanBSChar 0 chars input
-	= ScanEndOfChar n chars input
+	= ScanBSChar 0 chars input ScanEndOfChar
 
-ScanBSChar :: !Int ![Char] !Input -> (![Char], !Int, !Input)
-ScanBSChar n chars input
+ScanBSChar :: !Int ![Char] !Input (!Int ![Char] !Input -> (!Token, !Input)) -> (!Token, !Input)
+ScanBSChar n chars input cont
 	# (eof, c, input)		= ReadNormalChar input
-	| eof					= (chars, n, input)
+	| eof					= cont n chars input
 	= case c of
-		'n'		->	(['n','\\':chars], n + 2, input)
-		'r'		->	(['r','\\':chars], n + 2, input)
-		'f'		->	(['f','\\':chars], n + 2, input)
-// RWS ...		'b'		->	(['b','\\':chars], n + 2, input)
-		'b'		->	to_chars '\b' n input
-// ... RWS
-		't'		->	(['t','\\':chars], n + 2, input)
-// RWS ...		'v'		->	(['v','\\':chars], n + 2, input)
-		'v'		->	to_chars '\v' n input
-// ... RWS
-		'\\'	->	(['\\','\\':chars], n + 2, input)
-		'"'		->	(['"' ,'\\':chars], n + 2, input)
-		'\''	->	(['\'','\\':chars], n + 2, input)
-		'x'		#	(cc,input) = ScanNumChar Hex isHexDigit 2 0 input // max 2 characters
-				->	to_chars cc n input
-		'd'		#	(cc,input) = ScanNumChar Dec isDigit 3 0 input // max 3 characters
-				->	to_chars cc n input
+		'n'		->	cont (n+2) ['n','\\':chars] input		// (['n','\\':chars], n + 2, input)
+		'r'		->	cont (n+2) ['r','\\':chars] input		// (['r','\\':chars], n + 2, input)
+		'f'		->	cont (n+2) ['f','\\':chars] input		// (['f','\\':chars], n + 2, input)
+		'b'		->	to_chars '\b' input
+		't'		->	cont (n+2) ['t','\\':chars] input		// (['t','\\':chars], n + 2, input)
+		'v'		->	to_chars '\v' input
+		'\\'	->	cont (n+2) ['\\','\\':chars] input		// (['\\','\\':chars], n + 2, input)
+		'"'		->	cont (n+2) ['"','\\':chars] input		// (['"' ,'\\':chars], n + 2, input)
+		'\''	->	cont (n+2) ['\'','\\':chars] input		// (['\'','\\':chars], n + 2, input)
+		'x'		->	ScanNumChar Hex isHexDigit 2 0 input	// max 2 characters
+		'X'		->	ScanNumChar Hex isHexDigit 2 0 input	// max 2 characters
+		'd'		->	ScanNumChar Dec isDigit 3 0 input		// max 3 characters
+		'D'		->	ScanNumChar Dec isDigit 3 0 input		// max 3 characters
+		'0'		->	ScanNumChar Oct IsOct 3 0 input			// max 3 characters
 		c | IsOct c
-				#	(cc,input) = ScanNumChar Oct IsOct 2 (digitToInt c) input // max 3 characters, including current
-				->	to_chars cc n input
-				->	([c:chars], n + 1, input)
+				->	ScanNumChar Oct IsOct 2 (digitToInt c) input // max 2 more characters, 3 including current
+				->	cont (n+1) [c:chars] input
 where
 	ScanNumChar base valid 0 acc input
-		= (acc, input)
+		= to_chars acc input
 	ScanNumChar base valid n acc input
 		# (eof, c, input)		= ReadNormalChar input
-		| eof					= (acc, input)
-// RWS ...		| valid c				= ScanNumChar base valid (n-1) (base*acc+digitToInt c) input
+		| eof					= to_chars acc input
 		| valid c				= ScanNumChar base valid (n-1) (base*acc+hexDigitToInt c) input
-// ... RWS
-								= (acc, charBack input)	
+								= to_chars acc (charBack input)
 	Hex = 16
 	Oct = 8
 	Dec = 10
 	
-	to_chars cc n input
+	to_chars cc input
+	| toInt cc > 255
+	 = (ErrorToken "invalid char, value > 255", input)
 	 = case toChar cc of
-		'\n'	->	(['n','\\':chars], n + 2, input)
-		'\r'	->	(['r','\\':chars], n + 2, input)
-		'\f'	->	(['f','\\':chars], n + 2, input)
-// RWS \b not accepted in abc		'\b'	->	(['b','\\':chars], n + 2, input)
-		'\t'	->	(['t','\\':chars], n + 2, input)
-// RWS \v not accepted in abc		'\v'	->	(['v','\\':chars], n + 2, input)
-		'\\'	->	(['\\','\\':chars], n + 2, input)
-		'"'		->	(['"' ,'\\':chars], n + 2, input)
-		'\''	->	(['\'','\\':chars], n + 2, input)
-
-// RWS ...
+		'\n'	->	cont (n+2) ['n','\\':chars] input	// (['n','\\':chars], n + 2, input)
+		'\r'	->	cont (n+2) ['r','\\':chars] input	// (['r','\\':chars], n + 2, input)
+		'\f'	->	cont (n+2) ['f','\\':chars] input	// (['f','\\':chars], n + 2, input)
+		'\t'	->	cont (n+2) ['t','\\':chars] input	// (['t','\\':chars], n + 2, input)
+		'\\'	->	cont (n+2) ['\\','\\':chars] input	// (['\\','\\':chars], n + 2, input)
+		'"'		->	cont (n+2) ['"','\\':chars] input	// (['"' ,'\\':chars], n + 2, input)
+		'\''	->	cont (n+2) ['\'','\\':chars] input	// (['\'','\\':chars], n + 2, input)
+		// '\b' and '\v' not accepted in abc
 		// escape non-printable characters
 		c		| not (IsPrint c)
-					-> (more_chars, n+4, input)
+					-> cont (n+4) more_chars input
 					with
 						more_chars =
 							[	toChar (48 +  (toInt c       bitand 7))
@@ -1040,8 +1035,7 @@ where
 							,	'\\'
 							:	chars
 							]
-// ... RWS
-	 	c		->	([c:chars], n + 1, input)
+	 	c		->	cont (n+1) [c:chars] input
 
 ScanEndOfChar :: !Int ![Char] !Input -> (!Token, !Input)
 ScanEndOfChar n chars input
@@ -1058,21 +1052,19 @@ ScanCharList n chars input
 	= case c of
 		'\''		#	charList	= revCharListToString n chars % (1,n) // without '\''
 					->	(CharListToken charList, input)
-		'\\'		#	(chars, n, input)	= ScanBSChar n chars input
-					->	ScanCharList n chars input
+		'\\'		->	ScanBSChar n chars input ScanCharList
 		NewLineChar	->	(ErrorToken "newline in char list", input)
 		_			->	ScanCharList (n+1) [c:chars] input
 
-ScanString :: !Int !Input ![Char] -> (!Token, !Input)
-ScanString n input chars
+ScanString :: !Int ![Char] !Input -> (!Token, !Input)
+ScanString n chars input
  # (eof, c, input)		= ReadChar input
  | eof					= (ErrorToken "End of file inside String denotation", input)
  = case c of 
-	'\\' 		#  (chars, n, input) = ScanBSChar n chars input
-		 		-> ScanString n input chars
+	'\\' 		->	ScanBSChar n chars input ScanString
 	'\"' 		-> (StringToken (revCharListToString (n + 1) [c:chars]), input)
 	NewLineChar -> (ErrorToken ScanErrNLString, input)
-	_	 		-> ScanString (n + 1) input [c:chars]
+	_	 		-> ScanString (n + 1) [c:chars] input
 
 /*
 	some predicates on tokens
@@ -1096,12 +1088,6 @@ isEndGroupToken EndGroupToken = True
 isEndGroupToken CurlyCloseToken = True
 isEndGroupToken token = False
 /*
-contextDependent :: !Token -> Bool
-contextDependent HashToken 			= True
-//contextDependent (SeqLetToken _)	= True // Do not do this XXXXXX
-contextDependent _					= False
-*/
-/*
 	character functions
 */
 
@@ -1113,28 +1099,9 @@ IsDigit c :== isDigit c
 
 IsOct c :== '0' <= c && c <= '7'
 
-// RWS ...
-//IsDigit	:: Char	-> Bool
-// this assumes all 8 bit characters (>127) are not printable
+// IsPrint assumes all 8 bit characters (>127) are not printable
 IsPrint c
 	:== c >= ' ' && c <= '~'
-// ... RWS
-
-//IsHex c :== isDigit c || ('A' <= c && c <= 'F') || ('a' <= c && c <= 'f') 
-/*
-isHexDigit :: !Char -> Bool // Defined in StdChar
-isHexDigit c
-	| isDigit c
-		= True
-	| c < 'g'
-		= c >= 'a'
-	| c < 'G'
-		= c >= 'A'
-		= False
-*/
-//IsIdentChar	:: !Char !Context -> Bool
-//IsIdentChar c co
-// :== isAlphanum c ||	c == '_' ||	c == '`' ||	(c == '^' && co == TypeContext)
 
 hexDigitToInt :: !Char -> Int
 hexDigitToInt 'a' = 10
@@ -1263,145 +1230,8 @@ ReadChar {inp_stream = InFile file, inp_pos, inp_tabsize, inp_filename}
 				inp_tabsize=inp_tabsize,inp_filename=inp_filename,inp_pos=inp_pos,
 				inp_stream = OldLine 0 s (InFile file)
 				}
-/*
-		// otherwise // s <> ""
-	//		# chars = fromString s
-			# chars = string_to_list s
-			= ReadChar {
-	//				input &
-					inp_tabsize=inp_tabsize,inp_filename=inp_filename,inp_pos=inp_pos,
-					inp_stream = OldChars chars (InFile file)
-					}
-*/
-
-	/*
-			#! (eof, file) = fend file								// old, too slow
-			| eof
-				#	c	= NewLineChar
-					pos	= NextPos c inp_pos inp_tabsize
-				=	(	eof
-					,	c
-					,	{	input
-						&	inp_stream		= InFile file
-						,	inp_pos			= pos
-						}
-					) // -->> ("EOF in " + input.inp_filename + " found in ReadChar")
-			#! (ok, c, file) = freadc file
-			| ok
-				#	pos			= NextPos c inp_pos inp_tabsize
-					(c,input`)	= correctNewline c pos inp_tabsize (InFile file)
-				=	(	False
-					,	c
-					,	{	input
-						&	inp_stream		= input`
-						,	inp_pos			= pos
-						}
-					)
-				= abort "ReadChar failure"
-	*/
-/*
-	ReadChar input=:{inp_stream = InFile file, inp_pos, inp_tabsize, inp_filename}
-			#! (ok, c, file) = freadc file
-			| ok
-				| c==LFChar || c==CRChar || c=='\t'
-					#	pos			= NextPos c inp_pos inp_tabsize
-						(c,input`)	= correctNewline c pos inp_tabsize (InFile file)
-					=	(	False
-						,	c
-						,	{
-		//						input &
-								inp_filename=inp_filename,inp_tabsize=inp_tabsize,						
-								inp_stream		= input`
-							,	inp_pos			= pos
-							}
-						)
-						#	pos = {inp_pos & fp_col = inp_pos.fp_col + 1}
-						=	(	False
-							,	c
-							,	{
-			//						input &
-									inp_filename=inp_filename,inp_tabsize=inp_tabsize,						
-									inp_stream		= InFile file
-								,	inp_pos			= pos
-								}
-							)
-				#	c	= NewLineChar
-					pos	= NextPos c inp_pos inp_tabsize
-				=	(	True
-					,	c
-					,	{
-	//						input &
-							inp_filename=inp_filename,inp_tabsize=inp_tabsize,
-							inp_stream		= InFile file
-						,	inp_pos			= pos
-						}
-					)
-	//ReadChar input =: {inp_stream = OldChars [c] stream, inp_pos, inp_tabsize, /*, inp_curToken*/}
-	ReadChar {inp_stream = OldChars [c] stream, inp_pos, inp_tabsize, inp_filename}
-		#	pos			= NextPos c inp_pos inp_tabsize
-			(c,input`)	= correctNewline_OldChars c pos inp_tabsize[] stream
-		=	(	False
-			,	c
-			,	{
-	//				input &
-					inp_filename=inp_filename,inp_tabsize=inp_tabsize,
-					inp_stream		= input`
-				,	inp_pos			= pos
-				}
-			)
-	//ReadChar input =: {inp_stream = OldChars [c:rest] stream,inp_pos,inp_tabsize}
-	ReadChar {inp_stream = OldChars [c:rest] stream,inp_pos,inp_tabsize,inp_filename}
-		| c==LFChar || c==CRChar || c=='\t'
-			#	pos			= NextPos c inp_pos inp_tabsize
-				(c,input`)	= correctNewline_OldChars c pos inp_tabsize rest stream
-			=	(	False
-				,	c
-				,	{
-	//					input &
-						inp_filename=inp_filename,inp_tabsize=inp_tabsize,
-						inp_stream		= input`
-					,	inp_pos			= pos
-					}
-				)
-			#	pos = {inp_pos & fp_col = inp_pos.fp_col + 1}
-			=	(	False
-				,	c
-				,	{
-	//					input &
-						inp_filename=inp_filename,inp_tabsize=inp_tabsize,
-						inp_stream		= OldChars rest stream
-					,	inp_pos			= pos
-					}
-				)
-	
-	//	ReadChar input =: {inp_stream = OldChars [] stream, inp_pos}
-	ReadChar {inp_stream = OldChars [] stream, inp_pos,inp_filename,inp_tabsize}
-	//		= ReadChar {input & inp_stream = stream}
-			= ReadChar {inp_filename=inp_filename,inp_tabsize=inp_tabsize,inp_pos=inp_pos,
-									inp_stream = stream}
-*/
-
-/*
-	//ReadChar input =: {inp_stream = OldChar c pos oldfile}
-	ReadChar {inp_stream = OldChar c pos oldfile, inp_tabsize,inp_filename}
-		=	(	False
-				,	c
-				,	{
-	//					input &
-						inp_filename=inp_filename,inp_tabsize=inp_tabsize,
-						inp_stream		= oldfile
-					,	inp_pos			= pos
-					}
-				)
-*/
 
 ReadLine :: !Input -> (!String, !Input)
-/*
-ReadLine input=:{inp_stream = OldChars cs oldfile, inp_pos}
-	# input			= {input & inp_stream = oldfile, inp_pos = NextPos CRChar inp_pos 0}
-	| isEmpty cs	= ReadLine input
-	| otherwise		= (toString cs, input)
-*/
 ReadLine input=:{inp_stream = OldLine i line oldfile, inp_pos}
 	# input			= {input & inp_stream = oldfile, inp_pos = NextPos CRChar inp_pos 0}
 	| i<size line
@@ -1415,13 +1245,6 @@ ReadLine input=:{inp_stream = InFile infile,inp_pos}
 //MW8 was	# (l, file )	= freadline file
 	# (l, file )	= (SwitchPreprocessor freadPreprocessedLine freadline) file
 	= (l,  {input & inp_stream = InFile file, inp_pos = NextPos CRChar inp_pos 0})
-/*
-	ReadLine input=:{inp_stream = OldChar c p oldfile}
-		# input			= {input & inp_stream = oldfile}
-		| c==NewLineChar= ("\n", input)
-		# (line, input)	= ReadLine input
-		= (toString c + line, input)
-*/
 ReadLine input		= ("", input)
 
 NextPos :: !Char !FilePosition !Int -> FilePosition
@@ -1432,44 +1255,6 @@ NextPos c pos=:{fp_line, fp_col} t
 		'\t'	-> {pos & fp_col = t * (fp_col / t + 1)}
 		_		-> {pos & fp_col = fp_col + 1}
 
-/*
-	correctNewline :: !Char !FilePosition !Int !InputStream -> (!Char, !InputStream)
-	correctNewline c pos tab_size (InFile file) 	// Correct newline convention: Mac: CR, Unix: LF, DOS CR LF
-		=	case c of
-				LFChar	-> (NewLineChar,InFile file)	//-->> "UNIX newline"
-				CRChar 
-					# (ok,c2,file) =  freadc file
-					| ok
-						| c2 == LFChar -> (NewLineChar,InFile file) // -->> "DOS newline corrected"
-						               -> (NewLineChar,OldChar c2 (NextPos c2 pos tab_size) (InFile file))
-						-> (NewLineChar, InFile file)
-				_		-> (c, InFile file)
-	correctNewline c pos tab_size (OldChars [] input)
-		= correctNewline c pos tab_size input
-	correctNewline c pos tab_size (OldChars chars input)
-		=	case c of
-				LFChar	-> (NewLineChar,OldChars chars input)	//-->> "UNIX newline"
-				CRChar
-					# [c2:rest] = chars
-					| c2 == LFChar -> (NewLineChar,OldChars rest input) // -->> "DOS newline corrected"
-					               -> (NewLineChar,OldChars [c2:rest]/*chars*/ input)
-				_		-> (c,OldChars chars input)
-	correctNewline c _ _ input = (c, input)
-	
-	correctNewline_OldChars :: !Char !FilePosition !Int ! *[Char] ! InputStream -> (!Char, !InputStream)
-	correctNewline_OldChars c pos tab_size [] input
-		= correctNewline c pos tab_size input
-	correctNewline_OldChars c pos tab_size chars input
-		=	case c of
-				LFChar
-					-> (NewLineChar,OldChars chars input)	//-->> "UNIX newline"
-				CRChar
-					# [c2:rest] = chars
-					| c2 == LFChar
-						-> (NewLineChar,OldChars rest input) // -->> "DOS newline corrected"
-					    -> (NewLineChar,OldChars [c2:rest]/*chars*/ input)
-				_		-> (c,OldChars chars input)
-*/
 correctNewline_OldLine :: !Char !Int !Int !{#Char} ! InputStream -> (!Char, !InputStream)
 correctNewline_OldLine c i tab_size line input
 	=	case c of
@@ -1488,16 +1273,6 @@ charBack {inp_stream=OldLine i line stream,inp_pos,inp_tabsize,inp_filename}
 			inp_pos = {inp_pos & fp_col = inp_pos.fp_col - 1},
 			inp_tabsize=inp_tabsize,inp_filename=inp_filename
 		}
-/*
-charBack input=:{inp_stream,inp_charBuffer}
-	|	isEmptyBuffer inp_charBuffer
-		=	abort "charBack with empty character buffer"
-	#	((c,p),rest) = get inp_charBuffer
-	=	{	input
-		&	inp_stream		= OldChar c p inp_stream
-		,	inp_charBuffer	= rest
-		}
-*/
 
 GetPreviousChar :: !Input -> (!Char,!Input)
 GetPreviousChar input=:{inp_stream=OldLine i line stream}
@@ -1536,6 +1311,8 @@ where
 	toString CurlyCloseToken		= "}"
 	toString SquareOpenToken		= "["
 	toString SquareCloseToken		= "]"
+	toString ExistsToken			= "E."
+	toString ForAllToken			= "A."
 	toString GenericOpenToken		= "{|"
 	toString GenericCloseToken		= "|}"	
 	toString DotToken				= "."
@@ -1572,6 +1349,9 @@ where
 	toString RealTypeToken			= "Real"
 	toString BoolTypeToken			= "Bool"
 	toString StringTypeToken		= "String"
+	toString FileTypeToken			= "File"
+	toString WorldTypeToken			= "World"
+	toString VoidTypeToken			= "Void"
 	toString LeftAssocToken			= "left"
 	toString RightAssocToken		= "right"
 	toString ClassToken				= "class"
@@ -1736,8 +1516,6 @@ closeScanner_ {ss_input=Input {inp_stream}} files
 		No			->	files
 where
 	get_file (InFile file)			= Yes file
-//	get_file (OldChar _ _ stream)	= get_file stream
-//	get_file (OldChars _ stream)	= get_file stream
 	get_file (OldLine _ _ stream)	= get_file stream
 
 NewLineChar	:== '\n'
@@ -1768,8 +1546,8 @@ setUseLayout b (ScanState ss) = ScanState { ss & ss_useLayout = b }
 setUseLayout_ :: !Bool !RScanState -> RScanState
 setUseLayout_ b ss = { ss & ss_useLayout = b } // -->> ("uselayout set to ",b)
 
-checkOffside :: !FilePosition !Token !RScanState -> (Token,RScanState)
-checkOffside pos token scanState=:{ss_offsides,ss_useLayout,ss_input}
+checkOffside :: !FilePosition !Int !Token !RScanState -> (Token,RScanState)
+checkOffside pos index token scanState=:{ss_offsides,ss_useLayout,ss_input}
 	| ~ ss_useLayout
 		=	(token, scanState)  //-->> (token,pos,"No layout rule applied")
 	| isEmpty ss_offsides
@@ -1784,9 +1562,9 @@ checkOffside pos token scanState=:{ss_offsides,ss_useLayout,ss_input}
 			& ss_tokenBuffer
 				= store 
 					{	lt_position		= pos
+					,	lt_index		= index
 					,	lt_token		= newToken
-				//	,	lt_chars		= []
-				//	,	lt_context		= FunctionContext
+					,	lt_context		= FunctionContext
 					}
 					scanState.ss_tokenBuffer
 			}
@@ -1803,14 +1581,14 @@ checkOffside pos token scanState=:{ss_offsides,ss_useLayout,ss_input}
 								& ss_tokenBuffer
 									= store 
 										{	lt_position		= pos
+										,	lt_index		= index
 										,	lt_token		= newToken
-									//	,	lt_chars		= []
-									//	,	lt_context		= FunctionContext
+										,	lt_context		= FunctionContext
 										}
 										scanState.ss_tokenBuffer
 								} -->> ("new definition generated",token)
-  						False
-  							->	scanState
+						False
+							->	scanState
 		= gen_end_groups n scanState
 		with
 			newToken		= EndGroupToken
@@ -1826,9 +1604,9 @@ checkOffside pos token scanState=:{ss_offsides,ss_useLayout,ss_input}
 								& ss_tokenBuffer
 									= store 
 										{	lt_position		= pos
+										,	lt_index		= index
 										,	lt_token		= newToken
-									//	,	lt_chars		= []
-									//	,	lt_context		= FunctionContext
+										,	lt_context		= FunctionContext
 										}
 										scanState.ss_tokenBuffer
 								} -->> ("end group generated",pos) // insert EndGroupToken 
@@ -1846,7 +1624,6 @@ checkOffside pos token scanState=:{ss_offsides,ss_useLayout,ss_input}
 				= store 
 					{	lt_position		= pos
 					,	lt_token		= newToken
-				//	,	lt_chars		= []
 				//	,	lt_context		= FunctionContext
 					}
 					scanState.ss_tokenBuffer
@@ -1869,9 +1646,9 @@ where
 					& ss_tokenBuffer
 						= store 
 							{	lt_position		= pos
+							,	lt_index		= index
 							,	lt_token		= newToken
-						//	,	lt_chars		= ['groups should not start in column 1']
-						//	,	lt_context		= FunctionContext
+							,	lt_context		= FunctionContext
 							}
 							scanState.ss_tokenBuffer
 					}
@@ -1900,13 +1677,7 @@ needsNewDefinitionToken OfToken			= True
 //needsNewDefinitionToken WithToken		= True
 needsNewDefinitionToken SpecialToken	= True
 needsNewDefinitionToken _				= False
-/*
-repeatedOffside :: !Token -> Bool
-repeatedOffside BarToken		= True
-repeatedOffside EqualToken		= True
-repeatedOffside (SeqLetToken _)	= True
-repeatedOffside _				= False
-*/
+
 canBeOffside :: !Token -> Bool
 canBeOffside EqualToken			= False
 canBeOffside ColonDefinesToken	= False
@@ -1926,23 +1697,6 @@ dropOffsidePosition (ScanState s) = ScanState (dropOffsidePosition_ s)
 dropOffsidePosition_ :: !RScanState -> RScanState
 dropOffsidePosition_ scanState=:{ss_offsides} = { scanState & ss_offsides = drop 1 ss_offsides }
 
-/*
-addOffsidePosition :: !RScanState -> (Int, RScanState)
-addOffsidePosition scanState=:{ss_useLayout}
-	| ss_useLayout
-		# (position,scanState=:{ss_offsides})	= getPosition scanState
-		  new_offside							= position.fp_col
-		= (new_offside, { scanState & ss_offsides = [(new_offside,False): ss_offsides] })
-	| otherwise
-		= (1, scanState)
-
-atOffsidePosition :: !RScanState -> (!Bool, !RScanState)
-atOffsidePosition scanState=:{ss_offsides=[(col,_):_]}
-	#	(position, scanState)	= getPosition scanState
-	=	(position.fp_col == col, scanState) -->> ("atOffsidePosition",position.fp_col,col)
-atOffsidePosition scanState
-	=	(False, scanState)
-*/
   //-----------------------//
  //--- Buffer handling ---//
 //-----------------------//