diff --git a/src/vfs/_vfscommon/modules/punk-0.1.tm b/src/vfs/_vfscommon/modules/punk-0.1.tm index 20b7967..2d6e61d 100644 --- a/src/vfs/_vfscommon/modules/punk-0.1.tm +++ b/src/vfs/_vfscommon/modules/punk-0.1.tm @@ -295,16 +295,59 @@ namespace eval punk { return [twapi::new_uuid] } } - proc ::punk::var {varname {= {}} args} { + + #get last command result that was run through the repl + proc ::punk::get_runchunk {args} { + set argd [punk::args::get_dict { + *opts + -1 -optional 1 -type none + -2 -optional 1 -type none + *values -min 0 -max 0 + } $args] + #todo - make this command run without truncating previous runchunks + set runchunks [tsv::array names repl runchunks-*] + + set sortlist [list] + foreach cname $runchunks { + set num [lindex [split $cname -] 1] + lappend sortlist [list $num $cname] + } + set sorted [lsort -index 0 -integer $sortlist] + set chunkname [lindex $sorted end-1 1] + set runlist [tsv::get repl $chunkname] + #puts stderr "--$runlist" + if {![llength $runlist]} { + return "" + } else { + return [lindex [lsearch -inline -index 0 $runlist result] 1] + } + } + interp alias {} _ {} ::punk::get_runchunk + + + proc ::punk::var {varname {= _=.=_} args} { upvar $varname the_var - if {${=} == "="} { - if {[llength $args] > 1} { - set the_var [uplevel 1 $args] - } else { - set the_var [lindex $args 0] + switch -exact -- ${=} { + = { + if {[llength $args] > 1} { + set the_var $args + } else { + set the_var [lindex $args 0] + } + } + .= { + if {[llength $args] > 1} { + set the_var [uplevel 1 $args] + } else { + set the_var [uplevel 1 [lindex $args 0]] + } + } + _=.=_ { + set the_var + } + default { + set the_var [list ${=} {*}$args] } - } else { - set the_var } } proc src {args} { @@ -7302,6 +7345,15 @@ namespace eval punk { set usetable 1 if {$usetable} { set t [textblock::class::table new -show_hseps 0 -show_header 1 -ansiborder_header [a+ web-green]] + if {"windows" eq $::tcl_platform(platform)} { + #If any env vars have been set to empty string - this is considered a deletion of the variable on windows. + #The Tcl ::env array is linked to the underlying process view of the environment + #- but info exists ::env(var) can misreport as true if it has been deleted by setting to empty string rather than using unset. + #an 'array get' will resynchronise. + #Even if an env variable didn't exist before - setting it to empty string can get it to this inconsistent state. + array get ::env + } + #do an array read on ::env foreach {v vinfo} $punkenv_config { if {[info exists ::env($v)]} { set c2 [set ::env($v)] diff --git a/src/vfs/_vfscommon/modules/punk/config-0.1.tm b/src/vfs/_vfscommon/modules/punk/config-0.1.tm index dd7ae87..206b560 100644 --- a/src/vfs/_vfscommon/modules/punk/config-0.1.tm +++ b/src/vfs/_vfscommon/modules/punk/config-0.1.tm @@ -333,17 +333,30 @@ tcl::namespace::eval punk::config { error "No such global configuration item '$varname' found in startup config" } - proc get {whichconfig} { + proc get {whichconfig {globfor *}} { variable startup variable running switch -- $whichconfig { config - startup - startup-config - startup-configuration { #show *startup* config - different behaviour may be confusing to those used to router startup and running configs - return $startup + set configdata $startup } running - running-config - running-configuration { - return $running + set configdata $running } + default { + error "Unknown config name '$whichconfig' - try startup or running" + } + } + if {$globfor eq "*"} { + return $configdata + } else { + set keys [dict keys $configdata [string tolower $globfor]] + set filtered [dict create] + foreach k $keys { + dict set filtered $k [dict get $configdata $k] + } + return $filtered } } @@ -355,20 +368,10 @@ tcl::namespace::eval punk::config { } - proc show {whichconfig} { + proc show {whichconfig {globfor *}} { #todo - tables for console - variable startup - variable running - switch -- $whichconfig { - config - startup - startup-config - startup-configuration { - #show *startup* config - different behaviour may be confusing to those used to router startup and running configs - return [punk::lib::showdict $startup] - } - running - running-config - running-configuration { - return [punk::lib::showdict $running] - } - } - + set configdata [punk::config::get $whichconfig $globfor] + return [punk::lib::showdict $configdata] } #e.g diff --git a/src/vfs/_vfscommon/modules/punk/repl-0.1.tm b/src/vfs/_vfscommon/modules/punk/repl-0.1.tm index 937988c..9debf04 100644 --- a/src/vfs/_vfscommon/modules/punk/repl-0.1.tm +++ b/src/vfs/_vfscommon/modules/punk/repl-0.1.tm @@ -1992,6 +1992,9 @@ proc repl::repl_process_data {inputchan chunktype chunk stdinlines prompt_config set repl_runid [tsv::incr repl runid] tsv::set repl runchunks-$repl_runid [list] ;#last_run_display + catch { + tsv::unset repl runchunks-[expr {$repl_runid - 10}] + } #set ::repl::last_unknown "" tsv::set repl last_unknown "" @@ -2260,6 +2263,10 @@ proc repl::repl_process_data {inputchan chunktype chunk stdinlines prompt_config #we have copied rawresult using append with empty string - so our string interaction with result var here shouldn't affect the returned value #empty-string result handled in other branch + if {![tsv::llength repl runchunks-$repl_runid]} { + #write back to tsv var for use by punk::get_runchunks (underscore command) + tsv::set repl runchunks-$repl_runid [list [list result $result]] + } set flat [string map [list \r\n "" \n ""] $result] if {[string length $flat] == [string length $result]} { #no line-endings in data diff --git a/src/vfs/_vfscommon/modules/test/tomlish-1.1.1.tm b/src/vfs/_vfscommon/modules/test/tomlish-1.1.1.tm index 4ea2ce3..ae68fad 100644 Binary files a/src/vfs/_vfscommon/modules/test/tomlish-1.1.1.tm and b/src/vfs/_vfscommon/modules/test/tomlish-1.1.1.tm differ diff --git a/src/vfs/_vfscommon/modules/tomlish-1.1.1.tm b/src/vfs/_vfscommon/modules/tomlish-1.1.1.tm index d85d441..617f7f9 100644 --- a/src/vfs/_vfscommon/modules/tomlish-1.1.1.tm +++ b/src/vfs/_vfscommon/modules/tomlish-1.1.1.tm @@ -81,7 +81,7 @@ namespace eval tomlish { # inline tables are fully defined between their braces, as are dotted-key subtables defined within # No additional subtables or arrays of tables may be defined within an inline table after the ending brace - they must be entirely self-contained - set tags [list TOMLISH ARRAY TABLE ITABLE ANONTABLE WS NEWLINE COMMENT KEYVAL QKEYVAL STRING MULTISTRING LITSTRING MULTILITSTRING INT FLOAT BOOL DATETIME] + set tags [list TOMLISH ARRAY TABLE ITABLE ANONTABLE WS NEWLINE COMMENT KEYVAL QKEYVAL SQKEYVAL STRING STRINGPART MULTISTRING STRINGLIT MULTISTRINGLIT INT FLOAT BOOL DATETIME] #tomlish v1.0 should accept arbitrary 64-bit signed ints (from -2^63 to 2^63-1) #we will restrict to this range for compatibility for now - although Tcl can handle larger (arbitrarily so?) set min_int -9223372036854775808 ;#-2^63 @@ -117,7 +117,7 @@ namespace eval tomlish { foreach sub [lrange $keyval_element 2 end] { #note that a barekey/quotedkey won't occur directly inside a barekey/quotedkey switch -exact -- [lindex $sub 0] { - STRING - MULTISTRING - INT - FLOAT - BOOL - DATETIME - TABLE - ARRAY - ITABLE { + STRING - STRINGLIT - MULTISTRING - INT - FLOAT - BOOL - DATETIME - TABLE - ARRAY - ITABLE { set type [lindex $sub 0] set value [lindex $sub 1] set found_sub $sub @@ -141,9 +141,9 @@ namespace eval tomlish { STRING - STRINGPART { set result [list type $type value [::tomlish::utils::unescape_string $value]] } - LITSTRING { + STRINGLIT { #REVIEW - set result [list type $type value $value] + set result [list type STRINGLIT value $value] } TABLE - ITABLE - ARRAY - MULTISTRING { #jmn2024 - added ITABLE - review @@ -193,9 +193,12 @@ namespace eval tomlish { set tag [lindex $item 0] #puts "...> item:'$item' tag:'$tag'" switch -exact -- $tag { - KEYVAL - QKEYVAL { + KEYVAL - QKEYVAL - SQKEYVAL { log::debug "--> processing $tag: $item" set key [lindex $item 1] + if {$tag eq "QKEYVAL"} { + set key [::tomlish::utils::unescape_string $key] + } #!todo - normalize key. (may be quoted/doublequoted) if {[dict exists $datastructure $key]} { @@ -286,10 +289,20 @@ namespace eval tomlish { } + #ensure empty tables are still represented in the datastructure + set subkey [list] + foreach k $key_hierarchy { + lappend subkey $k + if {![dict exists $datastructure {*}$subkey]} { + dict set datastructure {*}$subkey [list] + } else { + tomlish::log::notice "get_dict datastructure at subkey $subkey already had data: [dict get $datastructure {*}$subkey]" + } + } #We must do this after the key-collision test above! lappend tablenames_seen $tablename - + log::debug ">>>>>>>>>>>>>>>>>>>>key_hierarchy : $key_hierarchy" log::debug ">>>>>>>>>>>>>>>>>>>>key_hierarchy_raw: $key_hierarchy_raw" @@ -298,8 +311,11 @@ namespace eval tomlish { foreach element [lrange $item 2 end] { set type [lindex $element 0] switch -exact -- $type { - KEYVAL - QKEYVAL { + KEYVAL - QKEYVAL - SQKEYVAL { set keyval_key [lindex $element 1] + if {$type eq "QKEYVAL"} { + set keyval_key [::tomlish::utils::unescape_string $keyval_key] + } set keyval_dict [_get_keyval_value $element] dict set datastructure {*}$key_hierarchy $keyval_key $keyval_dict } @@ -307,7 +323,7 @@ namespace eval tomlish { #ignore } default { - error "Sub element of type '$type' not understood in table context. Expected only KEYVAL,QKEYVAL,NEWLINE,COMMENT,WS" + error "Sub element of type '$type' not understood in table context. Expected only KEYVAL,QKEYVAL,SQKEYVAL,NEWLINE,COMMENT,WS" } } } @@ -320,16 +336,19 @@ namespace eval tomlish { foreach element [lrange $item 1 end] { set type [lindex $element 0] switch -exact -- $type { - KEYVAL - QKEYVAL { + KEYVAL - QKEYVAL - SQKEYVAL { set keyval_key [lindex $element 1] set keyval_dict [_get_keyval_value $element] + if {$type eq "QKEYVAL"} { + set keyval_key [::tomlish::utils::unescape_string $keyval_key] + } dict set datastructure $keyval_key $keyval_dict } NEWLINE - COMMENT - WS { #ignore } default { - error "Sub element of type '$type' not understood in ITABLE context. Expected only KEYVAL,QKEYVAL,NEWLINE,COMMENT,WS" + error "Sub element of type '$type' not understood in ITABLE context. Expected only KEYVAL,QKEYVAL,SQKEYVAL,NEWLINE,COMMENT,WS" } } } @@ -350,12 +369,16 @@ namespace eval tomlish { set value [lindex $element 1] lappend datastructure [list type $type value [::tomlish::utils::unescape_string $value]] } - TABLE - ARRAY - MULTISTRING { + STRINGLIT { + set value [lindex $element 1] + lappend datastructure [list type $type value $value] + } + ITABLE - TABLE - ARRAY - MULTISTRING { set value [lindex $element 1] lappend datastructure [list type $type value [::tomlish::get_dict [list $element]]] } - WS - SEP { - #ignore whitespace and commas + WS - SEP - NEWLINE - COMMENT { + #ignore whitespace, commas, newlines and comments } default { error "Unexpected value type '$type' found in array" @@ -553,13 +576,13 @@ namespace eval tomlish::encode { proc boolean {b} { #convert any Tcl-acceptable boolean to boolean as accepted by toml - lower case true/false - if {![string is boolean -strict $b]} { + if {![tcl::string::is boolean -strict $b]} { error "Unable to convert '$b' to Toml boolean true|false. [::tomlish::parse::report_line]" } else { if {[expr {$b && 1}]} { - return [list BOOL true] + return [::list BOOL true] } else { - return [list BOOL false] + return [::list BOOL false] } } } @@ -610,12 +633,13 @@ namespace eval tomlish::encode { #Handle invalid tag nestings switch -- $context { QKEYVAL - + SQKEYVAL - KEYVAL { - if {$tag in {KEYVAL QKEYVAL}} { + if {$tag in {KEYVAL QKEYVAL SQKEYVAL}} { error "Invalid tag '$tag' encountered within '$context'" } } - MULTISTRING { + MULTISTRING - MULTILITERAL { #explicitly list the valid child tags if {$tag ni {STRING STRINGPART WS NEWLINE CONT}} { error "Invalid tag '$tag' encountered within a MULTISTRING" @@ -631,9 +655,12 @@ namespace eval tomlish::encode { #optional root tag. Ignore. } QKEYVAL - + SQKEYVAL - KEYVAL { if {$tag eq "KEYVAL"} { append toml [lindex $item 1] ;#Key + } elseif {$tag eq "SQKEYVAL"} { + append toml '[lindex $item 1]' ;#SQuoted Key } else { append toml \"[lindex $item 1]\" ;#Quoted Key } @@ -691,10 +718,10 @@ namespace eval tomlish::encode { STRING { #simple double quoted strings only # - return \"[lindex $item 1]\" + append toml \"[lindex $item 1]\" } STRINGPART { - return [lindex $item 1] + append toml [lindex $item 1] } MULTISTRING { #Tripple quoted string which is a container for newlines,whitespace and multiple strings/stringparts @@ -704,17 +731,17 @@ namespace eval tomlish::encode { } append toml "\"\"\"$multistring\"\"\"" } - LITSTRING { + STRINGLIT { #Single Quoted string(literal string) append toml '[lindex $item 1]' } - MULTILITSTRING { - #review - multilitstring can be handled as a single string? - set litstring "" + MULTISTRINGLIT { + #review - multistringlit can be handled as a single string? + set stringlit "" foreach part [lrange $item 1 end] { - append litstring [::tomlish::encode::tomlish [list $part] $nextcontext] + append stringlit [::tomlish::encode::tomlish [list $part] $nextcontext] } - append toml '''$litstring''' + append toml '''$stringlit''' } INT - BOOL - @@ -853,11 +880,11 @@ namespace eval tomlish::decode { set prevstate $state ##### set nextstate [::tomlish::parse::getNextState $tokenType $prevstate] - ::tomlish::log::info "tok: $tok STATE TRANSITION tokenType: '$tokenType' triggering '$state' -> '$nextstate' last_space_action:$last_space_action" + ::tomlish::log::info "STATE TRANSITION tokenType: '$tokenType' tok: $tok triggering '$state' -> '$nextstate' last_space_action:$last_space_action" set state $nextstate if {$state eq "err"} { - error "State error - aborting parse. [tomlish::parse::report_line]" + error "State error for tokenType: $tokenType tok: $tok - aborting parse. [tomlish::parse::report_line]" } if {$last_space_action eq "pop"} { @@ -885,10 +912,10 @@ namespace eval tomlish::decode { lappend v($nest) "SEP" } endinlinetable { - puts stderr "endinlinetable" + ::tomlish::log::debug "endinlinetable for last_space_action pop" } endmultiquote { - puts stderr "endmultiquote for last_space_action 'pop'" + ::tomlish::log::debug "endmultiquote for last_space_action 'pop'" } default { error "unexpected tokenType '$tokenType' for last_space_action 'pop'" @@ -909,6 +936,9 @@ namespace eval tomlish::decode { quotedkey - itablequotedkey { set v($nest) [list QKEYVAL $tok] ;#$tok is the keyname } + squotedkey - itablesquotedkey { + set v($nest) [list SQKEYVAL $tok] ;#$tok is the keyname + } tablename { #note: we do not use the output of tomlish::tablename_trim to produce a tablename for storage in the tomlish list! #The tomlish list is intended to preserve all whitespace (and comments) - so a roundtrip from toml file to tomlish @@ -940,12 +970,8 @@ namespace eval tomlish::decode { set v($nest) [list ITABLE] ;#$tok is just the opening curly brace - don't output. } startmultiquote { - puts stderr "push trigger tokenType startmultiquote (todo)" + ::tomlish::log::debug "push trigger tokenType startmultiquote" set v($nest) [list MULTISTRING] ;#container for STRINGPART, NEWLINE - #JMN ??? - #set next_tokenType_known 1 - #::tomlish::parse::set_tokenType "multistring" - #set tok "" } default { error "push trigger tokenType '$tokenType' not yet implemented" @@ -969,7 +995,7 @@ namespace eval tomlish::decode { #no output into the tomlish list for this token } startinlinetable { - puts stderr "decode::toml error. did not expect startlinetable without space level change" + puts stderr "decode::toml error. did not expect startinlinetable without space level change" } startquote { switch -exact -- $nextstate { @@ -993,6 +1019,28 @@ namespace eval tomlish::decode { } } } + startsquote { + switch -exact -- $nextstate { + stringlit { + set next_tokenType_known 1 + ::tomlish::parse::set_tokenType "stringlit" + set tok "" + } + squotedkey { + set next_tokenType_known 1 + ::tomlish::parse::set_tokenType "squotedkey" + set tok "" + } + itablesquotedkey { + set next_tokenType_known 1 + ::tomlish::parse::set_tokenType "itablesquotedkey" + set tok "" + } + default { + error "startsquote switch case not implemented for nextstate: $nextstate" + } + } + } startmultiquote { #review puts stderr "no space level change - got startmultiquote" @@ -1004,6 +1052,9 @@ namespace eval tomlish::decode { #nothing to do? set tok "" } + endsquote { + set tok "" + } endmultiquote { #JMN!! set tok "" @@ -1023,6 +1074,9 @@ namespace eval tomlish::decode { } itablequotedkey { + } + stringlit { + lappend v($nest) [list STRINGLIT $tok] } untyped-value { #we can't determine the type of unquoted values (int,float,datetime,bool) until the entire token was read. @@ -1074,6 +1128,7 @@ namespace eval tomlish::decode { } if {!$next_tokenType_known} { + ::tomlish::log::notice "tomlish::decode::toml - current tokenType:$tokenType Next token type not known" ::tomlish::parse::set_tokenType "" set tok "" } @@ -1108,8 +1163,6 @@ namespace eval tomlish::decode { } finally { set is_parsing 0 } - - return $v(0) } @@ -1136,22 +1189,76 @@ namespace eval tomlish::utils { set segments [tablename_split $tablename false] set trimmed_segments [list] foreach seg $segments { - lappend trimmed_segments [::string trim $seg [list " " \t]] + lappend trimmed_segments [::string trim $seg " \t"] } return [join $trimmed_segments .] } + #basic generic quote matching for single and double quotes + #note for example that {[o'malley]} will return sq - as the single quote is not closed or wrapped in double quotes + proc tok_in_quotedpart {tok} { + set sLen [::string length $tok] + set quote_type "" + set had_slash 0 + for {set i 0} {$i < $sLen} {incr i} { + set c [tcl::string::index $tok $i] + if {$quote_type eq ""} { + if {$had_slash} { + #don't enter quote mode + #leave slash_mode because even if current char is slash - it is escaped + set had_slash 0 + } else { + set ctype [string map [list {"} dq {'} sq \\ bsl] $c] + switch -- $ctype { + dq { + set quote_type dq + } + sq { + set quote_type sq + } + bsl { + set had_slash 1 + } + } + } + } else { + if {$had_slash} { + #don't leave quoted mode + #leave slash_mode because even if current char is slash - it is escaped + set had_slash 0 + } else { + set ctype [string map [list {"} dq {'} sq \\ bsl] $c] + switch -- $ctype { + dq { + if {$quote_type eq "dq"} { + set quote_type "" + } + } + sq { + if {$quote_type eq "sq"} { + set quote_type "" + } + } + bsl { + set had_slash 1 + } + } + } + } + } + return $quote_type ;#dq | sq + } + #utils::tablename_split proc tablename_split {tablename {normalize false}} { #we can't just split on . because we have to handle quoted segments which may contain a dot. #eg {dog."tater.man"} - set i 0 set sLen [::string length $tablename] set segments [list] set mode "unknown" ;#5 modes: unknown, quoted,litquoted, unquoted, syntax #quoted is for double-quotes, litquoted is for single-quotes (string literal) set seg "" - for {} {$i < $sLen} {} { + for {set i 0} {$i < $sLen} {incr i} { if {$i > 0} { set lastChar [::string index $tablename [expr {$i - 1}]] @@ -1160,7 +1267,6 @@ namespace eval tomlish::utils { } set c [::string index $tablename $i] - incr i if {$c eq "."} { switch -exact -- $mode { @@ -1240,7 +1346,7 @@ namespace eval tomlish::utils { } append seg $c } - if {$i == $sLen} { + if {$i == $sLen-1} { #end of data ::tomlish::log::debug "End of data: mode='$mode'" switch -exact -- $mode { @@ -1275,14 +1381,14 @@ namespace eval tomlish::utils { } } foreach seg $segments { - set trimmed [::string trim $seg [list " " \t]] + set trimmed [::string trim $seg " \t"] #note - we explicitly allow 'empty' quoted strings '' & "" # (these are 'discouraged' but valid toml keys) #if {$trimmed in [list "''" "\"\""]} { # puts stderr "tablename_split. warning - Empty quoted string as tablename segment" #} if {$trimmed eq "" } { - error "tablename_split. Empty segment found. tablename: '$tablename'" + error "tablename_split. Empty segment found. tablename: '$tablename' segments [llength $segments] ($segments)" } } return $segments @@ -1640,18 +1746,26 @@ namespace eval tomlish::utils { } proc is_datetime {str} { - #e.g 1979-05-27T00:32:00-07:00 + #e.g 1979-05-27 + #e.g 1979-05-27T00:32:00Z + #e.g 1979-05-27 00:32:00-07:00 + #e.g 1979-05-27 00:32:00+10:00 + #e.g 1979-05-27 00:32:00.999999-07:00 + set matches [regexp -all {[zZtT0-9\-\+\.:]} $str] if {[::string length $str] == $matches} { #all characters in legal range #!todo - use full RFC 3339 parser? lassign [split $str T] datepart timepart #!todo - what if the value is 'time only'? - - if {[catch {clock scan $datepart} err]} { - puts stderr "tcl clock scan failed err:'$err'" - return 0 - } + + #Tcl's free-form clock scan (no -format option) is deprecated + # + #if {[catch {clock scan $datepart} err]} { + # puts stderr "tcl clock scan failed err:'$err'" + # return 0 + #} + #!todo - verify time part is reasonable } else { return 0 @@ -1692,20 +1806,35 @@ namespace eval tomlish::parse { set stateMatrix [dict create] dict set stateMatrix\ - key-space {whitespace "key-space" newline "key-space" bom "key-space" barekey {pushspace "keyval-space"} eof "end" startquote "quotedkey" startmultiquote "err" endquote "err" comment "key-space" comma "err" starttablename "tablename" starttablearrayname "tablearrayname"} - + key-space { + whitespace "key-space"\ + newline "key-space"\ + bom "key-space"\ + barekey {pushspace "keyval-space"}\ + startquote "quotedkey"\ + startsquote "squotedkey"\ + comment "key-space"\ + starttablename "tablename"\ + starttablearrayname "tablearrayname"\ + startmultiquote "err"\ + endquote "err"\ + comma "err"\ + eof "end"\ + } dict set stateMatrix\ curly-space {\ - whitespace "curly-space"\ - newline "curly-space"\ - barekey {pushspace "itablekeyval-space"}\ - itablequotedkey "itablekeyval-space"\ - endinlinetable "popspace"\ - startquote "itablequotedkey"\ - comma "curly-space"\ - eof "err"\ - comment "err"\ + whitespace "curly-space"\ + newline "curly-space"\ + barekey {pushspace "itablekeyval-space"}\ + itablequotedkey "itablekeyval-space"\ + itablesquotedkey "itablekeyval-space"\ + endinlinetable "popspace"\ + startquote "itablequotedkey"\ + startsquote "itablesquotedkey"\ + comma "curly-space"\ + comment "err"\ + eof "err"\ } #REVIEW @@ -1713,97 +1842,195 @@ namespace eval tomlish::parse { #https://github.com/toml-lang/toml/issues/781 dict set stateMatrix\ curly-syntax {\ - whitespace "curly-syntax"\ - newline "curly-syntax"\ - barekey {pushspace "itablekeyval-space"}\ - itablequotedkey "itablekeyval-space"\ - endinlinetable "popspace"\ - startquote "itablequotedkey"\ - comma "curly-space"\ - eof "err"\ - comment "err"\ + whitespace "curly-syntax"\ + newline "curly-syntax"\ + barekey {pushspace "itablekeyval-space"}\ + itablequotedkey "itablekeyval-space"\ + endinlinetable "popspace"\ + startquote "itablequotedkey"\ + comma "curly-space"\ + comment "curly-space"\ + eof "err"\ } + #review comment "err" vs comment "curly-space" - see if TOML 1.1 comes out and allows comments in multiline ITABLES + #We currently allow multiline ITABLES (also with comments) in the tokenizer. + #if we want to disallow as per TOML 1.0 - we should do so when attempting to get structure? dict set stateMatrix\ value-expected {\ - whitespace "value-expected"\ - newline "err"\ - eof "err"\ - untyped-value "samespace"\ - startquote "string"\ - startmultiquote {pushspace "multistring-space"}\ - startinlinetable {pushspace curly-space}\ - comment "err"\ - comma "err"\ - startarray {pushspace array-space}\ + whitespace "value-expected"\ + untyped-value "samespace"\ + startquote "string"\ + startsquote "stringlit"\ + startmultiquote {pushspace "multistring-space"}\ + startinlinetable {pushspace curly-space}\ + startarray {pushspace array-space}\ + comment "err"\ + comma "err"\ + newline "err"\ + eof "err"\ } dict set stateMatrix\ array-space {\ - whitespace "array-space"\ - newline "array-space"\ - eof "err"\ - untyped-value "samespace"\ - startarray {pushspace "array-space"}\ - endarray "popspace"\ - startquote "string"\ - startmultiquote "multistring"\ - comma "array-space"\ - comment "array-space"\ + whitespace "array-space"\ + newline "array-space"\ + untyped-value "samespace"\ + startarray {pushspace "array-space"}\ + endarray "popspace"\ + startmultiquote {pushspace multistring-space}\ + startinlinetable {pushspace curly-space}\ + startquote "string"\ + startsquote "stringlit"\ + comma "array-space"\ + comment "array-space"\ + eof "err"\ } dict set stateMatrix\ array-syntax {\ - whitespace "array-syntax"\ - newline "array-syntax"\ - untyped-value "samespace"\ - startarray {pushspace array-space}\ - endarray "popspace"\ - startquote "string"\ - startmultiquote "multistring"\ - comma "array-space"\ - comment "err"\ + whitespace "array-syntax"\ + newline "array-syntax"\ + untyped-value "samespace"\ + startarray {pushspace array-space}\ + endarray "popspace"\ + startmultiquote {pushspace multistring-space}\ + startquote "string"\ + startsquote "stringlit"\ + comma "array-space"\ + comment "err"\ } - - dict set stateMatrix\ - itablekeyval-syntax {whitespace "itablekeyval-syntax" endquote "itablekeyval-syntax" newline "err" equal "value-expected" eof "err"} - #dict set stateMatrix\ - # itablekeytail {whitespace "itablekeytail" endinlinetable "popspace" comma "popspace" newline "err" comment "err" eof "err"} dict set stateMatrix\ - itablevaltail {whitespace "itablevaltail" endinlinetable "popspace" comma "popspace" newline "err" comment "err" eof "err"} + itablekeyval-syntax {\ + whitespace "itablekeyval-syntax"\ + endquote "itablekeyval-syntax"\ + endsquote "itablekeyval-syntax"\ + newline "err"\ + equal "value-expected"\ + eof "err"\ + } dict set stateMatrix\ itablekeyval-space {} + + dict set stateMatrix\ + itablevaltail {\ + whitespace "itablevaltail"\ + endinlinetable "popspace"\ + comma "popspace"\ + newline "itablevaltail"\ + comment "itablevaltail"\ + eof "err"\ + } dict set stateMatrix\ - itablequotedkey {whitespace "NA" itablequotedkey {pushspace "itablekeyval-space"} newline "err" endquote "itablekeyval-syntax"} + itablequotedkey {\ + whitespace "NA"\ + itablequotedkey {pushspace "itablekeyval-space"}\ + newline "err"\ + endquote "itablekeyval-syntax"\ + } + dict set stateMatrix\ + itablesquotedkey {\ + whitespace "NA"\ + itablesquotedkey {pushspace "itablekeyval-space"}\ + newline "err"\ + endsquote "itablekeyval-syntax"\ + } dict set stateMatrix\ - keyval-syntax {whitespace "keyval-syntax" endquote "keyval-syntax" comma "err" newline "err" equal "value-expected" eof "err"} + keyval-space {\ + } + dict set stateMatrix\ + keyval-syntax {\ + whitespace "keyval-syntax"\ + endquote "keyval-syntax"\ + endsquote "keyval-syntax"\ + equal "value-expected"\ + comma "err"\ + newline "err"\ + eof "err"\ + } dict set stateMatrix\ keytail {whitespace "keytail" newline "popspace" comment "keytail" eof "end"} - dict set stateMatrix\ - keyval-space {} + #quotedkey & squotedkey need to pushspace from self to keyval-space + dict set stateMatrix\ + quotedkey {\ + whitespace "NA"\ + quotedkey {pushspace "keyval-space"}\ + newline "err"\ + endquote "keyval-syntax"\ + } dict set stateMatrix\ - quotedkey {whitespace "NA" quotedkey {pushspace "keyval-space"} newline "err" endquote "keyval-syntax"} + squotedkey {\ + whitespace "NA"\ + squotedkey {pushspace "keyval-space"}\ + newline "err"\ + endsquote "keyval-syntax"\ + } + dict set stateMatrix\ - string {whitespace "NA" newline "err" string "string" endquote "samespace" eof "err"} + string {\ + whitespace "NA"\ + string "string"\ + endquote "samespace"\ + newline "err"\ + eof "err"\ + } dict set stateMatrix\ - stringpart {eof "err" continuation "samespace" endmultiquote "popspace"} + stringlit {\ + whitespace "NA"\ + stringlit "stringlit"\ + endsquote "samespace"\ + newline "err"\ + eof "err"\ + } + + dict set stateMatrix\ - multistring {whitespace "NA" newline "NA" multistring "multistring" endmultiquote "samespace" endquote "err" eof "err"} + stringpart {\ + continuation "samespace"\ + endmultiquote "popspace"\ + eof "err"\ + } + #dict set stateMatrix\ + # multistring {whitespace "NA" newline "NA" multistring "multistring" endmultiquote "samespace" endquote "err" eof "err"} dict set stateMatrix\ - multistring-space {whitespace "multistring-space" continuation "multistring-space" stringpart "multistring-space" multistring "multistring-space" newline "multistring-space" eof "err" endmultiquote "popspace"} + multistring-space {\ + whitespace "multistring-space"\ + continuation "multistring-space"\ + stringpart "multistring-space"\ + newline "multistring-space"\ + endmultiquote "popspace"\ + eof "err"\ + } + #multistring "multistring-space" + + + dict set stateMatrix\ - tablename {whitespace "NA" tablename {zeropoppushspace key-space} tablename2 {pushspace key-space} newline "err" endtablename "tablenametail"} + tablename {\ + whitespace "NA"\ + tablename {zeropoppushspace key-space}\ + tablename2 {pushspace key-space}\ + endtablename "tablenametail"\ + comma "err"\ + newline "err"\ + } dict set stateMatrix\ - baretablename {whitespace "NA" newline "err" equal "value-expected"} + tablearrayname {\ + whitespace "NA"\ + tablearrayname {zeropoppushspace key-space}\ + tablearrayname2 {pushspace key-space}\ + endtablearray "tablearraynametail"\ + comma "err"\ + newline "err"\ + } + dict set stateMatrix\ tablenametail {whitespace "tablenametail" newline "key-space" comment "tablenametail" eof "end"} - dict set stateMatrix\ - tablearrayname {whitespace "NA" tablearrayname {zeropoppushspace key-space} tablearrayname2 {pushspace key-space} newline "err" endtablearray "tablearraynametail"} dict set stateMatrix\ tablearraynametail {whitespace "tablearraynametail" newline "key-space" comment "tablearraynametail" eof "end"} dict set stateMatrix\ @@ -1854,17 +2081,17 @@ namespace eval tomlish::parse { # e.g "string {array-space array-syntax}" means when transitioning from string to array-space, jump to array-syntax instead. #this is useful as we often don't know state $b. e.g when it is decided by 'popspace' variable spacePopTransitions { - array-space array-syntax - curly-space curly-syntax - keyval-space keytail - itablekeyval-space itablevaltail + array-space array-syntax + curly-space curly-syntax + keyval-space keytail + itablekeyval-space itablevaltail } variable spacePushTransitions { - keyval-space keyval-syntax - itablekeyval-space itablekeyval-syntax - array-space array-space - curly-space curly-space - key-space tablename + keyval-space keyval-syntax + itablekeyval-space itablekeyval-syntax + array-space array-space + curly-space curly-space + key-space tablename } @@ -1879,8 +2106,8 @@ namespace eval tomlish::parse { variable spacePopTransitions variable spacePushTransitions - variable last_space_action "none" - variable last_space_type "none" + variable last_space_action "none" + variable last_space_type "none" variable state_list set result "" @@ -1988,7 +2215,7 @@ namespace eval tomlish::parse { foreach el $list { if { [lindex $el 0] eq "NEWLINE"} { append prettier "[list $el]\n" - } elseif {([llength $el] > 1) && ([lindex $el 0] in {KEYVAL QKEYVAL TABLE ARRAY})} { + } elseif {([llength $el] > 1) && ([lindex $el 0] in {KEYVAL QKEYVAL SQKEYVAL TABLE ARRAY})} { append prettier [nest_pretty1 $el] } else { append prettier "[list $el] " @@ -2029,6 +2256,7 @@ namespace eval tomlish::parse { incr i -1 return -level 2 1 } elseif {$toklen == 2} { + puts stderr "_shortcircuit_startquotesequence toklen 2" set_tokenType "startquote" set tok "\"" incr i -2 @@ -2036,7 +2264,7 @@ namespace eval tomlish::parse { } } - #return a list of 0 1 or 2 tokens + #returns 0 or 1 #tomlish::parse::tok proc tok {s} { variable nest @@ -2085,22 +2313,30 @@ namespace eval tomlish::parse { } set c [string index $s $i] + tomlish::log::debug "- tokloop char <$c> index $i tokenType:$tokenType tok:<$tok>" #puts "got char $c during tokenType '$tokenType'" incr i ;#must incr here because we do'returns'inside the loop - set ctest [string map {\{ lc \} rc \[ lb \] rb \" dq \\ bsl \r cr \n lf \t tab \uFEFF bom} $c] + set ctest [string map {\{ lc \} rc \[ lb \] rb \" dq ' sq \\ bsl \r cr \n lf \t tab \uFEFF bom} $c] switch -exact -- $ctest { # { set dquotes $multi_dquote - set multi_dquote "" ;#!! - if {$slash_active} {append tok "\\"} ;#if tokentype not appropriate for \, we would already have errored out. + set multi_dquote "" + set had_slash $slash_active set slash_active 0 + + if {$had_slash} {append tok "\\"} ;#if tokentype not appropriate for \, we would already have errored out. if {[::string length $tokenType]} { switch -exact -- $tokenType { startquotesequence { _shortcircuit_startquotesequence } + startsquotesequence { + incr i -[tcl::string::length $tok] + set_tokenType "startsquote" + return 1 + } barekey { error "Unexpected character '$c' during bare key. Only \[a-zA-Z_-\] allowed. [tomlish::parse::report_line]" } @@ -2118,43 +2354,80 @@ namespace eval tomlish::parse { incr i -1 return 1 } + starttablename - starttablearrayname { + #fix! + error "Character '#' is invalid first character for $tokenType. [tomlish::parse::report_line]" + } + tablename - tablearrayname { + #invalid in bare parts - but allowed in quoted parts - let tablename parser sort it out + append tok $c + } default { - #quotedkey, string, multistring + #quotedkey, itablequotedkey, string,stringlit, multistring append tok $c } } } else { - #$slash_active not relevant when no tokenType - #start of token if we're not in a token - set_tokenType comment - set tok "" ;#The hash is not part of the comment data + switch -- $state { + multistring-space - multiliteral-space { + if {$state eq "multistring-space"} { + set_tokenType stringpart + } else { + set_tokenType stringlit ;#review + } + set tok "" + if {$had_slash} { + append tok "\\" + } + append tok "$dquotes#" + } + default { + #start of token if we're not in a token + set_tokenType comment + set tok "" ;#The hash is not part of the comment data + } + } } } lc { - set multi_dquote "" ;#!! - #test jmn2024 #left curly brace + set dquotes $multi_dquote + set multi_dquote "" + set had_slash $slash_active + set slash_active 0 + #test jmn2024 try { if {[::string length $tokenType]} { switch -exact -- $tokenType { startquotesequence { _shortcircuit_startquotesequence } - string - stringpart { - if {$slash_active} {append tok "\\"} + startsquotesequence { + incr i -[tcl::string::length $tok] + set_tokenType "startsquote" + return 1 + } + stringlit - squotedkey - itablesquotedkey { append tok $c } - starttablename { - error "unexpected tablename problem" - #$slash_active not relevant to this tokentype - #change the tokenType - switch_tokenType "starttablearrayname" - set tok "" ;#no output into the tomlish list for this token - #any following whitespace is part of the tablearrayname, so return now - return 1 + string - quotedkey - itablequotedkey { + if {$had_slash} {append tok "\\"} + append tok $c + } + stringpart - stringlit { + if {$had_slash} {append tok "\\"} + append tok $dquotes$c + } + starttablename - starttablearrayname { + #*bare* tablename can only contain letters,digits underscores + error "Invalid tablename first character \{ [tomlish::parse::report_line]" + } + tablename - tablearrayname { + #valid in quoted parts + append tok $c } comment { - if {$slash_active} {append tok "\\"} + if {$had_slash} {append tok "\\"} append tok "\[" } default { @@ -2171,14 +2444,6 @@ namespace eval tomlish::parse { set tok "\{" return 1 } - multistring-space { - set_tokenType "stringpart" - if {$slash_active} { - set tok "\\\{" - } else { - set tok "\{" - } - } key-space { #invalid - but allow parser statemachine to report it. ? set_tokenType "startinlinetable" @@ -2191,6 +2456,18 @@ namespace eval tomlish::parse { set tok "\{" return 1 } + multistring-space - multiliteral-space { + if {$state eq "multistring-space"} { + set_tokenType stringpart + } else { + set_tokenType stringlit ;#review + } + set tok "" + if {$had_slash} { + append tok "\\" + } + append tok "$dquotes\{" + } default { error "state: '$state'. left brace case not implemented [tomlish::parse::report_line]" } @@ -2204,34 +2481,45 @@ namespace eval tomlish::parse { } rc { - set multi_dquote "" ;#!! #right curly brace - try { + set dquotes $multi_dquote + set multi_dquote "" + set had_slash $slash_active + set slash_active 0 + if {[string length $tokenType]} { switch -exact -- $tokenType { startquotesequence { _shortcircuit_startquotesequence } - string - stringpart - comment { - if {$slash_active} {append tok "\\"} + stringlit - squotedkey - itablesquotedkey { append tok $c } - tablename { - if {$slash_active} {append tok "\\"} + string - quotedkey - itablequotedkey - comment { + if {$had_slash} {append tok "\\"} + append tok $c + } + stringpart { + if {$had_slash} {append tok "\\"} + append tok $dquotes$c + } + starttablename - tablename { + if {$had_slash} {append tok "\\"} #invalid! - but leave for datastructure loading stage to catch dict set token_waiting type endinlinetable dict set token_waiting tok "" return 1 } - tablearrayname { - if {$slash_active} {append tok "\\"} + starttablearrayname - tablearrayname { + if {$had_slash} {append tok "\\"} #invalid! - but leave for datastructure loading stage to catch dict set token_waiting type endtablearrayname dict set token_waiting tok "" return 1 } itablevaltail { - + #review + error "right-curly in itablevaltail" } default { #end any other token @@ -2257,7 +2545,7 @@ namespace eval tomlish::parse { tablename { #e.g [] - empty tablename - allowed or not? #empty tablename/tablearrayname ? - error "unexpected tablename problem" + #error "unexpected tablename problem" set_tokenType "endinlinetable" set tok "" ;#no output into the tomlish list for this token @@ -2290,42 +2578,80 @@ namespace eval tomlish::parse { itablekeyval-syntax { error "endinlinetable unexpected at this point. Expecting key=val syntax [tomlish::parse::report_line]" } + multistring-space - multiliteral-space { + if {$state eq "multistring-space"} { + set_tokenType "stringpart" + } else { + set_tokenType "stringlit" ;#review + } + set tok "" + if {$had_slash} { + append tok "\\" + } + append tok "$dquotes\}" + } default { #JMN2024b keytail? error "state '$state'. endinlinetable case not implemented [tomlish::parse::report_line]" } } } - } on error {em} { - error $em - } finally { - set slash_active 0 - } } lb { - set multi_dquote "" ;#!! #left square bracket - try { + set dquotes $multi_dquote + set multi_dquote "" + set had_slash $slash_active + set slash_active 0 + if {[::string length $tokenType]} { switch -exact -- $tokenType { startquotesequence { _shortcircuit_startquotesequence } - string - stringpart { - if {$slash_active} {append tok "\\"} + startsquotesequence { + incr i -[tcl::string::length $tok] + set_tokenType "startsquote" + return 1 + } + stringlit - squotedkey - itablesquotedkey { + append tok $c + } + string - quotedkey - itablequotedkey { + if {$had_slash} {append tok "\\"} append tok $c } + stringpart { + if {$had_slash} {append tok "\\"} + append tok $dquotes$c + } starttablename { - #$slash_active not relevant to this tokentype #change the tokenType switch_tokenType "starttablearrayname" set tok "" ;#no output into the tomlish list for this token #any following whitespace is part of the tablearrayname, so return now return 1 } + tablename { + #e.g a."x[0]".c is valid table name sequence - so we need to track quoting to know if rb is an end token + if {$had_slash} { + #resultant tablename may be invalid - but leave for datastructure loading stage to catch + append tok "\\[" + } else { + if {[tomlish::utils::tok_in_quotedpart $tok] eq ""} { + #invalid at this point - state machine should disallow table -> starttablearrayname + dict set token_waiting type starttablearrayname + dict set token_waiting tok "" + return 1 + } else { + #we appear to still be in single or double quoted section + append tok "\[" + } + } + } comment { - if {$slash_active} {append tok "\\"} + if {$had_slash} {append tok "\\"} append tok "\[" } default { @@ -2357,94 +2683,148 @@ namespace eval tomlish::parse { return 1 #error "state: array-space. startarray case not implemented [tomlish::parse::report_line]" } + multistring-space - multiliteral-space { + if {$state eq "multistring-space"} { + set_tokenType "stringpart" + } else { + set_tokenType "stringlit" ;#review + } + set tok "" + if {$had_slash} { + append tok "\\" + } + append tok "$dquotes\[" + } default { error "state: '$state'. startarray case not implemented [tomlish::parse::report_line]" } } } - } on error {em} { - error $em - } finally { - set slash_active 0 - } } rb { - set multi_dquote "" ;#!! #right square bracket - try { + set dquotes $multi_dquote + set multi_dquote "" + set had_slash $slash_active + set slash_active 0 - if {[string length $tokenType]} { - switch -exact -- $tokenType { - startquotesequence { - _shortcircuit_startquotesequence - } - string - stringpart - comment { - if {$slash_active} {append tok "\\"} - append tok $c - } - tablename { - if {$slash_active} {append tok "\\"} - #invalid! - but leave for datastructure loading stage to catch - dict set token_waiting type endtablename - dict set token_waiting tok "" - return 1 - } - tablearraynames { - if {$slash_active} {append tok "\\"} - #invalid! - but leave for datastructure loading stage to catch - dict set token_waiting type endtablearrayname - dict set token_waiting tok "" + if {[string length $tokenType]} { + switch -exact -- $tokenType { + startquotesequence { + _shortcircuit_startquotesequence + } + startsquotesequence { + incr i -[tcl::string::length $tok] + set_tokenType "startsquote" + return 1 + } + stringlit - squotedkey - itablesquotedkey { + append tok $c + } + string - quotedkey - itablequotedkey { + if {$had_slash} {append tok "\\"} + append tok $c + } + comment { + if {$had_slash} {append tok "\\"} + append tok $c + } + stringpart { + if {$had_slash} {append tok "\\"} + append tok $dquotes$c + } + whitespace { + if {$state eq "multistring-space"} { + #???? + incr i -1 + if {$had_slash} {incr i -1} ;#reprocess return 1 - } - default { + } else { incr i -1 + if {$had_slash} {incr i -1} ;#reprocess return 1 } } - } else { - #$slash_active not relevant when no tokenType - switch -exact -- $state { - value-expected { - #invalid - but allow parser statemachine to report it. - set_tokenType "endarray" - set tok "\]" - return 1 - } - key-space { - #invalid - but allow parser statemachine to report it. ? - set_tokenType "endarray" - set tok "\]" - return 1 + tablename { + #e.g a."x[0]".c is valid table name sequence - so we need to track quoting to know if rb is an end token + if {$had_slash} { + #resultant tablename may be invalid - but leave for datastructure loading stage to catch + append tok "\\]" + } else { + if {[tomlish::utils::tok_in_quotedpart $tok] eq ""} { + dict set token_waiting type endtablename + dict set token_waiting tok "" + return 1 + } else { + #we appear to still be in single or double quoted section + append tok "]" + } } - tablename { - #e.g [] - empty tablename - allowed or not? - #empty tablename/tablearrayname ? - error "unexpected tablename problem" + } + tablearraynames { + #todo? + if {$had_slash} {append tok "\\"} + #invalid! - but leave for datastructure loading stage to catch + dict set token_waiting type endtablearrayname + dict set token_waiting tok "" + return 1 + } + default { + incr i -1 + return 1 + } + } + } else { + #$slash_active not relevant when no tokenType + switch -exact -- $state { + value-expected { + #invalid - but allow parser statemachine to report it. + set_tokenType "endarray" + set tok "\]" + return 1 + } + key-space { + #invalid - but allow parser statemachine to report it. ? + set_tokenType "endarray" + set tok "\]" + return 1 + } + tablename { + #e.g [] - empty tablename - allowed or not? + #empty tablename/tablearrayname ? + #error "unexpected tablename problem" - set_tokenType "endtablename" - set tok "" ;#no output into the tomlish list for this token - return 1 - } - tablearrayname { - error "unexpected tablearrayname problem" - set_tokenType "endtablearray" - set tok "" ;#no output into the tomlish list for this token - return 1 - } - array-syntax - array-space { - set_tokenType "endarray" - set tok "\]" - return 1 + set_tokenType "endtablename" + set tok "" ;#no output into the tomlish list for this token + return 1 + } + tablearrayname { + error "unexpected tablearrayname problem" + set_tokenType "endtablearray" + set tok "" ;#no output into the tomlish list for this token + return 1 + } + array-syntax - array-space { + set_tokenType "endarray" + set tok "\]" + return 1 + } + multistring-space - multiliteral-space { + if {$state eq "multistring-space"} { + set_tokenType "stringpart" + } else { + set_tokenType "stringlit" ;#review } - default { - error "state '$state'. endarray case not implemented [tomlish::parse::report_line]" + set tok "" + if {$had_slash} { + append tok "\\" } + append tok "$dquotes\]" + } + default { + error "state '$state'. endarray case not implemented [tomlish::parse::report_line]" } } - } on error {em} { - error $em - } finally { - set slash_active 0 } } bsl { @@ -2456,7 +2836,26 @@ namespace eval tomlish::parse { startquotesequence { _shortcircuit_startquotesequence } - string - litstring - multilitstring - comment - tablename - tablearrayname { + startsquotesequence { + incr i -[tcl::string::length $tok] + set_tokenType "startsquote" + return 1 + } + whitespace { + if {$state eq "multistring-space"} { + #end whitespace token + incr i -1 ;#reprocess bsl in next run + return 1 + } else { + error "Unexpected backslash during whitespace. [tomlish::parse::report_line]" + } + } + stringlit - squotedkey - itablesquotedkey { + #never need to set slash_active true when in stringlit + append tok "\\" + set slash_active 0 + } + string - quotedkey - itablequotedkey - comment { if {$slash_active} { set slash_active 0 append tok "\\\\" @@ -2474,13 +2873,15 @@ namespace eval tomlish::parse { set slash_active 1 } } - whitespace { - if {$state eq "multistring-space"} { - #end whitespace token - incr i -1 - return 1 + starttablename - starttablearrayname { + error "backslash is invalid as first character of $tokenType [tomlish::parse::report_line]" + } + tablename - tablearrayname { + if {$slash_active} { + set slash_active 0 + append tok "\\\\" } else { - error "Unexpected backslash during whitespace. [tomlish::parse::report_line]" + set slash_active 1 } } barekey { @@ -2492,181 +2893,314 @@ namespace eval tomlish::parse { } } else { if {$state eq "multistring-space"} { - set slash_active 1 + if {$slash_active} { + set_tokenType "stringpart" + set tok "\\\\" + set slash_active 0 + } else { + if {$dquotes ne ""} { + set_tokenType "stringpart" + set tok $dquotes + } + set slash_active 1 + } } else { error "Unexpected backslash when no token is active. [tomlish::parse::report_line]" } } } - dq { - #double quote - try { - if {[::string length $tokenType]} { - switch -exact -- $tokenType { - startquotesequence { - set toklen [::string length $tok] - if {$toklen == 1} { + sq { + #single quote + set had_slash $slash_active + set slash_active 0 + if {[::string length $tokenType]} { + switch -exact -- $tokenType { + whitespace { + #end whitespace + incr i -1 ;#reprocess sq + return 1 + } + startquotesequence { + _shortcircuit_startquotesequence + } + startsquotesequence { + switch -- [tcl::string::length $tok] { + 1 { append tok $c - } elseif {$toklen == 2} { + } + 2 { + #switch? append tok $c - set_tokenType "startmultiquote" + set_tokenType startmultisquote return 1 - } else { - error "unexpected token length in 'startquotesequence'" + } + default { + error "unexpected token length [tcl::string::length $tok] in 'startsquotesequence'" } } - endquotesequence { - set toklen [::string length $tok] - if {$toklen == 1} { - append tok $c - } elseif {$toklen == 2} { - append tok $c - set_tokenType "endmultiquote" + } + stringlit { + #slash_active always false + #terminate the stringlit + dict set token_waiting type endsquote + dict set token_waiting tok "'" + return 1 + } + squotedkey - itablesquotedkey { + dict set token_waiting type endsquote + dict set token_waiting tok "'" + return 1 + } + starttablename - starttablearrayname { + #!!! + incr i -1 + return 1 + } + tablename - tablearrayname { + append tok $c + } + default { + append tok $c + } + } + } else { + switch -exact -- $state { + value-expected - array-space { + #todo - multilitstring startsquotesequence? + set_tokenType "startsquotesequence" + set tok "'" + } + key-space { + set_tokenType "startsquote" + set tok $c + return 1 + } + curly-space { + set_tokenType "startsquote" + set tok $c + return 1 + } + tablename - tablearrayname { + #first char in tablename/tablearrayname state + set_tokenType $state ;#token name matches state name for tablename/tablearrayname + append tok "'" + } + stringlit { + tomlish::log::debug "sq during stringlit state with no tokentype - empty stringlit?" + set_tokenType stringlit + incr -1 + return 1 + } + multistring-space { + + } + default { + error "unhandled squote during state '$state'. [tomlish::parse::report_line]" + } + } + } + + } + dq { + #double quote + set had_slash $slash_active + set slash_active 0 + + if {[::string length $tokenType]} { + switch -exact -- $tokenType { + startquotesequence { + set toklen [::string length $tok] + if {$toklen == 1} { + append tok $c + } elseif {$toklen == 2} { + append tok $c + #switch vs set? + set_tokenType "startmultiquote" + return 1 + } else { + error "unexpected token length $toklen in 'startquotesequence'" + } + } + startsquotesequence { + set toklen [tcl::string::length $tok] + switch -- $toklen { + 1 { + set_tokenType "startsquote" + incr i -1 return 1 - } else { - error "unexpected token length in 'endquotesequence'" } - } - string { - if {$slash_active} { - append tok "\\" - append tok $c - } else { - #unescaped quote always terminates a string? - dict set token_waiting type endquote - dict set token_waiting tok "\"" + 2 { + set_tokenType "startsquote" + incr i -2 return 1 } + default { + error "unexpected startsquotesequence length $toklen" + } } - stringpart { - #sub element of multistring - if {$slash_active} { - append tok "\\" - append tok $c + } + stringlit { + append tok $c + } + string { + if {$had_slash} { + append tok "\\" $c + } else { + #unescaped quote always terminates a string? + dict set token_waiting type endquote + dict set token_waiting tok "\"" + return 1 + } + } + stringpart { + #sub element of multistring + if {$had_slash} { + append tok "\\" $c + } else { + #incr i -1 + + if {$multi_dquote eq "\"\""} { + dict set token_waiting type endmultiquote + dict set token_waiting tok "\"\"\"" + set multi_dquote "" + return 1 } else { - #incr i -1 - + append multi_dquote "\"" + } + } + } + whitespace { + switch -exact -- $state { + multistring-space { + #REVIEW + if {$had_slash} { + incr i -2 + return 1 + } else { + switch -- [tcl::string::length $multi_dquote] { + 2 { + dict set token_waiting type endmultiquote + dict set token_waiting tok "\"\"\"" + set multi_dquote "" + return 1 + } + 1 { + incr i -2 + return 1 + } + 0 { + incr i -1 + return 1 + } + } + } + } + value-expected { if {$multi_dquote eq "\"\""} { - dict set token_waiting type endmultiquote + dict set token_waiting type startmultiquote dict set token_waiting tok "\"\"\"" set multi_dquote "" return 1 } else { - append multi_dquote "\"" - } - } - } - whitespace { - switch -exact -- $state { - multistring-space { - #REVIEW - if {$multi_dquote eq "\"\""} { - dict set token_waiting type endmultiquote - dict set token_waiting tok "\"\"\"" - set multi_dquote "" - return 1 - } else { - append multi_dquote "\"" - } - } - value-expected { - if {$multi_dquote eq "\"\""} { - dict set token_waiting type startmultiquote - dict set token_waiting tok "\"\"\"" - set multi_dquote "" - return 1 - } else { - #end whitespace token and reprocess - incr i -1 - return 1 - #append multi_dquote "\"" - } - } - default { - dict set token_waiting type startquote - dict set token_waiting tok "\"" - return 1 + #end whitespace token and reprocess + incr i -1 + return 1 } } - } - comment { - if {$slash_active} {append tok "\\"} - append tok $c - } - quotedkey - itablequotedkey { - if {$slash_active} { - append tok "\\" - append tok $c - } else { - dict set token_waiting type endquote + default { + dict set token_waiting type startquote dict set token_waiting tok "\"" - return 1 + return 1 } } - tablename - tablearrayname { - if {$slash_active} {append tok "\\"} + } + comment { + if {$had_slash} {append tok "\\"} + append tok $c + } + quotedkey - itablequotedkey { + if {$had_slash} { + append tok "\\" append tok $c - } - starttablename - starttablearrayname { - incr i -1 ;## + } else { + dict set token_waiting type endquote + dict set token_waiting tok "\"" return 1 } - default { - error "got quote during tokenType '$tokenType' [tomlish::parse::report_line]" - } } - } else { - #$slash_active not relevant when no tokenType - #token is string only if we're expecting a value at this point - switch -exact -- $state { - value-expected - array-space { - #!? start looking for possible multistartquote - #set_tokenType startquote - #set tok $c - #return 1 - set_tokenType startquotesequence ;#one or more quotes in a row - either startquote or multistartquote - set tok $c - } - multistring-space { - #REVIEW + squotedkey - itablesquotedkey { + append tok $c + } + tablename - tablearrayname { + if {$had_slash} {append tok "\\"} + append tok $c + } + starttablename - starttablearrayname { + incr i -1 ;## + return 1 + } + default { + error "got quote during tokenType '$tokenType' [tomlish::parse::report_line]" + } + } + } else { + #$slash_active not relevant when no tokenType + #token is string only if we're expecting a value at this point + switch -exact -- $state { + value-expected - array-space { + #!? start looking for possible multistartquote + #set_tokenType startquote + #set tok $c + #return 1 + set_tokenType "startquotesequence" ;#one or more quotes in a row - either startquote or multistartquote + set tok $c + } + multistring-space { + #TODO - had_slash!!! + #REVIEW + if {$had_slash} { + set_tokenType "stringpart" + set tok "\\\"" + set multi_dquote "" + } else { if {$multi_dquote eq "\"\""} { - dict set token_waiting type endmultiquote - dict set token_waiting tok "\"\"\"" - set multi_dquote "" + tomlish::log::debug "---> endmultiquote" + set_tokenType "endmultiquote" + set tok "\"\"\"" return 1 + #dict set token_waiting type endmultiquote + #dict set token_waiting tok "\"\"\"" + #set multi_dquote "" + #return 1 } else { append multi_dquote "\"" } } - key-space { - set tokenType startquote - set tok $c - return 1 - } - curly-space { - set tokenType startquote - set tok $c - return 1 - } - tablename - tablearrayname { - set_tokenType $state - set tok $c - } - default { - error "Unexpected quote during state '$state' [tomlish::parse::report_line]" - } + } + key-space { + set_tokenType "startquote" + set tok $c + return 1 + } + curly-space { + set_tokenType "startquote" + set tok $c + return 1 + } + tablename - tablearrayname { + set_tokenType $state + set tok $c + } + default { + error "Unexpected quote during state '$state' [tomlish::parse::report_line]" } } - } on error {em} { - error $em - } finally { - set slash_active 0 } } = { set dquotes $multi_dquote set multi_dquote "" ;#!! - if {$slash_active} {append tok "\\"} ;#if tokentype not appropriate for \, we would already have errored out. + set had_slash $slash_active set slash_active 0 if {[::string length $tokenType]} { @@ -2674,23 +3208,47 @@ namespace eval tomlish::parse { startquotesequence { _shortcircuit_startquotesequence } - string - comment - quotedkey { + startsquotesequence { + incr i -[tcl::string::length $tok] + set_tokenType "startsquote" + return 1 + } + stringlit - squotedkey { + #assertion had_slash 0, multi_dquote "" + append tok $c + } + string - comment - quotedkey - itablequotedkey { #for these tokenTypes an = is just data. + if {$had_slash} {append tok "\\"} append tok $c } stringpart { + if {$had_slash} {append tok "\\"} append tok $dquotes$c } whitespace { - dict set token_waiting type equal - dict set token_waiting tok = - return 1 + if {$state in {multistring-space multiliteral-space}} { + set backlen [expr {[tcl::string::length $dquotes] + 1}] + incr i -$backlen + return 1 + } else { + dict set token_waiting type equal + dict set token_waiting tok = + return 1 + } } barekey { dict set token_waiting type equal dict set token_waiting tok = return 1 } + starttablename - starttablearrayname { + error "Character '=' is invalid first character for $tokenType. [tomlish::parse::report_line]" + } + tablename - tablearrayname { + #invalid in bare name - but valid in quoted parts - leave for tablename parser to sort out + append tok $c + } default { error "unexpected = character during tokentype $tokenType. case not implemented. [tomlish::parse::report_line]" } @@ -2698,11 +3256,16 @@ namespace eval tomlish::parse { } else { switch -exact -- $state { multistring-space { - set_tokenType stringpart - set tok ${dquotes}= + set_tokenType "stringpart" + set tok "" + if {$had_slash} { + append tok "\\" + } + append tok ${dquotes}= } + default { - set_tokenType equal + set_tokenType "equal" set tok = return 1 } @@ -2710,6 +3273,7 @@ namespace eval tomlish::parse { } } cr { + #REVIEW! set dquotes $multi_dquote set multi_dquote "" ;#!! # \r carriage return @@ -2720,9 +3284,25 @@ namespace eval tomlish::parse { startquotesequence { _shortcircuit_startquotesequence } + startsquotesequence { + incr i -[tcl::string::length $tok] + set_tokenType "startsquote" + return 1 + } + stringlit { + append tok $c + } stringpart { append tok $dquotes$c } + starttablename - starttablearrayname { + error "Character is invalid first character for $tokenType. [tomlish::parse::report_line]" + } + tablename - tablearrayname { + #could in theory be valid in quoted part of name + #review - might be better just to disallow here + append tok $c + } default { #!todo - error out if cr inappropriate for tokenType append tok $c @@ -2731,21 +3311,30 @@ namespace eval tomlish::parse { } else { #lf may be appended if next #review - lone cr as newline? - this is uncommon - but so is lone cr in a string(?) - set_tokenType newline + set_tokenType "newline" set tok cr } } lf { + # \n newline set dquotes $multi_dquote set multi_dquote "" ;#!! - # \n newline + set had_slash $slash_active + set slash_active 0 if {[::string length $tokenType]} { - if {$slash_active} {append tok "\\"} ;#if tokentype not appropriate for \, we would already have errored out. - set slash_active 0 switch -exact -- $tokenType { startquotesequence { _shortcircuit_startquotesequence } + startsquotesequence { + incr i -[tcl::string::length $tok] + set_tokenType "startsquote" + return 1 + } + stringlit { + #review + append tok $c + } newline { #this lf is the trailing part of a crlf append tok lf @@ -2757,11 +3346,22 @@ namespace eval tomlish::parse { incr i -1 return 1 } else { - dict set token_waiting type newline - dict set token_waiting tok lf - return 1 + if {$had_slash} { + #emit the stringpart (return 1), queue the continuation, go back 1 to reprocess the lf (incr i -1) + dict set token_waiting type continuation + dict set token_waiting tok \\ + incr i -1 + return 1 + } else { + dict set token_waiting type newline + dict set token_waiting tok lf + return 1 + } } } + starttablename - tablename - tablearrayname - starttablearrayname { + error "Character is invalid in $tokenType. [tomlish::parse::report_line]" + } default { #newline ends all other tokens. #note for string: we don't add (raw unescaped) newline to simple string. (must use multi-string for this) @@ -2771,59 +3371,115 @@ namespace eval tomlish::parse { #puts "-------------- newline lf during tokenType $tokenType" dict set token_waiting type newline - dict set token_waiting tok lf + dict set token_waiting tok lf return 1 } } } else { - set had_slash $slash_active - set slash_active 0 - if {$had_slash} { - set_tokenType "continuation" - set tok "\\" - incr i -1 - return 1 - } else { - set_tokenType newline - set tok lf - return 1 + switch -exact -- $state { + multistring-space { + if {$had_slash} { + set_tokenType "continuation" + set tok "\\" + incr i -1 + return 1 + } else { + if {$dquotes ne ""} { + #e.g one or 2 quotes just before nl + set_tokenType "stringpart" + set tok $dquotes + incr i -1 + return 1 + } + set_tokenType "newline" + set tok lf + return 1 + } + } + default { + #ignore slash? error? + set_tokenType "newline" + set tok lf + return 1 + } } + #if {$had_slash} { + # #CONT directly before newline - allows strings_5_byteequivalent test to pass + # set_tokenType "continuation" + # set tok "\\" + # incr i -1 + # return 1 + #} else { + # set_tokenType newline + # set tok lf + # return 1 + #} } } , { set dquotes $multi_dquote - set multi_dquote "" ;#!! - if {$slash_active} {append tok "\\"} ;#if tokentype not appropriate for \, we would already have errored out. + set multi_dquote "" + set had_slash $slash_active set slash_active 0 if {[::string length $tokenType]} { switch -exact -- $tokenType { startquotesequence { _shortcircuit_startquotesequence } - string - comment - quotedkey - tablename - tablearrayname { + startsquotesequence { + incr i -[tcl::string::length $tok] + set_tokenType "startsquote" + return 1 + } + comment - tablename - tablearrayname { + if {$had_slash} {append tok "\\"} + append tok , + } + string - quotedkey - itablequotedkey { + if {$had_slash} {append tok "\\"} + append tok $c + } + stringlit - squotedkey - itablesquotedkey { + #assert had_slash always 0, multi_dquote "" append tok $c } stringpart { + if {$had_slash} {append tok "\\"} append tok $dquotes$c } + whitespace { + if {$state eq "multistring-space"} { + set backlen [expr {[tcl::string::length $dquotes] + 1}] + incr i -$backlen + return 1 + } else { + dict set token_waiting type comma + dict set token_waiting tok "," + return 1 + } + } default { dict set token_waiting type comma dict set token_waiting tok "," + if {$had_slash} {append tok "\\"} return 1 } } } else { switch -exact -- $state { multistring-space { - set_tokenType stringpart - set tok "," + set_tokenType "stringpart" + set tok "" + if {$had_slash} {append tok "\\"} + append tok "$dquotes," } multiliteral-space { - set_tokenType literalpart + #assert had_slash 0, multi_dquote "" + set_tokenType "stringlit" set tok "," } default { - set_tokenType comma + set_tokenType "comma" set tok "," return 1 } @@ -2831,24 +3487,61 @@ namespace eval tomlish::parse { } } . { + set dquotes $multi_dquote set multi_dquote "" ;#!! - if {$slash_active} {append tok "\\"} ;#if tokentype not appropriate for \, we would already have errored out. + set had_slash $slash_active set slash_active 0 if {[::string length $tokenType]} { switch -exact -- $tokenType { startquotesequence { _shortcircuit_startquotesequence } - string - stringpart - comment - quotedkey - untyped-value { + startsquotesequence { + incr i -[tcl::string::length $tok] + set_tokenType "startsquote" + return 1 + } + comment - untyped-value { + if {$had_slash} {append tok "\\"} + append tok $c + } + string - quotedkey - itablequotedkey { + if {$had_slash} {append tok "\\"} append tok $c } - baretablename - tablename - tablearrayname { + stringlit - squotedkey - itablesquotedkey { + #assert had_slash always 0, multi_dquote "" + append tok $c + } + stringpart { + if {$had_slash} {append tok "\\"} + append tok $dquotes$c + } + whitespace { + if {$state eq "multistring-space"} { + set backchars [expr {[tcl::string::length $dquotes] + 1}] + if {$had_slash} { + incr backchars 1 + } + incr i -$backchars + return 1 + } else { + error "Received period during tokenType 'whitespace' [tomlish::parse::report_line]" + } + } + starttablename - starttablearrayname { + #This would correspond to an empty table name + error "Character '.' is not allowed as first character ($tokenType). [tomlish::parse::report_line]" + } + tablename - tablearrayname { #subtable - split later - review append tok $c } barekey { + #e.g x.y = 1 #we need to transition the barekey to become a structured table name ??? review - switch_tokenType tablename + #x is the tablename y is the key + switch_tokenType tablenamepluskey incr i -1 #error "barekey period unimplemented" @@ -2863,15 +3556,19 @@ namespace eval tomlish::parse { } else { switch -exact -- $state { multistring-space { - set_tokenType stringpart - set tok "." + set_tokenType "stringpart" + set tok "" + if {$had_slash} {append tok "\\"} + append tok "$dquotes." } multiliteral-space { - set_tokenType literalpart - set tok "." + set_tokenType "literalpart" + set tok "" + if {$had_slash} {append tok "\\"} + append tok "$dquotes." } default { - set_tokenType untyped-value + set_tokenType "untyped-value" set tok "." } } @@ -2888,7 +3585,13 @@ namespace eval tomlish::parse { startquotesequence { _shortcircuit_startquotesequence } + startsquotesequence { + incr i -[tcl::string::length $tok] + set_tokenType "startsquote" + return 1 + } barekey { + #todo had_slash - emit token or error #whitespace is a terminator for bare keys #dict set token_waiting type whitespace #dict set token_waiting tok $c @@ -2906,22 +3609,34 @@ namespace eval tomlish::parse { if {$had_slash} { append tok "\\" } + append tok $dquotes$c + } + string - quotedkey - itablequotedkey { + if {$had_slash} { append tok "\\" } append tok $c } - quotedkey - string { - if {$had_slash} { - append tok "\\" - } - #if {$dquotes eq "\""} { - #} + stringlit - squotedkey - itablesquotedkey { append tok $c } whitespace { - append tok $c + if {$state eq "multistring-space"} { + if {$dquotes ne ""} { + #end whitespace token + #go back by the number of quotes plus this space char + set backchars [expr {[tcl::string::length $dquotes] + 1}] + incr i -$backchars + return 1 + } else { + append tok $c + } + } else { + append tok $c + } } stringpart { if {$had_slash} { #REVIEW + #emit the stringpart - go back to the slash incr i -2 return 1 } else { @@ -2932,11 +3647,7 @@ namespace eval tomlish::parse { return 1 } } - starttablename { - incr i -1 - return 1 - } - starttablearrayname { + starttablename - starttablearrayname { incr i -1 return 1 } @@ -2951,9 +3662,7 @@ namespace eval tomlish::parse { } } else { set had_slash $slash_active - if {$slash_active} { - set slash_active 0 - } + set slash_active 0 switch -exact -- $state { tablename - tablearrayname { #tablename can have leading,trailing and interspersed whitespace! @@ -2976,7 +3685,7 @@ namespace eval tomlish::parse { set_tokenType "stringpart" set tok $dquotes incr i -1 - return + return 1 } set_tokenType "whitespace" append tok $c @@ -2997,12 +3706,20 @@ namespace eval tomlish::parse { set multi_dquote "" ;#!! if {[::string length $tokenType]} { - if {$slash_active} {append tok "\\"} ;#if tokentype not appropriate for \, we would already have errored out. + if {$slash_active} {append tok "\\"} ;#if tokentype not appropriate for \, we would already have errored out (?review) set slash_active 0 switch -exact -- $tokenType { startquotesequence { _shortcircuit_startquotesequence } + startsquotesequence { + incr i -[tcl::string::length $tok] + set_tokenType "startsquote" + return 1 + } + stringlit { + append tok $c + } barekey { #whitespace is a terminator for bare keys incr i -1 @@ -3017,7 +3734,7 @@ namespace eval tomlish::parse { incr i -1 return 1 } - quotedkey { + quotedkey - itablequotedkey { append tok $c } string - comment - whitespace { @@ -3078,9 +3795,27 @@ namespace eval tomlish::parse { } bom { #BOM (Byte Order Mark) - ignored by token consumer - set_tokenType "bom" - set tok "\uFEFF" - return 1 + if {[string length $tokenType]} { + switch -exact -- $tokenType { + startsquotesequence { + incr i -[tcl::string::length $tok] + set_tokenType "startsquote" + return 1 + } + stringlit { + append tok $c + } + default { + set_tokenType "bom" + set tok "\uFEFF" + return 1 + } + } + } else { + set_tokenType "bom" + set tok "\uFEFF" + return 1 + } } default { set dquotes $multi_dquote @@ -3093,12 +3828,27 @@ namespace eval tomlish::parse { startquotesequence { _shortcircuit_startquotesequence } - endquotesequence { - puts stderr "endquotesequence: $tok" + startsquotesequence { + puts stdout "HERE $c" + incr i -[tcl::string::length $tok] + set_tokenType "startsquote" + return 1 } whitespace { - incr i -1 ;#We don't have a full token to add to the token_waiting dict - so leave this char for next run. - return 1 + if {$state eq "multistring-space"} { + if {$dquotes ne ""} { + set backlen [expr {[tcl::string::length $dquotes] + 1}] + incr i -$backlen + return 1 + } else { + incr i -1 + return 1 + } + } else { + #review + incr i -1 ;#We don't have a full token to add to the token_waiting dict - so leave this char for next run. + return 1 + } } barekey { if {[tomlish::utils::is_barekey $c]} { @@ -3116,7 +3866,7 @@ namespace eval tomlish::parse { append tok $dquotes$c } default { - #e.g comment/string/untyped-value/starttablename/starttablearrayname/tablename/tablearrayname + #e.g comment/string/stringlit/untyped-value/starttablename/starttablearrayname/tablename/tablearrayname append tok $c } } @@ -3151,6 +3901,7 @@ namespace eval tomlish::parse { set tok $c } default { + tomlish::log::debug "char '$c' setting to untyped-value while state:$state" set_tokenType "untyped-value" set tok $c } @@ -3167,20 +3918,39 @@ namespace eval tomlish::parse { #if {$state eq "err"} { # error "Reached end of data whilst tokenType = '$tokenType'. INVALID" #} - if {$tokenType eq "startquotesequence"} { - set toklen [::string length $tok] - if {$toklen == 1} { - #invalid - #eof with open string - eror "eof reached without closing quote for string. [tomlish::parse::report_line]" - } elseif {$toklen == 2} { - #valid - #we ended in a double quote, not actually a startquoteseqence - effectively an empty string - switch_tokenType "startquote" - incr i -1 - #dict set token_waiting type "string" - #dict set token_waiting tok "" - return 1 + switch -exact -- $tokenType { + startquotesequence { + set toklen [::string length $tok] + if {$toklen == 1} { + #invalid + #eof with open string + error "eof reached without closing quote for string. [tomlish::parse::report_line]" + } elseif {$toklen == 2} { + #valid + #we ended in a double quote, not actually a startquoteseqence - effectively an empty string + switch_tokenType "startquote" + incr i -1 + #dict set token_waiting type "string" + #dict set token_waiting tok "" + return 1 + } + } + startsquotesequence { + set toklen [::string length $tok] + switch -- $toklen { + 1 { + #invalid eof with open stringlit + error "eof reached without closing single quote for string literal. [tomlish::parse::report_line]" + } + 2 { + dict set token_waiting type endsquote + dict set token_waiting tok "'" + ### + set_tokenType "stringlit" + set tok "" + return 1 + } + } } } dict set token_waiting type "eof"