Browse Source

add punk::imap4 pkg, promise pkg, console/cesu/args updates

master
Julian Noble 5 days ago
parent
commit
ab43137c29
  1. 2
      callbacks/dispatch.tcl
  2. 69
      scriptlib/thread_interp.tcl
  3. 2
      src/Tcl9icon.six
  4. 11
      src/Tcl9icon.svg
  5. 1
      src/bootsupport/modules/include_modules.config
  6. 1311
      src/bootsupport/modules/promise-1.2.0.tm
  7. 13
      src/bootsupport/modules/punk/args-0.1.0.tm
  8. 60
      src/bootsupport/modules/punk/console-0.1.1.tm
  9. 12
      src/defaultconfigs/Adventure.toml
  10. 167
      src/lib/app-shellspy/shellspy.tcl
  11. 13
      src/modules/punk/args-999999.0a1.0.tm
  12. 215
      src/modules/punk/cesu-999999.0a1.0.tm
  13. 60
      src/modules/punk/console-999999.0a1.0.tm
  14. 2168
      src/modules/punk/icomm-999999.0a1.0.tm
  15. 3
      src/modules/punk/icomm-buildversion.txt
  16. 3412
      src/modules/punk/imap4-999999.0a1.0.tm
  17. 3
      src/modules/punk/imap4-buildversion.txt
  18. 44
      src/modules/punk/jtest.tcl
  19. 3444
      src/modules/punk/repl-999999.0a1.0.tm
  20. 3
      src/modules/punk/repl-buildversion.txt
  21. 1
      src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/include_modules.config
  22. 1311
      src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/promise-1.2.0.tm
  23. 13
      src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/args-0.1.0.tm
  24. 60
      src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/console-0.1.1.tm
  25. 1
      src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/include_modules.config
  26. 1311
      src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/promise-1.2.0.tm
  27. 13
      src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/args-0.1.0.tm
  28. 60
      src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/console-0.1.1.tm
  29. 5
      src/runtime/mapvfs.config
  30. 8
      src/vendormodules/commandstack-0.3.tm
  31. 2
      src/vendormodules/include_modules.config
  32. 195
      src/vendormodules/oolib-0.1.tm
  33. 45
      src/vendormodules/overtype-1.6.5.tm
  34. 1311
      src/vendormodules/promise-1.2.0.tm
  35. 9
      src/vendormodules/tomlish-1.1.1.tm
  36. 167
      src/vfs/_vfscommon.vfs/lib/app-shellspy/shellspy.tcl
  37. 1311
      src/vfs/_vfscommon.vfs/modules/promise-1.2.0.tm
  38. 13
      src/vfs/_vfscommon.vfs/modules/punk/args-0.1.0.tm
  39. 215
      src/vfs/_vfscommon.vfs/modules/punk/cesu-0.1.0.tm
  40. 60
      src/vfs/_vfscommon.vfs/modules/punk/console-0.1.1.tm
  41. 2168
      src/vfs/_vfscommon.vfs/modules/punk/icomm-0.1.0.tm
  42. 3412
      src/vfs/_vfscommon.vfs/modules/punk/imap4-0.9.tm
  43. 44
      src/vfs/_vfscommon.vfs/modules/punk/jtest.tcl
  44. 67
      src/vfs/_vfscommon.vfs/modules/punk/repl-0.1.1.tm
  45. 12
      src/vfs/_vfscommon.vfs/modules/punk/repl-0.1.tm

2
callbacks/dispatch.tcl

@ -2,7 +2,7 @@ namespace eval shellspy::callbacks {
package require shellfilter
#each member of args - ist not itself a list - and cannot be treated as one.
#each member of args - is not itself a list - and cannot be treated as one.
#things like [concat {*}args] will generall break things further down the line
proc cmdshellb {args} {
shellfilter::log::open callback_cmdb {-syslog 127.0.0.1:514}

69
scriptlib/thread_interp.tcl

@ -0,0 +1,69 @@
set arg1 [lindex $::argv 0]
interp create code1
interp create code2
puts stderr "loading Thread package in all 3 interps"
package require Thread
code1 eval {package require Thread}
code2 eval {package require Thread}
puts stderr "establishing ::testfunc proc in all 3 interps"
code1 eval {proc ::testfunc {args} {puts stderr "evaluated in code1 interp: $args"}}
code2 eval {proc ::testfunc {args} {puts stderr "evaluated in code2 interp: $args"}}
proc ::testfunc {args} {puts stderr "evaluated in parent interp: $args"}
puts stderr "Calling a thread function in nominated interp '$arg1' first"
#1st use of thread function makes that interp the one to receive all subsequent messages
switch -- $arg1 {
parent {
thread::id
}
code1 {
code1 eval {thread::id}
}
code2 {
code2 eval {thread::id}
}
default {
puts stderr "Usage thread_interp.tcl parent|code1|code2"
exit 1
}
}
puts stderr "sending scripts"
thread::send -async [thread::id] {
::testfunc script sent from parent interp
}
code1 eval {
thread::send -async [thread::id] {
::testfunc script sent from code1 interp
}
}
code2 eval {
thread::send -async [thread::id] {
::testfunc script sent from code2 interp
}
}
#test
after 0 {::testfunc after script in parent interp}
code1 eval {after 0 {::testfunc after script in code1 interp}}
code2 eval {after 0 {::testfunc after script in code2 interp}}
code1 eval {
set workertid [thread::create]
thread::send $workertid {package require Thread}
thread::send $workertid [list thread::send -async [thread::id] {
::testfunc script sent from code1 interp via worker
}]
}
after idle {set ::done 1}
vwait ::done

2
src/Tcl9icon.six

File diff suppressed because one or more lines are too long

11
src/Tcl9icon.svg

@ -0,0 +1,11 @@
<?xml version="1.0"?>
<svg width="256" height="256" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg" xml:space="preserve">
<g class="layer">
<title>Tcl9 Icon</title>
<g id="svg_8" transform="scale(0.04654545, 0.04654545)">
<path d="m5355,2741c0,1437 -1165,2601 -2602,2601" fill="none" id="svg_9" stroke="#696969" stroke-width="14"/>
<path d="m152,2741c0,-1437 1164,-2602 2601,-2602c303,0 550,55 820,150l42,-6c629,-187 1016,-213 1213,-208c110,3 161,16 161,16c-368,41 -1396,274 -2236,1150c-429,398 -1008,1473 -1162,2376c0,0 318,-554 753,-1097c394,-458 833,-915 1133,-1022c-259,253 -1527,1570 -2261,3342c-645,-474 -1064,-1238 -1064,-2099z" fill="#BADA55" id="svg_10"/>
<path d="m5355,2741c0,1437 -1165,2601 -2602,2601c-466,0 -904,-122 -1283,-337c0,0 -314,460 -321,480c2,-7 1,-42 8,-63c7,-22 209,-621 259,-695c51,-75 226,-609 518,-591c291,18 326,209 348,266c0,0 165,-302 -178,-480c0,0 128,28 181,42c52,15 243,89 540,-74c296,-162 531,-457 595,-552c65,-96 85,-117 91,-160c0,0 -526,-51 -550,-198c0,0 326,52 481,29c155,-24 293,-11 599,-285s605,-531 670,-815c0,0 -364,333 -835,243c0,0 385,-90 535,-180c149,-90 320,-164 458,-389c0,0 98,-160 103,-171c235,391 383,839 383,1329z" fill="#696969" id="svg_11"/>
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 1.2 KiB

1
src/bootsupport/modules/include_modules.config

@ -23,6 +23,7 @@ set bootsupport_modules [list\
src/vendormodules patterncmd\
src/vendormodules patternlib\
src/vendormodules patternpredator2\
src/vendormodules promise\
src/vendormodules sha1\
src/vendormodules tomlish\
src/vendormodules test::tomlish\

1311
src/bootsupport/modules/promise-1.2.0.tm

File diff suppressed because it is too large Load Diff

13
src/bootsupport/modules/punk/args-0.1.0.tm

@ -4001,7 +4001,17 @@ tcl::namespace::eval punk::args {
set choice_in_list 1
set choice_exact_match 1
} elseif {$v_test in $choices_test} {
set chosen $v_test
#assert - if we're here, nocase must be true
#we know choice is present as full-length match except for case
#now we want to select the case from the choice list - not the supplied value
#we don't set choice_exact_match - because we will need to override the optimistic existing val below
#review
foreach avail [lsort -unique $allchoices] {
if {[string match -nocase $c $avail]} {
set chosen $avail
}
}
#assert chosen will always get set
set choice_in_list 1
} else {
#PREFIX check required - any 'chosen' here is not an exact match or it would have matched above.
@ -4046,6 +4056,7 @@ tcl::namespace::eval punk::args {
}
}
#override the optimistic existing val
if {$choice_in_list && !$choice_exact_match} {
if {$choicemultiple_max != -1 && $choicemultiple_max < 2} {
if {$is_multiple} {

60
src/bootsupport/modules/punk/console-0.1.1.tm

@ -740,18 +740,27 @@ namespace eval punk::console {
set was_raw 1
set timeoutid($callid) [after $expected [list set $waitvarname timedout]]
}
#write before console enableRaw vs after??
#There seem to be problems (e.g on WSL) if we write too early - the output ends up on screen but we don't read it
puts -nonewline $output $query;flush $output
chan configure $input -blocking 0
set tslaunch($callid) [clock millis] ;#time of launch - may be delay before first event depending on what's going on
set tsclock($callid) $tslaunch($callid)
#write before console enableRaw vs after??
#There seem to be problems (e.g on WSL) if we write too early - the output ends up on screen but we don't read it
puts -nonewline $output $query;flush $output
#after 0
#------------------
#trying alternatives to get faster read and maintain reliability..REVIEW
#we should care more about performance in raw mode - as ultimately that's the one we prefer for full features
#------------------
# 1) faster - races?
$this_handler $input $callid $capturingendregex
$this_handler $input $callid $capturingendregex
if {$ignoreok || $waitvar($callid) ne "ok"} {
chan event $input readable [list $this_handler $input $callid $capturingendregex]
}
# 2) more reliable?
#chan event $input readable [list $this_handler $input $callid $capturingendregex]
#------------------
#response from terminal
@ -794,7 +803,7 @@ namespace eval punk::console {
if {$waitvar($callid) ne "timedout"} {
after cancel $timeoutid($callid)
} else {
puts stderr "timeout (timediff [expr {[clock millis] - $tslaunch($callid)}]ms) in get_ansi_response_payload. callid $callid Ansi request was:[ansistring VIEW -lf 1 -vt 1 $query]"
puts stderr "timeout (timediff [expr {[clock millis] - $tslaunch($callid)}]ms) in get_ansi_response_payload. callid $callid Ansi request was:'[ansistring VIEW -lf 1 -vt 1 $query]'"
}
if {$was_raw == 0} {
@ -956,9 +965,10 @@ namespace eval punk::console {
set sofar [append chunks($callid) $bytes]
#puts stderr [ansistring VIEW $chunks($callid)]
#review - what is min length of any ansiresponse?
#we know there is at least one of only 3 chars, vt52 response to ESC Z: ESC / Z
#endregex is capturing - but as we are only testing the match here
#it should perform the same as if it were non-capturing
if {[string length $sofar] > 3 && [regexp $endregex $sofar]} {
if {[string length $sofar] > 2 && [regexp $endregex $sofar]} {
#puts stderr "matched - setting ansi_response_wait($callid) ok"
chan event $chan readable {}
set waits($callid) ok
@ -1438,7 +1448,8 @@ namespace eval punk::console {
-inoutchannels -default {stdin stdout} -type list
@values -min 0 -max 1
newsize -default "" -help\
"character cell pixel dimensions WxH"
"character cell pixel dimensions WxH
or omit to query cell size."
}
proc cell_size {args} {
set argd [punk::args::get_by_id ::punk::console::cell_size $args]
@ -1474,6 +1485,31 @@ namespace eval punk::console {
}
set cell_size ${w}x${h}
}
punk::args::define {
@id -id ::punk::console::test_is_vt52
@cmd -name punk::console::test_is_vt52 -help\
"in development.. broken"
-inoutchannels -default {stdin stdout} -type list
@values -min 0 -max 0
}
#only works in raw mode for windows terminal - (esc in output stripped?) why?
# works in line mode for alacrity and wezterm
proc test_is_vt52 {args} {
set argd [punk::args::get_by_id ::punk::console::test_is_vt52 $args]
set inoutchannels [dict get $argd opts -inoutchannels]
#ESC / K VT52 without printer
#ESC / M VT52 with printer
#ESC / Z VT52 emulator?? review
#TODO
set capturingregex {(.*)(?:(\x1b\/(Z))|(\x1b\/(K))|(\x1b\/(M))|(\x1b\[\?([0-9;]+)c))$} ;#must capture prefix,entire-response,response-payload
#set capturingregex {(.*)(\x1b\[([0-9;]+)c)$} ;#must capture prefix,entire-response,response-payload
set request "\x1bZ"
set payload [punk::console::internal::get_ansi_response_payload -terminal $inoutchannels $request $capturingregex]
#puts -->$payload<--
return [expr {$payload in {Z K M}}]
}
#todo - determine cursor on/off state before the call to restore properly.
proc get_size {{inoutchannels {stdin stdout}}} {
@ -1587,7 +1623,6 @@ namespace eval punk::console {
}
proc get_mode_line_wrap {{inoutchannels {stdin stdout}}} {
set capturingregex {(.*)(\x1b\[\?7;([0-9]+)\$y)$} ;#must capture prefix,entire-response,response-payload
set request "\x1b\[?7\$p"
@ -1683,7 +1718,14 @@ namespace eval punk::console {
return
}
puts -nonewline stdout $char_or_string
#On tcl9 - we could get an 'invalid or incomplete multibye or wide character' error
#e.g contains surrogate pair
if {[catch {
puts -nonewline stdout $char_or_string
} errM]} {
puts stderr "test_char_width couldn't emit this string - \nerror: $errM"
}
set response [punk::console::get_cursor_pos]
lassign [split $response ";"] _row2 col2
if {![string is integer -strict $col2]} {

12
src/defaultconfigs/Adventure.toml

@ -0,0 +1,12 @@
# Adventure
[colors]
foreground = "#feffff"
background = "#040404"
cursor_bg = "#feffff"
cursor_border = "#feffff"
cursor_fg = "#000000"
selection_bg = "#606060"
selection_fg = "#ffffff"
ansi = ["#040404","#d84a33","#5da602","#eebb6e","#417ab3","#e5c499","#bdcfe5","#dbded8"]
brights = ["#685656","#d76b42","#99b52c","#ffb670","#97d7ef","#aa7900","#bdcfe5","#e4d5c7"]

167
src/lib/app-shellspy/shellspy.tcl

@ -233,77 +233,77 @@ namespace eval shellspy {
proc get_channel_config {config} {
#note tcl script being called from wrong place.. configs don't affect: todo - move it.
set params [dict create]
if {$config == 0} {
#bad for: everything. extra cr
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation auto ;#default
dict set params -outtranslation auto
}
if {$config == 1} {
#ok for: cmd, cmd/u/c,raw,pwsh, sh,raw, tcl script process
#not ok for: bash,wsl, tcl script
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation auto ;#default
dict set params -outtranslation lf
}
if {$config == 2} {
#ok for: cmd, cmd/uc,pwsh,sh , tcl script process
#not ok for: tcl script, bash, wsl
dict set params -inbuffering none ;#default
dict set params -outbuffering none ;#default
dict set params -readprocesstranslation auto ;#default
dict set params -outtranslation lf ;#default
}
if {$config == 3} {
#ok for: cmd
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation lf
dict set params -outtranslation lf
}
if {$config == 4} {
#ok for: cmd,cmd/uc,raw,sh
#not ok for pwsh,bash,wsl, tcl script, tcl script process
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation lf
dict set params -outtranslation lf
}
if {$config == 5} {
#ok for: pwsh,cmd,cmd/u/c,raw,sh, tcl script process
#not ok for bash,wsl
#ok for vim cmd/u/c but only with to_unix filter on stdout (works in gvim and console)
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation crlf
dict set params -outtranslation lf
}
if {$config == 6} {
#ok for: cmd,cmd/u/c,pwsh,raw,sh,bash
#not ok for: vim with cmd /u/c (?)
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation crlf
dict set params -outtranslation lf
}
if {$config == 7} {
#ok for: sh,bash
#not ok for: wsl (display ok but extra cr), cmd,cmd/u/c,pwsh, tcl script, tcl script process, raw
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation crlf
dict set params -outtranslation crlf
}
if {$config == 8} {
#not ok for anything..all have extra cr
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation lf
dict set params -outtranslation crlf
switch -- $config {
0 {
#bad for: everything. extra cr
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation auto ;#default
dict set params -outtranslation auto
}
1 {
#ok for: cmd, cmd/u/c,raw,pwsh, sh,raw, tcl script process
#not ok for: bash,wsl, tcl script
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation auto ;#default
dict set params -outtranslation lf
}
2 {
#ok for: cmd, cmd/uc,pwsh,sh , tcl script process
#not ok for: tcl script, bash, wsl
dict set params -inbuffering none ;#default
dict set params -outbuffering none ;#default
dict set params -readprocesstranslation auto ;#default
dict set params -outtranslation lf ;#default
}
3 {
#ok for: cmd
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation lf
dict set params -outtranslation lf
}
4 {
#ok for: cmd,cmd/uc,raw,sh
#not ok for pwsh,bash,wsl, tcl script, tcl script process
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation lf
dict set params -outtranslation lf
}
5 {
#ok for: pwsh,cmd,cmd/u/c,raw,sh, tcl script process
#not ok for bash,wsl
#ok for vim cmd/u/c but only with to_unix filter on stdout (works in gvim and console)
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation crlf
dict set params -outtranslation lf
}
6 {
#ok for: cmd,cmd/u/c,pwsh,raw,sh,bash
#not ok for: vim with cmd /u/c (?)
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation crlf
dict set params -outtranslation lf
}
7 {
#ok for: sh,bash
#not ok for: wsl (display ok but extra cr), cmd,cmd/u/c,pwsh, tcl script, tcl script process, raw
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation crlf
dict set params -outtranslation crlf
}
8 {
#not ok for anything..all have extra cr
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation lf
dict set params -outtranslation crlf
}
}
return $params
}
@ -653,10 +653,27 @@ namespace eval shellspy {
set script [string map [list %a% $args %s% $scriptpath %m% $modulesdir] {
::tcl::tm::add %m%
set scriptname %s%
set ::argv [list %a%]
set ::argc [llength $::argv]
source [file normalize $scriptname]
set normscript [file normalize $scriptname]
#save values
set prevscript [info script]
set prevglobal [dict create]
foreach g [list ::argv ::argc ::argv0] {
if {[info exists $g]} {
dict set prevglobal $g [set $g]
}
}
#setup and run
set ::argv [list %a%]
set ::argc [llength $::argv]
set ::argv0 $normscript
info script $normscript
source $normscript
#restore values
info script $prevscript
dict with prevglobal {}
}]
set repl_lines ""

13
src/modules/punk/args-999999.0a1.0.tm

@ -4001,7 +4001,17 @@ tcl::namespace::eval punk::args {
set choice_in_list 1
set choice_exact_match 1
} elseif {$v_test in $choices_test} {
set chosen $v_test
#assert - if we're here, nocase must be true
#we know choice is present as full-length match except for case
#now we want to select the case from the choice list - not the supplied value
#we don't set choice_exact_match - because we will need to override the optimistic existing val below
#review
foreach avail [lsort -unique $allchoices] {
if {[string match -nocase $c $avail]} {
set chosen $avail
}
}
#assert chosen will always get set
set choice_in_list 1
} else {
#PREFIX check required - any 'chosen' here is not an exact match or it would have matched above.
@ -4046,6 +4056,7 @@ tcl::namespace::eval punk::args {
}
}
#override the optimistic existing val
if {$choice_in_list && !$choice_exact_match} {
if {$choicemultiple_max != -1 && $choicemultiple_max < 2} {
if {$is_multiple} {

215
src/modules/punk/cesu-999999.0a1.0.tm

@ -70,7 +70,7 @@ package require Tcl 8.6-
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
tcl::namespace::eval punk::cesu {
tcl::namespace::export {[a-z]*} ;# Convention: export all lowercase
#variable xyz
variable PUNKARGS
#*** !doctools
#[subsection {Namespace punk::cesu}]
@ -96,6 +96,8 @@ tcl::namespace::eval punk::cesu {
}
proc mapReply string {
package rquire http
http::config
variable ::http::formMap
set string [encoding convertto utf-8 $string]
@ -104,19 +106,21 @@ tcl::namespace::eval punk::cesu {
}
#where did original come from? wiki?
proc cesu2utf str {
#hacked by JMN - as original seemed broken and intention as to input is unclear
if {[regexp {\xED([\xA0-\xAF])([\x80-\xBF])\xED([\xB0-\xBF])([\x80-\xBF])} $str]} {
#set str [string map {\ \\ \[ \\\[ \] \\\]} $str] ;#original -broken - unsure of usecase/intention
set str [string map {\\ \\\\ \[ \\\[ \] \\\]} $str] ;#guess intention is to stop premature substitution of escapes and commands
#return [subst -novariables [regsub -all {^\xED([\xA0-\xAF])([\x80-\xBF])\xED([\xB0-\xBF])([\x80-\xBF])$} $str {[cesu2utfR \1 \2 \3 \4]} ]] ;#original. anchoring seems unlikely to be desirable
#capture the relevant 4 of the 6 bytes
return [subst -novariables [regsub -all {\xED([\xA0-\xAF])([\x80-\xBF])\xED([\xB0-\xBF])([\x80-\xBF])} $str {[cesu2utfR \1 \2 \3 \4]} ]]
} else {
return $str
}
}
#4 captured bytes (excludes the 2 \xED leaders)
proc cesu2utfR {1 2 3 4} {
# UTF-8: 11110xxx 10xx xxxx 10xx xxxx 10xxxxxx
# CESU-8: 11101101 1010 yy yy 10xxxx xx 11101101 1011xxxx 10xxxxxx
@ -125,7 +129,7 @@ tcl::namespace::eval punk::cesu {
binary scan $3 c 3
puts [list $1 $2 $3]
#binary scan $4 c 4
incr 1
incr 1 ;#// Effectively adds 0x10000 to the codepoint ?
return [binary format ccca \
[expr {0xF0 | (($1 & 0xC) >> 2)}] \
@ -171,17 +175,106 @@ tcl::namespace::eval punk::cesu {
encoding convertfrom utf-8 $x
}
#e.g test2 "note \ud83f\udd1e etc"
#e.g test2 "faces \ud83d\ude10 \ud83d\ude21 \ud83d\ude31"
#note: test2 \U1f600 returns a mouse (\U1f400) instead of smiley
# but test2 \U1f400 returns a mouse.
# Either surrogated_string shouldn't include non BMP chars anyway (G.I.G.O?).. or we're doing something wrong.
proc test2 {surrogated_string} {
#JMN
#e.g from_surrogatestring "note \ud83f\udd1e etc"
#e.g from_surrogatestring "faces \ud83d\ude10 \ud83d\ude21 \ud83d\ude31"
#note: from_surrogatestring \U1f600 returns a mouse (\U1f400) instead of smiley
# but from_surrogatestring \U1f400 returns a mouse.
# Tcl bug - fixed some time in 9.x
# surrogated_string shouldn't include non BMP chars anyway (G.I.G.O?)
lappend PUNKARGS [list {
@id -id ::punk::cesu::from_surrogatestring
@cmd -name punk::cesu::from_surrogatestring -help\
"Convert a string containing surrogate pairs
to string with pairs converted to unicode non-BMP
characters"
@values
surrogated_string -help\
"May contain a mix of surrogate pairs and other
characters - only the surrogate pairs will be converted."
}]
proc from_surrogatestring {surrogated_string} {
set cesu [encoding convertto cesu-8 $surrogated_string]
set x [cesu2utf $cesu]
encoding convertfrom utf-8 $x
}
proc _to_test {emoji} {
puts stderr "_to_test incomplete"
set cesu [encoding convertto cesu-8 $e]
puts stderr "cesu-8: $cesu"
}
lappend PUNKARGS [list {
@id -id ::punk::cesu::to_surrogatestring
@opts
-format -default escape -choices {raw escape} -choicelabels {
raw\
" emit raw surrogate pairs
may not be writable to
output channels"
escape\
" emit unprocessed backslash hex
escape sequences for surrogate
pairs created for non-BMP chars.
(Does not convert existing surrogates
in the input into escape sequences!)"
}
@values -min 1 -max 1
string -help\
"String possibly containing non-BMP codepoints to be converted
e.g
>to_surrogatestring -format escape \"mouse: \\U1f400\"
mouse: \\uD83D\\uDC00
"
}]
proc to_surrogatestring {args} {
set argd [punk::args::parse $args withid ::punk::cesu::to_surrogatestring]
lassign [dict values $argd] leaders opts values received
set opt_format [dict get $opts -format]
set string [dict get $values string]
set out ""
foreach c [split $string ""] {
set dec [scan $c %c]
if {$dec < 65536} {
append out $c
#if {$opt_format eq "escape"} {
#todo - detect existing surrogates in input?
#}
} else {
set pairinfo [nonbmp_surrogate_info $c]
if {$opt_format eq "raw"} {
append out [dict get $pairinfo raw]
} else {
append out [dict get $pairinfo escapes]
}
}
}
return $out
}
proc nonbmp_surrogate_info {char} {
#set cinfo [punk::char::char_info $char]
#set dec [dict get $cinfo dec]
lassign [scan $char %c%s] dec remainder
if {$remainder ne "" || $dec < 65536} {
error "nonbmp_surrogate_info takes a single non-BMP char (codepoint in the range U+10000 to U+10FFFF)"
}
#U - 0x10000
set less [expr {$dec - 0x10000}]
set lsb10 [expr {$less & 0b11111_11111}] ;#Least significant 10 bits of 20
set msb10 [expr {($less & 0b11111_11111_00000_00000) >> 10}] ;#most significant 10 bits of 20
#apply 'base' values
set msbfinal [expr {$msb10 + 0xd800}]
set lsbfinal [expr {$lsb10 + 0xdc00}]
set msbhex [format %4.4llX $msbfinal]
#set msbinfo [punk::char::char_info_dec $msbfinal -fields all -except testwidth] ;#don't use all/testwidth will try to emit the char and fail/show error
set lsbhex [format %4.4llX $lsbfinal]
#set lsbinfo [punk::char::char_info_dec $lsbfinal -fields all -except testwidth] ;#don't use all/testwidth will try to emit the char and fail/show error
set esc "\\u$msbhex\\u$lsbhex"
set raw [format %c $msbfinal][format %c $lsbfinal]
return [dict create escapes $esc msbdec $msbfinal msbhex $msbhex lsbdec $lsbfinal lsbhex $lsbhex raw $raw]
}
#
#test_enc_equivalency \U1f400 \U1f600
@ -191,7 +284,7 @@ tcl::namespace::eval punk::cesu {
foreach enc [lsort [encoding names]] {
puts stdout "testing $enc"
if {$enc in "iso2022 iso2022-jp iso2022-kr"} {
puts stderr "skipping $enc - crashes tcl9 on non BMP codepoints"
puts stderr "skipping $enc - crashes (early versions?) tcl9 on non BMP codepoints"
continue
}
if {[catch {
@ -253,6 +346,106 @@ tcl::namespace::eval punk::cesu::lib {
#}
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
# == === === === === === === === === === === === === === ===
# Sample 'about' function with punk::args documentation
# == === === === === === === === === === === === === === ===
tcl::namespace::eval punk::cesu {
tcl::namespace::export {[a-z]*} ;# Convention: export all lowercase
variable PUNKARGS
variable PUNKARGS_aliases
lappend PUNKARGS [list {
@id -id "(package)punk::cesu"
@package -name "punk::cesu" -help\
"experimental cesu conversions + surrogate pair processing"
}]
namespace eval argdoc {
#namespace for custom argument documentation
proc package_name {} {
return punk::cesu
}
proc about_topics {} {
#info commands results are returned in an arbitrary order (like array keys)
set topic_funs [info commands [namespace current]::get_topic_*]
set about_topics [list]
foreach f $topic_funs {
set tail [namespace tail $f]
lappend about_topics [string range $tail [string length get_topic_] end]
}
#Adjust this function or 'default_topics' if a different order is required
return [lsort $about_topics]
}
proc default_topics {} {return [list Description *]}
# -------------------------------------------------------------
# get_topic_ functions add more to auto-include in about topics
# -------------------------------------------------------------
proc get_topic_Description {} {
punk::args::lib::tstr [string trim {
package punk::cesu
description to come..
} \n]
}
proc get_topic_License {} {
return "MIT"
}
proc get_topic_Version {} {
return "$::punk::cesu::version"
}
proc get_topic_Contributors {} {
set authors {"Julian Noble <julian@precisium.com.au>"}
set contributors ""
foreach a $authors {
append contributors $a \n
}
if {[string index $contributors end] eq "\n"} {
set contributors [string range $contributors 0 end-1]
}
return $contributors
}
proc get_topic_custom-topic {} {
punk::args::lib::tstr -return string {
nothing to see here
}
}
# -------------------------------------------------------------
}
# we re-use the argument definition from punk::args::standard_about and override some items
set overrides [dict create]
dict set overrides @id -id "::punk::cesu::about"
dict set overrides @cmd -name "punk::cesu::about"
dict set overrides @cmd -help [string trim [punk::args::lib::tstr {
About punk::cesu
}] \n]
dict set overrides topic -choices [list {*}[punk::cesu::argdoc::about_topics] *]
dict set overrides topic -choicerestricted 1
dict set overrides topic -default [punk::cesu::argdoc::default_topics] ;#if -default is present 'topic' will always appear in parsed 'values' dict
set newdef [punk::args::resolved_def -antiglobs -package_about_namespace -override $overrides ::punk::args::package::standard_about *]
lappend PUNKARGS [list $newdef]
proc about {args} {
package require punk::args
#standard_about accepts additional choices for topic - but we need to normalize any abbreviations to full topic name before passing on
set argd [punk::args::parse $args withid ::punk::cesu::about]
lassign [dict values $argd] _leaders opts values _received
punk::args::package::standard_about -package_about_namespace ::punk::cesu::argdoc {*}$opts {*}[dict get $values topic]
}
}
# end of sample 'about' function
# == === === === === === === === === === === === === === ===
# -----------------------------------------------------------------------------
# register namespace(s) to have PUNKARGS,PUNKARGS_aliases variables checked
# -----------------------------------------------------------------------------
# variable PUNKARGS
# variable PUNKARGS_aliases
namespace eval ::punk::args::register {
#use fully qualified so 8.6 doesn't find existing var in global namespace
lappend ::punk::args::register::NAMESPACES ::punk::cesu
}
# -----------------------------------------------------------------------------
## Ready
package provide punk::cesu [tcl::namespace::eval punk::cesu {
variable pkg punk::cesu

60
src/modules/punk/console-999999.0a1.0.tm

@ -740,18 +740,27 @@ namespace eval punk::console {
set was_raw 1
set timeoutid($callid) [after $expected [list set $waitvarname timedout]]
}
#write before console enableRaw vs after??
#There seem to be problems (e.g on WSL) if we write too early - the output ends up on screen but we don't read it
puts -nonewline $output $query;flush $output
chan configure $input -blocking 0
set tslaunch($callid) [clock millis] ;#time of launch - may be delay before first event depending on what's going on
set tsclock($callid) $tslaunch($callid)
#write before console enableRaw vs after??
#There seem to be problems (e.g on WSL) if we write too early - the output ends up on screen but we don't read it
puts -nonewline $output $query;flush $output
#after 0
#------------------
#trying alternatives to get faster read and maintain reliability..REVIEW
#we should care more about performance in raw mode - as ultimately that's the one we prefer for full features
#------------------
# 1) faster - races?
$this_handler $input $callid $capturingendregex
$this_handler $input $callid $capturingendregex
if {$ignoreok || $waitvar($callid) ne "ok"} {
chan event $input readable [list $this_handler $input $callid $capturingendregex]
}
# 2) more reliable?
#chan event $input readable [list $this_handler $input $callid $capturingendregex]
#------------------
#response from terminal
@ -794,7 +803,7 @@ namespace eval punk::console {
if {$waitvar($callid) ne "timedout"} {
after cancel $timeoutid($callid)
} else {
puts stderr "timeout (timediff [expr {[clock millis] - $tslaunch($callid)}]ms) in get_ansi_response_payload. callid $callid Ansi request was:[ansistring VIEW -lf 1 -vt 1 $query]"
puts stderr "timeout (timediff [expr {[clock millis] - $tslaunch($callid)}]ms) in get_ansi_response_payload. callid $callid Ansi request was:'[ansistring VIEW -lf 1 -vt 1 $query]'"
}
if {$was_raw == 0} {
@ -956,9 +965,10 @@ namespace eval punk::console {
set sofar [append chunks($callid) $bytes]
#puts stderr [ansistring VIEW $chunks($callid)]
#review - what is min length of any ansiresponse?
#we know there is at least one of only 3 chars, vt52 response to ESC Z: ESC / Z
#endregex is capturing - but as we are only testing the match here
#it should perform the same as if it were non-capturing
if {[string length $sofar] > 3 && [regexp $endregex $sofar]} {
if {[string length $sofar] > 2 && [regexp $endregex $sofar]} {
#puts stderr "matched - setting ansi_response_wait($callid) ok"
chan event $chan readable {}
set waits($callid) ok
@ -1438,7 +1448,8 @@ namespace eval punk::console {
-inoutchannels -default {stdin stdout} -type list
@values -min 0 -max 1
newsize -default "" -help\
"character cell pixel dimensions WxH"
"character cell pixel dimensions WxH
or omit to query cell size."
}
proc cell_size {args} {
set argd [punk::args::get_by_id ::punk::console::cell_size $args]
@ -1474,6 +1485,31 @@ namespace eval punk::console {
}
set cell_size ${w}x${h}
}
punk::args::define {
@id -id ::punk::console::test_is_vt52
@cmd -name punk::console::test_is_vt52 -help\
"in development.. broken"
-inoutchannels -default {stdin stdout} -type list
@values -min 0 -max 0
}
#only works in raw mode for windows terminal - (esc in output stripped?) why?
# works in line mode for alacrity and wezterm
proc test_is_vt52 {args} {
set argd [punk::args::get_by_id ::punk::console::test_is_vt52 $args]
set inoutchannels [dict get $argd opts -inoutchannels]
#ESC / K VT52 without printer
#ESC / M VT52 with printer
#ESC / Z VT52 emulator?? review
#TODO
set capturingregex {(.*)(?:(\x1b\/(Z))|(\x1b\/(K))|(\x1b\/(M))|(\x1b\[\?([0-9;]+)c))$} ;#must capture prefix,entire-response,response-payload
#set capturingregex {(.*)(\x1b\[([0-9;]+)c)$} ;#must capture prefix,entire-response,response-payload
set request "\x1bZ"
set payload [punk::console::internal::get_ansi_response_payload -terminal $inoutchannels $request $capturingregex]
#puts -->$payload<--
return [expr {$payload in {Z K M}}]
}
#todo - determine cursor on/off state before the call to restore properly.
proc get_size {{inoutchannels {stdin stdout}}} {
@ -1587,7 +1623,6 @@ namespace eval punk::console {
}
proc get_mode_line_wrap {{inoutchannels {stdin stdout}}} {
set capturingregex {(.*)(\x1b\[\?7;([0-9]+)\$y)$} ;#must capture prefix,entire-response,response-payload
set request "\x1b\[?7\$p"
@ -1683,7 +1718,14 @@ namespace eval punk::console {
return
}
puts -nonewline stdout $char_or_string
#On tcl9 - we could get an 'invalid or incomplete multibye or wide character' error
#e.g contains surrogate pair
if {[catch {
puts -nonewline stdout $char_or_string
} errM]} {
puts stderr "test_char_width couldn't emit this string - \nerror: $errM"
}
set response [punk::console::get_cursor_pos]
lassign [split $response ";"] _row2 col2
if {![string is integer -strict $col2]} {

2168
src/modules/punk/icomm-999999.0a1.0.tm

File diff suppressed because it is too large Load Diff

3
src/modules/punk/icomm-buildversion.txt

@ -0,0 +1,3 @@
0.1.0
#First line must be a semantic version number
#all other lines are ignored.

3412
src/modules/punk/imap4-999999.0a1.0.tm

File diff suppressed because it is too large Load Diff

3
src/modules/punk/imap4-buildversion.txt

@ -0,0 +1,3 @@
0.9
#First line must be a semantic version number
#all other lines are ignored.

44
src/modules/punk/jtest.tcl

@ -0,0 +1,44 @@
set a b
set x {a b}
set x []
set x {
a
{b c}
}
array set comm {
debug 0
chans {}
localhost 1.2
x {}
y jb
j aa
blah "xxxb"
defaulg 0
}
#test
if {"x" eq max(2,3)} {
}
if {"x" eq min(1)} {}
set x [dict create {a b c {x} e f }]
zlib adler32 "abc"
dict get $x "a"
#dict create {a b}
set x []
#test
array set test1 {blah etc}
array set comm {
debug 0 chans {} localhost 127.0.0.1
offerVers {3 2}
acceptVers {3 2}
defaultEncoding "utf-8"
defaultSilent 0
}
#test
set x blah

3444
src/modules/punk/repl-999999.0a1.0.tm

File diff suppressed because it is too large Load Diff

3
src/modules/punk/repl-buildversion.txt

@ -0,0 +1,3 @@
0.1.1
#First line must be a semantic version number
#all other lines are ignored.

1
src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/include_modules.config

@ -23,6 +23,7 @@ set bootsupport_modules [list\
src/vendormodules patterncmd\
src/vendormodules patternlib\
src/vendormodules patternpredator2\
src/vendormodules promise\
src/vendormodules sha1\
src/vendormodules tomlish\
src/vendormodules test::tomlish\

1311
src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/promise-1.2.0.tm

File diff suppressed because it is too large Load Diff

13
src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/args-0.1.0.tm

@ -4001,7 +4001,17 @@ tcl::namespace::eval punk::args {
set choice_in_list 1
set choice_exact_match 1
} elseif {$v_test in $choices_test} {
set chosen $v_test
#assert - if we're here, nocase must be true
#we know choice is present as full-length match except for case
#now we want to select the case from the choice list - not the supplied value
#we don't set choice_exact_match - because we will need to override the optimistic existing val below
#review
foreach avail [lsort -unique $allchoices] {
if {[string match -nocase $c $avail]} {
set chosen $avail
}
}
#assert chosen will always get set
set choice_in_list 1
} else {
#PREFIX check required - any 'chosen' here is not an exact match or it would have matched above.
@ -4046,6 +4056,7 @@ tcl::namespace::eval punk::args {
}
}
#override the optimistic existing val
if {$choice_in_list && !$choice_exact_match} {
if {$choicemultiple_max != -1 && $choicemultiple_max < 2} {
if {$is_multiple} {

60
src/project_layouts/custom/_project/punk.project-0.1/src/bootsupport/modules/punk/console-0.1.1.tm

@ -740,18 +740,27 @@ namespace eval punk::console {
set was_raw 1
set timeoutid($callid) [after $expected [list set $waitvarname timedout]]
}
#write before console enableRaw vs after??
#There seem to be problems (e.g on WSL) if we write too early - the output ends up on screen but we don't read it
puts -nonewline $output $query;flush $output
chan configure $input -blocking 0
set tslaunch($callid) [clock millis] ;#time of launch - may be delay before first event depending on what's going on
set tsclock($callid) $tslaunch($callid)
#write before console enableRaw vs after??
#There seem to be problems (e.g on WSL) if we write too early - the output ends up on screen but we don't read it
puts -nonewline $output $query;flush $output
#after 0
#------------------
#trying alternatives to get faster read and maintain reliability..REVIEW
#we should care more about performance in raw mode - as ultimately that's the one we prefer for full features
#------------------
# 1) faster - races?
$this_handler $input $callid $capturingendregex
$this_handler $input $callid $capturingendregex
if {$ignoreok || $waitvar($callid) ne "ok"} {
chan event $input readable [list $this_handler $input $callid $capturingendregex]
}
# 2) more reliable?
#chan event $input readable [list $this_handler $input $callid $capturingendregex]
#------------------
#response from terminal
@ -794,7 +803,7 @@ namespace eval punk::console {
if {$waitvar($callid) ne "timedout"} {
after cancel $timeoutid($callid)
} else {
puts stderr "timeout (timediff [expr {[clock millis] - $tslaunch($callid)}]ms) in get_ansi_response_payload. callid $callid Ansi request was:[ansistring VIEW -lf 1 -vt 1 $query]"
puts stderr "timeout (timediff [expr {[clock millis] - $tslaunch($callid)}]ms) in get_ansi_response_payload. callid $callid Ansi request was:'[ansistring VIEW -lf 1 -vt 1 $query]'"
}
if {$was_raw == 0} {
@ -956,9 +965,10 @@ namespace eval punk::console {
set sofar [append chunks($callid) $bytes]
#puts stderr [ansistring VIEW $chunks($callid)]
#review - what is min length of any ansiresponse?
#we know there is at least one of only 3 chars, vt52 response to ESC Z: ESC / Z
#endregex is capturing - but as we are only testing the match here
#it should perform the same as if it were non-capturing
if {[string length $sofar] > 3 && [regexp $endregex $sofar]} {
if {[string length $sofar] > 2 && [regexp $endregex $sofar]} {
#puts stderr "matched - setting ansi_response_wait($callid) ok"
chan event $chan readable {}
set waits($callid) ok
@ -1438,7 +1448,8 @@ namespace eval punk::console {
-inoutchannels -default {stdin stdout} -type list
@values -min 0 -max 1
newsize -default "" -help\
"character cell pixel dimensions WxH"
"character cell pixel dimensions WxH
or omit to query cell size."
}
proc cell_size {args} {
set argd [punk::args::get_by_id ::punk::console::cell_size $args]
@ -1474,6 +1485,31 @@ namespace eval punk::console {
}
set cell_size ${w}x${h}
}
punk::args::define {
@id -id ::punk::console::test_is_vt52
@cmd -name punk::console::test_is_vt52 -help\
"in development.. broken"
-inoutchannels -default {stdin stdout} -type list
@values -min 0 -max 0
}
#only works in raw mode for windows terminal - (esc in output stripped?) why?
# works in line mode for alacrity and wezterm
proc test_is_vt52 {args} {
set argd [punk::args::get_by_id ::punk::console::test_is_vt52 $args]
set inoutchannels [dict get $argd opts -inoutchannels]
#ESC / K VT52 without printer
#ESC / M VT52 with printer
#ESC / Z VT52 emulator?? review
#TODO
set capturingregex {(.*)(?:(\x1b\/(Z))|(\x1b\/(K))|(\x1b\/(M))|(\x1b\[\?([0-9;]+)c))$} ;#must capture prefix,entire-response,response-payload
#set capturingregex {(.*)(\x1b\[([0-9;]+)c)$} ;#must capture prefix,entire-response,response-payload
set request "\x1bZ"
set payload [punk::console::internal::get_ansi_response_payload -terminal $inoutchannels $request $capturingregex]
#puts -->$payload<--
return [expr {$payload in {Z K M}}]
}
#todo - determine cursor on/off state before the call to restore properly.
proc get_size {{inoutchannels {stdin stdout}}} {
@ -1587,7 +1623,6 @@ namespace eval punk::console {
}
proc get_mode_line_wrap {{inoutchannels {stdin stdout}}} {
set capturingregex {(.*)(\x1b\[\?7;([0-9]+)\$y)$} ;#must capture prefix,entire-response,response-payload
set request "\x1b\[?7\$p"
@ -1683,7 +1718,14 @@ namespace eval punk::console {
return
}
puts -nonewline stdout $char_or_string
#On tcl9 - we could get an 'invalid or incomplete multibye or wide character' error
#e.g contains surrogate pair
if {[catch {
puts -nonewline stdout $char_or_string
} errM]} {
puts stderr "test_char_width couldn't emit this string - \nerror: $errM"
}
set response [punk::console::get_cursor_pos]
lassign [split $response ";"] _row2 col2
if {![string is integer -strict $col2]} {

1
src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/include_modules.config

@ -23,6 +23,7 @@ set bootsupport_modules [list\
src/vendormodules patterncmd\
src/vendormodules patternlib\
src/vendormodules patternpredator2\
src/vendormodules promise\
src/vendormodules sha1\
src/vendormodules tomlish\
src/vendormodules test::tomlish\

1311
src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/promise-1.2.0.tm

File diff suppressed because it is too large Load Diff

13
src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/args-0.1.0.tm

@ -4001,7 +4001,17 @@ tcl::namespace::eval punk::args {
set choice_in_list 1
set choice_exact_match 1
} elseif {$v_test in $choices_test} {
set chosen $v_test
#assert - if we're here, nocase must be true
#we know choice is present as full-length match except for case
#now we want to select the case from the choice list - not the supplied value
#we don't set choice_exact_match - because we will need to override the optimistic existing val below
#review
foreach avail [lsort -unique $allchoices] {
if {[string match -nocase $c $avail]} {
set chosen $avail
}
}
#assert chosen will always get set
set choice_in_list 1
} else {
#PREFIX check required - any 'chosen' here is not an exact match or it would have matched above.
@ -4046,6 +4056,7 @@ tcl::namespace::eval punk::args {
}
}
#override the optimistic existing val
if {$choice_in_list && !$choice_exact_match} {
if {$choicemultiple_max != -1 && $choicemultiple_max < 2} {
if {$is_multiple} {

60
src/project_layouts/custom/_project/punk.shell-0.1/src/bootsupport/modules/punk/console-0.1.1.tm

@ -740,18 +740,27 @@ namespace eval punk::console {
set was_raw 1
set timeoutid($callid) [after $expected [list set $waitvarname timedout]]
}
#write before console enableRaw vs after??
#There seem to be problems (e.g on WSL) if we write too early - the output ends up on screen but we don't read it
puts -nonewline $output $query;flush $output
chan configure $input -blocking 0
set tslaunch($callid) [clock millis] ;#time of launch - may be delay before first event depending on what's going on
set tsclock($callid) $tslaunch($callid)
#write before console enableRaw vs after??
#There seem to be problems (e.g on WSL) if we write too early - the output ends up on screen but we don't read it
puts -nonewline $output $query;flush $output
#after 0
#------------------
#trying alternatives to get faster read and maintain reliability..REVIEW
#we should care more about performance in raw mode - as ultimately that's the one we prefer for full features
#------------------
# 1) faster - races?
$this_handler $input $callid $capturingendregex
$this_handler $input $callid $capturingendregex
if {$ignoreok || $waitvar($callid) ne "ok"} {
chan event $input readable [list $this_handler $input $callid $capturingendregex]
}
# 2) more reliable?
#chan event $input readable [list $this_handler $input $callid $capturingendregex]
#------------------
#response from terminal
@ -794,7 +803,7 @@ namespace eval punk::console {
if {$waitvar($callid) ne "timedout"} {
after cancel $timeoutid($callid)
} else {
puts stderr "timeout (timediff [expr {[clock millis] - $tslaunch($callid)}]ms) in get_ansi_response_payload. callid $callid Ansi request was:[ansistring VIEW -lf 1 -vt 1 $query]"
puts stderr "timeout (timediff [expr {[clock millis] - $tslaunch($callid)}]ms) in get_ansi_response_payload. callid $callid Ansi request was:'[ansistring VIEW -lf 1 -vt 1 $query]'"
}
if {$was_raw == 0} {
@ -956,9 +965,10 @@ namespace eval punk::console {
set sofar [append chunks($callid) $bytes]
#puts stderr [ansistring VIEW $chunks($callid)]
#review - what is min length of any ansiresponse?
#we know there is at least one of only 3 chars, vt52 response to ESC Z: ESC / Z
#endregex is capturing - but as we are only testing the match here
#it should perform the same as if it were non-capturing
if {[string length $sofar] > 3 && [regexp $endregex $sofar]} {
if {[string length $sofar] > 2 && [regexp $endregex $sofar]} {
#puts stderr "matched - setting ansi_response_wait($callid) ok"
chan event $chan readable {}
set waits($callid) ok
@ -1438,7 +1448,8 @@ namespace eval punk::console {
-inoutchannels -default {stdin stdout} -type list
@values -min 0 -max 1
newsize -default "" -help\
"character cell pixel dimensions WxH"
"character cell pixel dimensions WxH
or omit to query cell size."
}
proc cell_size {args} {
set argd [punk::args::get_by_id ::punk::console::cell_size $args]
@ -1474,6 +1485,31 @@ namespace eval punk::console {
}
set cell_size ${w}x${h}
}
punk::args::define {
@id -id ::punk::console::test_is_vt52
@cmd -name punk::console::test_is_vt52 -help\
"in development.. broken"
-inoutchannels -default {stdin stdout} -type list
@values -min 0 -max 0
}
#only works in raw mode for windows terminal - (esc in output stripped?) why?
# works in line mode for alacrity and wezterm
proc test_is_vt52 {args} {
set argd [punk::args::get_by_id ::punk::console::test_is_vt52 $args]
set inoutchannels [dict get $argd opts -inoutchannels]
#ESC / K VT52 without printer
#ESC / M VT52 with printer
#ESC / Z VT52 emulator?? review
#TODO
set capturingregex {(.*)(?:(\x1b\/(Z))|(\x1b\/(K))|(\x1b\/(M))|(\x1b\[\?([0-9;]+)c))$} ;#must capture prefix,entire-response,response-payload
#set capturingregex {(.*)(\x1b\[([0-9;]+)c)$} ;#must capture prefix,entire-response,response-payload
set request "\x1bZ"
set payload [punk::console::internal::get_ansi_response_payload -terminal $inoutchannels $request $capturingregex]
#puts -->$payload<--
return [expr {$payload in {Z K M}}]
}
#todo - determine cursor on/off state before the call to restore properly.
proc get_size {{inoutchannels {stdin stdout}}} {
@ -1587,7 +1623,6 @@ namespace eval punk::console {
}
proc get_mode_line_wrap {{inoutchannels {stdin stdout}}} {
set capturingregex {(.*)(\x1b\[\?7;([0-9]+)\$y)$} ;#must capture prefix,entire-response,response-payload
set request "\x1b\[?7\$p"
@ -1683,7 +1718,14 @@ namespace eval punk::console {
return
}
puts -nonewline stdout $char_or_string
#On tcl9 - we could get an 'invalid or incomplete multibye or wide character' error
#e.g contains surrogate pair
if {[catch {
puts -nonewline stdout $char_or_string
} errM]} {
puts stderr "test_char_width couldn't emit this string - \nerror: $errM"
}
set response [punk::console::get_cursor_pos]
lassign [split $response ";"] _row2 col2
if {![string is integer -strict $col2]} {

5
src/runtime/mapvfs.config

@ -35,8 +35,9 @@ tclkit-win64-dyn.exe {punk86bawt.vfs punksys kit}
#TCL9
#tclsh90b2 {punk9win.vfs punk90b2 zip}
#tclsh90b4_piperepl.exe {punk9win.vfs punk90b4 zip}
#tclsh901.exe {punk9win.vfs punk901 zip}
tclsh901t.exe {punk9win.vfs punk901t zipcat}
#tclsh901.exe {punk9win.vfs punk901 zip}
tclsh901t.exe {punk9win.vfs punk901t zipcat}
tclsh90magic.exe {punk9magicsplat.vfs punkmagic zipcat}
#tclsh901k.exe {mkzipfix.vfs punktest zip}

8
src/vendormodules/commandstack-0.3.tm

@ -211,7 +211,7 @@ namespace eval commandstack {
set new_code [string trim $procbody]
if {$current_code eq $new_code} {
puts stderr "(commandstack::rename_command) WARNING - renamer '$renamer' has already renamed the '$command' command with same procbody - Aborting rename."
puts stderr [show_stack $command]
puts stderr [::commandstack::show_stack $command]
} else {
puts stderr "(commandstack::rename_command) WARNING - renamer '$renamer' has already renamed the '$command' command - but appears to be with new code - proceeding."
puts stdout "----------"
@ -236,8 +236,7 @@ namespace eval commandstack {
set next_target ::commandstack::renamed_commands::${mungedcommand}_${munged_next_implementor}-$mungedrenamer-$tokenid
set do_rename 1
} elseif {$next_implementor in {unspecified undetermined}} {
#review - probably don't need a warning anyway
puts stderr "(commandstack::rename_command) WARNING - Something may have renamed the '$command' command. Attempting to cooperate.(untested)"
#could be a standard tcl proc, or from application or package
set next_target ::commandstack::renamed_commands::${mungedcommand}_${munged_next_implementor}-$mungedrenamer-$tokenid
set do_rename 1
} else {
@ -380,7 +379,8 @@ namespace eval commandstack {
#if caller is attempting exact match - use the calling context to resolve in case they didn't supply namespace
set commandname_glob [uplevel 1 [list namespace which $commandname_glob]]
}
if {[package provide punk::lib] ne ""} {
if {[package provide punk::lib] ne "" && [package provide punk] ne ""} {
#punk pipeline also needed for patterns
return [punk::lib::pdict -channel none all_stacks $commandname_glob/@*/@*.@*]
} else {
set result ""

2
src/vendormodules/include_modules.config

@ -1,3 +1,5 @@
#todo - change to include_modules.toml
#aim is to be programatically editable whilst retaining comments
set local_modules [list\
c:/repo/nonexistant/tclmodules/blah/modules blah\

195
src/vendormodules/oolib-0.1.tm

@ -1,195 +0,0 @@
#JMN - api should be kept in sync with package patternlib where possible
#
package provide oolib [namespace eval oolib {
variable version
set version 0.1
}]
namespace eval oolib {
oo::class create collection {
variable o_data ;#dict
variable o_alias
constructor {} {
set o_data [dict create]
}
method info {} {
return [dict info $o_data]
}
method count {} {
return [dict size $o_data]
}
method isEmpty {} {
expr {[dict size $o_data] == 0}
}
method names {{globOrIdx {}}} {
if {[llength $globOrIdx]} {
if {[string is integer -strict $globOrIdx]} {
if {$idx < 0} {
set idx "end-[expr {abs($idx + 1)}]"
}
if {[catch {lindex [dict keys $o_data] $idx} result]} {
error "[self object] no such index : '$idx'"
} else {
return $result
}
} else {
#glob
return [lsearch -glob -all -inline [dict keys $o_data] $globOrIdx]
}
} else {
return [dict keys $o_data]
}
}
#like names but without globbing
method keys {} {
dict keys $o_data
}
method key {{posn 0}} {
if {$posn < 0} {
set posn "end-[expr {abs($posn + 1)}]"
}
if {[catch {lindex [dict keys $o_data] $posn} result]} {
error "[self object] no such index : '$posn'"
} else {
return $result
}
}
method hasKey {key} {
dict exists $o_data $key
}
method get {} {
return $o_data
}
method items {} {
return [dict values $o_data]
}
method item {key} {
if {[string is integer -strict $key]} {
if {$key > 0} {
set valposn [expr {(2*$key) +1}]
return [lindex $o_data $valposn]
} else {
set key "end-[expr {abs($key + 1)}]"
return [lindex [dict keys $o_data] $key]
}
}
if {[dict exists $o_data $key]} {
return [dict get $o_data $key]
}
}
#inverse lookup
method itemKeys {value} {
set value_indices [lsearch -all [dict values $o_data] $value]
set keylist [list]
foreach i $value_indices {
set idx [expr {(($i + 1) *2) -2}]
lappend keylist [lindex $o_data $idx]
}
return $keylist
}
method search {value args} {
set matches [lsearch {*}$args [dict values $o_data] $value]
if {"-inline" in $args} {
return $matches
} else {
set keylist [list]
foreach i $matches {
set idx [expr {(($i + 1) *2) -2}]
lappend keylist [lindex $o_data $idx]
}
return $keylist
}
}
#review - see patternlib. Is the intention for aliases to be configurable independent of whether the target exists?
method alias {newAlias existingKeyOrAlias} {
if {[string is integer -strict $newAlias]} {
error "[self object] collection key alias cannot be integer"
}
if {[string length $existingKeyOrAlias]} {
set o_alias($newAlias) $existingKeyOrAlias
} else {
unset o_alias($newAlias)
}
}
method aliases {{key ""}} {
if {[string length $key]} {
set result [list]
foreach {n v} [array get o_alias] {
if {$v eq $key} {
lappend result $n $v
}
}
return $result
} else {
return [array get o_alias]
}
}
#if the supplied index is an alias, return the underlying key; else return the index supplied.
method realKey {idx} {
if {[catch {set o_alias($idx)} key]} {
return $idx
} else {
return $key
}
}
method add {value key} {
if {[string is integer -strict $key]} {
error "[self object] collection key must not be an integer. Use another structure if integer keys required"
}
if {[dict exists $o_data $key]} {
error "[self object] col_processors object error: key '$key' already exists in collection"
}
dict set o_data $key $value
return [expr {[dict size $o_data] - 1}] ;#return index of item
}
method remove {idx {endRange ""}} {
if {[string length $endRange]} {
error "[self object] collection error: ranged removal not yet implemented.. remove one item at a time"
}
if {[string is integer -strict $idx]} {
if {$idx < 0} {
set idx "end-[expr {abs($idx+1)}]"
}
set key [lindex [dict keys $o_data] $idx]
set posn $idx
} else {
set key $idx
set posn [lsearch -exact [dict keys $o_data] $key]
if {$posn < 0} {
error "[self object] no such index: '$idx' in this collection"
}
}
dict unset o_data $key
return
}
method clear {} {
set o_data [dict create]
return
}
method reverse {} {
set dictnew [dict create]
foreach k [lreverse [dict keys $o_data]] {
dict set dictnew $k [dict get $o_data $k]
}
set o_data $dictnew
return
}
#review - cmd as list vs cmd as script?
method map {cmd} {
set seed [list]
dict for {k v} $o_data {
lappend seed [uplevel #0 [list {*}$cmd $v]]
}
return $seed
}
method objectmap {cmd} {
set seed [list]
dict for {k v} $o_data {
lappend seed [uplevel #0 [list $v {*}$cmd]]
}
return $seed
}
}
}

45
src/vendormodules/overtype-1.6.5.tm

@ -216,7 +216,9 @@ tcl::namespace::eval overtype {
}
set optargs [lrange $args 0 end-2]
if {[llength $optargs] % 2 == 0} {
lassign [lrange $args end-1 end] underblock overblock
set overblock [lindex $args end]
set underblock [lindex $args end-1]
#lassign [lrange $args end-1 end] underblock overblock
set argsflags [lrange $args 0 end-2]
} else {
set optargs [lrange $args 0 end-1]
@ -1810,8 +1812,10 @@ tcl::namespace::eval overtype {
if {[llength $args] < 2} {
error {usage: ?-info 0|1? ?-startcolumn <int>? ?-cursor_column <int>? ?-cursor_row <int>|""? ?-transparent [0|1|<regexp>]? ?-expand_right [1|0]? undertext overtext}
}
lassign [lrange $args end-1 end] under over
if {[string first \n $under] >= 0} {
set under [lindex $args end-1]
set over [lindex $args end]
#lassign [lrange $args end-1 end] under over
if {[string last \n $under] >= 0} {
error "overtype::renderline not allowed to contain newlines in undertext"
}
#if {[string first \n $over] >=0 || [string first \n $under] >= 0} {
@ -2920,6 +2924,7 @@ tcl::namespace::eval overtype {
set leadernorm [tcl::string::range [tcl::string::map [list\
\x1b\[< 1006\
\x1b\[ 7CSI\
\x1bY 7MAP\
\x1bP 7DCS\
\x90 8DCS\
\x9b 8CSI\
@ -2948,6 +2953,10 @@ tcl::namespace::eval overtype {
#8-bit Device Control String
set codenorm [tcl::string::cat $leadernorm [tcl::string::range $code 1 end]]
}
7MAP {
#map to another type of code to share implementation branch
set codenorm $leadernorm[tcl::string::range $code 1 end]
}
7ESC {
#set codenorm [tcl::string::cat $leadernorm [tcl::string::range $code 1 end]]
set codenorm $leadernorm[tcl::string::range $code 1 end]
@ -2964,6 +2973,30 @@ tcl::namespace::eval overtype {
}
}
switch -- $leadernorm {
7MAP {
switch -- [lindex $codenorm 4] {
Y {
#vt52 movement. we expect 2 chars representing position (limited range)
set params [tcl::string::range $codenorm 5 end]
if {[tcl::string::length $params] != 2} {
#shouldn't really get here or need this branch if ansi splitting was done correctly
puts stderr "overtype::renderline ESC Y recognised as vt52 move, but incorrect parameters length ([string length $params] vs expected 2) [ansistring VIEW -lf 1 -vt 1 -nul 1 $code] not implemented codenorm:[ansistring VIEW -lf 1 -vt 1 -nul 1 $codenorm]"
}
set line [tcl::string::index $params 5]
set column [tcl::string::index $params 1]
set r [expr {[scan $line %c] -31}]
set c [expr {[scan $column %c] -31}]
#MAP to:
#CSI n;m H - CUP - Cursor Position
set leadernorm 7CSI
set codenorm "$leadernorm${r}\;${c}H"
}
}
}
}
#we've mapped 7 and 8bit escapes to values we can handle as literals in switch statements to take advantange of jump tables.
switch -- $leadernorm {
1006 {
@ -2982,7 +3015,8 @@ tcl::namespace::eval overtype {
{7CSI} - {8CSI} {
set param [tcl::string::range $codenorm 4 end-1]
#puts stdout "--> CSI [tcl::string::index $leadernorm 0] bit param:$param"
set code_end [tcl::string::index $codenorm end] ;#used for e.g h|l set/unset mode
set code_end [tcl::string::index $codenorm end] ;#used for e.g h|l set/unset mode
switch -exact -- $code_end {
A {
#Row move - up
@ -3875,6 +3909,7 @@ tcl::namespace::eval overtype {
7ESC {
#
#re_other_single {\x1b(D|M|E)$}
#also vt52 Y..
#also PM \x1b^...(ST)
switch -- [tcl::string::index $codenorm 4] {
c {
@ -4586,6 +4621,8 @@ tcl::namespace::eval overtype::priv {
set o [lreplace $o $i $i]
set ustacks [lreplace $ustacks $i $i]
set gxstacks [lreplace $gxstacks $i $i]
} elseif {$i == 0 || $i == $nxt} {
#nothing to do
} else {
puts stderr "render_delchar - attempt to delchar at index $i >= number of outcols $nxt - shouldn't happen"
}

1311
src/vendormodules/promise-1.2.0.tm

File diff suppressed because it is too large Load Diff

9
src/vendormodules/tomlish-1.1.1.tm

@ -716,6 +716,7 @@ namespace eval tomlish {
set toml [::tomlish::to_toml $tomlish]
}
#TODO use huddle?
proc from_json {json} {
set jstruct [::tomlish::json_struct $json]
return [::tomlish::from_json_struct $jstruct]
@ -1080,11 +1081,13 @@ namespace eval tomlish::decode {
# For this reason, we also do absolutely no line-ending transformations based on platform.
# All line-endings are maintained as is, and even a file with mixed cr crlf line-endings will be correctly interpreted and can be 'roundtripped'
proc toml {s} {
proc toml {args} {
#*** !doctools
#[call [fun toml] [arg s]]
#[call [fun toml] [arg arg...]]
#[para] return a Tcl list of tomlish tokens
set s [join $args \n]
namespace upvar ::tomlish::parse is_parsing is_parsing
set is_parsing 1
@ -2380,7 +2383,7 @@ namespace eval tomlish::parse {
squotedkey {PUSHSPACE "itable-keyval-space" state "itable-keyval-syntax"}\
endinlinetable "POPSPACE"\
startquote "quoted-key"\
startsquote {TOSTATE "squoted-key" comment "jn-ok"}\
startsquote {TOSTATE "squoted-key" comment "jn-testing"}\
comma "itable-space"\
comment "err-state"\
eof "err-state"\

167
src/vfs/_vfscommon.vfs/lib/app-shellspy/shellspy.tcl

@ -233,77 +233,77 @@ namespace eval shellspy {
proc get_channel_config {config} {
#note tcl script being called from wrong place.. configs don't affect: todo - move it.
set params [dict create]
if {$config == 0} {
#bad for: everything. extra cr
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation auto ;#default
dict set params -outtranslation auto
}
if {$config == 1} {
#ok for: cmd, cmd/u/c,raw,pwsh, sh,raw, tcl script process
#not ok for: bash,wsl, tcl script
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation auto ;#default
dict set params -outtranslation lf
}
if {$config == 2} {
#ok for: cmd, cmd/uc,pwsh,sh , tcl script process
#not ok for: tcl script, bash, wsl
dict set params -inbuffering none ;#default
dict set params -outbuffering none ;#default
dict set params -readprocesstranslation auto ;#default
dict set params -outtranslation lf ;#default
}
if {$config == 3} {
#ok for: cmd
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation lf
dict set params -outtranslation lf
}
if {$config == 4} {
#ok for: cmd,cmd/uc,raw,sh
#not ok for pwsh,bash,wsl, tcl script, tcl script process
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation lf
dict set params -outtranslation lf
}
if {$config == 5} {
#ok for: pwsh,cmd,cmd/u/c,raw,sh, tcl script process
#not ok for bash,wsl
#ok for vim cmd/u/c but only with to_unix filter on stdout (works in gvim and console)
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation crlf
dict set params -outtranslation lf
}
if {$config == 6} {
#ok for: cmd,cmd/u/c,pwsh,raw,sh,bash
#not ok for: vim with cmd /u/c (?)
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation crlf
dict set params -outtranslation lf
}
if {$config == 7} {
#ok for: sh,bash
#not ok for: wsl (display ok but extra cr), cmd,cmd/u/c,pwsh, tcl script, tcl script process, raw
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation crlf
dict set params -outtranslation crlf
}
if {$config == 8} {
#not ok for anything..all have extra cr
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation lf
dict set params -outtranslation crlf
switch -- $config {
0 {
#bad for: everything. extra cr
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation auto ;#default
dict set params -outtranslation auto
}
1 {
#ok for: cmd, cmd/u/c,raw,pwsh, sh,raw, tcl script process
#not ok for: bash,wsl, tcl script
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation auto ;#default
dict set params -outtranslation lf
}
2 {
#ok for: cmd, cmd/uc,pwsh,sh , tcl script process
#not ok for: tcl script, bash, wsl
dict set params -inbuffering none ;#default
dict set params -outbuffering none ;#default
dict set params -readprocesstranslation auto ;#default
dict set params -outtranslation lf ;#default
}
3 {
#ok for: cmd
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation lf
dict set params -outtranslation lf
}
4 {
#ok for: cmd,cmd/uc,raw,sh
#not ok for pwsh,bash,wsl, tcl script, tcl script process
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation lf
dict set params -outtranslation lf
}
5 {
#ok for: pwsh,cmd,cmd/u/c,raw,sh, tcl script process
#not ok for bash,wsl
#ok for vim cmd/u/c but only with to_unix filter on stdout (works in gvim and console)
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation crlf
dict set params -outtranslation lf
}
6 {
#ok for: cmd,cmd/u/c,pwsh,raw,sh,bash
#not ok for: vim with cmd /u/c (?)
dict set params -inbuffering line
dict set params -outbuffering line
dict set params -readprocesstranslation crlf
dict set params -outtranslation lf
}
7 {
#ok for: sh,bash
#not ok for: wsl (display ok but extra cr), cmd,cmd/u/c,pwsh, tcl script, tcl script process, raw
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation crlf
dict set params -outtranslation crlf
}
8 {
#not ok for anything..all have extra cr
dict set params -inbuffering none
dict set params -outbuffering none
dict set params -readprocesstranslation lf
dict set params -outtranslation crlf
}
}
return $params
}
@ -653,10 +653,27 @@ namespace eval shellspy {
set script [string map [list %a% $args %s% $scriptpath %m% $modulesdir] {
::tcl::tm::add %m%
set scriptname %s%
set ::argv [list %a%]
set ::argc [llength $::argv]
source [file normalize $scriptname]
set normscript [file normalize $scriptname]
#save values
set prevscript [info script]
set prevglobal [dict create]
foreach g [list ::argv ::argc ::argv0] {
if {[info exists $g]} {
dict set prevglobal $g [set $g]
}
}
#setup and run
set ::argv [list %a%]
set ::argc [llength $::argv]
set ::argv0 $normscript
info script $normscript
source $normscript
#restore values
info script $prevscript
dict with prevglobal {}
}]
set repl_lines ""

1311
src/vfs/_vfscommon.vfs/modules/promise-1.2.0.tm

File diff suppressed because it is too large Load Diff

13
src/vfs/_vfscommon.vfs/modules/punk/args-0.1.0.tm

@ -4001,7 +4001,17 @@ tcl::namespace::eval punk::args {
set choice_in_list 1
set choice_exact_match 1
} elseif {$v_test in $choices_test} {
set chosen $v_test
#assert - if we're here, nocase must be true
#we know choice is present as full-length match except for case
#now we want to select the case from the choice list - not the supplied value
#we don't set choice_exact_match - because we will need to override the optimistic existing val below
#review
foreach avail [lsort -unique $allchoices] {
if {[string match -nocase $c $avail]} {
set chosen $avail
}
}
#assert chosen will always get set
set choice_in_list 1
} else {
#PREFIX check required - any 'chosen' here is not an exact match or it would have matched above.
@ -4046,6 +4056,7 @@ tcl::namespace::eval punk::args {
}
}
#override the optimistic existing val
if {$choice_in_list && !$choice_exact_match} {
if {$choicemultiple_max != -1 && $choicemultiple_max < 2} {
if {$is_multiple} {

215
src/vfs/_vfscommon.vfs/modules/punk/cesu-0.1.0.tm

@ -70,7 +70,7 @@ package require Tcl 8.6-
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
tcl::namespace::eval punk::cesu {
tcl::namespace::export {[a-z]*} ;# Convention: export all lowercase
#variable xyz
variable PUNKARGS
#*** !doctools
#[subsection {Namespace punk::cesu}]
@ -96,6 +96,8 @@ tcl::namespace::eval punk::cesu {
}
proc mapReply string {
package rquire http
http::config
variable ::http::formMap
set string [encoding convertto utf-8 $string]
@ -104,19 +106,21 @@ tcl::namespace::eval punk::cesu {
}
#where did original come from? wiki?
proc cesu2utf str {
#hacked by JMN - as original seemed broken and intention as to input is unclear
if {[regexp {\xED([\xA0-\xAF])([\x80-\xBF])\xED([\xB0-\xBF])([\x80-\xBF])} $str]} {
#set str [string map {\ \\ \[ \\\[ \] \\\]} $str] ;#original -broken - unsure of usecase/intention
set str [string map {\\ \\\\ \[ \\\[ \] \\\]} $str] ;#guess intention is to stop premature substitution of escapes and commands
#return [subst -novariables [regsub -all {^\xED([\xA0-\xAF])([\x80-\xBF])\xED([\xB0-\xBF])([\x80-\xBF])$} $str {[cesu2utfR \1 \2 \3 \4]} ]] ;#original. anchoring seems unlikely to be desirable
#capture the relevant 4 of the 6 bytes
return [subst -novariables [regsub -all {\xED([\xA0-\xAF])([\x80-\xBF])\xED([\xB0-\xBF])([\x80-\xBF])} $str {[cesu2utfR \1 \2 \3 \4]} ]]
} else {
return $str
}
}
#4 captured bytes (excludes the 2 \xED leaders)
proc cesu2utfR {1 2 3 4} {
# UTF-8: 11110xxx 10xx xxxx 10xx xxxx 10xxxxxx
# CESU-8: 11101101 1010 yy yy 10xxxx xx 11101101 1011xxxx 10xxxxxx
@ -125,7 +129,7 @@ tcl::namespace::eval punk::cesu {
binary scan $3 c 3
puts [list $1 $2 $3]
#binary scan $4 c 4
incr 1
incr 1 ;#// Effectively adds 0x10000 to the codepoint ?
return [binary format ccca \
[expr {0xF0 | (($1 & 0xC) >> 2)}] \
@ -171,17 +175,106 @@ tcl::namespace::eval punk::cesu {
encoding convertfrom utf-8 $x
}
#e.g test2 "note \ud83f\udd1e etc"
#e.g test2 "faces \ud83d\ude10 \ud83d\ude21 \ud83d\ude31"
#note: test2 \U1f600 returns a mouse (\U1f400) instead of smiley
# but test2 \U1f400 returns a mouse.
# Either surrogated_string shouldn't include non BMP chars anyway (G.I.G.O?).. or we're doing something wrong.
proc test2 {surrogated_string} {
#JMN
#e.g from_surrogatestring "note \ud83f\udd1e etc"
#e.g from_surrogatestring "faces \ud83d\ude10 \ud83d\ude21 \ud83d\ude31"
#note: from_surrogatestring \U1f600 returns a mouse (\U1f400) instead of smiley
# but from_surrogatestring \U1f400 returns a mouse.
# Tcl bug - fixed some time in 9.x
# surrogated_string shouldn't include non BMP chars anyway (G.I.G.O?)
lappend PUNKARGS [list {
@id -id ::punk::cesu::from_surrogatestring
@cmd -name punk::cesu::from_surrogatestring -help\
"Convert a string containing surrogate pairs
to string with pairs converted to unicode non-BMP
characters"
@values
surrogated_string -help\
"May contain a mix of surrogate pairs and other
characters - only the surrogate pairs will be converted."
}]
proc from_surrogatestring {surrogated_string} {
set cesu [encoding convertto cesu-8 $surrogated_string]
set x [cesu2utf $cesu]
encoding convertfrom utf-8 $x
}
proc _to_test {emoji} {
puts stderr "_to_test incomplete"
set cesu [encoding convertto cesu-8 $e]
puts stderr "cesu-8: $cesu"
}
lappend PUNKARGS [list {
@id -id ::punk::cesu::to_surrogatestring
@opts
-format -default escape -choices {raw escape} -choicelabels {
raw\
" emit raw surrogate pairs
may not be writable to
output channels"
escape\
" emit unprocessed backslash hex
escape sequences for surrogate
pairs created for non-BMP chars.
(Does not convert existing surrogates
in the input into escape sequences!)"
}
@values -min 1 -max 1
string -help\
"String possibly containing non-BMP codepoints to be converted
e.g
>to_surrogatestring -format escape \"mouse: \\U1f400\"
mouse: \\uD83D\\uDC00
"
}]
proc to_surrogatestring {args} {
set argd [punk::args::parse $args withid ::punk::cesu::to_surrogatestring]
lassign [dict values $argd] leaders opts values received
set opt_format [dict get $opts -format]
set string [dict get $values string]
set out ""
foreach c [split $string ""] {
set dec [scan $c %c]
if {$dec < 65536} {
append out $c
#if {$opt_format eq "escape"} {
#todo - detect existing surrogates in input?
#}
} else {
set pairinfo [nonbmp_surrogate_info $c]
if {$opt_format eq "raw"} {
append out [dict get $pairinfo raw]
} else {
append out [dict get $pairinfo escapes]
}
}
}
return $out
}
proc nonbmp_surrogate_info {char} {
#set cinfo [punk::char::char_info $char]
#set dec [dict get $cinfo dec]
lassign [scan $char %c%s] dec remainder
if {$remainder ne "" || $dec < 65536} {
error "nonbmp_surrogate_info takes a single non-BMP char (codepoint in the range U+10000 to U+10FFFF)"
}
#U - 0x10000
set less [expr {$dec - 0x10000}]
set lsb10 [expr {$less & 0b11111_11111}] ;#Least significant 10 bits of 20
set msb10 [expr {($less & 0b11111_11111_00000_00000) >> 10}] ;#most significant 10 bits of 20
#apply 'base' values
set msbfinal [expr {$msb10 + 0xd800}]
set lsbfinal [expr {$lsb10 + 0xdc00}]
set msbhex [format %4.4llX $msbfinal]
#set msbinfo [punk::char::char_info_dec $msbfinal -fields all -except testwidth] ;#don't use all/testwidth will try to emit the char and fail/show error
set lsbhex [format %4.4llX $lsbfinal]
#set lsbinfo [punk::char::char_info_dec $lsbfinal -fields all -except testwidth] ;#don't use all/testwidth will try to emit the char and fail/show error
set esc "\\u$msbhex\\u$lsbhex"
set raw [format %c $msbfinal][format %c $lsbfinal]
return [dict create escapes $esc msbdec $msbfinal msbhex $msbhex lsbdec $lsbfinal lsbhex $lsbhex raw $raw]
}
#
#test_enc_equivalency \U1f400 \U1f600
@ -191,7 +284,7 @@ tcl::namespace::eval punk::cesu {
foreach enc [lsort [encoding names]] {
puts stdout "testing $enc"
if {$enc in "iso2022 iso2022-jp iso2022-kr"} {
puts stderr "skipping $enc - crashes tcl9 on non BMP codepoints"
puts stderr "skipping $enc - crashes (early versions?) tcl9 on non BMP codepoints"
continue
}
if {[catch {
@ -253,6 +346,106 @@ tcl::namespace::eval punk::cesu::lib {
#}
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
# == === === === === === === === === === === === === === ===
# Sample 'about' function with punk::args documentation
# == === === === === === === === === === === === === === ===
tcl::namespace::eval punk::cesu {
tcl::namespace::export {[a-z]*} ;# Convention: export all lowercase
variable PUNKARGS
variable PUNKARGS_aliases
lappend PUNKARGS [list {
@id -id "(package)punk::cesu"
@package -name "punk::cesu" -help\
"experimental cesu conversions + surrogate pair processing"
}]
namespace eval argdoc {
#namespace for custom argument documentation
proc package_name {} {
return punk::cesu
}
proc about_topics {} {
#info commands results are returned in an arbitrary order (like array keys)
set topic_funs [info commands [namespace current]::get_topic_*]
set about_topics [list]
foreach f $topic_funs {
set tail [namespace tail $f]
lappend about_topics [string range $tail [string length get_topic_] end]
}
#Adjust this function or 'default_topics' if a different order is required
return [lsort $about_topics]
}
proc default_topics {} {return [list Description *]}
# -------------------------------------------------------------
# get_topic_ functions add more to auto-include in about topics
# -------------------------------------------------------------
proc get_topic_Description {} {
punk::args::lib::tstr [string trim {
package punk::cesu
description to come..
} \n]
}
proc get_topic_License {} {
return "MIT"
}
proc get_topic_Version {} {
return "$::punk::cesu::version"
}
proc get_topic_Contributors {} {
set authors {"Julian Noble <julian@precisium.com.au>"}
set contributors ""
foreach a $authors {
append contributors $a \n
}
if {[string index $contributors end] eq "\n"} {
set contributors [string range $contributors 0 end-1]
}
return $contributors
}
proc get_topic_custom-topic {} {
punk::args::lib::tstr -return string {
nothing to see here
}
}
# -------------------------------------------------------------
}
# we re-use the argument definition from punk::args::standard_about and override some items
set overrides [dict create]
dict set overrides @id -id "::punk::cesu::about"
dict set overrides @cmd -name "punk::cesu::about"
dict set overrides @cmd -help [string trim [punk::args::lib::tstr {
About punk::cesu
}] \n]
dict set overrides topic -choices [list {*}[punk::cesu::argdoc::about_topics] *]
dict set overrides topic -choicerestricted 1
dict set overrides topic -default [punk::cesu::argdoc::default_topics] ;#if -default is present 'topic' will always appear in parsed 'values' dict
set newdef [punk::args::resolved_def -antiglobs -package_about_namespace -override $overrides ::punk::args::package::standard_about *]
lappend PUNKARGS [list $newdef]
proc about {args} {
package require punk::args
#standard_about accepts additional choices for topic - but we need to normalize any abbreviations to full topic name before passing on
set argd [punk::args::parse $args withid ::punk::cesu::about]
lassign [dict values $argd] _leaders opts values _received
punk::args::package::standard_about -package_about_namespace ::punk::cesu::argdoc {*}$opts {*}[dict get $values topic]
}
}
# end of sample 'about' function
# == === === === === === === === === === === === === === ===
# -----------------------------------------------------------------------------
# register namespace(s) to have PUNKARGS,PUNKARGS_aliases variables checked
# -----------------------------------------------------------------------------
# variable PUNKARGS
# variable PUNKARGS_aliases
namespace eval ::punk::args::register {
#use fully qualified so 8.6 doesn't find existing var in global namespace
lappend ::punk::args::register::NAMESPACES ::punk::cesu
}
# -----------------------------------------------------------------------------
## Ready
package provide punk::cesu [tcl::namespace::eval punk::cesu {
variable pkg punk::cesu

60
src/vfs/_vfscommon.vfs/modules/punk/console-0.1.1.tm

@ -740,18 +740,27 @@ namespace eval punk::console {
set was_raw 1
set timeoutid($callid) [after $expected [list set $waitvarname timedout]]
}
#write before console enableRaw vs after??
#There seem to be problems (e.g on WSL) if we write too early - the output ends up on screen but we don't read it
puts -nonewline $output $query;flush $output
chan configure $input -blocking 0
set tslaunch($callid) [clock millis] ;#time of launch - may be delay before first event depending on what's going on
set tsclock($callid) $tslaunch($callid)
#write before console enableRaw vs after??
#There seem to be problems (e.g on WSL) if we write too early - the output ends up on screen but we don't read it
puts -nonewline $output $query;flush $output
#after 0
#------------------
#trying alternatives to get faster read and maintain reliability..REVIEW
#we should care more about performance in raw mode - as ultimately that's the one we prefer for full features
#------------------
# 1) faster - races?
$this_handler $input $callid $capturingendregex
$this_handler $input $callid $capturingendregex
if {$ignoreok || $waitvar($callid) ne "ok"} {
chan event $input readable [list $this_handler $input $callid $capturingendregex]
}
# 2) more reliable?
#chan event $input readable [list $this_handler $input $callid $capturingendregex]
#------------------
#response from terminal
@ -794,7 +803,7 @@ namespace eval punk::console {
if {$waitvar($callid) ne "timedout"} {
after cancel $timeoutid($callid)
} else {
puts stderr "timeout (timediff [expr {[clock millis] - $tslaunch($callid)}]ms) in get_ansi_response_payload. callid $callid Ansi request was:[ansistring VIEW -lf 1 -vt 1 $query]"
puts stderr "timeout (timediff [expr {[clock millis] - $tslaunch($callid)}]ms) in get_ansi_response_payload. callid $callid Ansi request was:'[ansistring VIEW -lf 1 -vt 1 $query]'"
}
if {$was_raw == 0} {
@ -956,9 +965,10 @@ namespace eval punk::console {
set sofar [append chunks($callid) $bytes]
#puts stderr [ansistring VIEW $chunks($callid)]
#review - what is min length of any ansiresponse?
#we know there is at least one of only 3 chars, vt52 response to ESC Z: ESC / Z
#endregex is capturing - but as we are only testing the match here
#it should perform the same as if it were non-capturing
if {[string length $sofar] > 3 && [regexp $endregex $sofar]} {
if {[string length $sofar] > 2 && [regexp $endregex $sofar]} {
#puts stderr "matched - setting ansi_response_wait($callid) ok"
chan event $chan readable {}
set waits($callid) ok
@ -1438,7 +1448,8 @@ namespace eval punk::console {
-inoutchannels -default {stdin stdout} -type list
@values -min 0 -max 1
newsize -default "" -help\
"character cell pixel dimensions WxH"
"character cell pixel dimensions WxH
or omit to query cell size."
}
proc cell_size {args} {
set argd [punk::args::get_by_id ::punk::console::cell_size $args]
@ -1474,6 +1485,31 @@ namespace eval punk::console {
}
set cell_size ${w}x${h}
}
punk::args::define {
@id -id ::punk::console::test_is_vt52
@cmd -name punk::console::test_is_vt52 -help\
"in development.. broken"
-inoutchannels -default {stdin stdout} -type list
@values -min 0 -max 0
}
#only works in raw mode for windows terminal - (esc in output stripped?) why?
# works in line mode for alacrity and wezterm
proc test_is_vt52 {args} {
set argd [punk::args::get_by_id ::punk::console::test_is_vt52 $args]
set inoutchannels [dict get $argd opts -inoutchannels]
#ESC / K VT52 without printer
#ESC / M VT52 with printer
#ESC / Z VT52 emulator?? review
#TODO
set capturingregex {(.*)(?:(\x1b\/(Z))|(\x1b\/(K))|(\x1b\/(M))|(\x1b\[\?([0-9;]+)c))$} ;#must capture prefix,entire-response,response-payload
#set capturingregex {(.*)(\x1b\[([0-9;]+)c)$} ;#must capture prefix,entire-response,response-payload
set request "\x1bZ"
set payload [punk::console::internal::get_ansi_response_payload -terminal $inoutchannels $request $capturingregex]
#puts -->$payload<--
return [expr {$payload in {Z K M}}]
}
#todo - determine cursor on/off state before the call to restore properly.
proc get_size {{inoutchannels {stdin stdout}}} {
@ -1587,7 +1623,6 @@ namespace eval punk::console {
}
proc get_mode_line_wrap {{inoutchannels {stdin stdout}}} {
set capturingregex {(.*)(\x1b\[\?7;([0-9]+)\$y)$} ;#must capture prefix,entire-response,response-payload
set request "\x1b\[?7\$p"
@ -1683,7 +1718,14 @@ namespace eval punk::console {
return
}
puts -nonewline stdout $char_or_string
#On tcl9 - we could get an 'invalid or incomplete multibye or wide character' error
#e.g contains surrogate pair
if {[catch {
puts -nonewline stdout $char_or_string
} errM]} {
puts stderr "test_char_width couldn't emit this string - \nerror: $errM"
}
set response [punk::console::get_cursor_pos]
lassign [split $response ";"] _row2 col2
if {![string is integer -strict $col2]} {

2168
src/vfs/_vfscommon.vfs/modules/punk/icomm-0.1.0.tm

File diff suppressed because it is too large Load Diff

3412
src/vfs/_vfscommon.vfs/modules/punk/imap4-0.9.tm

File diff suppressed because it is too large Load Diff

44
src/vfs/_vfscommon.vfs/modules/punk/jtest.tcl

@ -0,0 +1,44 @@
set a b
set x {a b}
set x []
set x {
a
{b c}
}
array set comm {
debug 0
chans {}
localhost 1.2
x {}
y jb
j aa
blah "xxxb"
defaulg 0
}
#test
if {"x" eq max(2,3)} {
}
if {"x" eq min(1)} {}
set x [dict create {a b c {x} e f }]
zlib adler32 "abc"
dict get $x "a"
#dict create {a b}
set x []
#test
array set test1 {blah etc}
array set comm {
debug 0 chans {} localhost 127.0.0.1
offerVers {3 2}
acceptVers {3 2}
defaultEncoding "utf-8"
defaultSilent 0
}
#test
set x blah

67
src/modules/punk/repl-0.1.tm → src/vfs/_vfscommon.vfs/modules/punk/repl-0.1.1.tm

@ -80,7 +80,17 @@ namespace eval repl {
#(this is an example of a deaddrop)
variable post_script
}
namespace eval punk::repl::class {
oo::class create con {
variable o_data ;#dict
constructor {} {
set o_data [dict create]
}
method info {} {
return [dict info $o_data]
}
}
}
namespace eval punk::repl {
tsv::set repl runid 0
@ -2659,16 +2669,18 @@ namespace eval repl {
set codethread_mutex [thread::mutex create]
set init_script [string map [list %args% [list $opts]\
%argv0% [list $::argv0]\
%argv% [list $::argv]\
%argc% [list $::argc]\
%replthread% [thread::id]\
%replthread_cond% $codethread_cond\
%replthread_interp% [list $opt_callback_interp]\
%tmlist% [list [tcl::tm::list]]\
%autopath% [list $::auto_path]\
] {
set scriptmap [list %args% [list $opts] \
%argv0% [list $::argv0] \
%argv% [list $::argv] \
%argc% [list $::argc] \
%replthread% [thread::id] \
%replthread_cond% $codethread_cond \
%replthread_interp% [list $opt_callback_interp] \
%tmlist% [list [tcl::tm::list]] \
%autopath% [list $::auto_path] \
]
#scriptmap applied at end to satisfy silly editor highlighting.
set init_script {
set ::argv0 %argv0%
set ::argv %argv%
set ::argc %argc%
@ -2698,6 +2710,30 @@ namespace eval repl {
package require punk::packagepreference
punk::packagepreference::install
package require punk::args
package require Thread
package require snit
if {[catch {package require punk::icomm} errM]} {
puts stdout "---icomm $errM"
}
namespace eval ::punk::repl::codethread {}
#todo - review. According to fifo2 docs Memchan involves one less thread (may offer better performance/resource use)
catch {package require tcl::chan::fifo2}
if {[catch {
#first use can raise error being a version number e.g 0.1.0 - why?
lassign [tcl::chan::fifo2] ::punk::repl::codethread::repltalk replside
} errMsg]} {
#puts stdout "---tcl::chan::fifo2 error: $errM"
} else {
#puts stdout "transferring chan $replside to thread %replthread%"
#flush stdout
if {[catch {
#after 0 [list thread::transfer %replthread% $replside]
} errMsg]} {
#puts stdout "---thread::transfer error: $errMsg"
}
}
package require punk::console
package require punk::repl::codethread
package require shellfilter
@ -2945,6 +2981,7 @@ namespace eval repl {
interp create code
}
punkisland {
interp create code
#todo
#when no island paths specified - should be like safebase, but without folder hiding and with expanded read to ::auto_path folders
}
@ -3378,7 +3415,9 @@ namespace eval repl {
#puts stderr [thread::id]
thread::id
}]
}
set init_script [string map $scriptmap $init_script]
#thread::send $codethread $init_script
if {![catch {
@ -3395,8 +3434,8 @@ namespace eval repl {
#init - don't auto init - require init with possible options e.g -safe
}
package provide punk::repl [namespace eval punk::repl {
variable version
set version 0.1
variable version
set version 0.1.1
}]
#repl::start $program_read_stdin_pipe

12
src/vfs/_vfscommon.vfs/modules/punk/repl-0.1.tm

@ -80,7 +80,17 @@ namespace eval repl {
#(this is an example of a deaddrop)
variable post_script
}
namespace eval punk::repl::class {
oo::class create con {
variable o_data ;#dict
constructor {} {
set o_data [dict create]
}
method info {} {
return [dict info $o_data]
}
}
}
namespace eval punk::repl {
tsv::set repl runid 0

Loading…
Cancel
Save