ansi control string functions and terminal tests

src/modules/punk-0.1.tm

@@ -6706,7 +6706,7 @@ namespace eval punk {
     #e.g
     #= {a z c} |> inspect -label input_dict |> lsort |> {inspect $data}
     proc inspect {args} {
-        set defaults [list -label "" -limit 20 -channel stderr -showcount 1]
+        set defaults [list -label "" -limit 20 -channel stderr -showcount 1 -ansi 1]
         set flags [list]
         set endoptsposn [lsearch $args --] ;#first --  if data expected to contain --, then should always be called with --.  e.g inspect --
         if {$endoptsposn >= 0} {
@@ -6732,8 +6732,8 @@ namespace eval punk {
                 error "inspect: unknown option $k. Known options: [dict keys $defaults].  If data contains flaglike elements, consider calling with end-of-opts marker. e.g inspect --"
             }
         }
-
         set opts [dict merge $defaults $flags]
+        # -- --- --- --- ---
         set label [dict get $opts -label]
         set channel [dict get $opts -channel]
         set showcount [dict get $opts -showcount]
@@ -6741,6 +6741,15 @@ namespace eval punk {
             set label "${label}: "
         }
         set limit [dict get $opts -limit]
+        set opt_ansi [dict get $opts -ansi]
+        if {[string tolower $opt_ansi] ni [list 0 1 2 view]} {
+            error "inspect -ansi 0|1|2|view - received -ansi $opt_ansi"
+        }
+        if {[string tolower $opt_ansi] eq "view"} {
+            set opt_ansi 2
+        }
+        # -- --- --- --- ---
+
         set more ""
         if {[llength $pipeargs] == 1} {
             #usual case is data as a single element
@@ -6755,7 +6764,8 @@ namespace eval punk {
             return $val
         }
         set displayval $val ;#default - may be overridden based on -limit
-        if {![catch {llength $val} llen]} {
+
+        if {$count > 1} {
             #val is a list
             if {$limit > 0 && ($limit < $llen)} {
                 set displayval [lrange $val 0 $limit-1]
@@ -6786,6 +6796,11 @@ namespace eval punk {
         } else {
             set displaycount ""
         }
+        if {$opt_ansi == 0} {
+            set displayval [punk::ansi::stripansi $displayval]
+        } elseif {$opt_ansi == 2} {
+            set displayval [ansistring VIEW $displayval]
+        }
         if {![string length $more]} {
             puts $channel "$displaycount$label[a green bold]$displayval[a]"
         } else {
@@ -6868,17 +6883,49 @@ namespace eval punk {
             append warningblock \n "minor warning: punk::repl::has_script_var_bug returned true! (string rep for list variable in script generated when script changed)"
         }
 
-        set hidden_width_pm [punk::console::test_char_width [punk::ansi::controlstring_PM "hidden"]]
+        lappend cstring_tests [dict create\
-        if {$hidden_width_pm != 0} {
+            type "PM "\
-            append warningblock \n "WARNING: terminal doesn't hide PM 'privacy message' (ESC ^) control strings"
+            msg "PRIVACY MESSAGE"\
+            f7  punk::ansi::controlstring_PM\
+            f7desc "7bit ESC ^"\
+            f8  punk::ansi::controlstring_PM8\
+            f8desc "8bit \\x9e"\
+        ]
+        lappend cstring_tests [dict create\
+            type SOS\
+            msg "STRING"\
+            f7  punk::ansi::controlstring_SOS\
+            f7desc "7bit ESC X"\
+            f8  punk::ansi::controlstring_SOS8\
+            f8desc "8bit \\x98"\
+        ]
+        lappend cstring_tests [dict create\
+            type APC\
+            msg "APPLICATION PROGRAM COMMAND"\
+            f7  punk::ansi::controlstring_APC\
+            f7desc "7bit ESC _"\
+            f8  punk::ansi::controlstring_APC8\
+            f8desc "8bit \\x9f"\
+        ]
+
+        foreach test $cstring_tests {
+            set m [[dict get $test f7] [dict get $test msg]]
+            set hidden_width_m [punk::console::test_char_width $m]
+            set m8 [[dict get $test f8] [dict get $test msg]]
+            set hidden_width_m8 [punk::console::test_char_width $m8]
+            if {$hidden_width_m != 0 || $hidden_width_m8 != 0} {
+                if {$hidden_width_m == 0} {
+                    set d "[a+ green bold][dict get $test f7desc] [a red]${m}[a]"
+                } else {
+                    set d "[a+ yellow bold][dict get $test f7desc] [a red]$m[a]"
+                }
+                if {$hidden_width_m8 == 0} {
+                    set d8 "[a+ green ][dict get $test f8desc] [a red]$m8[a]"
+                } else {
+                    set d8 "[a+ yellow bold][dict get $test f8desc]  [a red]$m8[a]"
                 }
-        set hidden_width_sos [punk::console::test_char_width [punk::ansi::controlstring_SOS "hidden"]]
+                append warningblock \n "WARNING: terminal doesn't hide all [dict get $test type] control strings: $d  $d8"
-        if {$hidden_width_sos != 0} {
-            append warningblock \n "WARNING: terminal doesn't hide SOS 'start of string' (ESC X) control strings"
             }
-        set hidden_width_apc [punk::console::test_char_width [punk::ansi::controlstring_APC "hidden"]]
-        if {$hidden_width_apc != 0} {
-            append warningblock \n "WARNING: terminal doesn't hide APC 'application program command' (ESC _) control strings"
         }
 
         lappend chunks [list stdout $introblock]

src/modules/punk/ansi-999999.0a1.0.tm

@@ -152,12 +152,21 @@ namespace eval punk::ansi {
     proc controlstring_PM {text} {
         return "\x1b^${text}\033\\"
     }
+    proc controlstring_PM8 {text} {
+        return "\x9e${text}\x9c"
+    }
     proc controlstring_SOS {text} {
         return "\x1bX${text}\033\\"
     }
+    proc controlstring_SOS8 {text} {
+        return "\x98${text}\x9c"
+    }
     proc controlstring_APC {text} {
         return "\x1b_${text}\033\\"
     }
+    proc controlstring_APC8 {text} {
+        return "\x9f${text}\x9c"
+    }
 
     #candidate for zig/c implementation?
     proc stripansi {text} {
@@ -828,7 +837,7 @@ namespace eval punk::ansi::ta {
     variable re_standalones {(?:\x1bc|\x1b7|\x1b8|\x1bM|\x1bE|\x1bD|\x1bD|\x1bH|\x1b=|\x1b>|\x1b#3|\x1b#4|\x1b#5|\x1b#6|\x1b#8)}
 
     #see stripansi
-    set re_start_ST {^(?:\033X|\u0098|\033\^|\u009E|\033_|\u009F)} 
+    set re_start_ST {^(?:\033X|\u0098|\033\^|\u009e|\033_|\u009f)} 
     #ST terminators [list \007 \033\\ \u009c]
 
     #regex to capture the start of string/privacy message/application command block including the contents and string terminator (ST)
@@ -1003,8 +1012,45 @@ namespace eval punk::ansi::ansistring {
     #todo - expose _splits_ methods so caller can work efficiently with the splits themselves
     #we need to consider whether these can be agnostic towards splits from split_codes vs split_codes_single 
 
+    #\UFFFD  - replacement char or \U2426
+
+    #using ISO 2047 graphical representations of control characters
+    #00	NUL	Null	⎕	U+2395	NU
+    #01	TC1, SOH	Start of Heading	⌈	U+2308	SH
+    #02	TC2, STX	Start of Text	⊥	U+22A5	SX
+    #03	TC3, ETX	End of Text	⌋	U+230B	EX
+    #04	TC4, EOT	End of Transmission	⌁	U+2301[9]	ET
+    #05	TC5, ENQ	Enquiry	⊠[a]	U+22A0	EQ
+    #06	TC6, ACK	Acknowledge	✓	U+2713	AK
+    #07	BEL	Bell	⍾	U+237E[9]	BL
+    #08	FE0, BS	Backspace	⤺	—[b]	BS
+    #09	FE1, HT	Horizontal Tabulation	⪫	U+2AAB	HT
+    #0A	FE2, LF	Line Feed	≡	U+2261	LF
+    #0B	FE3, VT	Vertical Tabulation	⩛	U+2A5B	VT
+    #0C	FE4, FF	Form Feed	↡	U+21A1	FF
+    #0D	FE5, CR	Carriage Return	⪪	U+2AAA	CR
+    #0E	SO	Shift Out	⊗	U+2297	SO
+    #0F	SI	Shift In	⊙	U+2299	SI
+    #10	TC7, DLE	Data Link Escape	⊟	U+229F	DL
+    #11	DC1, XON, CON[10]	Device Control 1	◷	U+25F7	D1
+    #12	DC2, RPT,[10] TAPE[c]	Device Control 2	◶	U+25F6	D2
+    #13	DC3, XOF, XOFF	Device Control 3	◵	U+25F5	D3
+    #14	DC4, COF, KMC,[10] TAPE[c]	Device Control 4	◴	U+25F4	D4
+    #15	TC8, NAK	Negative Acknowledge	⍻	U+237B[9]	NK
+    #16	TC9, SYN	Synchronization	⎍	U+238D	SY
+    #17	TC10, ETB	End of Transmission Block	⊣	U+22A3	EB
+    #18	CAN	Cancel	⧖	U+29D6	CN
+    #19	EM	End of Medium	⍿	U+237F[9]	EM
+    #1A	SUB	Substitute Character	␦	U+2426[12]	SB
+    #1B	ESC	Escape	⊖	U+2296	EC
+    #1C	IS4, FS	File Separator	◰	U+25F0	FS
+    #1D	IS3, GS	Group Separator	◱	U+25F1	GS
+    #1E	IS2, RS	Record Separator	◲	U+25F2	RS
+    #1F	IS1 US	Unit Separator	◳	U+25F3	US
+    #20	SP	Space	△	U+25B3	SP
+    #7F	DEL	Delete	▨	—[d]	DT
     proc VIEW {string} {
-        return [string map [list \033 \uFFFD] $string]
+        return [string map [list \033 \U2296 \007 \U237E] $string]
     }
 
     proc length {string} {