diff --git a/src/bootsupport/include_modules.config b/src/bootsupport/include_modules.config index 8544d90c..aa6c50f8 100644 --- a/src/bootsupport/include_modules.config +++ b/src/bootsupport/include_modules.config @@ -1,4 +1,5 @@ +#each entry - base module set bootsupport_modules [list\ src/vendormodules cksum\ src/vendormodules natsort\ @@ -30,11 +31,12 @@ set bootsupport_modules [list\ modules punk::mix::commandset::scriptwrap\ modules punk::overlay\ modules punk::repo\ + modules punk::encmime\ modules punk::tdl\ modules punk::winpath\ ] - -#Don't include punk/mix/templates - recursive bootsupport problem! +#each entry - base subpath set bootsupport_module_folders [list\ + modules punk/mix/templates ] diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/mime-1.7.0.tm b/src/bootsupport/modules/mime-1.7.1.tm similarity index 99% rename from src/mixtemplates/layouts/basic/src/bootsupport/modules/mime-1.7.0.tm rename to src/bootsupport/modules/mime-1.7.1.tm index fa460769..b4b0d61d 100644 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/mime-1.7.0.tm +++ b/src/bootsupport/modules/mime-1.7.1.tm @@ -23,7 +23,7 @@ # new string features and inline scan are used, requiring 8.3. package require Tcl 8.5 -package provide mime 1.7.0 +package provide mime 1.7.1 package require tcl::chan::memchan @@ -681,7 +681,7 @@ proc ::mime::parsepart {token} { } if {!$blankP && [string match *\r $line]} { - set line [string range $line 0 $x-2]] + set line [string range $line 0 $x-2] if {$x == 1} { set blankP 1 } @@ -1340,12 +1340,19 @@ proc ::mime::getsize {token} { proc ::mime::getContentType token { variable $token upvar 0 $token state - set boundary {} set res $state(content) + + set boundary {} foreach {k v} $state(params) { - set boundary $v + if {$k eq {boundary}} { + set boundary $v + } append res ";\n $k=\"$v\"" } + + # Save boundary separate from the params + set state(boundary) $boundary + if {([string match multipart/* $state(content)]) \ && ($boundary eq {})} { # we're doing everything in one pass... @@ -1356,6 +1363,8 @@ proc ::mime::getContentType token { } set boundary "----- =_[string trim [base64 -mode encode -- $key]]" + set state(boundary) $boundary + append res ";\n boundary=\"$boundary\"" } return $res @@ -1397,7 +1406,7 @@ proc ::mime::getheader {token {key {}}} { lappend result MIME-Version $state(version) foreach lower $state(lowerL) mixed $state(mixedL) { foreach value $header($lower) { - lappend result $mixed $value + lappend result $mixed $value } } set tencoding [getTransferEncoding $token] @@ -1878,18 +1887,12 @@ proc ::mime::copymessageaux {token channel} { array set header $state(header) - set boundary {} - set result {} foreach {mixed value} [getheader $token] { puts $channel "$mixed: $value" } - foreach {k v} $state(params) { - if {$k eq {boundary}} { - set boundary $v - } - } + set boundary $state(boundary) ;# computed by `getheader` set converter {} set encoding {} @@ -1916,17 +1919,6 @@ proc ::mime::copymessageaux {token channel} { } } } - } elseif {([string match multipart/* $state(content)]) \ - && ($boundary eq {})} { - # we're doing everything in one pass... - set key [clock seconds]$token[info hostname][array get state] - set seqno 8 - while {[incr seqno -1] >= 0} { - set key [md5 -- $key] - } - set boundary "----- =_[string trim [base64 -mode encode -- $key]]" - - puts $channel ";\n boundary=\"$boundary\"" } if {[info exists state(error)]} { @@ -2072,7 +2064,7 @@ proc ::mime::copymessageaux {token channel} { # token The MIME token to parse. # # Results: -# The message. +# The message. proc ::mime::buildmessage token { global errorCode errorInfo diff --git a/src/bootsupport/modules/natsort-0.1.1.5.tm b/src/bootsupport/modules/natsort-0.1.1.5.tm index 0dcf57e7..0e4260b8 100644 --- a/src/bootsupport/modules/natsort-0.1.1.5.tm +++ b/src/bootsupport/modules/natsort-0.1.1.5.tm @@ -238,8 +238,13 @@ namespace eval natsort { #considered using hex to make large numbers more compact for viewing in debug output - but it's not that much shorter and probably obscures more than it helps. proc hex2dec {largeHex} { + #todo - use punk::lib::hex2dec - (scan supports ll so can do larger hex values directly) set res 0 set largeHex [string map [list _ ""] $largeHex] + if {[string length $largeHex] <=7} { + #scan can process up to FFFFFFF and does so quickly + return [scan $largeHex %x] + } foreach hexDigit [split $largeHex {}] { set new 0x$hexDigit set res [expr {16*$res + $new}] @@ -249,6 +254,8 @@ namespace eval natsort { proc dec2hex {decimalNumber} { format %4.4llX $decimalNumber } + + #punk::lib::trimzero proc trimzero {number} { set trimmed [string trimleft $number 0] if {[string length $trimmed] == 0} { @@ -382,6 +389,7 @@ namespace eval natsort { } proc get_char_count {str char} { + #faster than lsearch on split for str of a few K expr {[string length $str]-[string length [string map [list $char {}] $str]]} } diff --git a/src/vendormodules/overtype-1.5.0.tm b/src/bootsupport/modules/overtype-1.5.1.tm similarity index 74% rename from src/vendormodules/overtype-1.5.0.tm rename to src/bootsupport/modules/overtype-1.5.1.tm index f4e466f3..92f2464a 100644 --- a/src/vendormodules/overtype-1.5.0.tm +++ b/src/bootsupport/modules/overtype-1.5.1.tm @@ -7,23 +7,69 @@ # (C) Julian Noble 2003-2023 # # @@ Meta Begin -# Application overtype 1.5.0 +# Application overtype 1.5.1 # Meta platform tcl # Meta license BSD # @@ Meta End +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +# doctools header +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +#*** !doctools +#[manpage_begin overtype_module_overtype 0 1.5.1] +#[copyright "2024"] +#[titledesc {overtype text layout - ansi aware}] [comment {-- Name section and table of contents description --}] +#[moddesc {overtype text layout}] [comment {-- Description at end of page heading --}] +#[require overtype] +#[keywords module text ansi] +#[description] +#[para] - + +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ + +#*** !doctools +#[section Overview] +#[para] overview of overtype +#[subsection Concepts] +#[para] - + # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ ## Requirements -##e.g package require frobz +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ + +#*** !doctools +#[subsection dependencies] +#[para] packages used by overtype +#[list_begin itemized] + +package require Tcl 8.6 package require textutil package require punk::ansi ;#required to detect, split, strip and calculate lengths package require punk::char ;#box drawing - and also unicode character width determination for proper layout of text with double-column-width chars +#*** !doctools +#[item] [package {Tcl 8.6}] +#[item] [package textutil] +#[item] [package punk::ansi] +#[para] - required to detect, split, strip and calculate lengths of text possibly containing ansi codes +#[item] [package punk::char] +#[para] - box drawing - and also unicode character width determination for proper layout of text with double-column-width chars + +# #package require frobz +# #*** !doctools +# #[item] [package {frobz}] + +#*** !doctools +#[list_end] + # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +#*** !doctools +#[section API] + #Julian Noble - 2003 #Released under standard 'BSD license' conditions. @@ -83,183 +129,69 @@ namespace eval overtype { } -#candidate for zig/c implementation? -proc overtype::stripansi {text} { - variable escape_terminals ;#dict - variable ansi_2byte_codes_dict - #important that we don't spend too much time on this for plain text that doesn't contain any escapes anyway - if {[string first \033 $text] <0 && [string first \009c $text] <0} { - #\033 same as \x1b - return $text - } - - set text [convert_g0 $text] - - #we process char by char - line-endings whether \r\n or \n should be processed as per any other character. - #line endings can theoretically occur within an ansi escape sequence (review e.g title?) - set inputlist [split $text ""] - set outputlist [list] - - set 2bytecodes [dict values $ansi_2byte_codes_dict] - - set in_escapesequence 0 - #assumption - undertext already 'rendered' - ie no backspaces or carriagereturns or other cursor movement controls - set i 0 - foreach u $inputlist { - set v [lindex $inputlist $i+1] - set uv ${u}${v} - if {$in_escapesequence eq "2b"} { - #2nd byte - done. - set in_escapesequence 0 - } elseif {$in_escapesequence != 0} { - set escseq [dict get $escape_terminals $in_escapesequence] - if {$u in $escseq} { - set in_escapesequence 0 - } elseif {$uv in $escseq} { - set in_escapseequence 2b ;#flag next byte as last in sequence - } - } else { - #handle both 7-bit and 8-bit CSI and OSC - if {[regexp {^(?:\033\[|\u009b)} $uv]} { - set in_escapesequence CSI - } elseif {[regexp {^(?:\033\]|\u009c)} $uv]} { - set in_escapesequence OSC - } elseif {$uv in $2bytecodes} { - #self-contained e.g terminal reset - don't pass through. - set in_escapesequence 2b - } else { - lappend outputlist $u - } - } - incr i - } - return [join $outputlist ""] -} - -#review -#todo - map other chars to unicode equivs -proc overtype::convert_g0 {text} { - #using not \033 inside to stop greediness - review how does it compare to ".*?" - set re {\033\(0[^\033]*\033\(B} - set re2 {\033\(0(.*)\033\(B} ;#capturing - set parts [ta::_perlish_split $re $text] - set out "" - foreach {pt g} $parts { - append out $pt - if {$g ne ""} { - #puts --$g-- - #box sample - #lqk - #x x - #mqj - #m = boxd_lur - #set map [list l \u250f k \u2513] ;#heavy - set map [list l \u250c q \u2500 k \u2510 x \u2502 m \u2514 j \u2518] ;#light - - regexp $re2 $g _match contents - append out [string map $map $contents] - } - } - return $out -} - -#todo - convert esc(0 graphics sequences to single char unicode equivalents e.g box drawing set -# esc) ?? -proc overtype::stripansi_gx {text} { - #e.g "\033(0" - select VT100 graphics for character set G0 - #e.g "\033(B" - reset - #e.g "\033)0" - select VT100 graphics for character set G1 - #e.g "\033)X" - where X is any char other than 0 to reset ?? - return [convert_g0 $text] -} - - -#This shouldn't be called on text containing ansi codes! -proc overtype::strip_nonprinting_ascii {str} { - #review - some single-byte 'control' chars have visual representations e.g ETX as heart - #It is currently used for screen display width calculations - #equivalent for various unicode combining chars etc? - set map [list\ - \007 ""\ - [format %c 0] ""\ - [format %c 0x7f] ""\ - ] - return [string map $map $str] -} - -#length of text for printing characters only -#review - unicode and other non-printing chars and combining sequences? -#certain unicode chars are full-width (single char 2 columns wide) e.g see "Halfwdith and fullwidth forms" and ascii_fuillwidth blocks in punk::char::charset_names -#review - is there an existing library or better method? print to a terminal and query cursor position? -#Note this length calculation is only suitable for lines being appended to other strings if the line is pre-processed to account for backspace and carriage returns first -#If the raw line is appended to another string without such processing - the backspaces & carriage returns can affect data prior to the start of the string. -proc overtype::printing_length {line} { - if {[string first \n $line] >= 0} { - error "line_print_length must not contain newline characters" - } +#proc overtype::stripansi {text} { +# variable escape_terminals ;#dict +# variable ansi_2byte_codes_dict +# #important that we don't spend too much time on this for plain text that doesn't contain any escapes anyway +# if {[string first \033 $text] <0 && [string first \009c $text] <0} { +# #\033 same as \x1b +# return $text +# } +# +# set text [convert_g0 $text] +# +# #we process char by char - line-endings whether \r\n or \n should be processed as per any other character. +# #line endings can theoretically occur within an ansi escape sequence (review e.g title?) +# set inputlist [split $text ""] +# set outputlist [list] +# +# set 2bytecodes [dict values $ansi_2byte_codes_dict] +# +# set in_escapesequence 0 +# #assumption - undertext already 'rendered' - ie no backspaces or carriagereturns or other cursor movement controls +# set i 0 +# foreach u $inputlist { +# set v [lindex $inputlist $i+1] +# set uv ${u}${v} +# if {$in_escapesequence eq "2b"} { +# #2nd byte - done. +# set in_escapesequence 0 +# } elseif {$in_escapesequence != 0} { +# set escseq [dict get $escape_terminals $in_escapesequence] +# if {$u in $escseq} { +# set in_escapesequence 0 +# } elseif {$uv in $escseq} { +# set in_escapseequence 2b ;#flag next byte as last in sequence +# } +# } else { +# #handle both 7-bit and 8-bit CSI and OSC +# if {[regexp {^(?:\033\[|\u009b)} $uv]} { +# set in_escapesequence CSI +# } elseif {[regexp {^(?:\033\]|\u009c)} $uv]} { +# set in_escapesequence OSC +# } elseif {$uv in $2bytecodes} { +# #self-contained e.g terminal reset - don't pass through. +# set in_escapesequence 2b +# } else { +# lappend outputlist $u +# } +# } +# incr i +# } +# return [join $outputlist ""] +#} - #review - - set line [stripansi $line] - set line [strip_nonprinting_ascii $line] ;#only strip nonprinting after stripansi - some like BEL are part of ansi - #backspace 0x08 only erases* printing characters anyway - so presumably order of processing doesn't matter - #(* more correctly - moves cursor back) - #backspace will not move beyond a preceding newline - but we have disallowed newlines for this function already - #leading backspaces will eat into any string (even prompt in non-standard tclsh shell) that is prepended to the line - # - but for the purposes of overtype we wouldn't want that - so we strip it here in the length calculation and should strip leading backspaces in the actual data concatenation operations too. - #curiously - a backspace sequence at the end of a string also doesn't reduce the printing width - so we can also strip from RHS - #Note that backspace following a \t will only shorten the string by one (ie it doesn't move back the whole tab width like it does interactively in the terminal) - #for this reason - it would seem best to normalize the tabs to spaces prior to performing the backspace calculation - otherwise we won't account for the 'short' tabs it effectivley produces - #normalize tabs to an appropriate* width - #*todo - handle terminal/context where tabwidth != the default 8 spaces - set line [textutil::tabify::untabify2 $line] - set bs [format %c 0x08] - #set line [string map [list "\r${bs}" "\r"] $line] ;#backsp following a \r will have no effect - set line [string trim $line $bs] - set n 0 - - set chars [split $line ""] - #build an output - set idx 0 - set outchars [list] - set outsizes [list] - foreach c $chars { - if {$c eq $bs} { - if {$idx > 0} { - incr idx -1 - } - } elseif {$c eq "\r"} { - set idx 0 - } else { - priv::printing_length_addchar $idx $c - incr idx - } - } - set line2 [join $outchars ""] - return [punk::char::string_width $line2] -} proc overtype::string_columns {text} { if {[punk::ansi::ta::detect $text]} { - error "error string_columns is for calculating character length of string - ansi codes must be stripped/rendered first e.g with overtype::stripansi. Alternatively try overtype::printing_length" + error "error string_columns is for calculating character length of string - ansi codes must be stripped/rendered first e.g with punk::ansi::stripansi. Alternatively try punk::ansi::printing_length" } return [punk::char::string_width $text] } -namespace eval overtype::priv { - proc printing_length_addchar {i c} { - upvar outchars outc - upvar outsizes outs - set nxt [llength $outc] - if {$i < $nxt} { - lset outc $i $c - } else { - lappend outc $c - } - } -} #string range should generally be avoided for both undertext and overtext which contain ansi escapes and other cursor affecting chars such as \b and \r proc overtype::left {args} { @@ -299,13 +231,13 @@ proc overtype::left {args} { set overblock [string map $norm $overblock] set underlines [split $underblock \n] - set colwidth [tcl::mathfunc::max {*}[lmap v $underlines {printing_length $v}]] + set colwidth [tcl::mathfunc::max {*}[lmap v $underlines {punk::ansi::printing_length $v}]] set overlines [split $overblock \n] set outputlines [list] foreach undertext $underlines overtext $overlines { - set undertext_printlen [printing_length $undertext] - set overlen [printing_length $overtext] + set undertext_printlen [punk::ansi::printing_length $undertext] + set overlen [punk::ansi::printing_length $overtext] set diff [expr {$overlen - $colwidth}] #review @@ -387,46 +319,53 @@ proc overtype::centre {args} { set overblock [string map $norm $overblock] set underlines [split $underblock \n] - set colwidth [tcl::mathfunc::max {*}[lmap v $underlines {printing_length $v}]] + set colwidth [tcl::mathfunc::max {*}[lmap v $underlines {punk::ansi::printing_length $v}]] set overlines [split $overblock \n] + set overblock_width [tcl::mathfunc::max {*}[lmap v $overlines {punk::ansi::printing_length $v}]] set outputlines [list] foreach undertext $underlines overtext $overlines { - set olen [printing_length $overtext] - set ulen [printing_length $undertext] + #set olen [punk::ansi::printing_length $overtext] + set ulen [punk::ansi::printing_length $undertext] if {$ulen < $colwidth} { set udiff [expr {$colwidth - $ulen}] set undertext "$undertext[string repeat { } $udiff]" } #review - append overtext "\033\[0m" + #append overtext "\033\[0m" - set diff [expr {$colwidth - $olen}] - if {$diff > 0} { + set under_exposed [expr {$colwidth - $overblock_width}] + if {$under_exposed > 0} { #background block is wider - set half [expr {round(int($diff / 2))}] - if {[string match right [dict get $opts -bias]]} { - if {[expr {2 * $half}] < $diff} { - incr half + if {$under_exposed % 2 == 0} { + #even left/right exposure + set left_exposed [expr {$under_exposed / 2}] + } else { + set beforehalf [expr {$under_exposed / 2}] ;#1 less than half due to integer division + if {[string tolower [dict get $opts -bias]] eq "left"} { + set left_exposed $beforehalf + } else { + #bias to the right + set left_exposed [expr {$beforehalf + 1}] } } + if 0 { set rhs [expr {$diff - $half - 1}] set lhs [expr {$half - 1}] set rhsoffset [expr {$rhs +1}] - if 0 { set a [string range $undertext 0 $lhs] set background [string range $undertext $lhs+1 end-$rhsoffset] set b [renderline -transparent $opt_transparent $background $overtext] set c [string range $undertext end-$rhs end] lappend outputlines $a$b$c } - lappend outputlines [renderline -start $lhs -transparent $opt_transparent -exposed1 $opt_exposed1 -exposed2 $opt_exposed2 $undertext $overtext] + lappend outputlines [renderline -start $left_exposed -transparent $opt_transparent -exposed1 $opt_exposed1 -exposed2 $opt_exposed2 $undertext $overtext] } else { #overlay wider or equal set rendered [renderline -transparent $opt_transparent -overflow [dict get $opts -overflow] $undertext $overtext] - if {$diff < 0} { + if {$under_exposed < 0} { #overlay is wider - trim if overflow not specified in opts if {![dict get $opts -overflow]} { #lappend outputlines [string range $overtext 0 [expr {$colwidth - 1}]] @@ -436,7 +375,7 @@ proc overtype::centre {args} { } } } else { - #widths match + #zero under_exposed - widths match } lappend outputlines $rendered #lappend outputlines [renderline -transparent $opt_transparent $undertext $overtext] @@ -485,13 +424,13 @@ proc overtype::right {args} { set overblock [string map $norm $overblock] set underlines [split $underblock \n] - set colwidth [tcl::mathfunc::max {*}[lmap v $underlines {printing_length $v}]] + set colwidth [tcl::mathfunc::max {*}[lmap v $underlines {punk::ansi::printing_length $v}]] set overlines [split $overblock \n] set outputlines [list] foreach undertext $underlines overtext $overlines { - set olen [printing_length $overtext] - set ulen [printing_length $undertext] + set olen [punk::ansi::printing_length $overtext] + set ulen [punk::ansi::printing_length $undertext] if {$ulen < $colwidth} { set udiff [expr {$colwidth - $ulen}] set undertext "$undertext[string repeat { } $udiff]" @@ -547,7 +486,7 @@ namespace eval overtype::piper { interp alias "" piper_renderline "" overtype::piper::renderline #-returnextra to enable returning of overflow and length -# todo - use ta::detect to short-circuit processing and do simple string calcs as an optimisation? +# todo - use punk::ansi::ta::detect to short-circuit processing and do simple string calcs as an optimisation? #review - DECSWL/DECDWL double width line codes - very difficult/impossible to align and compose with other elements #todo - review transparency issues with single/double width characters! proc overtype::renderline {args} { @@ -905,9 +844,10 @@ namespace eval overtype::priv { # -- --- --- --- --- --- --- --- --- --- --- +if 0 { namespace eval overtype::ta { namespace path ::overtype - #*based* on but not identical to: + # *based* on but not identical to: #https://github.com/perlancar/perl-Text-ANSI-Util/blob/master/lib/Text/ANSI/BaseUtil.pm #handle both 7-bit and 8-bit csi @@ -934,45 +874,28 @@ namespace eval overtype::ta { #detect any ansi escapes #review - only detect 'complete' codes - or just use the opening escapes for performance? - proc detect {text} { - variable re_ansi_detect - #variable re_csi_open - #variable re_esc_osc1 - #variable re_esc_osc2 - #todo - other escape sequences - #expr {[regexp $re_csi_open $text] || [regexp $re_esc_osc1 $text] || [regexp $re_esc_osc2 $text]} - expr {[regexp $re_ansi_detect $text]} - } + #proc detect {text} { + # variable re_ansi_detect + # #variable re_csi_open + # #variable re_esc_osc1 + # #variable re_esc_osc2 + # #todo - other escape sequences + # #expr {[regexp $re_csi_open $text] || [regexp $re_esc_osc1 $text] || [regexp $re_esc_osc2 $text]} + # expr {[regexp $re_ansi_detect $text]} + #} #not in perl ta - proc detect_csi {text} { - variable re_csi_colour - expr {[regexp $re_csi_colour $text]} - } + #proc detect_csi {text} { + # variable re_csi_colour + # expr {[regexp $re_csi_colour $text]} + #} proc strip {text} { - tailcall stripansi $text + tailcall punk::ansi::stripansi $text } #note this is character length after stripping ansi codes - not the printing length proc length {text} { - string length [overtype::stripansi $text] - } - #todo - handle newlines - #not in perl ta - proc printing_length {text} { - + string length [punk::ansi::stripansi $text] } - proc trunc {text width args} { - - } - - #not in perl ta - #returns just the plaintext portions in a list - proc split_at_codes {text} { - variable re_esc_osc1 - variable re_esc_osc2 - variable re_csi_code - textutil::splitx $text "${re_csi_code}|${re_esc_osc1}|${re_esc_osc2}" - } # -- --- --- --- --- --- #Split $text to a list containing alternating ANSI color codes and text. @@ -987,43 +910,41 @@ namespace eval overtype::ta { #ta_split_codes "\e[31ma\e[0mb" # => {"" "\e[31m" "a" "\e[0m", "b"} #ta_split_codes "\e[31m\e[0mb" # => {"" "\e[31m\e[0m" "b"} # - proc split_codes {text} { - variable re_esc_osc1 - variable re_esc_osc2 - variable re_csi_code - set re "(?:${re_csi_code}|${re_esc_osc1}|${re_esc_osc2})+" - return [_perlish_split $re $text] - } - #like split_codes - but each ansi-escape is split out separately (with empty string of plaintext between codes so odd/even plain ansi still holds) - proc split_codes_single {text} { - variable re_esc_osc1 - variable re_esc_osc2 - variable re_csi_code - set re "${re_csi_code}|${re_esc_osc1}|${re_esc_osc2}" - return [_perlish_split $re $text] - } - - #review - tcl greedy expressions may match multiple in one element - proc _perlish_split {re text} { - if {[string length $text] == 0} { - return {} - } - set list [list] - set start 0 - while {[regexp -start $start -indices -- $re $text match]} { - lassign $match matchStart matchEnd - lappend list [string range $text $start $matchStart-1] [string range $text $matchStart $matchEnd] - set start [expr {$matchEnd+1}] - } - lappend list [string range $text $start end] - return $list - } - proc _ws_split {text} { - regexp -all -inline {(?:\S+)|(?:\s+)} $text - } - # -- --- --- --- --- --- + #proc split_codes {text} { + # variable re_esc_osc1 + # variable re_esc_osc2 + # variable re_csi_code + # set re "(?:${re_csi_code}|${re_esc_osc1}|${re_esc_osc2})+" + # return [_perlish_split $re $text] + #} + ##like split_codes - but each ansi-escape is split out separately (with empty string of plaintext between codes so odd/even plain ansi still holds) + #proc split_codes_single {text} { + # variable re_esc_osc1 + # variable re_esc_osc2 + # variable re_csi_code + # set re "${re_csi_code}|${re_esc_osc1}|${re_esc_osc2}" + # return [_perlish_split $re $text] + #} + + ##review - tcl greedy expressions may match multiple in one element + #proc _perlish_split {re text} { + # if {[string length $text] == 0} { + # return {} + # } + # set list [list] + # set start 0 + # while {[regexp -start $start -indices -- $re $text match]} { + # lassign $match matchStart matchEnd + # lappend list [string range $text $start $matchStart-1] [string range $text $matchStart $matchEnd] + # set start [expr {$matchEnd+1}] + # } + # lappend list [string range $text $start end] + # return $list + #} + ## -- --- --- --- --- --- } +} ;# end if 0 # -- --- --- --- --- --- --- --- --- --- --- namespace eval overtype { @@ -1034,6 +955,9 @@ namespace eval overtype { ## Ready package provide overtype [namespace eval overtype { variable version - set version 1.5.0 + set version 1.5.1 }] -return \ No newline at end of file +return + +#*** !doctools +#[manpage_end] diff --git a/src/bootsupport/modules/overtype-1.5.3.tm b/src/bootsupport/modules/overtype-1.5.3.tm new file mode 100644 index 00000000..6b54a4ac --- /dev/null +++ b/src/bootsupport/modules/overtype-1.5.3.tm @@ -0,0 +1,1037 @@ +# -*- tcl -*- +# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt +# +# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. +# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +# (C) Julian Noble 2003-2023 +# +# @@ Meta Begin +# Application overtype 1.5.3 +# Meta platform tcl +# Meta license BSD +# @@ Meta End + + +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +# doctools header +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +#*** !doctools +#[manpage_begin overtype_module_overtype 0 1.5.3] +#[copyright "2024"] +#[titledesc {overtype text layout - ansi aware}] [comment {-- Name section and table of contents description --}] +#[moddesc {overtype text layout}] [comment {-- Description at end of page heading --}] +#[require overtype] +#[keywords module text ansi] +#[description] +#[para] - + +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ + +#*** !doctools +#[section Overview] +#[para] overview of overtype +#[subsection Concepts] +#[para] - + + +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +## Requirements +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ + +#*** !doctools +#[subsection dependencies] +#[para] packages used by overtype +#[list_begin itemized] + +package require Tcl 8.6 +package require textutil +package require punk::ansi ;#required to detect, split, strip and calculate lengths +package require punk::char ;#box drawing - and also unicode character width determination for proper layout of text with double-column-width chars +#*** !doctools +#[item] [package {Tcl 8.6}] +#[item] [package textutil] +#[item] [package punk::ansi] +#[para] - required to detect, split, strip and calculate lengths of text possibly containing ansi codes +#[item] [package punk::char] +#[para] - box drawing - and also unicode character width determination for proper layout of text with double-column-width chars + +# #package require frobz +# #*** !doctools +# #[item] [package {frobz}] + +#*** !doctools +#[list_end] + + + + +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +#*** !doctools +#[section API] + + +#Julian Noble - 2003 +#Released under standard 'BSD license' conditions. +# +#todo - ellipsis truncation indicator for center,right + +#v1.4 2023-07 - naive ansi color handling - todo - fix string range +# - need to extract and replace ansi codes? + +namespace eval overtype { + namespace export * + variable default_ellipsis_horizontal "..." ;#fallback + variable default_ellipsis_vertical "..." + namespace eval priv { + proc _init {} { + upvar ::overtype::default_ellipsis_horizontal e_h + upvar ::overtype::default_ellipsis_vertical e_v + set e_h [format %c 0x2026] ;#Unicode Horizontal Ellipsis + set e_v [format %c 0x22EE] + #The unicode ellipsis looks more natural than triple-dash which is centred vertically whereas ellipsis is at floorline of text + #Also - unicode ellipsis has semantic meaning that other processors can interpret + #unicode does also provide a midline horizontal ellipsis 0x22EF + + #set e [format %c 0x2504] ;#punk::char::charshort boxd_ltdshhz - Box Drawings Light Triple Dash Horizontal + #if {![catch {package require punk::char}]} { + # set e [punk::char::charshort boxd_ltdshhz] + #} + } + } + priv::_init +} +proc overtype::about {} { + return "Simple text formatting. Author JMN. BSD-License" +} + +namespace eval overtype { + variable escape_terminals + #single "final byte" in the range 0x40–0x7E (ASCII @A–Z[\]^_`a–z{|}~). + dict set escape_terminals CSI [list @ \\ ^ _ ` | ~ a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z "\{" "\}"] + #dict set escape_terminals CSI [list J K m n A B C D E F G s u] ;#basic + dict set escape_terminals OSC [list \007 \033\\] ;#note mix of 1 and 2-byte terminals + + #self-contained 2 byte ansi escape sequences - review more? + variable ansi_2byte_codes_dict + set ansi_2byte_codes_dict [dict create\ + "reset_terminal" "\u001bc"\ + "save_cursor_posn" "\u001b7"\ + "restore_cursor_posn" "\u001b8"\ + "cursor_up_one" "\u001bM"\ + "NEL - Next Line" "\u001bE"\ + "IND - Down one line" "\u001bD"\ + "HTS - Set Tab Stop" "\u001bH"\ + ] + + #debatable whether strip should reveal the somethinghidden - some terminals don't hide it anyway. + # "PM - Privacy Message" "\u001b^somethinghidden\033\\"\ +} + + +#proc overtype::stripansi {text} { +# variable escape_terminals ;#dict +# variable ansi_2byte_codes_dict +# #important that we don't spend too much time on this for plain text that doesn't contain any escapes anyway +# if {[string first \033 $text] <0 && [string first \009c $text] <0} { +# #\033 same as \x1b +# return $text +# } +# +# set text [convert_g0 $text] +# +# #we process char by char - line-endings whether \r\n or \n should be processed as per any other character. +# #line endings can theoretically occur within an ansi escape sequence (review e.g title?) +# set inputlist [split $text ""] +# set outputlist [list] +# +# set 2bytecodes [dict values $ansi_2byte_codes_dict] +# +# set in_escapesequence 0 +# #assumption - undertext already 'rendered' - ie no backspaces or carriagereturns or other cursor movement controls +# set i 0 +# foreach u $inputlist { +# set v [lindex $inputlist $i+1] +# set uv ${u}${v} +# if {$in_escapesequence eq "2b"} { +# #2nd byte - done. +# set in_escapesequence 0 +# } elseif {$in_escapesequence != 0} { +# set escseq [dict get $escape_terminals $in_escapesequence] +# if {$u in $escseq} { +# set in_escapesequence 0 +# } elseif {$uv in $escseq} { +# set in_escapseequence 2b ;#flag next byte as last in sequence +# } +# } else { +# #handle both 7-bit and 8-bit CSI and OSC +# if {[regexp {^(?:\033\[|\u009b)} $uv]} { +# set in_escapesequence CSI +# } elseif {[regexp {^(?:\033\]|\u009c)} $uv]} { +# set in_escapesequence OSC +# } elseif {$uv in $2bytecodes} { +# #self-contained e.g terminal reset - don't pass through. +# set in_escapesequence 2b +# } else { +# lappend outputlist $u +# } +# } +# incr i +# } +# return [join $outputlist ""] +#} + + + + + +proc overtype::string_columns {text} { + if {[punk::ansi::ta::detect $text]} { + error "error string_columns is for calculating character length of string - ansi codes must be stripped/rendered first e.g with punk::ansi::stripansi. Alternatively try punk::ansi::printing_length" + } + return [punk::char::string_width $text] +} + + +#string range should generally be avoided for both undertext and overtext which contain ansi escapes and other cursor affecting chars such as \b and \r +proc overtype::left {args} { + # @c overtype starting at left (overstrike) + # @c can/should we use something like this?: 'format "%-*s" $len $overtext + variable default_ellipsis_horizontal + + if {[llength $args] < 2} { + error {usage: ?-transparent [0|1]? ?-overflow [1|0]? ?-ellipsis [1|0]? ?-ellipsistext ...? undertext overtext} + } + lassign [lrange $args end-1 end] underblock overblock + set defaults [dict create\ + -bias ignored\ + -ellipsis 0\ + -ellipsistext $default_ellipsis_horizontal\ + -ellipsiswhitespace 0\ + -overflow 0\ + -transparent 0\ + -exposed1 \uFFFD\ + -exposed2 \uFFFD\ + ] + set known_opts [dict keys $defaults] + set argsflags [lrange $args 0 end-2] + dict for {k v} $argsflags { + if {$k ni $known_opts} { + error "overtype::left unknown option '$k'. Known options: $known_opts" + } + } + set opts [dict merge $defaults $argsflags] + # -- --- --- --- --- --- + set opt_transparent [dict get $opts -transparent] + set opt_ellipsistext [dict get $opts -ellipsistext] + set opt_ellipsiswhitespace [dict get $opts -ellipsiswhitespace] + set opt_exposed1 [dict get $opts -exposed1] ;#widechar_exposed_left - todo + set opt_exposed2 [dict get $opts -exposed2] ;#widechar_exposed_right - todo + # -- --- --- --- --- --- + + set norm [list \r\n \n] + set underblock [string map $norm $underblock] + set overblock [string map $norm $overblock] + + set underlines [split $underblock \n] + #set colwidth [tcl::mathfunc::max {*}[lmap v $underlines {punk::ansi::printing_length $v}]] + lassign [blocksize $underblock] _w colwidth _h colheight + set overlines [split $overblock \n] + #set overblock_width [tcl::mathfunc::max {*}[lmap v $overlines {punk::ansi::printing_length $v}]] + lassign [blocksize $overblock] _w overblock_width _h overblock_height + set under_exposed_max [expr {$colwidth - $overblock_width}] + set right_exposed $under_exposed_max + + set outputlines [list] + foreach undertext $underlines overtext $overlines { + set undertext_printlen [punk::ansi::printing_length $undertext] + if {$undertext_printlen < $colwidth} { + set udiff [expr {$colwidth - $undertext_printlen}] + set undertext "$undertext[string repeat { } $udiff]" + } + set overtext_printlen [punk::ansi::printing_length $overtext] + set overflowlength [expr {$overtext_printlen - $colwidth}] + + #review + #append overtext "\033\[0m" + + + if {$overflowlength > 0} { + #background line is narrower than data in line + set rendered [renderline -transparent $opt_transparent -exposed1 $opt_exposed1 -exposed2 $opt_exposed2 -overflow [dict get $opts -overflow] $undertext $overtext] + if {![dict get $opts -overflow]} { + #set overtext [string range $overtext 0 $colwidth-1] ;#string range won't be correct e.g if contains ansi codes or leading \r or \b etc + if {[dict get $opts -ellipsis]} { + set show_ellipsis 1 + if {!$opt_ellipsiswhitespace} { + #we don't want ellipsis if only whitespace was lost + set lostdata [string range $overtext end-[expr {$overflowlength-1}] end] + if {[string trim $lostdata] eq ""} { + set show_ellipsis 0 + } + } + if {$show_ellipsis} { + set rendered [overtype::right $rendered $opt_ellipsistext] + } + } + } + lappend outputlines $rendered + } else { + #we know overtext data is shorter or equal (for this line) + lappend outputlines [renderline -transparent $opt_transparent -exposed1 $opt_exposed1 -exposed2 $opt_exposed2 $undertext $overtext] + } + } + return [join $outputlines \n] + +} + +namespace eval overtype::piper { + proc overcentre {args} { + if {[llength $args] < 2} { + error {usage: ?-bias left|right? ?-transparent [0|1|]? ?-exposed1 ? ?-exposed2 ? ?-overflow [1|0]? overtext pipelinedata} + } + lassign [lrange $args end-1 end] over under + set argsflags [lrange $args 0 end-2] + tailcall overtype::centre {*}$argsflags $under $over + } + proc overleft {args} { + if {[llength $args] < 2} { + error {usage: ?-start ? ?-transparent [0|1|]? ?-exposed1 ? ?-exposed2 ? ?-overflow [1|0]? overtext pipelinedata} + } + lassign [lrange $args end-1 end] over under + set argsflags [lrange $args 0 end-2] + tailcall overtype::left {*}$argsflags $under $over + } +} +#todo - left-right ellipsis ? +proc overtype::centre {args} { + variable default_ellipsis_horizontal + if {[llength $args] < 2} { + error {usage: ?-transparent [0|1]? ?-bias [left|right]? ?-overflow [1|0]? undertext overtext} + } + + foreach {underblock overblock} [lrange $args end-1 end] break + + #todo - vertical vs horizontal overflow for blocks + set defaults [dict create\ + -bias left\ + -ellipsis 0\ + -ellipsistext $default_ellipsis_horizontal\ + -ellipsiswhitespace 0\ + -overflow 0\ + -transparent 0\ + -exposed1 \uFFFD\ + -exposed2 \uFFFD\ + ] + set known_opts [dict keys $defaults] + set argsflags [lrange $args 0 end-2] + dict for {k v} $argsflags { + if {$k ni $known_opts} { + error "overtype::centre unknown option '$k'. Known options: $known_opts" + } + } + set opts [dict merge $defaults $argsflags] + # -- --- --- --- --- --- + set opt_transparent [dict get $opts -transparent] + set opt_ellipsis [dict get $opts -ellipsis] + set opt_ellipsistext [dict get $opts -ellipsistext] + set opt_ellipsiswhitespace [dict get $opts -ellipsiswhitespace] + set opt_exposed1 [dict get $opts -exposed1] + set opt_exposed2 [dict get $opts -exposed2] + # -- --- --- --- --- --- + + + set norm [list \r\n \n] + set underblock [string map $norm $underblock] + set overblock [string map $norm $overblock] + + set underlines [split $underblock \n] + #set colwidth [tcl::mathfunc::max {*}[lmap v $underlines {punk::ansi::printing_length $v}]] + lassign [blocksize $underblock] _w colwidth _h colheight + set overlines [split $overblock \n] + #set overblock_width [tcl::mathfunc::max {*}[lmap v $overlines {punk::ansi::printing_length $v}]] + lassign [blocksize $overblock] _w overblock_width _h overblock_height + set under_exposed_max [expr {$colwidth - $overblock_width}] + if {$under_exposed_max > 0} { + #background block is wider + if {$under_exposed_max % 2 == 0} { + #even left/right exposure + set left_exposed [expr {$under_exposed_max / 2}] + } else { + set beforehalf [expr {$under_exposed_max / 2}] ;#1 less than half due to integer division + if {[string tolower [dict get $opts -bias]] eq "left"} { + set left_exposed $beforehalf + } else { + #bias to the right + set left_exposed [expr {$beforehalf + 1}] + } + } + } else { + set left_exposed 0 + } + + set outputlines [list] + foreach undertext $underlines overtext $overlines { + set overtext_datalen [punk::ansi::printing_length $overtext] + set ulen [punk::ansi::printing_length $undertext] + if {$ulen < $colwidth} { + set udiff [expr {$colwidth - $ulen}] + set undertext "$undertext[string repeat { } $udiff]" + } + + set overflowlength [expr {$overtext_datalen - $colwidth}] + #review - right-to-left langs should elide on left! - extra option required + + if {$overflowlength > 0} { + #overlay line wider or equal + set rendered [renderline -transparent $opt_transparent -overflow [dict get $opts -overflow] -exposed1 $opt_exposed1 -exposed2 $opt_exposed2 $undertext $overtext] + #overlay line data is wider - trim if overflow not specified in opts - and overtype an ellipsis at right if it was specified + if {![dict get $opts -overflow]} { + #lappend outputlines [string range $overtext 0 [expr {$colwidth - 1}]] + #set overtext [string range $overtext 0 $colwidth-1 ] + if {$opt_ellipsis} { + set show_ellipsis 1 + if {!$opt_ellipsiswhitespace} { + #we don't want ellipsis if only whitespace was lost + set lostdata [string range $overtext end-[expr {$overflowlength-1}] end] + if {[string trim $lostdata] eq ""} { + set show_ellipsis 0 + } + } + if {$show_ellipsis} { + set rendered [overtype::right $rendered $opt_ellipsistext] + } + } + } + lappend outputlines $rendered + #lappend outputlines [renderline -transparent $opt_transparent $undertext $overtext] + } else { + #background block is wider than or equal to data for this line + lappend outputlines [renderline -start $left_exposed -transparent $opt_transparent -exposed1 $opt_exposed1 -exposed2 $opt_exposed2 $undertext $overtext] + } + } + return [join $outputlines \n] +} + +proc overtype::right {args} { + #NOT the same as align-right - which should be done to the overblock first if required + variable default_ellipsis_horizontal + # @d !todo - implement overflow, length checks etc + + if {[llength $args] < 2} { + error {usage: ?-overflow [1|0]? undertext overtext} + } + foreach {underblock overblock} [lrange $args end-1 end] break + + set defaults [dict create\ + -bias ignored\ + -ellipsis 0\ + -ellipsistext $default_ellipsis_horizontal\ + -ellipsiswhitespace 0\ + -overflow 0\ + -transparent 0\ + -exposed1 \uFFFD\ + -exposed2 \uFFFD\ + ] + set known_opts [dict keys $defaults] + set argsflags [lrange $args 0 end-2] + dict for {k v} $argsflags { + if {$k ni $known_opts} { + error "overtype::centre unknown option '$k'. Known options: $known_opts" + } + } + set opts [dict merge $defaults $argsflags] + # -- --- --- --- --- --- + set opt_transparent [dict get $opts -transparent] + set opt_ellipsis [dict get $opts -ellipsis] + set opt_ellipsistext [dict get $opts -ellipsistext] + set opt_ellipsiswhitespace [dict get $opts -ellipsiswhitespace] + set opt_overflow [dict get $opts -overflow] + set opt_exposed1 [dict get $opts -exposed1] + set opt_exposed2 [dict get $opts -exposed2] + # -- --- --- --- --- --- + + set norm [list \r\n \n] + set underblock [string map $norm $underblock] + set overblock [string map $norm $overblock] + + set underlines [split $underblock \n] + #set colwidth [tcl::mathfunc::max {*}[lmap v $underlines {punk::ansi::printing_length $v}]] + lassign [blocksize $underblock] _w colwidth _h colheight + set overlines [split $overblock \n] + #set overblock_width [tcl::mathfunc::max {*}[lmap v $overlines {punk::ansi::printing_length $v}]] + lassign [blocksize $overblock] _w overblock_width _h overblock_height + set under_exposed_max [expr {$colwidth - $overblock_width}] + set left_exposed $under_exposed_max + + set outputlines [list] + foreach undertext $underlines overtext $overlines { + set overtext_datalen [punk::ansi::printing_length $overtext] + set ulen [punk::ansi::printing_length $undertext] + if {$ulen < $colwidth} { + set udiff [expr {$colwidth - $ulen}] + puts xxx + set undertext "$undertext[string repeat { } $udiff]" + } + if {$overtext_datalen < $overblock_width} { + set odiff [expr {$overblock_width - $overtext_datalen}] + #padding always on right - if alignment is required it should be done to block beforehand - not here + set overtextpadding "$overtext[string repeat { } $odiff]" + } + + set overflowlength [expr {$overtext_datalen - $colwidth}] + if {$overflowlength > 0} { + #raw overtext wider than undertext column + set rendered [renderline -transparent $opt_transparent -exposed1 $opt_exposed1 -exposed2 $opt_exposed2 -overflow $opt_overflow -start 0 $undertext $overtext] + if {!$opt_overflow} { + if {$opt_ellipsis} { + set show_ellipsis 1 + if {!$opt_ellipsiswhitespace} { + #we don't want ellipsis if only whitespace was lost + set lostdata [string range $overtext end-[expr {$overflowlength-1}] end] + if {[string trim $lostdata] eq ""} { + set show_ellipsis 0 + } + } + if {$show_ellipsis} { + set rendered [overtype::right $rendered $opt_ellipsistext] + } + } + } + lappend outputlines $rendered + } else { + #padded overtext + lappend outputlines [renderline -transparent $opt_transparent -start $left_exposed $undertext $overtext] + } + } + + return [join $outputlines \n] +} + +# -- --- --- --- --- --- --- --- --- --- --- +proc overtype::transparentline {args} { + foreach {under over} [lrange $args end-1 end] break + set argsflags [lrange $args 0 end-2] + set defaults [dict create\ + -transparent 1\ + -exposed 1 " "\ + -exposed 2 " "\ + ] + set newargs [dict merge $defaults $argsflags] + tailcall overtype::renderline {*}$newargs $under $over +} +#renderline may not make sense as it is in the long run for blocks of text - but is handy in the single-line-handling form anyway. +# We are trying to handle ansi codes in a block of text which is acting like a mini-terminal in some sense. +#We can process standard cursor moves such as \b \r - but no way to respond to other cursor movements e.g moving to other lines. +# +namespace eval overtype::piper { + proc renderline {args} { + if {[llength $args] < 2} { + error {usage: ?-start ? ?-transparent [0|1|]? ?-overflow [1|0]? overtext pipelinedata} + } + foreach {over under} [lrange $args end-1 end] break + set argsflags [lrange $args 0 end-2] + tailcall overtype::renderline {*}$argsflags $under $over + } +} +interp alias "" piper_renderline "" overtype::piper::renderline + +#-returnextra to enable returning of overflow and length +# todo - use punk::ansi::ta::detect to short-circuit processing and do simple string calcs as an optimisation? +#review - DECSWL/DECDWL double width line codes - very difficult/impossible to align and compose with other elements +#todo - review transparency issues with single/double width characters! +proc overtype::renderline {args} { + if {[llength $args] < 2} { + error {usage: ?-start ? ?-transparent [0|1|]? ?-overflow [1|0]? undertext overtext} + } + lassign [lrange $args end-1 end] under over + if {[string first \n $under] >=0 || [string first \n $over] >= 0} { + error "overtype::renderline not allowed to contain newlines" + } + set defaults [dict create\ + -overflow 0\ + -transparent 0\ + -start 0\ + -returnextra 0\ + -exposed1 \uFFFD\ + -exposed2 \uFFFD\ + ] + #exposed1 and exposed2 for first and second col of underying 2wide char which is truncated by transparency or overflow + + set known_opts [dict keys $defaults] + set argsflags [lrange $args 0 end-2] + dict for {k v} $argsflags { + if {$k ni $known_opts} { + error "overtype::renderline unknown option '$k'. Known options: $known_opts" + } + } + set opts [dict merge $defaults $argsflags] + # -- --- --- --- --- --- --- --- --- --- --- --- + set opt_overflow [dict get $opts -overflow] + set opt_colstart [dict get $opts -start] + # -- --- --- --- --- --- --- --- --- --- --- --- + set opt_transparent [dict get $opts -transparent] + if {$opt_transparent eq "0"} { + set do_transparency 0 + } else { + set do_transparency 1 + if {$opt_transparent eq "1"} { + set opt_transparent {[\s]} + } + } + # -- --- --- --- --- --- --- --- --- --- --- --- + set opt_returnextra [dict get $opts -returnextra] + # -- --- --- --- --- --- --- --- --- --- --- --- + set opt_exposed1 [dict get $opts -exposed1] + set opt_exposed2 [dict get $opts -exposed2] + # -- --- --- --- --- --- --- --- --- --- --- --- + + #----- + # + if {[string first \t $under] >= 0} { + #set under [textutil::tabify::untabify2 $under] + set under [textutil::tabify::untabifyLine $under 8] ;#8 is default for untabify2 - review + } + set overdata $over + if {[string first \t $over] >= 0} { + #set overdata [textutil::tabify::untabify2 $over] + set overdata [textutil::tabify::untabifyLine $over 8] + } + #------- + + #ta_detect ansi and do simpler processing? + + + # -- --- --- --- --- --- --- --- + set undermap [punk::ansi::ta::split_codes_single $under] + set understacks [dict create] + + set i_u -1 + set i_o 0 + set out [list] + set u_codestack [list] + set pt_underchars "" ;#for string_columns length calculation for overflow 0 truncation + set remainder [list] ;#for returnextra + foreach {pt code} $undermap { + #pt = plain text + append pt_underchars $pt + foreach ch [split $pt ""] { + set width [punk::char::string_width $ch] + incr i_u + dict set understacks $i_u $u_codestack + lappend out $ch + if {$width > 1} { + #presumably there are no triple-column or wider unicode chars.. until the aliens arrive.(?) + incr i_u + dict set understacks $i_u $u_codestack + lappend out "" + } + } + #underlay should already have been rendered and not have non-sgr codes - but let's check for and not stack them if other codes are here + if {[priv::is_sgr $code]} { + if {[priv::has_sgr_leadingreset $code]} { + set u_codestack [list $code] + } else { + lappend u_codestack $code + } + } + #consider also other codes that should be stacked..? + } + #trailing codes in effect for underlay + if {[llength $undermap]} { + dict set understacks [expr {$i_u + 1}] $u_codestack + } + + + # -- --- --- --- --- --- --- --- + #### + #if opt_colstart - we need to build a space (or any singlewidth char really) padding on the left of the right number of columns. + #this will be processed as transparent - and handle doublewidth underlay characters appropriately + set startpad [string repeat " " $opt_colstart] + append startpad $overdata ;#overdata with left padding spaces based on col-start under will show through for left-padding portion regardless of -transparency + set overmap [punk::ansi::ta::split_codes_single $startpad] + #### + + + + set overstacks [dict create] + set o_codestack [list] + set pt_overchars "" + foreach {pt code} $overmap { + append pt_overchars $pt + foreach ch [split $pt ""] { + dict set overstacks $i_o $o_codestack + incr i_o + } + if {[priv::is_sgr $code]} { + #only stack SGR (graphics rendition) codes - not title sets, cursor moves etc + if {[priv::has_sgr_leadingreset $code]} { + #m code which has sgr reset at start - no need to replay prior sgr codes + set o_codestack [list $code] + } else { + lappend o_codestack $code + } + } + } + # -- --- --- --- --- --- --- --- + + + + + set bs [format %c 0x08] + set idx 0 ;# line index (cursor - 1) + set idx_over -1 + foreach {pt code} $overmap { + set ptchars [split $pt ""] ;#for lookahead + #emit plaintext chars first using existing SGR codes from under/over stack as appropriate + #then check if the following code is a cursor movement within the line and adjust index if so + foreach ch $ptchars { + incr idx_over + if {$ch eq "\r"} { + set idx $opt_colstart + } elseif {$ch eq "\b"} { + #review - backspace effect on double-width chars + if {$idx > $opt_colstart} { + incr idx -1 + } + } elseif {($idx < $opt_colstart)} { + incr idx + } elseif {($do_transparency && [regexp $opt_transparent $ch])} { + #pre opt_colstart is effectively transparent (we have applied padding of required number of columns to left of overlay) + set owidth [punk::char::string_width $ch] + if {$idx > [llength $out]-1} { + lappend out " " + dict set understacks $idx [list] ;#review - use idx-1 codestack? + incr idx + } else { + set uwidth [punk::char::string_width [lindex $out $idx]] + if {[lindex $out $idx] eq ""} { + #2nd col of 2-wide char in underlay + incr idx + } elseif {$uwidth == 0} { + #e.g combining diacritic + incr idx + } elseif {$uwidth == 1} { + incr idx + if {$owidth > 1} { + incr idx + } + } elseif {$uwidth > 1} { + if {[punk::char::string_width $ch] == 1} { + #normal singlewide transparency + set next_pt_overchar [string index $pt_overchars $idx_over+1] ;#lookahead of next plain-text char in overlay + if {$next_pt_overchar eq ""} { + #special-case trailing transparent - no next_pt_overchar + incr idx + } else { + if {[regexp $opt_transparent $next_pt_overchar]} { + incr idx + } else { + #next overlay char is not transparent.. first-half of underlying 2wide char is exposed + priv::render_addchar $idx $opt_exposed1 [dict get $overstacks $idx_over] + incr idx + } + } + } else { + #2wide transparency over 2wide in underlay + incr idx + } + } + } + } else { + #non-transparent char in overlay + set owidth [punk::char::string_width $ch] + set uwidth [punk::char::string_width [lindex $out $idx]] + if {[lindex $out $idx] eq ""} { + #2nd col of 2wide char in underlay + priv::render_addchar $idx $ch [dict get $overstacks $idx_over] + incr idx + } elseif {$uwidth == 0} { + #e.g combining diacritic - increment before over char REVIEW + #arguably the previous overchar should have done this - ie lookahead for combiners? + priv::render_addchar $idx "" [dict get $overstacks $idx_over] + incr idx + priv::render_addchar $idx $ch [dict get $overstacks $idx_over] + incr idx + + } elseif {$uwidth == 1} { + if {$owidth == 1} { + priv::render_addchar $idx $ch [dict get $overstacks $idx_over] + incr idx + } else { + priv::render_addchar $idx $ch [dict get $overstacks $idx_over] + incr idx + priv::render_addchar $idx "" [dict get $overstacks $idx_over] + } + } elseif {$uwidth > 1} { + if {$owidth == 1} { + priv::render_addchar $idx $ch [dict get $overstacks $idx_over] + incr idx + priv::render_addchar $idx $opt_exposed2 [dict get $overstacks $idx_over] + #don't incr idx - we are just putting a broken-indication in the underlay - which may get overwritten by next overlay char + } else { + #2wide over 2wide + priv::render_addchar $idx $ch [dict get $overstacks $idx_over] + incr idx + } + } + } + } + #check following code + if {![priv::is_sgr $code]} { + + } + } + + if {$opt_overflow == 0} { + #need to truncate to the width of the original undertext + #review - string_width vs printing_length here. undertext requirement to be already rendered therefore punk::char::string_width ok? + set num_under_columns [punk::char::string_width $pt_underchars] ;#plaintext underchars + } + + #coalesce and replay codestacks for out char list + set outstring "" + set remstring "" ;#remainder after overflow point reached + set i 0 + set cstack [list] + set prevstack [list] + set out_rawchars ""; #for overflow counting + set output_to "outstring" ;#var in effect depending on overflow + set in_overflow 0 ;#used to stop char-width scanning once in overflow + foreach ch $out { + append out_rawchars $ch + if {$opt_overflow == 0 && !$in_overflow} { + if {[set nextvisualwidth [punk::char::string_width $out_rawchars]] < $num_under_columns} { + } else { + #todo - check if we overflowed with a double-width char ? + #store visualwidth which may be short + set in_overflow 1 + } + } + set cstack [dict get $understacks $i] + if {$cstack ne $prevstack} { + if {[llength $prevstack]} { + append $output_to \033\[m + } + foreach code $cstack { + append $output_to $code + } + } + append $output_to $ch + set prevstack $cstack + incr i + if {$in_overflow} { + set output_to "remstring" + } + } + if {[dict size $understacks] > 0} { + append $output_to [join [dict get $understacks [expr {[dict size $understacks]-1}]] ""] ;#tail codes + } + if {[string length $remstring]} { + #puts stderr "remainder:$remstring" + } + #pdict $understacks + if {$opt_returnextra} { + return [list $outstring $visualwidth [string length $outstring] $remstring] + } else { + return $outstring + } + #return [join $out ""] +} +proc overtype::test_renderline {} { + set t \uFF5E ;#2-wide tilde + set u \uFF3F ;#2-wide underscore + set missing \uFFFD + return [list $t $u A${t}B] +} + +#same as textblock::size - but we don't want that circular dependency +proc overtype::blocksize {textblock} { + if {$textblock eq ""} { + return [dict create width 0 height 1] ;#no such thing as zero-height block - for consistency with non-empty strings having no line-endings + } + set textblock [textutil::tabify::untabify2 $textblock] + #stripansi on entire block in one go rather than line by line - result should be the same - review - make tests + set textblock [punk::ansi::stripansi $textblock] + if {[string first \n $textblock] >= 0} { + set width [tcl::mathfunc::max {*}[lmap v [lines_as_list -- $textblock] {::punk::char::string_width $v}]] + } else { + set width [punk::char::string_width $textblock] + } + set num_le [expr {[string length $textblock]-[string length [string map [list \n {}] $textblock]]}] ;#faster than splitting into single-char list + #our concept of block-height is likely to be different to other line-counting mechanisms + set height [expr {$num_le + 1}] ;# one line if no le - 2 if there is one trailing le even if no data follows le + + return [dict create width $width height $height] ;#maintain order in 'image processing' standard width then height - caller may use lassign [dict values [blocksize ]] width height +} + +namespace eval overtype::priv { + + + #todo - move to punk::ansi::codetype + proc is_sgr {code} { + #SGR (Select Graphic Rendition) - codes ending in 'm' - e.g colour/underline + #we will accept and pass through the less common colon separator (ITU Open Document Architecture) + #Terminals should generally ignore it if they don't use it + regexp {\033\[[0-9;:]*m$} $code + } + proc is_cursor_move_in_line {code} { + #review - what about CSI n : m H where row n happens to be current line? + regexp {\033\[[0-9]*(:?C|D|G)$} + } + #pure SGR reset + proc is_sgr_reset {code} { + #todo 8-bit csi + regexp {\033\[0*m$} $code + } + #whether this code has 0 (or equivalently empty) parameter (but may set others) + #if an SGR code as a reset in it - we don't need to carry forward any previous SGR codes + #it generally only makes sense for the reset to be the first entry - otherwise the code has ineffective portions + #However - detecting zero or empty parameter in other positions requires knowing all other codes that may allow zero or empty params. + #We will only look at initial parameter as this is the well-formed normal case. + #Review - consider normalizing sgr codes to remove other redundancies such as setting fg or bg color twice in same code + proc has_sgr_leadingreset {code} { + set params "" + regexp {\033\[(.*)m} $code _match params + set plist [split $params ";"] + if {[string trim [lindex $plist 0] 0] eq ""} { + #e.g \033\[m \033\[0\;...m \033\[0000...m + return 1 + } else { + return 0 + } + } + #has_sgr_reset - rather than support this - create an sgr normalize function that removes dead params and brings reset to front of param list + proc render_addchar {i c stack} { + upvar out o + upvar understacks ustacks + set nxt [llength $o] + if {$i < $nxt} { + lset o $i $c + } else { + lappend o $c + } + dict set ustacks $i $stack + } + +} + + +# -- --- --- --- --- --- --- --- --- --- --- +if 0 { +namespace eval overtype::ta { + namespace path ::overtype + # *based* on but not identical to: + #https://github.com/perlancar/perl-Text-ANSI-Util/blob/master/lib/Text/ANSI/BaseUtil.pm + + #handle both 7-bit and 8-bit csi + #review - does codepage affect this? e.g ebcdic has 8bit csi in different position + + #CSI + #variable re_csi_open {(?:\033\[|\u009b)[0-9;]+} ;#too specific - doesn't detect \033\[m + variable re_csi_open {(?:\033\[|\u009b])} + + #colour and style + variable re_csi_colour {(?:\033\[|\u009b)[0-9;]*m} ;#e.g \033\[31m \033\[m \033\[0m \033\[m0000m + #single "final byte" in the range 0x40–0x7E (ASCII @A–Z[\]^_`a–z{|}~). + variable re_csi_code {(?:\033\[|\u009b])[0-9;]*[a-zA-Z\\@^_|~`]} + + #OSC - termnate with BEL (\a \007) or ST (string terminator \033\\) + #variable re_esc_osc1 {(?:\033\]|\u009c).*\007} + #variable re_esc_osc2 {(?:\033\]|\u009c).*\033\\} + + #test - non-greedy + variable re_esc_osc1 {(?:\033\]|\u009c).*?\007} + variable re_esc_osc2 {(?:\033\]|\u009c).*?\033\\} + + variable re_ansi_detect "${re_csi_open}|${re_esc_osc1}|${re_esc_osc2}" + + #detect any ansi escapes + #review - only detect 'complete' codes - or just use the opening escapes for performance? + #proc detect {text} { + # variable re_ansi_detect + # #variable re_csi_open + # #variable re_esc_osc1 + # #variable re_esc_osc2 + # #todo - other escape sequences + # #expr {[regexp $re_csi_open $text] || [regexp $re_esc_osc1 $text] || [regexp $re_esc_osc2 $text]} + # expr {[regexp $re_ansi_detect $text]} + #} + #not in perl ta + #proc detect_csi {text} { + # variable re_csi_colour + # expr {[regexp $re_csi_colour $text]} + #} + proc strip {text} { + tailcall punk::ansi::stripansi $text + } + #note this is character length after stripping ansi codes - not the printing length + proc length {text} { + string length [punk::ansi::stripansi $text] + } + + + # -- --- --- --- --- --- + #Split $text to a list containing alternating ANSI color codes and text. + #ANSI color codes are always on the second element, fourth, and so on. + #(ie plaintext on odd list-indices ansi on even indices) + # Example: + #ta_split_codes "" # => "" + #ta_split_codes "a" # => "a" + #ta_split_codes "a\e[31m" # => {"a" "\e[31m"} + #ta_split_codes "\e[31ma" # => {"" "\e[31m" "a"} + #ta_split_codes "\e[31ma\e[0m" # => {"" "\e[31m" "a" "\e[0m"} + #ta_split_codes "\e[31ma\e[0mb" # => {"" "\e[31m" "a" "\e[0m", "b"} + #ta_split_codes "\e[31m\e[0mb" # => {"" "\e[31m\e[0m" "b"} + # + #proc split_codes {text} { + # variable re_esc_osc1 + # variable re_esc_osc2 + # variable re_csi_code + # set re "(?:${re_csi_code}|${re_esc_osc1}|${re_esc_osc2})+" + # return [_perlish_split $re $text] + #} + ##like split_codes - but each ansi-escape is split out separately (with empty string of plaintext between codes so odd/even plain ansi still holds) + #proc split_codes_single {text} { + # variable re_esc_osc1 + # variable re_esc_osc2 + # variable re_csi_code + # set re "${re_csi_code}|${re_esc_osc1}|${re_esc_osc2}" + # return [_perlish_split $re $text] + #} + + ##review - tcl greedy expressions may match multiple in one element + #proc _perlish_split {re text} { + # if {[string length $text] == 0} { + # return {} + # } + # set list [list] + # set start 0 + # while {[regexp -start $start -indices -- $re $text match]} { + # lassign $match matchStart matchEnd + # lappend list [string range $text $start $matchStart-1] [string range $text $matchStart $matchEnd] + # set start [expr {$matchEnd+1}] + # } + # lappend list [string range $text $start end] + # return $list + #} + ## -- --- --- --- --- --- + +} +} ;# end if 0 + +# -- --- --- --- --- --- --- --- --- --- --- +namespace eval overtype { + interp alias {} ::overtype::center {} ::overtype::centre +} + +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +## Ready +package provide overtype [namespace eval overtype { + variable version + set version 1.5.3 +}] +return + +#*** !doctools +#[manpage_end] diff --git a/src/bootsupport/modules/punk/cap-0.1.0.tm b/src/bootsupport/modules/punk/cap-0.1.0.tm index 1c70a227..8eeef7fe 100644 --- a/src/bootsupport/modules/punk/cap-0.1.0.tm +++ b/src/bootsupport/modules/punk/cap-0.1.0.tm @@ -232,6 +232,7 @@ namespace eval punk::cap { #such unregistered capabilitynames may be used just to flag something, or have datamembers significant to callers cooperatively interested in that capname. #we allow registering a capability with an empty handler (capnamespace) - but this means another handler could be registered later. proc register_capabilityname {capname capnamespace} { + puts stderr "REGISTER_CAPABILITYNAME $capname $capnamespace" variable caps variable pkgcapsdeclared variable pkgcapsaccepted @@ -245,7 +246,8 @@ namespace eval punk::cap { #as handlers can be used to validate during provider registration - ideally handlers should be registered before any pkgs call register_package #we allow loading a handler later though - but will need to validate existing data from pkgs that have already registered as providers if {[set hdlr [capability_get_handler $capname]] ne ""} { - error "register_capabilityname cannot register capability:$capname with handler:$capnamespace. There is already a registered handler:$hdlr" + puts stderr "register_capabilityname cannot register capability:$capname with handler:$capnamespace. There is already a registered handler:$hdlr" + return } #assert: capnamespace may or may not be empty string, capname may or may not already exist in caps dict, caps $capname providers may have existing entries. dict set caps $capname handler $capnamespace @@ -262,7 +264,13 @@ namespace eval punk::cap { if {$cn ne $capname} { continue } - set do_register [$capreg pkg_register $pkg $capdict $fullcapabilitylist] + if {[catch {$capreg pkg_register $pkg $capdict $fullcapabilitylist} do_register]} { + puts stderr "punk::cap::register_capabilityname '$capname' '$capnamespace' failed to register provider package '$pkg' - possible error in handler or provider" + puts stderr "error message:" + puts stderr $do_register + set do_register 0 + } + set list_accepted [dict get $pkgcapsaccepted $pkg] if {$do_register} { if {$capspec ni $list_accepted} { @@ -337,7 +345,7 @@ namespace eval punk::cap { } #register package with arbitrary capnames from capabilitylist - #The registered pkg is a module that provides some service to that capname. Possibly just data members, that the capability will use. + #The registered pkg is a module that provides some service to that capname. Possibly just data members or possibly an implementation of an API, that the capability will use. proc register_package {pkg capabilitylist args} { variable pkgcapsdeclared variable pkgcapsaccepted @@ -370,6 +378,10 @@ namespace eval punk::cap { #for each capability # - ensure 1st element is a single word # - ensure that if 2nd element (capdict) is present - it is dict shaped + set capabilitylist_count [llength $capabilitylist] + set accepted_count 0 + set errorlist [list];# list of dicts + set warninglist [list] foreach capspec $capabilitylist { lassign $capspec capname capdict @@ -379,16 +391,23 @@ namespace eval punk::cap { } } if {[llength $capname] !=1} { - error "register_package error. pkg: '$pkg' An entry in the capability list doesn't appear to have a single-word name. Problematic entry:'$capspec'" + puts stderr "register_package error. pkg: '$pkg' An entry in the capability list doesn't appear to have a single-word name. Problematic entry:'$capspec'" + set reason "First element of capspec not a single-word name" + lappend errorlist [dict create msg $reason capspec $capspec] + continue } if {[expr {[llength $capdict] %2 != 0}]} { - error "register_package error. pkg:'$pkg' The second element for capname:'$capname' doesn't appear to be a valid dict. Problematic entry: '$capspec'" + puts stderr "register_package error. pkg:'$pkg' The second element for capname:'$capname' doesn't appear to be a valid dict. Problematic entry: '$capspec'" + set reason "The second element of the capspec isn't a valid dict" + lappend errorlist [dict create msg $reason capspec $capspec] + continue } if {$capspec in $pkg_already_accepted} { #review - multiple handlers? if so - will need to record which handler(s) accepted the capspec if {$warnings} { puts stderr "WARNING: register_package pkg $pkg already has capspec marked as accepted: $capspec" } + lappend warninglist [dict create msg "pkg $pkg already has this capspec marked as accepted" capspec $capspec] continue } if {[dict exists $caps $capname]} { @@ -424,6 +443,14 @@ namespace eval punk::cap { } else { dict set pkgcapsdeclared $pkg $capabilitylist } + set resultdict [list num_capabilities $capabilitylist_count num_accepted $accepted_count] + if {[llength $errorlist]} { + dict set resultdict errors $errorlist + } + if {[llength $warninglist]} { + dict set resultdict warnings $warninglist + } + return $resultdict } #todo! @@ -460,7 +487,7 @@ namespace eval punk::cap { } } - proc pkgcap {pkg} { + proc pkgcap {pkg {capsearch}} { variable pkgcapsdeclared variable pkgcapsaccepted if {[string match ::* $pkg]} { @@ -469,9 +496,9 @@ namespace eval punk::cap { if {[dict exists $pkgcapsdeclared $pkg]} { set accepted "" if {[dict exists $pkgcapsaccepted $pkg]} { - set accepted [dict get $pkgcapsaccepted $pkg] + set accepted [lsearch -all -inline -glob -index 0 [dict get $pkgcapsaccepted $pkg] $capsearch] } - return [dict create declared [dict get $pkgcapsdeclared $pkg] accepted $accepted] + return [dict create declared [lsearch -all -inline -glob -index 0 [dict get $pkgcapsdeclared $pkg] $capsearch] accepted $accepted] } else { return } diff --git a/src/bootsupport/modules/punk/cap/handlers/templates-0.1.0.tm b/src/bootsupport/modules/punk/cap/handlers/templates-0.1.0.tm index 75a925dd..9c0b9469 100644 --- a/src/bootsupport/modules/punk/cap/handlers/templates-0.1.0.tm +++ b/src/bootsupport/modules/punk/cap/handlers/templates-0.1.0.tm @@ -18,7 +18,7 @@ ## Requirements ##e.g package require frobz - +package require punk::repo # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ @@ -43,39 +43,163 @@ namespace eval punk::cap::handlers::templates { # -- --- --- --- --- --- --- ---- --- # validation of capdict # -- --- --- --- --- --- --- ---- --- - if {![dict exists $capdict relpath]} { - puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability, but is missing 'relpath' key" + if {![dict exists $capdict vendor]} { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability, but is missing the 'vendor' key" return 0 } - set provide_statement [package ifneeded $pkg [package require $pkg]] - set tmfile [lindex $provide_statement end] - if {![file exists $tmfile]} { - puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - unable to determine base folder for package '$pkg' which is attempting to register with punk::cap as a provider of '$capname' capability" + if {![dict exists $capdict path] || ![dict exists $capdict pathtype]} { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability, but is missing the 'path' or 'pathtype' key" return 0 } - set tpath [file normalize [file join $tmfile [dict get $capdict relpath]]] ;#relpath is relative to the tm *file* - not it's containing folder - if {![file isdirectory $tpath]} { - puts stderr "punk::cap::handlers::templates::capsystem pkg_register WARNING - unable to validate relpath location [dict get $capdict relpath] ($tpath) for package '$pkg' which is attempting to register with punk::cap as a provider of '$capname' capability" + set pathtype [dict get $capdict pathtype] + set vendor [dict get $capdict vendor] + set known_pathtypes [list adhoc currentproject_multivendor currentproject shellproject_multivendor shellproject module absolute] + if {$pathtype ni $known_pathtypes} { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability, but 'pathtype' value '$pathtype' is not recognised. Known type: $known_pathtypes" return 0 } + set path [dict get $capdict path] + + set cname [string map [list . _] $capname] + + set multivendor_package_whitelist [list punk::mix::templates] + + + #for template pathtype module & shellproject* we can resolve whether it's within a project at registration time and store the projectbase rather than rechecking it each time the templates handler api is called + #for template pathtype absolute - we can do the same. + #There is a small chance for a long-running shell that a project is later created which makes the absolute path within a project - but it seems an unlikely case, and probably won't surprise the user that they need to relaunch the shell or reload the capsystem to see the change. + + #adhoc and currentproject* paths are relative to cwd - so no projectbase information can be stored at registration time. + #not all template item types will need projectbase information - as the item data may be self-contained within the template structure - + #but project_layout will need it - or at least need to know if there is no project - because project_layout data is never stored in the template folder structure directly. + + if {$pathtype eq "adhoc"} { + if {[file pathtype $path] ne "relative"} { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability but provided a path '$path' of type $pathtype which doesn't seem to be a relative path" + return 0 + } + set extended_capdict $capdict + dict set extended_capdict vendor $vendor + } elseif {$pathtype eq "module"} { + set provide_statement [package ifneeded $pkg [package require $pkg]] + set tmfile [lindex $provide_statement end] + if {![file exists $tmfile]} { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - unable to determine base folder for package '$pkg' which is attempting to register with punk::cap as a provider of '$capname' capability" + return 0 + } + + if {[file pathtype $path] ne "relative"} { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability but provided a path '$path' of type $pathtype which doesn't seem to be a relative path" + } + set tmfolder [file dirname $tmfile] + #todo - handle wrapped or unwrapped tarjar files - in which case we have to adjust tmfolder appropriately + #set tpath [file normalize [file join $tmfile [dict get $capdict relpath]]] ;#relpath is relative to the tm *file* - not it's containing folder + + set projectinfo [punk::repo::find_repos $tmfolder] + set projectbase [dict get $projectinfo closest] + #store the projectbase even if it's empty string + set extended_capdict $capdict + set resolved_path [file join $tmfolder $path] + dict set extended_capdict resolved_path $resolved_path + dict set extended_capdict projectbase $projectbase + } elseif {$pathtype eq "currentproject_multivendor"} { + #currently only intended for punk::mix::templates - review if 3rd party _multivendor trees even make sense + if {$pkg ni $multivendor_package_whitelist} { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability but package is not in whitelist $multivendor_package_whitelist - 3rd party _multivendor tree not supported" + return 0 + } + if {[file pathtype $path] ne "relative"} { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability but provided a path '$path' of type $pathtype which doesn't seem to be a relative path" + return 0 + } + + set extended_capdict $capdict + dict set extended_capdict vendor $vendor ;#vendor key still required.. controlling vendor? + } elseif {$pathtype eq "currentproject"} { + if {[file pathtype $path] ne "relative"} { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability but provided a path '$path' of type $pathtype which doesn't seem to be a relative path" + return 0 + } + #verify that the relative path is within the relative path of a currentproject_multivendor tree + #todo - api for the _multivendor tree controlling package to validate + + + set extended_capdict $capdict + dict set extended_capdict vendor $vendor + } elseif {$pathtype eq "shellproject"} { + if {[file pathtype $path] ne "relative"} { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability but provided a path '$path' of type $pathtype which doesn't seem to be a relative path" + return 0 + } + set shellbase [file dirname [file dirname [file normalize [set ::argv0]/__]]] ;#review + set projectinfo [punk::repo::find_repos $shellbase] + set projectbase [dict get $projectinfo closest] + + set extended_capdict $capdict + dict set extended_capdict vendor $vendor + dict set extended_capdict projectbase $projectbase + } elseif {$pathtype eq "shellproject_multivendor"} { + #currently only intended for punk::templates - review if 3rd party _multivendor trees even make sense + if {$pkg ni $multivendor_package_whitelist} { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability but package is not in whitelist $multivendor_package_whitelist - 3rd party _multivendor tree not supported" + return 0 + } + if {[file pathtype $path] ne "relative"} { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability but provided a path '$path' of type $pathtype which doesn't seem to be a relative path" + return 0 + } + set shellbase [file dirname [file dirname [file normalize [set ::argv0]/__]]] ;#review + set projectinfo [punk::repo::find_repos $shellbase] + set projectbase [dict get $projectinfo closest] + + set extended_capdict $capdict + dict set extended_capdict vendor $vendor + dict set extended_capdict projectbase $projectbase + } elseif {$pathtype eq "absolute"} { + if {[file pathtype $path] ne "absolute"} { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability but provided a path '$path' of type $pathtype which doesn't seem to be absolute" + return 0 + } + set normpath [file normalize $path] + if {!file exists $normpath} { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability but provided a path '$path' which doesn't seem to exist" + return 0 + } + set projectinfo [punk::repo::find_repos $normpath] + set projectbase [dict get $projectinfo closest] + + #todo - verify no other provider has registered same absolute path - if sharing a project-external location is needed - they need their own subfolder + set extended_capdict $capdict + dict set extended_capdict resolved_path $normpath + dict set extended_capdict vendor $vendor + dict set extended_capdict projectbase $projectbase + } else { + puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability but provided a path '$path' with unrecognised type $pathtype" + return 0 + } # -- --- --- --- --- --- --- ---- --- # update package internal data # -- --- --- --- --- --- --- ---- --- + upvar ::punk::cap::handlers::templates::provider_info_$cname provider_info + if {$capname ni $::punk::cap::handlers::templates::handled_caps} { lappend ::punk::cap::handlers::templates::handled_caps $capname } - set cname [string map [list . _] $capname] - upvar ::punk::cap::handlers::templates::pkg_folders_$cname pfolders - dict lappend pfolders $pkg $tpath + if {![info exists provider_info] || $extended_capdict ni [dict get $provider_info $pkg]} { + #this checks for duplicates from the same provider - but not if other providers already added the path + #review - + dict lappend provider_info $pkg $extended_capdict + } # -- --- --- --- --- --- --- ---- --- # instantiation of api at punk::cap::handlers::templates::api_$capname # -- --- --- --- --- --- --- ---- --- - if {[info commands ::punk::cap::handlers::templates::$capname] eq ""} { - punk::cap::handlers::templates::class::api create ::punk::cap::handlers::templates::api_$capname $capname + set apicmd "::punk::cap::handlers::templates::api_$capname" + if {[info commands $apicmd] eq ""} { + punk::cap::handlers::templates::class::api create $apicmd $capname } return 1 @@ -84,8 +208,8 @@ namespace eval punk::cap::handlers::templates { upvar ::punk::cap::handlers::templates::handled_caps hcaps foreach capname $hcaps { set cname [string map [list . _] $capname] - upvar ::punk::cap::handlers::templates::pkg_folders_$cname pfolders - dict unset pfolders $pkg + upvar ::punk::cap::handlers::templates::provider_info_$cname my_provider_info + dict unset my_provider_info $pkg #destroy api objects? } } @@ -109,24 +233,491 @@ namespace eval punk::cap::handlers::templates { set cname [string map [list . _] $capname] set capabilityname $capname } - method folders {} { + method folders {args} { + lassign [punk::args::opts_values { + -startdir -default "" + } $args -maxvalues 0] _o opts + set opt_startdir [dict get $opts -startdir] + if {$opt_startdir eq ""} { + set startdir [pwd] + } else { + if {[file pathtype $opt_startdir] eq "relative"} { + set startdir [file join [pwd] $opt_startdir] + } else { + set startdir $opt_startdir + } + } + + variable capabilityname variable cname - upvar punk::cap::handlers::templates::pkg_folders_$cname pkg_folders + upvar ::punk::cap::handlers::templates::provider_info_$cname my_provider_info package require punk::cap set capinfo [punk::cap::capability $capabilityname] # e.g {punk.templates {handler punk::mix::templates providers ::somepkg}} #use the order of pkgs as registered with punk::cap - may have been modified with punk::cap::promote_package/demote_package - set pkgs [dict get $capinfo providers] + set providerpkg [dict get $capinfo providers] set folderdict [dict create] - foreach pkg $pkgs { - foreach pfolder [dict get $pkg_folders $pkg] { - dict set folderdict $pfolder [list source $pkg sourcetype package] + + #maintain separate paths for different override levels - all keyed on vendor (or pseudo-vendor '_project') + set found_paths_adhoc [dict create] + set found_paths_module [dict create] + set found_paths_currentproject_multivendor [dict create] + set found_paths_currentproject [dict create] + set found_paths_shellproject_multivendor [dict create] + set found_paths_shellproject [dict create] + set found_paths_absolute [list] + + + foreach pkg $providerpkg { + set found_paths [list] + #set acceptedlist [dict get [punk::cap::pkgcap $pkg $capabilityname] accepted] + + foreach capdecl_extended [dict get $my_provider_info $pkg] { + #basic validation and extension was done when accepted - so we can trust the capdecl_extended dictionary has the right entries + + set path [dict get $capdecl_extended path] + set pathtype [dict get $capdecl_extended pathtype] + set vendor [dict get $capdecl_extended vendor] + # projectbase not present in capdecl_extended for all template pathtypes + if {$pathtype eq "adhoc"} { + #e.g (cwd)/templates + set targetpath [file join $startdir [dict get $capdecl_extended path]] + if {[file isdirectory $targetpath]} { + dict lappend found_paths_adhoc $vendor [list pkg $pkg path $targetpath pathtype $pathtype] + } + } elseif {$pathtype eq "module"} { + set module_projectroot [dict get $capdecl_extended projectbase] + dict lappend found_paths_module $vendor [list pkg $pkg path [dict get $capdecl_extended resolved_path] pathtype $pathtype projectbase $module_projectroot] + } elseif {$pathtype eq "currentproject_multivendor"} { + set searchbase $startdir + set pathinfo [punk::repo::find_repos $searchbase] + set pwd_projectroot [dict get $pathinfo closest] + if {$pwd_projectroot ne ""} { + set deckbase [file join $pwd_projectroot $path] + if {![file exists $deckbase]} { + continue + } + #add vendor/x folders first - earlier in list is lower priority + set vendorbase [file join $deckbase vendor] + if {[file isdirectory $vendorbase]} { + set vendorfolders [glob -nocomplain -dir $vendorbase -type d -tails *] + foreach vf $vendorfolders { + if {$vf ne "_project"} { + dict lappend found_paths_currentproject_multivendor $vf [list pkg $pkg path [file join $vendorbase $vf] pathtype $pathtype] + } + } + if {[file isdirectory [file join $vendorbase _project]]} { + dict lappend found_paths_currentproject_multivendor _project [list pkg $pkg path [file join $vendorbase _project] pathtype $pathtype] + } + } + set custombase [file join $deckbase custom] + if {[file isdirectory $custombase]} { + set customfolders [glob -nocomplain -dir $custombase -type d -tails *] + foreach cf $customfolders { + if {$cf ne "_project"} { + dict lappend found_paths_currentproject_multivendor $cf [list pkg $pkg path [file join $custombase $cf] pathtype $pathtype] + } + } + if {[file isdirectory [file join $custombase _project]]} { + dict lappend found_paths_currentproject_multivendor _project [list pkg $pkg path [file join $custombase _project] pathtype $pathtype] + } + } + } + } elseif {$pathtype eq "currentproject"} { + set searchbase $startdir + set pathinfo [punk::repo::find_repos $searchbase] + set pwd_projectroot [dict get $pathinfo closest] + if {$pwd_projectroot ne ""} { + #path relative to projectroot already validated by handler as being within a currentproject_multivendor tree + set targetfolder [file join $pwd_projectroot $path] + if {[file isdirectory $targetfolder]} { + dict lappend found_paths_currentproject $vendor [list pkg $pkg path $targetfolder pathtype $pathtype] + } + } + } elseif {$pathtype eq "shellproject_multivendor"} { + #review - consider also [info script] - but it can be empty if we just start a tclsh, load packages and start a repl + #set shellbase [file dirname [file dirname [file normalize [set ::argv0]/__]]] ;#review + #set pathinfo [punk::repo::find_repos $shellbase] + #set pwd_projectroot [dict get $pathinfo closest] + + set shell_projectroot [dict get $capdecl_extended projectbase] + if {$shell_projectroot ne ""} { + set deckbase [file join $shell_projectroot $path] + if {![file exists $deckbase]} { + continue + } + #add vendor/x folders first - earlier in list is lower priority + set vendorbase [file join $deckbase vendor] + if {[file isdirectory $vendorbase]} { + set vendorfolders [glob -nocomplain -dir $vendorbase -type d -tails *] + foreach vf $vendorfolders { + if {$vf ne "_project"} { + dict lappend found_paths_shellproject_multivendor $vf [list pkg $pkg path [file join $vendorbase $vf] pathtype $pathtype projectbase $shell_projectroot] + } + } + if {[file isdirectory [file join $vendorbase _project]]} { + dict lappend found_paths_shellproject_multivendor _project [list pkg $pkg path [file join $vendorbase _project] pathtype $pathtype projectbase $shell_projectroot] + } + } + set custombase [file join $deckbase custom] + if {[file isdirectory $custombase]} { + set customfolders [glob -nocomplain -dir $custombase -type d -tails *] + foreach cf $customfolders { + if {$cf ne "_project"} { + dict lappend found_paths_shellproject_multivendor $cf [list pkg $pkg path [file join $custombase $cf] pathtype $pathtype projectbase $shell_projectroot] + } + } + if {[file isdirectory [file join $custombase _project]]} { + dict lappend found_paths_shellproject_multivendor _project [list pkg $pkg path [file join $custombase _project] pathtype $pathtype projectbase $shell_projectroot] + } + } + + } + + } elseif {$pathtype eq "shellproject"} { + #review - consider also [info script] - but it can be empty if we just start a tclsh, load packages and start a repl + #set shellbase [file dirname [file dirname [file normalize [set ::argv0]/__]]] ;#review + #set pathinfo [punk::repo::find_repos $shellbase] + #set pwd_projectroot [dict get $pathinfo closest] + + set shell_projectroot [dict get $capdecl_extended projectbase] + if {$shell_projectroot ne ""} { + set targetfolder [file join $shell_projectroot $path] + if {[file isdirectory $targetfolder]} { + dict lappend found_paths_shellproject $vendor [list pkg $pkg path $targetfolder pathtype $pathtype projectbase $shell_projectroot] + } + } + } elseif {$pathtype eq "absolute"} { + #lappend found_paths [dict get $capdecl_extended resolved_path] + set abs_projectroot [dict get $capdecl_extended projectbase] + dict lappend found_paths_absolute $vendor [list pkg $pkg path [dict get $capdecl_extended resolved_path] pathtype $pathtype projectbase $abs_projectroot] + } + + } + + #todo - ensure vendor pkg capdict elements such source and allowupdates override any existing entry from a _multivendor pkg? + #currently relying on order in which loaded? review + #foreach pfolder $found_paths { + # dict set folderdict $pfolder [list source $pkg sourcetype package] + #} + } + + #add in order of preference low priority to high + + dict for {vendor pathinfolist} $found_paths_module { + foreach pathinfo $pathinfolist { + dict set folderdict [dict get $pathinfo path] [list source [dict get $pathinfo pkg] sourcetype package pathtype [dict get $pathinfo pathtype] projectbase [dict get $pathinfo projectbase] vendor $vendor] + } + } + + #Templates within project of shell we launched with has lower priority than 'currentproject' (which depends on our CWD) + dict for {vendor pathinfolist} $found_paths_shellproject_multivendor { + foreach pathinfo $pathinfolist { + dict set folderdict [dict get $pathinfo path] [list source [dict get $pathinfo pkg] sourcetype package pathtype [dict get $pathinfo pathtype] projectbase [dict get $pathinfo projectbase] vendor $vendor] + } + } + dict for {vendor pathinfolist} $found_paths_shellproject { + foreach pathinfo $pathinfolist { + dict set folderdict [dict get $pathinfo path] [list source [dict get $pathinfo pkg] sourcetype package pathtype [dict get $pathinfo pathtype] projectbase [dict get $pathinfo projectbase] vendor $vendor] + } + } + + dict for {vendor pathinfolist} $found_paths_currentproject_multivendor { + foreach pathinfo $pathinfolist { + dict set folderdict [dict get $pathinfo path] [list source [dict get $pathinfo pkg] sourcetype package pathtype [dict get $pathinfo pathtype] vendor $vendor] + } + } + dict for {vendor pathinfolist} $found_paths_currentproject { + foreach pathinfo $pathinfolist { + dict set folderdict [dict get $pathinfo path] [list source [dict get $pathinfo pkg] sourcetype package pathtype [dict get $pathinfo pathtype] vendor $vendor] + } + } + dict for {vendor pathinfolist} $found_paths_absolute { + foreach pathinfo $pathinfolist { + dict set folderdict [dict get $pathinfo path] [list source [dict get $pathinfo pkg] sourcetype package pathtype [dict get $pathinfo pathtype] projectbase [dict get $pathinfo projectbase] vendor $vendor] + } + } + #adhoc paths relative to cwd (or specified -startdir) can override any + dict for {vendor pathinfolist} $found_paths_adhoc { + foreach pathinfo $pathinfolist { + dict set folderdict [dict get $pathinfo path] [list source [dict get $pathinfo pkg] sourcetype package pathtype [dict get $pathinfo pathtype] vendor $vendor] } } return $folderdict } + method get_itemdict_projectlayouts {args} { + lassign [punk::args::opts_values { + -startdir -default "" + #peek -startdir while allowing all other opts/vals to be verified down-the-line instead of here + } $args -maxvalues -1 -anyopts 1] _o opts _v values + set opt_startdir [dict get $opts -startdir] + if {$opt_startdir eq ""} { + set searchbase [pwd] + } else { + set searchbase $opt_startdir + } + + set refdict [my get_itemdict_projectlayoutrefs {*}$args] + set layoutdict [dict create] + + set projectinfo [punk::repo::find_repos $searchbase] + set projectroot [dict get $projectinfo closest] + + dict for {layoutname refinfo} $refdict { + set templatepathtype [dict get $refinfo sourceinfo pathtype] + set sourceinfo [dict get $refinfo sourceinfo] + set path [dict get $refinfo path] + set reftail [file tail $path] + set atparts [split [file rootname $reftail] @] + #may be two @s if referencing a renamed layout override? + # e.g ref may be @vendor+punks+othersample@sample-0.1 or layoutalias-1.1@vendor+punk+othersample@sample-0.1 + #there must always be an @ before vendor or custom . There is either a template-name alias or empty string before this first @ + #trim off first @ part + set tailats [join [lrange $atparts 1 end] @] + # @ parts after the first are part of the path within the project_layouts structure + set subpathlist [split $tailats +] + if {[dict exists $refinfo sourceinfo projectbase]} { + #some template pathtypes refer to the projectroot from the template - not the cwd + set projectroot [dict get $refinfo sourceinfo projectbase] + } + + if {$projectroot ne ""} { + set layoutroot [file join $projectroot src/project_layouts] + set layoutfolder [file join $layoutroot {*}$subpathlist] + if {[file isdirectory $layoutfolder]} { + #todo - check if layoutname already in layoutdict append .ref path to list of refs that linked to this layout? + set layoutinfo [list path $layoutfolder basefolder $layoutroot sourceinfo $sourceinfo] + dict set layoutdict $layoutname $layoutinfo + } + } + } + return $layoutdict + } + method get_itemdict_projectlayoutrefs {args} { + set config { + -templatefolder_subdir "layout_refs"\ + -command_get_items_from_base {apply {{base} { + set matched_files [glob -nocomplain -dir $base -type f *@*.ref] + set items [list] + foreach rf $matched_files { + #puts stderr "--> $rf" + if {[string match ignore* $rf]} { + continue + } + #we silently skip .ref files that don't match - todo - more verification - and warn of invalid .refs? + if {[string match *@vendor+* $rf] || [string match *@custom+* $rf]} { + lappend items $rf + } + } + return $items + }}}\ + -command_get_item_name {apply {{vendor basefolder itempath} { + set itemtail [file rootname [file tail $itempath]] + set alias [lindex [split $itemtail @] 0] + if {$alias eq ""} { + set itemname [lindex [split $itemtail +] end] + } else { + set itemname $alias + } + if {$vendor ne "_project"} { + set itemname $vendor.$itemname + } + return $itemname + }}} + } + set arglist [concat $config $args] + my _get_itemdict {*}$arglist + } + method get_itemdict_scriptappwrappers {args} { + set config { + -templatefolder_subdir "utility/scriptappwrappers"\ + -command_get_items_from_base {apply {{base} { + + set matched_files [punk::path::treefilenames -dir $base *] + set wrappers [list] + foreach tf $matched_files { + if {[string match ignore* $tf]} { + continue + } + set ext [file extension $tf] + if {[string tolower $ext] in [list "" ".bat" ".cmd" ".sh" ".bash" ".pl" ".ps1" ".tcl"]} { + lappend wrappers $tf + } + } + return $wrappers + }}}\ + -command_get_item_name {apply {{vendor basefolder itempath} { + + set relativepath [punk::path::relative $basefolder $itempath] + set ftail [file tail $itempath] + set tname $relativepath + if {$vendor ne "_project"} { + set tname ${vendor}.$tname + } + return $tname + }}} + } + set arglist [concat $config $args] + my _get_itemdict {*}$arglist + } + method get_itemdict_moduletemplates {args} { + set config { + -templatefolder_subdir "modules"\ + -command_get_items_from_base {apply {{base} { + + set matched_files [punk::path::treefilenames -dir $base template_*.tm] + set tfiles [list] + foreach tf $matched_files { + if {[string match ignore* $tf]} { + continue + } + set ext [file extension $tf] + if {[string tolower $ext] in [list ".tm"]} { + #we will ignore any .tm files that don't have versions that tcl understands - but warn + #this reduces the cases we have to test later + set fname [file tail $tf] + lassign [split [punk::mix::cli::lib::split_modulename_version $fname]] mname ver + if {[catch {punk::mix::cli::lib::validate_modulename $mname} errM]} { + puts stderr "Invalid module name/version $tf - please rename with standard Tcl .tm module name and version (or leave out version)" + if {[string match *-* $mname]} { + puts stderr "Tcl module name cannot contain dash character - except between name and version" + } + } else { + lappend tfiles $tf + } + } + } + return $tfiles + + }}}\ + -command_get_item_name {apply {{vendor basefolder itempath} { + + set relativepath [punk::path::relative $basefolder $itempath] + set dirs [file dirname $relativepath] + if {$dirs eq "."} { + set dirs "" + } + set moduleprefix [join $dirs ::] + set ftail [file rootname [file tail $itempath]] + set tname [string range $ftail [string length template_] end] + if {$moduleprefix ne ""} { + set tname ${moduleprefix}::$tname + } + if {$vendor ne "_project"} { + set tname ${vendor}.$tname + } + return $tname + }}} + } + set arglist [concat $config $args] + my _get_itemdict {*}$arglist + } + + #shared algorithm for get_itemdict_* methods + #requires a -templatefolder_subdir indicating a directory within each template base folder in which to search + #and a file selection mechanism command -command_get_items_from_base + #and a name determining command -command_get_item_name + method _get_itemdict {args} { + lassign [punk::args::opts_values { + -startdir -default "" + -templatefolder_subdir -optional 0 + -command_get_items_from_base -optional 0 + -command_get_item_name -optional 0 + -not -default "" -multiple 1 + globsearches -default * -multiple 1 + } $args -maxvalues -1] _o opts _v values + set globsearches [dict get $values globsearches]; #note that in this case our globsearch won't reduce the machine's effort in scannning the filesystem - as we need to search on the renamed results + # -- --- --- --- --- --- --- --- --- + set opt_startdir [dict get $opts -startdir] + set opt_templatefolder_subdir [dict get $opts -templatefolder_subdir] + if {[file pathtype $opt_templatefolder_subdir] ne "relative"} { + error templates::_get_itemdict + } + # -- --- --- --- --- --- --- --- --- + set opt_command_get_items_from_base [dict get $opts -command_get_items_from_base] + set opt_command_get_item_name [dict get $opts -command_get_item_name] + set opt_not [dict get $opts -not] + # -- --- --- --- --- --- --- --- --- + set itembases [list] + #set tbasedict [punk::mix::base::lib::get_template_basefolders $opt_startdir] + set tbasedict [my folders -startdir $opt_startdir ] + #turn the dict into a list we can temporarily reverse sort while we expand the items from within each path + dict for {tbase folderinfo} $tbasedict { + lappend itembases [list basefolder [file join $tbase $opt_templatefolder_subdir] sourceinfo $folderinfo] + } + + set items [list] + set itemdict [dict create] + set seen_dict [dict create] + + #flip the priority order for layout folders encountered so we can set the trailing # dup/overridden indicators + foreach baseinfo [lreverse $itembases] { + set basefolder [dict get $baseinfo basefolder] + set sourceinfo [dict get $baseinfo sourceinfo] + set vendor [dict get $sourceinfo vendor] + #call the custom script from our caller which determines resultset of files we are interested in + set matches [{*}$opt_command_get_items_from_base $basefolder] + set items_here [dict create] ;#maintain a list keyed on name for sorting within this base only + foreach itempath $matches { + set itemname [{*}$opt_command_get_item_name $vendor $basefolder $itempath] + dict set items_here $itemname [list item $itempath baseinfo $baseinfo] + #lappend items [list item $itempath baseinfo $baseinfo] + } + set ordered_names [lsort [dict keys $items_here]] + #add to the outer items list + foreach nm $ordered_names { + set iteminfo [dict get $items_here $nm] + lappend items [list originalname $nm iteminfo $iteminfo] + } + } + + #append #n instance/duplicate name indicators based on cyling through entire list of found items + foreach itemrecord $items { + set oname [dict get $itemrecord originalname] + set iteminfo [dict get $itemrecord iteminfo] + set itempath [dict get $iteminfo item] + set baseinfo [dict get $iteminfo baseinfo] + if {![dict exists $seen_dict $oname]} { + dict set seen_dict $oname 1 + dict set itemdict $oname [list path $itempath {*}$baseinfo] ; #first seen of oname gets no number + } else { + set n [dict get $seen_dict $oname] + incr n + dict incr seen_dict $oname + dict set itemdict ${oname}#$n [list path $itempath {*}$baseinfo] + } + } + + #assert path is first key of itemdict {callers are allowed to rely on it being first} + #assert itemdict has keys path,basefolder,sourceinfo + set result [dict create] + set keys [lreverse [dict keys $itemdict]] + foreach k $keys { + set maybe "" + foreach g $globsearches { + if {[string match $g $k]} { + set maybe $k + break + } + } + set not "" + if {$maybe ne ""} { + foreach n $opt_not { + if {[string match $n $k]} { + set not $k + break + } + } + } + if {$maybe ne "" && $not eq ""} { + dict set result $k [dict get $itemdict $k] + } + + } + return $result + } } } diff --git a/src/bootsupport/modules/punk/du-0.1.0.tm b/src/bootsupport/modules/punk/du-0.1.0.tm index fa02b3a9..65115ec0 100644 --- a/src/bootsupport/modules/punk/du-0.1.0.tm +++ b/src/bootsupport/modules/punk/du-0.1.0.tm @@ -825,7 +825,7 @@ namespace eval punk::du { ] set errors [dict create] set known_opts [dict keys $defaults] - foreach k [dict keys $args] { + dict for {k -} $args { if {$k ni $known_opts} { error "du_dirlisting_generic unknown-option $k" } @@ -1250,7 +1250,7 @@ namespace eval punk::du { #experiment with csv as easy way to get column like format.. #The /r is somewhat cheating however.. as it messes up redirected output .. e.g if redirected to text file - interp alias {} du {} .=args>* punk::du::du |> .=>1 natsort::sort -cols 1 -outputformat csv -outputformatoptions {\r\t\t\t} |> list_as_lines * punk::du::du |> .=>1 natsort::sort -cols 1 -outputformat csv -outputformatoptions {\r\t\t\t} |> list_as_lines -- -buildversion.txt +# module template: punkshell/src/decktemplates/vendor/punk/modules/template_module-0.0.2.tm +# +# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. +# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +# (C) 2024 +# +# @@ Meta Begin +# Application punk::encmime 0.1.0 +# Meta platform tcl +# Meta license +# @@ Meta End + + +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +# doctools header +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +#*** !doctools +#[manpage_begin punkshell_module_punk::encmime 0 0.1.0] +#[copyright "2024"] +#[titledesc {mime encodings related subset of tcllib mime}] [comment {-- Name section and table of contents description --}] +#[moddesc {mime encoding names and aliases}] [comment {-- Description at end of page heading --}] +#[require punk::encmime] +#[keywords module encodings] +#[description] +#[para] This is a workaround package to provide the mime encoding names used in tcllib's mime package - without additional dependencies +#[para]tcllib mime loads either Trf or tcl::memchan functions. punk::encmime needs to work in a context where tcllib may not yet be loaded/available, and even these few dependencies are too much. +#[para]MAINTENANCE NOTE: The data in this module needs to be checked against the latest tcllib mime package +#[para]taken from tcllib mime version: 1.7.2 in 2024 + +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ + +#*** !doctools +#[section Overview] +#[para] overview of punk::encmime +#[subsection Concepts] +#[para] Where practical - the actual tcllib mime package should be used instead. +#[para]This set of encoding related functions is a snapshot of the data from the mime package - and may not be up to date. +#[para]This pseudo-package was created to minimize dependencies for punk::char and punk::overtype + + +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +## Requirements +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ + +#*** !doctools +#[subsection dependencies] +#[para] packages used by punk::encmime +#[list_begin itemized] + +package require Tcl 8.6 +#*** !doctools +#[item] [package {Tcl 8.6}] + +# #package require frobz +# #*** !doctools +# #[item] [package {frobz}] + +#*** !doctools +#[list_end] + +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ + +#*** !doctools +#[section API] + +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +# oo::class namespace +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +namespace eval punk::encmime::class { + #*** !doctools + #[subsection {Namespace punk::encmime::class}] + #[para] class definitions + if {[info commands [namespace current]::interface_sample1] eq ""} { + #*** !doctools + #[list_begin enumerated] + + # oo::class create interface_sample1 { + # #*** !doctools + # #[enum] CLASS [class interface_sample1] + # #[list_begin definitions] + + # method test {arg1} { + # #*** !doctools + # #[call class::interface_sample1 [method test] [arg arg1]] + # #[para] test method + # puts "test: $arg1" + # } + + # #*** !doctools + # #[list_end] [comment {-- end definitions interface_sample1}] + # } + + #*** !doctools + #[list_end] [comment {--- end class enumeration ---}] + } +} +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ + +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +# Base namespace +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +namespace eval punk::encmime { + namespace export * + + variable encList { + ascii US-ASCII + big5 Big5 + cp1250 Windows-1250 + cp1251 Windows-1251 + cp1252 Windows-1252 + cp1253 Windows-1253 + cp1254 Windows-1254 + cp1255 Windows-1255 + cp1256 Windows-1256 + cp1257 Windows-1257 + cp1258 Windows-1258 + cp437 IBM437 + cp737 {} + cp775 IBM775 + cp850 IBM850 + cp852 IBM852 + cp855 IBM855 + cp857 IBM857 + cp860 IBM860 + cp861 IBM861 + cp862 IBM862 + cp863 IBM863 + cp864 IBM864 + cp865 IBM865 + cp866 IBM866 + cp869 IBM869 + cp874 {} + cp932 {} + cp936 GBK + cp949 {} + cp950 {} + dingbats {} + ebcdic {} + euc-cn EUC-CN + euc-jp EUC-JP + euc-kr EUC-KR + gb12345 GB12345 + gb1988 GB1988 + gb2312 GB2312 + iso2022 ISO-2022 + iso2022-jp ISO-2022-JP + iso2022-kr ISO-2022-KR + iso8859-1 ISO-8859-1 + iso8859-2 ISO-8859-2 + iso8859-3 ISO-8859-3 + iso8859-4 ISO-8859-4 + iso8859-5 ISO-8859-5 + iso8859-6 ISO-8859-6 + iso8859-7 ISO-8859-7 + iso8859-8 ISO-8859-8 + iso8859-9 ISO-8859-9 + iso8859-10 ISO-8859-10 + iso8859-13 ISO-8859-13 + iso8859-14 ISO-8859-14 + iso8859-15 ISO-8859-15 + iso8859-16 ISO-8859-16 + jis0201 JIS_X0201 + jis0208 JIS_C6226-1983 + jis0212 JIS_X0212-1990 + koi8-r KOI8-R + koi8-u KOI8-U + ksc5601 KS_C_5601-1987 + macCentEuro {} + macCroatian {} + macCyrillic {} + macDingbats {} + macGreek {} + macIceland {} + macJapan {} + macRoman {} + macRomania {} + macThai {} + macTurkish {} + macUkraine {} + shiftjis Shift_JIS + symbol {} + tis-620 TIS-620 + unicode {} + utf-8 UTF-8 + } + variable encodings + array set encodings $encList + variable reversemap + variable encAliasList { + ascii ANSI_X3.4-1968 + ascii iso-ir-6 + ascii ANSI_X3.4-1986 + ascii ISO_646.irv:1991 + ascii ASCII + ascii ISO646-US + ascii us + ascii IBM367 + ascii cp367 + cp437 cp437 + cp437 437 + cp775 cp775 + cp850 cp850 + cp850 850 + cp852 cp852 + cp852 852 + cp855 cp855 + cp855 855 + cp857 cp857 + cp857 857 + cp860 cp860 + cp860 860 + cp861 cp861 + cp861 861 + cp861 cp-is + cp862 cp862 + cp862 862 + cp863 cp863 + cp863 863 + cp864 cp864 + cp865 cp865 + cp865 865 + cp866 cp866 + cp866 866 + cp869 cp869 + cp869 869 + cp869 cp-gr + cp936 CP936 + cp936 MS936 + cp936 Windows-936 + iso8859-1 ISO_8859-1:1987 + iso8859-1 iso-ir-100 + iso8859-1 ISO_8859-1 + iso8859-1 latin1 + iso8859-1 l1 + iso8859-1 IBM819 + iso8859-1 CP819 + iso8859-2 ISO_8859-2:1987 + iso8859-2 iso-ir-101 + iso8859-2 ISO_8859-2 + iso8859-2 latin2 + iso8859-2 l2 + iso8859-3 ISO_8859-3:1988 + iso8859-3 iso-ir-109 + iso8859-3 ISO_8859-3 + iso8859-3 latin3 + iso8859-3 l3 + iso8859-4 ISO_8859-4:1988 + iso8859-4 iso-ir-110 + iso8859-4 ISO_8859-4 + iso8859-4 latin4 + iso8859-4 l4 + iso8859-5 ISO_8859-5:1988 + iso8859-5 iso-ir-144 + iso8859-5 ISO_8859-5 + iso8859-5 cyrillic + iso8859-6 ISO_8859-6:1987 + iso8859-6 iso-ir-127 + iso8859-6 ISO_8859-6 + iso8859-6 ECMA-114 + iso8859-6 ASMO-708 + iso8859-6 arabic + iso8859-7 ISO_8859-7:1987 + iso8859-7 iso-ir-126 + iso8859-7 ISO_8859-7 + iso8859-7 ELOT_928 + iso8859-7 ECMA-118 + iso8859-7 greek + iso8859-7 greek8 + iso8859-8 ISO_8859-8:1988 + iso8859-8 iso-ir-138 + iso8859-8 ISO_8859-8 + iso8859-8 hebrew + iso8859-9 ISO_8859-9:1989 + iso8859-9 iso-ir-148 + iso8859-9 ISO_8859-9 + iso8859-9 latin5 + iso8859-9 l5 + iso8859-10 iso-ir-157 + iso8859-10 l6 + iso8859-10 ISO_8859-10:1992 + iso8859-10 latin6 + iso8859-14 iso-ir-199 + iso8859-14 ISO_8859-14:1998 + iso8859-14 ISO_8859-14 + iso8859-14 latin8 + iso8859-14 iso-celtic + iso8859-14 l8 + iso8859-15 ISO_8859-15 + iso8859-15 Latin-9 + iso8859-16 iso-ir-226 + iso8859-16 ISO_8859-16:2001 + iso8859-16 ISO_8859-16 + iso8859-16 latin10 + iso8859-16 l10 + jis0201 X0201 + jis0208 iso-ir-87 + jis0208 x0208 + jis0208 JIS_X0208-1983 + jis0212 x0212 + jis0212 iso-ir-159 + ksc5601 iso-ir-149 + ksc5601 KS_C_5601-1989 + ksc5601 KSC5601 + ksc5601 korean + shiftjis MS_Kanji + utf-8 UTF8 + } + + #*** !doctools + #[subsection {Namespace punk::encmime}] + #[para] Core API functions for punk::encmime + #[list_begin definitions] + + # ::mime::mapencoding -- + # + # mime::mapencodings maps tcl encodings onto the proper names for their + # MIME charset type. This is only done for encodings whose charset types + # were known. The remaining encodings return {} for now. + # + # Arguments: + # enc The tcl encoding to map. + # + # Results: + # Returns the MIME charset type for the specified tcl encoding, or {} + # if none is known. + proc mapencoding {enc} { + #*** !doctools + #[call mapencoding [arg enc]] + #[para]maps tcl encodings onto the proper names for their MIME charset type. + #[para]This is only done for encodings whose charset types were known. + #[para]The remaining encodings return {} for now. + #[para]NOTE: consider using tcllib's mime::mapencoding instead if mime package available + + variable encodings + if {[info exists encodings($enc)]} { + return $encodings($enc) + } + return {} + } + + proc reversemapencoding {mimeType} { + #*** !doctools + #[call reversemapencoding [arg mimeType]] + #[para]mime::reversemapencodings maps MIME charset types onto tcl encoding names. + #[para]Returns the tcl encoding name for the specified mime charset, or {} if none is known + #[para] Arguments: + # [list_begin arguments] + # [arg_def string mimeType] The MIME charset to convert into a tcl encoding type. + # [list_end] + #[para]NOTE: consider using tcllib's mime::reversemapencoding instead if mime package available + + variable reversemap + + set lmimeType [string tolower $mimeType] + if {[info exists reversemap($lmimeType)]} { + return $reversemap($lmimeType) + } + return {} + } + + + + #*** !doctools + #[list_end] [comment {--- end definitions namespace punk::encmime ---}] +} +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ + +::apply {{} { + variable encList + variable encAliasList + variable reversemap + foreach {enc mimeType} $encList { + if {$mimeType eq {}} continue + set reversemap([string tolower $mimeType]) $enc + } + foreach {enc mimeType} $encAliasList { + set reversemap([string tolower $mimeType]) $enc + } + # Drop the helper variables + unset encList encAliasList + +} ::punk::encmime} + + +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +# Secondary API namespace +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +namespace eval punk::encmime::lib { + namespace export * + namespace path [namespace parent] + #*** !doctools + #[subsection {Namespace punk::encmime::lib}] + #[para] Secondary functions that are part of the API + #[list_begin definitions] + + #proc utility1 {p1 args} { + # #*** !doctools + # #[call lib::[fun utility1] [arg p1] [opt {?option value...?}]] + # #[para]Description of utility1 + # return 1 + #} + + + + #*** !doctools + #[list_end] [comment {--- end definitions namespace punk::encmime::lib ---}] +} +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ + + + +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +#*** !doctools +#[section Internal] +namespace eval punk::encmime::system { + #*** !doctools + #[subsection {Namespace punk::encmime::system}] + #[para] Internal functions that are not part of the API + + + +} +# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +## Ready +package provide punk::encmime [namespace eval punk::encmime { + variable pkg punk::encmime + variable version + set version 0.1.0 +}] +return + +#*** !doctools +#[manpage_end] + diff --git a/src/bootsupport/modules/punk/mix-0.2.tm b/src/bootsupport/modules/punk/mix-0.2.tm index 482c79a4..f24deb65 100644 --- a/src/bootsupport/modules/punk/mix-0.2.tm +++ b/src/bootsupport/modules/punk/mix-0.2.tm @@ -5,7 +5,12 @@ package require punk::cap::handlers::templates ;#handler for templates cap punk::cap::register_capabilityname punk.templates ::punk::cap::handlers::templates package require punk::mix::templates ;#registers as provider pkg for 'punk.templates' capability with punk::cap -punk::mix::templates::provider register * +if {[catch {punk::mix::templates::provider register *} errM]} { + puts stderr "punk::mix failure during punk::mix::templates::provider register *" + puts stderr $errM + puts stderr "-----" + puts stderr $::errorInfo +} package require punk::mix::base package require punk::mix::cli diff --git a/src/bootsupport/modules/punk/mix/base-0.1.tm b/src/bootsupport/modules/punk/mix/base-0.1.tm index 47f6c9d8..e9977bec 100644 --- a/src/bootsupport/modules/punk/mix/base-0.1.tm +++ b/src/bootsupport/modules/punk/mix/base-0.1.tm @@ -69,6 +69,8 @@ namespace eval punk::mix::base { set d_commands [get_commands -extension $extension] set all_commands [list {*}[dict get $d_commands main] {*}[dict get $d_commands base]] + + error "Unknown subcommand \"[lindex $args 0]\": must be one of: $all_commands" "punk::mix::base _unknown $ns $args" [list unknown_ensemble_subcommand ensemble punk::mix::base] } proc _redirected {from_ns subcommand args} { @@ -93,6 +95,16 @@ namespace eval punk::mix::base { } tailcall [namespace current] $subcommand {*}$argvals {*}$args -extension $from_ns } else { + if {[regexp {.*[*?].*} $subcommand]} { + set d_commands [get_commands -extension $from_ns] + set all_commands [list {*}[dict get $d_commands main] {*}[dict get $d_commands base]] + set matched_commands [lsearch -all -inline $all_commands $subcommand] + set commands "" + foreach m $matched_commands { + append commands $m \n + } + return $commands + } tailcall [namespace current] $subcommand {*}$args -extension $from_ns } } @@ -269,7 +281,9 @@ namespace eval punk::mix::base { } #result for just 'pmix help' + puts stderr "-->$args" set helpstr "" + append helpstr "limit commandlist with a glob search such as *word*" append helpstr "commands:\n" foreach {source cmdlist} $command_info { @@ -357,59 +371,24 @@ namespace eval punk::mix::base { } #get_template_basefolders - # scriptpath - file or folder - # It represents the base point from which to search for mixtemplates folders either directly related to the scriptpath (../) or in the containing project if any + # startpath - file or folder + # It represents the base point from which to search for templates folders either directly related to the scriptpath (../) or in the containing project if any # The cwd will also be searched for project root - but with lower precedence in the resultset (later in list) - proc get_template_basefolders {{scriptpath ""}} { - #1 lowest precedence - templates from packages (ordered by order in which packages registered with punk::cap) - set folderdict [dict create] + proc get_template_basefolders {{startpath ""}} { + # templates from punk.templates provider packages (ordered by order in which packages registered with punk::cap) + if {[file isfile $startpath]} { + set startpath [file dirname $startpath] + } package require punk::cap if {[punk::cap::capability_has_handler punk.templates]} { - set template_folder_dict [punk::cap::call_handler punk.templates folders] - dict for {dir folderinfo} $template_folder_dict { - dict set folderdict $dir $folderinfo - } - } - - #2 middle precedence - mixtemplates folder relative to cwd - set searchbase [pwd] - set fld [file join $searchbase mixtemplates] - if {[file isdirectory $fld]} { - if {![dict exists $folderdict $fld]} { - dict set folderdict $fld [list source $searchbase sourcetype cwd] - } - } - set pathinfo [punk::repo::find_repos $searchbase] - set pwd_projectroot [dict get $pathinfo closest] - if {$pwd_projectroot ne ""} { - set fld [file join $pwd_projectroot src/mixtemplates] - if {![dict exists $folderdict $fld]} { - dict set folderdict $fld [list source $pwd_projectroot sourcetype project] - } + set template_folder_dict [punk::cap::call_handler punk.templates folders -startdir $startpath] + } else { + put stderr "get_template_basefolders WARNING - no handler available for the 'punk.templates' capability - template providers will be unable to provide template locations" } - #3 highest precedence - mixtemplates relative to scriptpath argument - if {$scriptpath ne ""} { - if {[file type $scriptpath] eq "file"} { - set searchbase [file dirname $scriptpath] - } else { - set searchbase $scriptpath - } - if {[file isdirectory [file join $searchbase mixtemplates]]} { - dict set folderdict [file join $searchbase mixtemplates] [list source $searchbase sourcetype pathsearch] - } - set pathinfo [punk::repo::find_repos $searchbase] - set scriptpath_projectroot [dict get $pathinfo closest] - if {$scriptpath_projectroot ne ""} { - set fld [file join $scriptpath_projectroot src/mixtemplates] - if {[file isdirectory $fld]} { - dict set folderdict $fld [list source $scriptpath_projectroot sourcetype project] - } - } - } #don't sort - order in which encountered defines the precedence - with later overriding earlier - return $folderdict + return $template_folder_dict } proc module_subpath {modulename} { diff --git a/src/bootsupport/modules/punk/mix/cli-0.3.tm b/src/bootsupport/modules/punk/mix/cli-0.3.tm index a845285c..13d75d78 100644 --- a/src/bootsupport/modules/punk/mix/cli-0.3.tm +++ b/src/bootsupport/modules/punk/mix/cli-0.3.tm @@ -1,5 +1,5 @@ # -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt +# Maintenance Instruction: leave the 999999.xxx.x as is and use 'deck make' or src/make.tcl to update from -buildversion.txt # # Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. # Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. @@ -128,7 +128,7 @@ namespace eval punk::mix::cli { #review - why can't we be anywhere in the project? if {([file tail $sourcefolder] ne "src") || (![file exists $sourcefolder/make.tcl])} { - puts stderr "pmix make must be run from src folder containing make.tcl - unable to proceed (cwd: [pwd])" + puts stderr "deck make must be run from src folder containing make.tcl - unable to proceed (cwd: [pwd])" if {[string length $project_base]} { if {[file exists $project_base/src] && [string tolower [pwd]] ne [string tolower $project_base/src]} { puts stderr "Try cd to $project_base/src" @@ -210,7 +210,7 @@ namespace eval punk::mix::cli { proc validate_modulename {modulename args} { set defaults [list\ - -name_description modulename\ + -errorprefix validate_modulename\ ] if {[llength $args] %2 != 0} {error "validate_modulename args must be name-value pairs: received '$args'"} set known_opts [dict keys $defaults] @@ -221,18 +221,18 @@ namespace eval punk::mix::cli { } set opts [dict merge $defaults $args] # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_name_description [dict get $opts -name_description] + set opt_errorprefix [dict get $opts -errorprefix] # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - validate_name_not_empty_or_spaced $modulename -name_description $opt_name_description + validate_name_not_empty_or_spaced $modulename -errorprefix $opt_errorprefix set testname [string map [list :: ""] $modulename] if {[string first : $testname] >=0} { - error "$opt_name_description '$modulename' can only contain paired colons" + error "$opt_errorprefix '$modulename' can only contain paired colons" } set badchars [list - "$" "?" "*"] foreach bc $badchars { if {[string first $bc $modulename] >= 0} { - error "$opt_name_description '$modulename' can not contain character '$bc'" + error "$opt_errorprefix '$modulename' can not contain character '$bc'" } } return $modulename @@ -240,7 +240,7 @@ namespace eval punk::mix::cli { proc validate_projectname {projectname args} { set defaults [list\ - -name_description projectname\ + -errorprefix projectname\ ] if {[llength $args] %2 != 0} {error "validate_modulename args must be name-value pairs: received '$args'"} set known_opts [dict keys $defaults] @@ -251,21 +251,21 @@ namespace eval punk::mix::cli { } set opts [dict merge $defaults $args] # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_name_description [dict get $opts -name_description] + set opt_errorprefix [dict get $opts -errorprefix] # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - validate_name_not_empty_or_spaced $projectname -name_description $opt_name_description + validate_name_not_empty_or_spaced $projectname -errorprefix $opt_errorprefix set reserved_words [list etc lib bin modules src doc vendorlib vendormodules embedded runtime _aside _build] if {$projectname in $reserved_words } { - error "$opt_name_description '$projectname' cannot be one of reserved_words: $reserved_words" + error "$opt_errorprefix '$projectname' cannot be one of reserved_words: $reserved_words" } if {[string first "::" $projectname] >= 0} { - error "$opt_name_description '$projectname' cannot contain namespace separator '::'" + error "$opt_errorprefix '$projectname' cannot contain namespace separator '::'" } return $projectname } proc validate_name_not_empty_or_spaced {name args} { set defaults [list\ - -name_description projectname\ + -errorprefix projectname\ ] if {[llength $args] %2 != 0} {error "validate_modulename args must be name-value pairs: received '$args'"} set known_opts [dict keys $defaults] @@ -276,13 +276,13 @@ namespace eval punk::mix::cli { } set opts [dict merge $defaults $args] # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_name_description [dict get $opts -name_description] + set opt_errorprefix [dict get $opts -errorprefix] # -- --- --- --- --- --- --- --- --- --- --- --- --- --- if {![string length $name]} { - error "$opt_name_description cannot be empty" + error "$opt_errorprefix cannot be empty" } if {[string length [string map [list " " "" \n "" \r "" \t ""] $name]] != [string length $name]} { - error "$opt_name_description cannot contain whitespace" + error "$opt_errorprefix cannot contain whitespace" } return $name } @@ -364,7 +364,7 @@ namespace eval punk::mix::cli { if {"project" in $repotypes} { #punk project if {![catch {package require textblock; package require patternpunk}]} { - set result [textblock::join [textblock::join [>punk . logo] " "] $result] + set result [textblock::join [>punk . logo] " " $result] append result \n } } @@ -727,8 +727,8 @@ namespace eval punk::mix::cli { proc kettle_reinit {} { variable kettle_reset_bodies variable kettle_reset_args - foreach p [dict keys $kettle_reset_bodies] { - set b [dict get $kettle_reset_bodies $p] + dict for {p b} $kettle_reset_bodies { + #set b [dict get $kettle_reset_bodies $p] set argl [dict get $kettle_reset_args $p] uplevel 1 [list ::proc $p $argl $b] } @@ -751,7 +751,7 @@ namespace eval punk::mix::cli { gui\ ] #set ::kettle::recipe::recipe [dict create] - foreach r [dict keys $::kettle::recipe::recipe] { + dict for {r -} $::kettle::recipe::recipe { if {$r ni $standard_recipes} { dict unset ::kettle::recipe::recipe $r } @@ -760,14 +760,14 @@ namespace eval punk::mix::cli { proc kettle_call {calltype args} { variable kettle_reset_bodies if {$calltype ni [list lib shell]} { - error "pmix kettle_call 1st argument must be one of: 'lib' for direct use of kettle module or 'shell' to call as separate process" + error "deck kettle_call 1st argument must be one of: 'lib' for direct use of kettle module or 'shell' to call as separate process" } if {$calltype eq "shell"} { set kettleappfile [file dirname [info nameofexecutable]]/kettle set kettlebatfile [file dirname [info nameofexecutable]]/kettle.bat if {(![file exists $kettleappfile]) && (![file exists $kettlebatfile])} { - error "pmix kettle_call unable to find installed kettle application file '$kettleappfile' (or '$kettlebatfile' if on windows)" + error "deck kettle_call unable to find installed kettle application file '$kettleappfile' (or '$kettlebatfile' if on windows)" } if {[file exists $kettleappfile]} { set kettlescript $kettleappfile @@ -780,7 +780,7 @@ namespace eval punk::mix::cli { } set startdir [pwd] if {![file exists $startdir/build.tcl]} { - error "pmix kettle must be run from a folder containing build.tcl (cwd: [pwd])" + error "deck kettle must be run from a folder containing build.tcl (cwd: [pwd])" } if {[package provide kettle] eq ""} { puts stdout "Loading kettle package - may be delay on first load ..." @@ -797,7 +797,7 @@ namespace eval punk::mix::cli { } set first [lindex $args 0] if {[string match @* $first]} { - error "pmix kettle doesn't support special operations - try calling tclsh kettle directly" + error "deck kettle doesn't support special operations - try calling tclsh kettle directly" } if {$first eq "-f"} { set args [lassign $args __ path] @@ -839,7 +839,7 @@ namespace eval punk::mix::cli { --* { #instead of using: kettle option known if {$o ni $knownopts} { - error "Unable to process unknown option $o." {} [list KETTLE (pmix)] + error "Unable to process unknown option $o." {} [list KETTLE (deck)] } lappend opts $o [lindex $args 1] #::kettle::option set $o [lindex $args 1] diff --git a/src/bootsupport/modules/punk/mix/commandset/doc-0.1.0.tm b/src/bootsupport/modules/punk/mix/commandset/doc-0.1.0.tm index 36a654b7..c742bb3a 100644 --- a/src/bootsupport/modules/punk/mix/commandset/doc-0.1.0.tm +++ b/src/bootsupport/modules/punk/mix/commandset/doc-0.1.0.tm @@ -44,7 +44,7 @@ namespace eval punk::mix::commandset::doc { } #user may delete the comment containing "--- punk::docgen::overwrites" and then manually edit, and we won't overwrite #we still generate output in src/docgen so user can diff and manually update if thats what they prefer - set oldfiles [punk::path::treefilenames $projectdir/src/doc _module_*.man] + set oldfiles [punk::path::treefilenames -dir $projectdir/src/doc _module_*.man] foreach maybedoomed $oldfiles { set fd [open $maybedoomed r] set data [read $fd] @@ -93,7 +93,10 @@ namespace eval punk::mix::commandset::doc { # -- --- --- --- --- --- puts stdout "BUILDING DOCS at $projectdir/src/embedded from src/doc" if {[catch { - + if {"::meta" eq [info commands ::meta]} { + puts stderr "There appears to be a leftover ::meta command which is presumed to be from doctools. Destroying object" + ::meta destroy + } punk::mix::cli::lib::kettle_call lib doc #Kettle doc @@ -213,7 +216,7 @@ namespace eval punk::mix::commandset::doc { } file mkdir $output_base - set matched_paths [punk::path::treefilenames $codesource_path *.tm -antiglob_paths {**/mix/templates/** **/mixtemplates/**}] + set matched_paths [punk::path::treefilenames -dir $codesource_path -antiglob_paths {**/mix/templates/** **/project_layouts/** **/decktemplates/**} *.tm] set count 0 set newdocs [list] set docgen_header_comments "" diff --git a/src/bootsupport/modules/punk/mix/commandset/layout-0.1.0.tm b/src/bootsupport/modules/punk/mix/commandset/layout-0.1.0.tm index 0a9ff2d4..9eede1f5 100644 --- a/src/bootsupport/modules/punk/mix/commandset/layout-0.1.0.tm +++ b/src/bootsupport/modules/punk/mix/commandset/layout-0.1.0.tm @@ -18,6 +18,7 @@ ## Requirements ##e.g package require frobz +package require punk::args #sort of a circular dependency when commandset loaded by punk::mix::cli - that's ok, but this could theoretically be loaded by another cli and with another base package require punk::mix package require punk::mix::base @@ -28,6 +29,7 @@ package require punk::mix::base namespace eval punk::mix::commandset::layout { namespace export * + #per layout functions proc files {layout} { set allfiles [lib::layout_all_files $layout] @@ -38,25 +40,17 @@ namespace eval punk::mix::commandset::layout { return [join $templatefiles \n] } proc templatefiles.relative {layout} { - set template_base_dict [punk::mix::base::lib::get_template_basefolders] - set bases_containing_layout [list] - dict for {tbase folderinfo} $template_base_dict { - if {[file exists $tbase/layouts/$layout]} { - lappend bases_containing_layout $tbase - } - } - if {![llength $bases_containing_layout]} { - puts stderr "Unable to locate folder for layout '$layout'" - puts stderr "searched [dict size $template_base_dict] template folders" + set layoutdict [lib::layouts_dict] + if {![dict exists $layoutdict $layout]} { + puts stderr "layout '$layout' not found." return } - set tpldir [lindex $bases_containing_layout end] + set layoutinfo [dict get $layoutdict $layout] + set layoutfolder [dict get $layoutinfo path] - set layout_base $tpldir/layouts - set layout_dir [file join $layout_base $layout] - set stripprefix [file normalize $layout_dir] + set stripprefix [file normalize $layoutfolder] set templatefiles [lib::layout_scan_for_template_files $layout] set tails [list] foreach templatefullpath $templatefiles { @@ -68,46 +62,150 @@ namespace eval punk::mix::commandset::layout { #layout collection functions - to be imported with punk::overlay::import_commandset separately namespace eval collection { namespace export * - proc _default {{glob {}}} { + proc _defaultx {{glob {}}} { if {![string length $glob]} { set glob * } set layouts [list] - #set tplfolderdict [punk::cap::call_handler punk.templates folders] - set tplfolderdict [punk::mix::base::lib::get_template_basefolders] - dict for {tdir folderinfo} $tplfolderdict { - set layout_base $tdir/layouts - #collect all layouts and use lsearch glob rather than the filesystem glob (avoid issues with dotted folder names) - set all_layouts [lsort [glob -nocomplain -dir $layout_base -type d -tail *]] - foreach match [lsearch -all -inline $all_layouts $glob] { - lappend layouts [list $match $folderinfo] - } + set layoutdict [punk::cap::call_handler punk.templates get_itemdict_projectlayouts] + #set tplfolderdict [punk::mix::base::lib::get_template_basefolders] + dict for {layoutname layoutinfo} $layoutdict { + lappend layouts [list $layoutname $layoutinfo] + } + #return [join [lsort -index 0 $layouts] \n] + return [join $layouts \n] + } + + proc _default {args} { + set tdict_low_to_high [as_dict {*}$args] + #convert to screen order - with higher priority at the top + set tdict [dict create] + foreach k [lreverse [dict keys $tdict_low_to_high]] { + dict set tdict $k [dict get $tdict_low_to_high $k] + } + + package require overtype + package require textblock + #set pathinfolist [dict values $tdict] + #set paths [lsearch -all -inline -index 1 -subindices $pathinfolist *] ;#relies on first key of templates_dict being path + + set names [dict keys $tdict] + set paths [list] + set pathtypes [list] + dict for {nm tinfo} $tdict { + lappend paths [dict get $tinfo path] + lappend pathtypes [dict get $tinfo sourceinfo pathtype] + } + + set title(path) "Path" + set widest(path) [tcl::mathfunc::max {*}[lmap v [concat [list $title(path)] $paths] {punk::strlen $v}]] + set col(path) [string repeat " " $widest(path)] + + set title(pathtype) "[a+ green]Path Type[a]" + set widest(pathtype) [tcl::mathfunc::max {*}[lmap v [concat [list $title(pathtype)] $pathtypes] {punk::strlen $v}]] + set col(pathtype) [string repeat " " $widest(pathtype)] + + set title(name) "Layout Name" + set widest(name) [tcl::mathfunc::max {*}[lmap v [concat [list $title(name)] $names] {punk::strlen $v}]] + set col(name) [string repeat " " $widest(name)] + + set vsep " | " + set vsep_w [string length $vsep] ;#unicode? + set tablewidth [expr {$widest(name) + $vsep_w + $widest(pathtype) + $vsep_w + $widest(path)}] + set table "" + append table [string repeat - $tablewidth] \n + append table "[textblock::join [overtype::left $col(name) $title(name)] $vsep [overtype::left $col(pathtype) $title(pathtype)] $vsep [overtype::left $col(path) $title(path)]]" \n + append table [string repeat - $tablewidth] \n + + foreach n $names pt $pathtypes p $paths { + append table "[textblock::join [overtype::left $col(name) $n] $vsep [overtype::left $col(pathtype) $pt] $vsep [overtype::left $col(path) $p]]" \n + } + + return $table + } + proc references {args} { + set tdict_low_to_high [references_as_dict {*}$args] + #convert to screen order - with higher priority at the top + set tdict [dict create] + foreach k [lreverse [dict keys $tdict_low_to_high]] { + dict set tdict $k [dict get $tdict_low_to_high $k] } - return [join [lsort -index 0 $layouts] \n] + + package require overtype + package require textblock + #set pathinfolist [dict values $tdict] + #set paths [lsearch -all -inline -index 1 -subindices $pathinfolist *] ;#relies on first key of templates_dict being path + + set names [dict keys $tdict] + set paths [list] + set pathtypes [list] + dict for {nm tinfo} $tdict { + lappend paths [dict get $tinfo path] + lappend pathtypes [dict get $tinfo sourceinfo pathtype] + } + + set title(path) "Path" + set widest(path) [tcl::mathfunc::max {*}[lmap v [concat [list $title(path)] $paths] {punk::strlen $v}]] + set col(path) [string repeat " " $widest(path)] + + set title(pathtype) "[a+ green]Path Type[a]" + set widest(pathtype) [tcl::mathfunc::max {*}[lmap v [concat [list $title(pathtype)] $pathtypes] {punk::strlen $v}]] + set col(pathtype) [string repeat " " $widest(pathtype)] + + set title(name) "Layout Name" + set widest(name) [tcl::mathfunc::max {*}[lmap v [concat [list $title(name)] $names] {punk::strlen $v}]] + set col(name) [string repeat " " $widest(name)] + + set vsep " | " + set vsep_w [string length $vsep] ;#unicode? + set tablewidth [expr {$widest(name) + $vsep_w + $widest(pathtype) + $vsep_w + $widest(path)}] + set table "" + append table [string repeat - $tablewidth] \n + append table "[textblock::join [overtype::left $col(name) $title(name)] $vsep [overtype::left $col(pathtype) $title(pathtype)] $vsep [overtype::left $col(path) $title(path)]]" \n + append table [string repeat - $tablewidth] \n + + foreach n $names pt $pathtypes p $paths { + append table "[textblock::join [overtype::left $col(name) $n] $vsep [overtype::left $col(pathtype) $pt] $vsep [overtype::left $col(path) $p]]" \n + } + + return $table } + proc as_dict {args} { + tailcall punk::mix::commandset::layout::lib::layouts_dict {*}$args + } + proc references_as_dict {args} { + package require punk::cap + if {[punk::cap::capability_has_handler punk.templates]} { + set ref_dict [punk::cap::call_handler punk.templates get_itemdict_projectlayoutrefs {*}$args] + } else { + put stderr "commandset::layout::lib::layouts_dict WARNING - no handler available for the 'punk.templates' capability - template providers will be unable to provide template locations" + } + return $ref_dict + } } namespace eval lib { + proc layouts_dict {args} { + package require punk::cap + if {[punk::cap::capability_has_handler punk.templates]} { + set layout_dict [punk::cap::call_handler punk.templates get_itemdict_projectlayouts {*}$args] + } else { + put stderr "commandset::layout::lib::layouts_dict WARNING - no handler available for the 'punk.templates' capability - template providers will be unable to provide template locations" + } + return $layout_dict + } + proc layout_all_files {layout} { - set tplbasedict [punk::mix::base::lib::get_template_basefolders] - set layouts_found [list] - dict for {tplbase folderinfo} $tplbasedict { - if {[file isdirectory $tplbase/layouts/$layout]} { - lappend layouts_found $tplbase/layouts/$layout - } - } - if {![llength $layouts_found]} { + #todo - allow versionless layout name to pick highest version found + set layoutdict [layouts_dict] + if {![dict exists $layoutdict $layout]} { puts stderr "layout '$layout' not found." - puts stderr "searched [dict size $tplbasedict] template folders" - dict for {tplbase pkg} $tplbasedict { - puts stderr " - $tplbase $pkg" - } return } - set layoutfolder [lindex $layouts_found end] - + set layoutinfo [dict get $layoutdict $layout] + set layoutfolder [dict get $layoutinfo path] if {![file isdirectory $layoutfolder]} { - puts stderr "layout '$layout' not found in /layouts within one of template_folders. (get_template_folder returned: $tplbasedict)" + puts stderr "layout '$layout' points to path $layoutfolder - but it doesn't seem to exist" } set file_list [list] util::foreach-file $layoutfolder path { @@ -118,25 +216,15 @@ namespace eval punk::mix::commandset::layout { } # - #todo - allow specifying which package the layout is from: e.g "punk::mix::templates project" ?? proc layout_scan_for_template_files {layout {tags {}}} { - #equivalent for projects? punk::mix::commandset::module::lib::templates_dict -scriptpath "" - set tplbasedict [punk::mix::base::lib::get_template_basefolders] - set layouts_found [list] - dict for {tpldir pkg} $tplbasedict { - if {[file isdirectory $tpldir/layouts/$layout]} { - lappend layouts_found $tpldir/layouts/$layout - } - } - if {![llength $layouts_found]} { + #todo JMN + set layoutdict [layouts_dict] + if {![dict exists $layoutdict $layout]} { puts stderr "layout '$layout' not found." - puts stderr "searched [dict size $tplbasedict] template folders" - dict for {tpldir pkg} $tplbasedict { - puts stderr " - $tpldir $pkg" - } return } - set layoutfolder [lindex $layouts_found end] + set layoutinfo [dict get $layoutdict $layout] + set layoutfolder [dict get $layoutinfo path] #use last matching layout found. review silent if multiple? if {![llength $tags]} { diff --git a/src/bootsupport/modules/punk/mix/commandset/loadedlib-0.1.0.tm b/src/bootsupport/modules/punk/mix/commandset/loadedlib-0.1.0.tm index 0028c439..3bbe8b47 100644 --- a/src/bootsupport/modules/punk/mix/commandset/loadedlib-0.1.0.tm +++ b/src/bootsupport/modules/punk/mix/commandset/loadedlib-0.1.0.tm @@ -419,7 +419,7 @@ namespace eval punk::mix::commandset::loadedlib { if {![file exists $source_file]} { error "Unable to verify source file existence at: $source_file" } - set source_data [fcat $source_file -translation binary] + set source_data [fcat -translation binary $source_file] if {![string match "*package provide*" $source_data]} { puts stderr "Sorry - unable to verify source file contains 'package provide' statement of some sort - copy manually" return false diff --git a/src/bootsupport/modules/punk/mix/commandset/module-0.1.0.tm b/src/bootsupport/modules/punk/mix/commandset/module-0.1.0.tm index 19fbadb3..013c3bb8 100644 --- a/src/bootsupport/modules/punk/mix/commandset/module-0.1.0.tm +++ b/src/bootsupport/modules/punk/mix/commandset/module-0.1.0.tm @@ -1,5 +1,5 @@ # -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt +# Maintenance Instruction: leave the 999999.xxx.x as is and use 'deck make' or src/make.tcl to update from -buildversion.txt # # Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. # Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. @@ -9,7 +9,7 @@ # @@ Meta Begin # Application punk::mix::commandset::module 0.1.0 # Meta platform tcl -# Meta license +# Meta license BSD # @@ Meta End @@ -17,8 +17,8 @@ # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ ## Requirements ##e.g package require frobz - - +package require punk::repo +# depends on punk,punk::mix::base,punk::mix::cli # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ @@ -76,45 +76,69 @@ namespace eval punk::mix::commandset::module { } #require current dir when calling to be the projectdir, or proc templates {args} { - set tdict [templates_dict {*}$args] + set tdict_low_to_high [templates_dict {*}$args] + #convert to screen order - with higher priority at the top + set tdict [dict create] + foreach k [lreverse [dict keys $tdict_low_to_high]] { + dict set tdict $k [dict get $tdict_low_to_high $k] + } + + package require overtype + package require textblock + #set pathinfolist [dict values $tdict] + #set paths [lsearch -all -inline -index 1 -subindices $pathinfolist *] ;#relies on first key of templates_dict being path - package require overtype - set paths [dict values $tdict] set names [dict keys $tdict] + set paths [list] + set pathtypes [list] + dict for {nm tinfo} $tdict { + lappend paths [dict get $tinfo path] + lappend pathtypes [dict get $tinfo sourceinfo pathtype] + } + + set title(path) "Path" + set widest(path) [tcl::mathfunc::max {*}[lmap v [concat [list $title(path)] $paths] {punk::strlen $v}]] + set col(path) [string repeat " " $widest(path)] - set title1 "Path" - set widest1 [tcl::mathfunc::max {*}[lmap v [concat [list $title1] $paths] {punk::strlen $v}]] - set col1 [string repeat " " $widest1] + set title(pathtype) "[a+ green]Path Type[a]" + set widest(pathtype) [tcl::mathfunc::max {*}[lmap v [concat [list $title(pathtype)] $pathtypes] {string length $v}]] + set col(pathtype) [string repeat " " $widest(pathtype)] - set title2 "Template Name" - set widest2 [tcl::mathfunc::max {*}[lmap v [concat [list $title2] $names] {punk::strlen $v}]] - set col2 [string repeat " " $widest2] + set title(name) "Template Name" + set widest(name) [tcl::mathfunc::max {*}[lmap v [concat [list $title(name)] $names] {string length $v}]] + set col(name) [string repeat " " $widest(name)] - set tablewidth [expr {$widest1 + 1 + $widest2}] + set tablewidth [expr {$widest(name) + 1 + $widest(pathtype) + 1 + $widest(name)}] set table "" append table [string repeat - $tablewidth] \n - append table "[overtype::left $col1 $title1] [overtype::left $col2 $title2]" \n + append table "[textblock::join [overtype::left $col(name) $title(name)] " " [overtype::left $col(pathtype) $title(pathtype)] " " [overtype::left $col(path) $title(path)]]" \n append table [string repeat - $tablewidth] \n - foreach p $paths n $names { - append table "[overtype::left $col1 $p] [overtype::left $col2 $n]" \n + foreach n $names pt $pathtypes p $paths { + append table "[overtype::left $col(name) $n] [overtype::left $col(pathtype) $pt] [overtype::left $col(path) $p]" \n } return $table } #return all module templates with repeated ones suffixed with .2 .3 etc proc templates_dict {args} { - tailcall lib::templates_dict {*}$args - } + package require punk::cap + if {[punk::cap::capability_has_handler punk.templates]} { + set template_folder_dict [punk::cap::call_handler punk.templates get_itemdict_moduletemplates {*}$args] + } else { + put stderr "get_template_basefolders WARNING - no handler available for the 'punk.templates' capability - template providers will be unable to provide template locations" + } + } proc new {module args} { set year [clock format [clock seconds] -format %Y] set defaults [list\ -project \uFFFF\ -version \uFFFF\ -license \ - -template module-0.0.1.tm\ + -template punk.module\ -type \uFFFF\ -force 0\ + -quiet 0\ ] set opts [dict merge $defaults $args] @@ -133,7 +157,7 @@ namespace eval punk::mix::commandset::module { } else { set opt_version $opt_version_supplied if {![util::is_valid_tm_version $opt_version]} { - error "pmix module.new error - supplied -version $opt_version doesn't appear to be a valid Tcl module version" + error "deck module.new error - supplied -version $opt_version doesn't appear to be a valid Tcl module version" } } # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- @@ -144,7 +168,7 @@ namespace eval punk::mix::commandset::module { #if it has a dash then version is required to be valid lassign [punk::mix::cli::lib::split_modulename_version $module] modulename mversion if {![util::is_valid_tm_version $mversion]} { - error "pmix module.new error - unable to determine modulename-version from supplied value '$module'" + error "deck module.new error - unable to determine modulename-version from supplied value '$module'" } set mversion_supplied $mversion ;#record as may need to compare to version from templatefile name set vcompare_is_mversion_bigger [package vcompare $mversion $opt_version] @@ -163,15 +187,15 @@ namespace eval punk::mix::commandset::module { } else { set modulename $module } - punk::mix::cli::lib::validate_modulename $modulename -name_description "mix module.new name" + punk::mix::cli::lib::validate_modulename $modulename -errorprefix "punk::mix::commandset::module::new" # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- #options # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_project [dict get $opts -project] + set opt_project [dict get $opts -project] set testdir [pwd] if {![string length [set projectdir [punk::repo::find_project $testdir]]]} { if {![string length [set projectdir [punk::repo::find_candidate $testdir]]]} { - set msg [punkc::repo::is_candidate_root_requirements_msg] + set msg [punk::repo::is_candidate_root_requirements_msg] error "module.new unable to create module in projectdir:$projectdir - directory doesn't appear to meet basic standards $msg" } } @@ -187,13 +211,60 @@ namespace eval punk::mix::commandset::module { set opt_license [dict get $opts -license] # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- set opt_template [dict get $opts -template] + if {[regexp {.*[?*].*} $opt_template]} { + error "module.new -template does not support glob chars. Use an exact full name including version (and optionally .tm) - or use just the name without version or .tm, and the latest version will be selected" + } - set templates_dict [templates_dict] ;#possibly suffixed with .2 .3 etc - #todo - allow versionless name - pick latest which isn't suffixed with .2 etc - if {![dict exists $templates_dict $opt_template]} { - error "module.new unable to find template '$opt_template'. Known templates: [dict keys $templates_dict]" + set templates_dict [templates_dict] ;#keys are possibly prefixed with . and/or suffixed with #2 #3 etc if there are collisions - the remaining unsuffixed being the one with highest preference + #todo - allow versionless name - pick latest which isn't suffixed with #2 etc + #if the user wants to exactly match an unversioned template, in the presence of versioned ones - they may need to include the trailing .tm + if {[dict exists $templates_dict $opt_template]} { + #exact long name (possibly including version) + #Note - an unversioned .tm template will be matched here - even though versioned templates of the same name may exist. + set templatefile [dict get $templates_dict $opt_template path] + set templatefile_info [dict get $templates_dict $opt_template sourceinfo] + } else { + #if it wasn't an exact match for opt_template - then opt_template now shouldn't contain a version (we have also ruled out glob chars * & ? above) + #(if it does - then we just won't find anything - which is fine) + #module file name could contain dots - but only one dash - if it is versioned + + set matches [lsearch -all -inline [dict keys $templates_dict] $opt_template-*] ;#the key is of form vendor.modulename-version(#suffix) (version optional, suffix if lower precedence with same name was found) + #only .tm (or .TM .Tm .tM) files make it into the templates_dict - they are allowed to be unversioned though. + set key_version_list [list] + foreach m $matches { + #vendorname could contain dashes or dots - so easiest way to split out is to examine the stored vendor value in sourceinfo + set vendor [dict get $templates_dict $m sourceinfo vendor] + if {$vendor ne "_project"} { + #_project special case - not included in module names + set module $m + } else { + set module [string range [string length $vendor.] end] + } + lassign [punk::mix::cli::lib::split_modulename_version $m] _tailmname mversion + lappend key_version_list [list $m $mversion] + } + if {[llength $matches]} { + set highest_m "" + set highest_v "" + foreach kv $key_version_list { + if {$highest_v eq ""} { + set highest_m [lindex $kv 0] + set highest_v [lindex $kv 1] + } else { + if {[package vcompare $highest_v [lindex $kv 1]] == -1} { + set highest_m [lindex $kv 0] + set highest_v [lindex $kv 1] + } + } + } + set templatefile [dict get $templates_dict $highest_m path] + set templatefile_info [dict get $templates_dict $highest_m sourceinfo] + } else { + error "module.new unable to find template '$opt_template'. [dict size $templates_dict] Known templates. Use deck module.templates to display" + } } - set templatefile [dict get $templates_dict $opt_template] + + set tpldir [file dirname $templatefile] ;#use same folder for modulename_buildversion.txt, modulename_description.txt if they exist # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- @@ -205,6 +276,8 @@ namespace eval punk::mix::commandset::module { error "module.new - error - unknown -type '$opt_type' known-types: [punk::mix::cli::lib::module_types]" } # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- + set opt_quiet [dict get $opts -quiet] + # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- @@ -253,6 +326,8 @@ namespace eval punk::mix::commandset::module { if {$opt_version_supplied ne "\uFFFF"} { set build_version $opt_version } else { + # + if {[util::is_valid_tm_version $t_version]} { if {$mversion_supplied eq ""} { set build_version $t_version @@ -273,7 +348,19 @@ namespace eval punk::mix::commandset::module { set infile_version $build_version } - set template_filedata [string map [list %project% $projectname %pkg% $modulename %year% $year %license% $opt_license %version% $infile_version] $template_filedata] + set moduletemplate [file join $projectname [punk::path::relative $projectdir $templatefile]] ;#if templatfile is on another volume - just $templatefile will be returned. + #moduletemplate should usually be a relative path - but could be absolute, or contain info about the relative locations of projectdir vs templatefile if template comes from another project or a module outside the project + #This path info may be undesired in the template output (%moduletemplate%) + #it is nevertheless useful information - and not the only way developer-machine/build-machine paths can leak + #for now the user has the option to override any templates and remove %moduletemplate% if it is a security/privacy concern + + #Don't put litera %x% in the code for the commandset::module itself - to stop them being seen by layout scanner as replacable tokens + set tagnames [list moduletemplate $moduletemplate project $projectname pkg $modulename year $year license $opt_license version $infile_version] + set strmap [list] + dict for {tag val} $tagnames { + lappend strmap %$tag% $val + } + set template_filedata [string map $strmap $template_filedata] set modulefile $modulefolder/${moduletail}-$infile_version.tm if {[file exists $modulefile]} { @@ -330,7 +417,9 @@ namespace eval punk::mix::commandset::module { } } - + if {!$opt_quiet} { + puts stdout "Creating $modulefile from template $moduletemplate" + } set fd [open $modulefile w] fconfigure $fd -translation binary puts -nonewline $fd $template_filedata @@ -347,50 +436,8 @@ namespace eval punk::mix::commandset::module { } namespace eval lib { - proc templates_dict {args} { - set defaults [list -scriptpath ""] - set opts [dict merge $defaults $args] - set opt_scriptpath [dict get $opts -scriptpath] - - set module_template_bases [list] - set tbasedict [punk::mix::base::lib::get_template_basefolders $opt_scriptpath] - dict for {tbase folderinfo} $tbasedict { - lappend module_template_bases [file join $tbase modules] - } - - set template_files [list] - foreach basefld $module_template_bases { - set matched_files [glob -nocomplain -dir $basefld -type f template_*] - foreach tf $matched_files { - if {[string match ignore* $tf]} { - continue - } - set ext [file extension $tf] - if {$ext in [list ".tm"]} { - lappend template_files $tf - } - } - } - - set tdict [dict create] - set seen_dict [dict create] - foreach fullpath $template_files { - set ftail [file tail $fullpath] - set tname [string range $ftail [string length template_] end] - if {![dict exists $seen_dict $tname]} { - dict set seen_dict $tname 1 - dict set tdict $tname $fullpath ; #first seen of filename gets no number - } else { - set n [dict get $seen_dict $tname] - incr n - dict incr seen_dict $tname - dict set tdict ${tname}.$n $fullpath - } - } - return $tdict - } } diff --git a/src/bootsupport/modules/punk/mix/commandset/project-0.1.0.tm b/src/bootsupport/modules/punk/mix/commandset/project-0.1.0.tm index d8da3a47..badfa87d 100644 --- a/src/bootsupport/modules/punk/mix/commandset/project-0.1.0.tm +++ b/src/bootsupport/modules/punk/mix/commandset/project-0.1.0.tm @@ -1,5 +1,5 @@ # -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt +# Maintenance Instruction: leave the 999999.xxx.x as is and use 'deck make' or src/make.tcl to update from -buildversion.txt # # Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. # Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. @@ -19,8 +19,8 @@ #*** !doctools #[manpage_begin punkshell_module_punk::mix::commandset::project 0 0.1.0] #[copyright "2023"] -#[titledesc {pmix commandset - project}] [comment {-- Name section and table of contents description --}] -#[moddesc {pmix CLI commandset - project}] [comment {-- Description at end of page heading --}] +#[titledesc {dec commandset - project}] [comment {-- Name section and table of contents description --}] +#[moddesc {deck CLI commandset - project}] [comment {-- Description at end of page heading --}] #[require punk::mix::commandset::project] #[description] @@ -129,7 +129,7 @@ namespace eval punk::mix::commandset::project { error "punk::mix::cli::new error: unable to determine containing folder for '$newprojectpath_or_name'" } - punk::mix::cli::lib::validate_projectname $projectname -name_description "punk mix project.new" + punk::mix::cli::lib::validate_projectname $projectname -errorprefix "punk mix project.new" set defaults [list\ @@ -139,7 +139,7 @@ namespace eval punk::mix::commandset::project { -update 0\ -confirm 1\ -modules \uFFFF\ - -layout project + -layout punk.project ] ;#todo set known_opts [dict keys $defaults] foreach {k v} $args { @@ -151,7 +151,7 @@ namespace eval punk::mix::commandset::project { # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- set opt_type [dict get $opts -type] if {$opt_type ni [punk::mix::cli::lib::module_types]} { - error "pmix new error - unknown type '$opt_type' known types: [punk::mix::cli::lib::module_types]" + error "deck new error - unknown type '$opt_type' known types: [punk::mix::cli::lib::module_types]" } # -- --- --- --- --- --- --- --- --- --- --- --- --- set opt_force [dict get $opts -force] @@ -170,10 +170,10 @@ namespace eval punk::mix::commandset::project { set fossil_prog [auto_execok fossil] if {![string length $fossil_prog]} { - puts stderr "The fossil program was not found. A fossil executable is required to use most pmix features." + puts stderr "The fossil program was not found. A fossil executable is required to use most deck features." if {[string length [set scoop_prog [auto_execok scoop]]]} { #restrict to windows? - set answer [util::askuser "scoop detected. Would you like pmix to install fossil now using scoop? Y|N"] + set answer [util::askuser "scoop detected. Would you like deck to install fossil now using scoop? Y|N"] if {[string tolower $answer] ne "y"} { puts stderr "mix new aborting due to user response '$answer' (required Y|y to proceed) use -confirm 0 to avoid prompts." return @@ -216,23 +216,31 @@ namespace eval punk::mix::commandset::project { puts stdout "This project will be searched for templates" puts stdout "-------------------------------------------" } - set template_base_dict [punk::mix::base::lib::get_template_basefolders] - set template_bases_containing_layout [list] - dict for {tbase folderinfo} $template_base_dict { - if {[file exists $tbase/layouts/$opt_layout]} { - lappend template_bases_containing_layout $tbase - } + + + package require punk::cap + if {[punk::cap::capability_has_handler punk.templates]} { + set layout_dict [punk::cap::call_handler punk.templates get_itemdict_projectlayouts] + } else { + put stderr "commandset::project::new WARNING - no handler available for the 'punk.templates' capability - template providers will be unable to provide layout locations" + return } - if {![llength $template_bases_containing_layout]} { - puts stderr "layout '$opt_layout' was not found in template dirs" - puts stderr "searched [dict size $template_base_dict] template folders" - dict for {tbase folderinfo} $template_base_dict { - puts stderr " - $tbase $folderinfo" - } + if {[dict exists $layout_dict $opt_layout]} { + set layout_name $opt_layout + set layout_info [dict get $layout_dict $layout_name] + set layout_path [dict get $layout_info path] + set layout_sourceinfo [dict get $layout_info sourceinfo] + } else { + puts stderr "commandset::project::new - no exact match for specified layout-name $opt_layout found" + puts stderr "layout names found: [dict keys $layout_dict]" return + + #todo - pick highest version layout that matches opt_layout if version not specified but multiple exist + + #set layout_name ... + #set layout_info .. + #set layout_path ... } - #review: silently use last entry which had the layout (?) - set templatebase [lindex $template_bases_containing_layout end] @@ -275,7 +283,7 @@ namespace eval punk::mix::commandset::project { puts stderr "Unable to create new project at $projectdir - file/folder already exists use -update 1 to fill in missing items from template use -force 1 to overwrite from template" return } elseif {$project_dir_exists && $opt_force} { - puts stderr "mix new WARNING: -force 1 was supplied. Will copy layout $templatebase/layouts/$opt_layout using -force option to overwrite from template" + puts stderr "mix new WARNING: -force 1 was supplied. Will copy layout $layout_path using -force option to overwrite from template" if {$opt_confirm ni [list 0 no false]} { set answer [util::askuser "Do you want to proceed to possibly overwrite existing files in $projectdir? Y|N"] if {[string tolower $answer] ne "y"} { @@ -284,7 +292,7 @@ namespace eval punk::mix::commandset::project { } } } elseif {$project_dir_exists && $opt_update} { - puts stderr "mix new WARNING: -update 1 was supplied. Will copy layout $templatebase/layouts/$opt_layout using -update option to add missing items" + puts stderr "mix new WARNING: -update 1 was supplied. Will copy layout $layout_path using -update option to add missing items" } set fossil_repo_file "" @@ -325,29 +333,38 @@ namespace eval punk::mix::commandset::project { } } + # + + + file mkdir $projectdir - set layout_dir $templatebase/layouts/$opt_layout - puts stdout ">>> about to call punkcheck::install $layout_dir $projectdir" + puts stdout ">>> about to call punkcheck::install $layout_path $projectdir" set resultdict [dict create] set antipaths [list\ src/doc/*\ src/doc/include/*\ + src/PROJECT_LAYOUTS_*\ + ] + + #set antiglob_dir [list\ + # _ignore_*\ + #] + set antiglob_dir [list\ ] #default antiglob_dir_core will stop .fossil* from being updated - which is generally desirable as these are likely to be customized if {$opt_force} { puts stdout "copying layout files - with force applied - overwrite all-targets" - set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -overwrite ALL-TARGETS -antiglob_paths $antipaths] - #file copy -force $layout_dir $projectdir + set resultdict [punkcheck::install $layout_path $projectdir -installer project.new -overwrite ALL-TARGETS -antiglob_paths $antipaths -antiglob_dir $antiglob_dir] } else { puts stdout "copying layout files - (if source file changed)" - set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -overwrite installedsourcechanged-targets -antiglob_paths $antipaths] + set resultdict [punkcheck::install $layout_path $projectdir -installer project.new -overwrite installedsourcechanged-targets -antiglob_paths $antipaths -antiglob_dir $antiglob_dir] } puts stdout [punkcheck::summarize_install_resultdict $resultdict] puts stdout "copying layout src/doc files (if target missing)" - set resultdict [punkcheck::install $layout_dir/src/doc $projectdir/src/doc -punkcheck_folder $projectdir -installer project.new -overwrite SYNCED-TARGETS] + set resultdict [punkcheck::install $layout_path/src/doc $projectdir/src/doc -punkcheck_folder $projectdir -installer project.new -overwrite SYNCED-TARGETS] puts stdout [punkcheck::summarize_install_resultdict $resultdict] #target folders .fossil-custom and .fossil-settings may not exist. use -createdir 1 to ensure existence. @@ -355,33 +372,33 @@ namespace eval punk::mix::commandset::project { ## default_antiglob_dir_core [list "#*" "_aside" ".git" ".fossil*"] set override_antiglob_dir_core [list #* _aside .git] puts stdout "copying layout src/.fossil-custom files (if target missing or uncustomised)" - set resultdict [punkcheck::install $layout_dir/.fossil-custom $projectdir/.fossil-custom -createdir 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS] + set resultdict [punkcheck::install $layout_path/.fossil-custom $projectdir/.fossil-custom -createdir 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS] puts stdout [punkcheck::summarize_install_resultdict $resultdict] puts stdout "copying layout src/.fossil-settings files (if target missing or uncustomised)" - set resultdict [punkcheck::install $layout_dir/.fossil-settings $projectdir/.fossil-settings -createdir 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS] + set resultdict [punkcheck::install $layout_path/.fossil-settings $projectdir/.fossil-settings -createdir 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS] puts stdout [punkcheck::summarize_install_resultdict $resultdict] - - - - #lappend substfiles $projectdir/README.md - #lappend substfiles $projectdir/src/README.md - #lappend substfiles $projectdir/src/doc/main.man - #expect this in all templates? - todo make these substitutions independent of specific paths and filenames? #scan all files in template # - #TODO - pmix command to substitute templates? + #TODO - deck command to substitute templates? set templatefiles [punk::mix::commandset::layout::lib::layout_scan_for_template_files $opt_layout] - set stripprefix [file normalize $layout_dir] + set stripprefix [file normalize $layout_path] + set tagmap [list [lib::template_tag project] $projectname] + if {[llength $templatefiles]} { + puts stdout "Filling template file placeholders with the following tag map:" + dict for {placeholder value} $tagmap { + puts stdout " $placeholder -> $value" + } + } foreach templatefullpath $templatefiles { set templatetail [punk::repo::path_strip_alreadynormalized_prefixdepth $templatefullpath $stripprefix] set fpath [file join $projectdir $templatetail] if {[file exists $fpath]} { set fd [open $fpath r]; fconfigure $fd -translation binary; set data [read $fd]; close $fd - set data2 [string map [list [lib::template_tag project] $projectname] $data] + set data2 [string map $tagmap $data] if {$data2 ne $data} { puts stdout "updated template file: $fpath" set fdout [open $fpath w]; fconfigure $fdout -translation binary; puts -nonewline $fdout $data2; close $fdout @@ -392,12 +409,12 @@ namespace eval punk::mix::commandset::project { } #todo - tag substitutions in src/doc tree - ::cd $projectdir if {[file exists $projectdir/src/modules]} { foreach m $opt_modules { if {![file exists $projectdir/src/modules/$m-[punk::mix::util::magic_tm_version].tm]} { + #todo - option for -module_template - and check existence at top? or change opt_modules to be a list of dicts with configuration info -template -type etc punk::mix::commandset::module::new $m -project $projectname -type $opt_type } else { if {$opt_force} { @@ -531,7 +548,7 @@ namespace eval punk::mix::commandset::project { set widest2 [tcl::mathfunc::max {*}[lmap v [concat [list $title2] $col2items] {punk::strlen $v}]] set col2 [string repeat " " $widest2] set title3 "Checkouts" - set widest3 [tcl::mathfunc::max {*}[lmap v [concat [list $title3] $col3items] {punk::strlen $v}]] + set widest3 [tcl::mathfunc::max {*}[lmap v [concat [list $title3] $col3items] {string length $v}]] set col3 [string repeat " " $widest3] set tablewidth [expr {$widest1 + 1 + $widest2 + 1 + $widest3}] @@ -617,19 +634,19 @@ namespace eval punk::mix::commandset::project { set widest2 [tcl::mathfunc::max {*}[lmap v [concat [list $title2] $col2items] {punk::strlen $v}]] set col2 [string repeat " " $widest2] set title3 "Checkouts" - set widest3 [tcl::mathfunc::max {*}[lmap v [concat [list $title3] $col3items] {punk::strlen $v}]] + set widest3 [tcl::mathfunc::max {*}[lmap v [concat [list $title3] $col3items] {string length $v}]] set col3 [string repeat " " $widest3] set title4 "Project Name" - set widest4 [tcl::mathfunc::max {*}[lmap v [concat [list $title4] $col4_pnames] {punk::strlen $v}]] + set widest4 [tcl::mathfunc::max {*}[lmap v [concat [list $title4] $col4_pnames] {string length $v}]] set col4 [string repeat " " $widest4] set title5 "Project Code" - set widest5 [tcl::mathfunc::max {*}[lmap v [concat [list $title5] $col5_pcodes] {punk::strlen $v}]] + set widest5 [tcl::mathfunc::max {*}[lmap v [concat [list $title5] $col5_pcodes] {string length $v}]] set col5 [string repeat " " $widest5] set title6 "Dup" - set widest6 [tcl::mathfunc::max {*}[lmap v [concat [list $title6] $col6_dupids] {punk::strlen $v}]] + set widest6 [tcl::mathfunc::max {*}[lmap v [concat [list $title6] $col6_dupids] {string length $v}]] set col6 [string repeat " " $widest6] set title7 "Description" - #set widest7 [tcl::mathfunc::max {*}[lmap v [concat [list $title4] $col7_pdescs] {punk::strlen $v}]] + #set widest7 [tcl::mathfunc::max {*}[lmap v [concat [list $title4] $col7_pdescs] {string length $v}]] set widest7 35 set col7 [string repeat " " $widest7] @@ -766,7 +783,12 @@ namespace eval punk::mix::commandset::project { puts stderr "Result is from a single repo db [dict keys $fosdb_cache]" } if {$opt_detail} { - puts stderr "Gathering file state for [llength $workdirs] checkout folder(s). Use -detail 0 to omit file state" + if {!$opt_detail_explicit_zero} { + set detailmsg "Use -detail 0 to omit file state" + } else { + set detailmsg "" + } + puts stderr "Gathering file state for [llength $workdirs] checkout folder(s). $detailmsg" set c_rev [list] set c_rev_iso [list] set c_unchanged [list] @@ -881,7 +903,7 @@ namespace eval punk::mix::commandset::project { set index [expr {$answer - 1}] set workingdir [lindex $workdirs $index] ::cd $workingdir - puts stdout [pmix stat] + puts stdout [deck stat] return $workingdir } } @@ -907,23 +929,8 @@ namespace eval punk::mix::commandset::project { } set fossil_prog [auto_execok fossil] - set fossilinfo [exec {*}$fossil_prog info] ;#will give us the necessary config-db info whether in a project folder or not - set matching_lines [punk::repo::grep {config-db:*} $fossilinfo] - if {[llength $matching_lines] != 1} { - puts stderr "Unable to find config-db info from fossil. Check your fossil installation." - puts stderr "Fossil output was:" - puts stderr "-------------" - puts stderr "$fossilinfo" - puts stderr "-------------" - puts stderr "config-db info:" - puts stderr "$matching_lines" - return - } - set ln [lindex $matching_lines 0] - set configdb [string trim [string range $ln [string length "config-db: "] end]] - if {![file exists $configdb]} { - error "config-db not found at path $configdb" - } + set configdb [punk::repo::fossil_get_configdb] + package require sqlite3 ::sqlite3 fosconf $configdb #set testresult [fosconf eval {select name,value from global_config;}] diff --git a/src/bootsupport/modules/punk/mix/commandset/repo-0.1.0.tm b/src/bootsupport/modules/punk/mix/commandset/repo-0.1.0.tm index abfb0e55..2b3ca282 100644 --- a/src/bootsupport/modules/punk/mix/commandset/repo-0.1.0.tm +++ b/src/bootsupport/modules/punk/mix/commandset/repo-0.1.0.tm @@ -68,6 +68,334 @@ namespace eval punk::mix::commandset::repo { } return $result } + proc fossil-move-repository {{path ""}} { + set searchbase [pwd] + set projectinfo [punk::repo::find_repos $searchbase] + set projectbase [dict get $projectinfo closest] + set is_fossil [expr {"fossil" in [dict get $projectinfo closest_types]}] + if {[catch { + package require sqlite3 + } errM]} { + puts stderr "sqlite3 package failed to load" + puts stderr "Try using 'fossil test-move-repository ' from within an open checkout folder, or ensure that the Tcl sqlite3 package is available." + return + } + set ansiprompt [a+ green bold] + set ansiwarn [a+ red bold] + set ansihighlight [a+ cyan bold] + set ansireset [a] + + set in_checkout 0 + set is_checkout_relink 0; #whether we are attempting to link a checkout that has lost its repo + #we may also encounter a different kind of relink candidate - other checkouts of the same repo that we examine and find don't point back. + if {$projectbase eq "" || !$is_fossil} { + set repodbs [glob -dir $searchbase -type f -tail *.fossil] + if {![llength $repodbs]} { + puts stderr "Current directory does not seem to be directly below a fossil checkout, and no .fossil files found" + puts stderr "Please move to a folder containing the .fossil repository database to move, or to a folder directly within a fossil checkout (and with no intermediate git/fossil repos)" + return + } + set choice_files [list] + set i 1 + set menu_message "" + append menu_message "${ansiprompt}Select the number of the fossil repo db to potentially move (confirmation will be requested before any action is taken)${ansireset}" \n + foreach db $repodbs { + sqlite3 dbinfo [file join $searchbase $db] + set ckouts [dbinfo eval {select name from config where name like 'ckout:%'}] + dbinfo close + lappend choice_files [list index $i repofile $db checkouts [llength $ckouts]] + append menu_message "$i $db checkouts: [llength $ckouts]" \n + incr i + } + puts stdout $menu_message + set max [llength $choice_files] + if {$max == 1} { + set rangemsg "the number 1" + } else { + set rangemsg "a number from 1 to $max" + } + set answer [punk::repo::askuser "${ansiprompt}Enter $rangemsg to select a .fossil repository database to show details and potentially move. (or N to abort)${ansireset}"] + if {![string is integer -strict $answer]} { + puts stderr "Aborting" + return + } + + set index [expr {int($answer) -1}] + if {$index >= 0 && $index <= $max-1} { + set repo_file_choice [lindex $choice_files $index] + set repo_file [dict get $repo_file_choice repofile] + set repo_file [file join $searchbase $repo_file] + puts stdout "Selected fossil repo database file: $repo_file" + } else { + puts stderr " No menu number matched - aborting." + return + } + } else { + if {[file exists $projectbase/_FOSSIL_]} { + set cdbfile [file join $projectbase/_FOSSIL_] + } elseif {[file exists $projectbase/.fslckout]} { + set cdbfile [file join $projectbase/.fslckout] + } else { + puts stderr "No checkout database (_FOSSIL_ or .fslckout) found in nearest repository folder $projectbase (looked upwards from $searchbase)" + puts stderr "Unable to locate repository databases for potential move. Please move to a checkout folder or a folder containing .fossil repositories" + puts stderr "If run from a location where repositories are found, fossil-move-repository will give you the option to select a repository or cancel the operation" + return + } + set in_checkout 1 + sqlite3 cdb $cdbfile + set repo_file [cdb eval {select value from vvar where name='repository'}] + cdb close + if {[string length [string trim $repo_file]] && [file pathtype $repo_file] eq "relative"} { + set repo_file [file join $projectbase $repo_file] + } + if {![string length [string trim $repo_file]] || ![file exists $repo_file]} { + puts stderr "${ansiwarn}Checkout at $projectbase points to repository '$repo_file' - but it doesn't seem to exist${ansireset}" + set answer [punk::repo::askuser "${ansiprompt}Do you want to link this to an existing repository file? (Y|N)${ansireset}"] + if {[string match y* [string tolower $answer]]} { + set is_checkout_relink 1 + } else { + puts stderr "Aborting - Unable to link this checkout dir to a repository database file" + return + } + } + } + + set pname [file rootname [file tail $repo_file]] + set full_path_repo_file [file join $searchbase $repo_file] + if {[file isfile $full_path_repo_file]} { + sqlite3 dbinfo [file join $searchbase $repo_file] + set ckouts [dbinfo eval {select name from config where name like 'ckout:%'}] + dbinfo close + if {![llength $ckouts]} { + puts stdout "Repository db at [file join $searchbase $repo_file] appears to have no open checkouts" + } else { + puts stdout "Repository db at [file join $searchbase $repo_file] appears to have [llength $ckouts] open checkouts:" + foreach ck $ckouts { + puts stdout [string range $ck 6 end] + } + } + } else { + puts stderr "${ansiwarn}Missing repository db at $full_path_repo_file${ansireset}" + } + puts stdout "${ansihighlight}Report for all projects with repository file name $pname${ansireset}" + puts stdout [punk::mix::commandset::project::collection::detail $pname] + puts stdout [punk::mix::commandset::project::collection::work $pname -detail 1] + + #todo + #ask user if they want to select a different pname + set wantrenameprompt "${ansiprompt}Would you like to rename the .fossil file? (Y|N)${ansireset}" + append wantrenameprompt \n "${ansiprompt}.eg change $pname.fossil to something else such as ${pname}_new.fossil${ansireset}" + set answer [punk::repo::askuser $wantrenameprompt] + set pname2 $pname + if {[string match y* [string tolower $answer]]} { + set dorenameprompt "${ansiprompt}Enter the new name and hit enter. (Just an alphanumeric name (possibly with dots/dashes/underscores) without .fossil and without any path)${ansireset}" + set namechoice [punk::repo::askuser $dorenameprompt] + if {[string length $namechoice]} { + set permittedmap [list . "" - "" _ ""] + if {[string is alnum -strict [string map $permittedmap $namechoice]]} { + set pname2 $namechoice + } else { + puts stderr "Entered name was invalid. Must be numbers,letters,underscore,dot,dash" + } + } + puts stdout "Continuing with name $pname2 - cancel at next prompt if this is incorrect" + } + + set target_repodb_folder [punk::repo::fossil_get_repository_folder_for_project $pname2 -parentfolder $searchbase -askpath 1] + #target_repodb_folder might be same as source folder - check for same file if name wasn't changed? + if {![string length $target_repodb_folder]} { + puts stderr "No usable repository database folder selected for $pname2.fossil file" + return + } + + set existing_target_repofile 0 + if {[file exists $target_repodb_folder/$pname2.fossil]} { + set existing_target_repofile 1 + puts stdout "${ansiwarn}NOTICE: $target_repodb_folder/$pname2.fossil already exists${ansireset}" + if {!$is_checkout_relink} { + set finalquestion "${ansiprompt}Are you sure you want to switch the repository $repo_file for the open checkout(s) to the existing file $target_repodb_folder/$pname2.fossil? (Y|N)${ansireset}" + } else { + set finalquestion "${ansiprompt}Are you sure you want to attempt to linke the repository (previously linked with '$repo_file') for the open checkout(s) to the existing file $target_repodb_folder/$pname2.fossil? (Y|N)${ansireset}" + } + } else { + if {!$is_checkout_relink} { + set finalquestion "${ansiprompt}Proceed to move repository $repo_file to the new file $target_repodb_folder/$pname2.fossil? Y|N${ansireset}" + } else { + set finalquestion "${ansiprompt}Proceed to attempt link for missing repo db $repo_file to the new file $target_repodb_folder/$pname2.fossil? Y|N${ansireset}" + } + } + set line "${ansiwarn}[string repeat - [string length $finalquestion]]${ansireset}" + set finalprompt $line\n + append finalprompt $finalquestion \n + append finalprompt $line \n + + set answer [punk::repo::askuser $finalprompt] + if {[string match y* [string tolower $answer]]} { + if {!$existing_target_repofile && !$is_checkout_relink} { + if {[catch { + file copy $repo_file $target_repodb_folder/$pname2.fossil + } errM]} { + puts stderr "${ansiwarn}FAILED to copy $repo_file to $target_repodb_folder/$pname2.fossil - aborting${ansireset}" + puts stderr "Error message was:\n $errM" + return + } + if {$in_checkout} { + #in_checkout means we can assume projectbase var exists + #there may be other checkouts on the old repo + #if so, we will remind the user of their existence + if {[catch {exec fossil test-move-repository $target_repodb_folder/$pname2.fossil} errM]} { + puts stderr "${ansiwarn}The fossil test-move-repository command appears to have failed${ansireset}" + puts stderr "$errM" + } else { + + sqlite3 oldrepo $repo_file + set ckouts [oldrepo eval {select name from config where name like 'ckout:%'}] + set pcode [oldrepo eval {select value from config where name = 'project-code'}] + oldrepo close + if {[string length $pcode] < 20} { + puts stderr "WARNING: Failed to get project-code from repo db $repo_file" + } + set other_checkouts [list] + set norm_projectbase [file normalize $projectbase] + foreach ck $ckouts { + set ckfolder [string trim [string range $ck 6 end]] + if {![file isdirectory $ckfolder]} { + #as the process was launched within a checkout - we won't bother user with reports of non-existant other checkouts + continue + } + if {[file normalize $ckfolder] ne $norm_projectbase} { + lappend other_checkouts $ckfolder + } + } + if {[llength $other_checkouts]} { + puts stderr "${ansiwarn}Other checkouts of $repo_file that may need consideration${ansireset}" + foreach other $other_checkouts { + puts stdout $other + } + } + } + } else { + #we aren't in a checkout - moving a repo to a new db location and/or name so there's no reason to prefer one checkout over another.. presumably the user either wants to move them all - or be asked.. + sqlite3 oldrepo $repo_file + set ckouts [oldrepo eval {select name from config where name like 'ckout:%'}] + oldrepo close + if {[llength $ckouts] > 1} { + puts stdout "There are [llength $ckouts] checkouts for the repository you are moving" + puts stdout "You will be asked for each checkout if you want to adjust it to point to $target_repodb_folder/$pname2.folder" + } + set original_cwd [pwd] + foreach ck $ckouts { + set ckfolder [string trim [string range $ck 6 end]] + if {![file isdirectory $ckfolder]} { + puts stderr "old repo shows a checkout at $ckfolder - but it doesn't seem to exist. Ignoring" + continue + } + cd $ckfolder + puts stdout [exec fossil info] + puts stdout [state] + set answer [punk::repo::askuser "${ansiprompt}Do you want to point this checkout to $target_repodb_folder/$pname2.folder? (Y|N) Q to stop processing checkouts${ansireset}"] + if {[string match q* [string tolower $answer]]} { + puts stderr "User aborting loop" + break + } + if {[string match y* [string tolower $answer]]} { + if {[catch {exec fossil test-move-repository $target_repodb_folder/$pname2.fossil} moveresult]} { + puts stderr "${ansiwarn}The fossil test-move-repository command appears to have failed${ansireset}" + puts stderr "$moveresult" + } else { + puts stdout "OK - move performed with result:" + puts stdout $moveresult + } + } + } + cd $original_cwd + + } + } else { + if {$is_checkout_relink} { + #relinking a lost checkout to an existing repo.. we should probably check it's other checkouts and see if they point back + if {[catch {exec fossil test-move-repository $target_repodb_folder/$pname2.fossil} errM]} { + puts stderr "${ansiwarn}The fossil test-move-repository command appears to have failed${ansireset}" + puts stderr "$errM" + } + } else { + if {$in_checkout} { + if {[catch {exec fossil test-move-repository $target_repodb_folder/$pname2.fossil} errM]} { + puts stderr "${ansiwarn}The fossil test-move-repository command appears to have failed${ansireset}" + puts stderr "$errM" + } + } else { + #not in checkout - we're wanting what pointed to one repo to point to a different existing one - presumably for all checkouts + sqlite3 newrepo $target_repodb_folder/$pname2.fossil + set newpname [newrepo eval {select value from config where name = 'project-name'}] + set newpcode [newrepo eval {select value from config where name = 'project-code'}] + set newckouts [newrepo eval {select name from config where name like 'ckout:%'}] + newrepo close + + sqlite3 oldrepo $repo_file + set oldpname [oldrepo eval {select value from config where name = 'project-name'}] + set oldpcode [oldrepo eval {select value from config where name = 'project-code'}] + set oldckouts [oldrepo eval {select name from config where name like 'ckout:%'}] + oldrepo close + if {$newpname eq $oldpname} { + set ansi_newpname [a+ green bold]$newpname[a] + set ansi_oldpname [a+ green bold]$oldpname[a] + } else { + set ansi_newpname [a+ cyan bold]$newpname[a] + set ansi_oldpname [a+ red bold]$oldpname[a] + } + if {$newpcode eq $oldpcode} { + set ansi_newpcode [a+ green bold]$newpcode[a] + set ansi_oldpcode [a+ green bold]$oldpcode[a] + } else { + set ansi_newpcode [a+ cyan bold]$newpcode[a] + set ansi_oldpcode [a+ red bold]$oldpcode[a] + } + puts stdout "Target repository $target_repodb_folder/$pname2.fossil has project-name: $ansi_newpname and [llength $newckouts] existing checkouts" + puts stdout "Target project code: $ansi_newpcode" + puts stdout "Source repository $repo_file has project-name: $ansi_oldpname and [llength $oldckouts] existing checkouts" + puts stdout "Source project code: $ansi_oldpcode" + if {[llength $oldckouts] > 1} { + puts stdout "You will be asked for each checkout if you want to adjust it to point to $target_repodb_folder/$pname2.folder" + } + set original_cwd [pwd] + foreach ck $oldckouts { + set ckfolder [string trim [string range $ck 6 end]] + if {![file isdirectory $ckfolder]} { + puts stderr "old repo shows a checkout at $ckfolder - but it doesn't seem to exist. Ignoring" + continue + } + cd $ckfolder + puts stdout [exec fossil info] + puts stdout [state] + set answer [punk::repo::askuser "${ansiprompt}Do you want to point this checkout to $target_repodb_folder/$pname2.folder? (Y|N) Q to stop processing checkouts${ansireset}"] + if {[string match q* [string tolower $answer]]} { + puts stderr "User aborting loop" + break + } + if {[string match y* [string tolower $answer]]} { + if {[catch {exec fossil test-move-repository $target_repodb_folder/$pname2.fossil} moveresult]} { + puts stderr "${ansiwarn}The fossil test-move-repository command appears to have failed${ansireset}" + puts stderr "$moveresult" + } else { + puts stdout "OK - move performed with result:" + puts stdout $moveresult + } + } + } + cd $original_cwd + + } + } + } + + + puts stdout "-done-" + } else { + puts stdout "-cancelled by user-" + } + + } } diff --git a/src/bootsupport/modules/punk/mix/commandset/scriptwrap-0.1.0.tm b/src/bootsupport/modules/punk/mix/commandset/scriptwrap-0.1.0.tm index e40bc899..b1c899da 100644 --- a/src/bootsupport/modules/punk/mix/commandset/scriptwrap-0.1.0.tm +++ b/src/bootsupport/modules/punk/mix/commandset/scriptwrap-0.1.0.tm @@ -20,6 +20,7 @@ package require punk::mix package require punk::mix::base +package require punk::fileline @@ -27,83 +28,734 @@ package require punk::mix::base namespace eval punk::mix::commandset::scriptwrap { namespace export * + namespace eval fileline { + namespace import ::punk::fileline::lib::* + namespace import ::punk::fileline::class::* + } - #scriptpath allows templates command to use same custom template set as when multishell pointed to a filepath - #it may or may not be within a project - #by using the same folder or path, the same project root will be discovered. REVIEW. - proc templates_dict {args} { - set defaults [list -scriptpath ""] - set opts [dict merge $defaults $args] - set opt_scriptpath [dict get $opts -scriptpath] - - set wrapper_folders [lib::get_wrapper_folders $opt_scriptpath] - - set wrapper_templates [list] - foreach fld $wrapper_folders { - set templates [glob -nocomplain -dir $fld -type f *] - foreach tf $templates { - if {[string match ignore* $tf]} { - continue - } - set ext [file extension $tf] - if {$ext in [list "" ".bat" ".cmd" ".sh"]} { - lappend wrapper_templates $tf - } - } + proc templates {args} { + package require overtype + package require textblock + set tdict_low_to_high [templates_dict {*}$args] + #convert to screen order - with higher priority at the top + set tdict [dict create] + foreach k [lreverse [dict keys $tdict_low_to_high]] { + dict set tdict $k [dict get $tdict_low_to_high $k] } + + #set pathinfolist [dict values $tdict] + set names [dict keys $tdict] - set tdict [dict create] - set seen_dict [dict create] - foreach fullpath $wrapper_templates { - set ftail [file tail $fullpath] - if {![dict exists $seen_dict $ftail]} { - dict set seen_dict $ftail 1 - dict set tdict $ftail $fullpath ; #first seen of filename gets no number - } else { - set n [dict get $seen_dict $ftail] - incr n - dict incr seen_dict $ftail - dict set tdict ${ftail}.$n $fullpath - } + #set paths [lsearch -all -inline -index 1 -subindices $pathinfolist *]; #first key of templates_dict is path + set paths [list] + set pathtypes [list] + dict for {nm tinfo} $tdict { + lappend paths [dict get $tinfo path] + lappend pathtypes [dict get $tinfo sourceinfo pathtype] } - return $tdict - } - proc templates {args} { - package require overtype - set tdict [templates_dict {*}$args] - - set paths [dict values $tdict] - set names [dict keys $tdict] + package require textblock + set title(name) "Template Name" + set widest(name) [tcl::mathfunc::max {*}[lmap v [concat [list $title(name)] $names] {string length $v}]] + set col(name) [string repeat " " $widest(name)] - set title1 "Path" - set widest1 [tcl::mathfunc::max {*}[lmap v [concat [list $title1] $paths] {punk::strlen $v}]] - set col1 [string repeat " " $widest1] + set title(pathtype) "[a+ green]Path\nType[a]" + set widest(pathtype) [tcl::mathfunc::max {*}[lmap v [concat [list $title(pathtype)] $pathtypes] {textblock::width $v}]] + set col(pathtype) [string repeat " " $widest(pathtype)] - set title2 "Template Name" - set widest2 [tcl::mathfunc::max {*}[lmap v [concat [list $title2] $names] {punk::strlen $v}]] - set col2 [string repeat " " $widest2] + set title(path) "Path" + set widest(path) [tcl::mathfunc::max {*}[lmap v [concat [list $title(path)] $paths] {string length $v}]] + set col(path) [string repeat " " $widest(path)] - set tablewidth [expr {$widest1 + 1 + $widest2}] + + + + set tablewidth [expr {$widest(name) + 1 + $widest(pathtype) + $widest(path)}] set table "" append table [string repeat - $tablewidth] \n - append table "[overtype::left $col1 $title1] [overtype::left $col2 $title2]" \n + append table [textblock::join [overtype::left $col(name) $title(name)] " " [overtype::left $col(pathtype) $title(pathtype)] " " [overtype::left $col(path) $title(path)]] \n append table [string repeat - $tablewidth] \n - foreach p $paths n $names { - append table "[overtype::left $col1 $p] [overtype::left $col2 $n]" \n + foreach n $names pt $pathtypes p $paths { + append table "[overtype::left $col(name) $n] [overtype::left $col(pathtype) $pt] [overtype::left $col(path) $p]" \n } return $table } - #specific filepath to just wrap one script at the tcl-payload or xxx-payload-pre-tcl site + proc templates_dict {args} { + package require punk::cap + if {[punk::cap::capability_has_handler punk.templates]} { + return [punk::cap::call_handler punk.templates get_itemdict_scriptappwrappers {*}$args] + } else { + put stderr "commandset::scriptwrap::templates_dict WARNING - no handler available for the 'punk.templates' capability - template providers will be unable to provide template locations" + } + return + } + + + #A batch file with unix line-endings is sensitive to label positioning. + #batch file with windows crlf line endings can exhibit this problem - but probably only if specifically crafted with long lines deliberately designed to trigger it. + #see: https://www.dostips.com/forum/viewtopic.php?t=8988#p58888 (Call and goto may fail when the batch file has Unix line endings) + #The windows batch file scanner appears to parse in 512 Byte chunks. + #If a label following a call/goto is at a position spanning a 512 byte block as counted from the call/goto site (callsite assumed to be EOL - works for basic cases at least) then the label won't be found. + #A label preceding a call/goto site can't span a 512 byte boundary as counted from the beginning of the file + #checkfile produces warnings and errors in ansi-coloured form (both to stdout and a summary in the return value) + #The script should then be adjusted with comments and/or whitespace and checkfile should be re-run to confirm there are no new boundary-spanning labels. + #checkfile needs to be run even on previously tested scriptwrapper templates because the final :exit label is beyond the payloads and so could span a 512 Byte block + #It is more likely to catch issues if adjustments are made to the initial batch-script code in a template. + # + #cmd allows labels at call sites to span lines with line continuation character ^ + #target labels can't span lines with ^ - cmd doesn't recognise them (They possibly do span in a way - but but the newlines may be included in the label - so they may be hard/impossible to call). + #Note that we can't filter obviously non-batch-script lines before processing - as the way batch label-scanning works it scans in chunks of 512 bytes so all lines are relevant. + #This means label-like things could be incorrectly found in other script data - that's partly the point of this check + #Note that we can't filter obviously non-batch-script lines before processing - as the way batch label-scanning works it scans in chunks of 512 bytes so all lines are relevant. + #This means label-like things could be incorrectly found in other script data - that's partly the point of this check. + proc checkfile {filepath args} { + if {![file exists $filepath]} { + error "punk::mix::commandset:scriptwrap error cannot find file '$filepath'" + } + set crlf_lf_replacements [list \uFFFF \uFFFE] ;#defaults - if already exist in file - error out with message + # -ignore_rems 1 allows testing of alignment state if rems were stripped - todo - lf/crlf-preserving rem strip function + set defaults [dict create\ + -ignore_rems 0\ + -substitutionmap {}\ + -crlf_lf_replacements $crlf_lf_replacements\ + ] + set known_opts [dict keys $defaults] + foreach {k v} $args { + if {$k ni $known_opts} { + error "checkfile error - unknown option '$k'. Known options: $known_opts" + } + } + set opts [dict merge $defaults $args] + # -- --- --- --- --- --- --- + set opt_ignore_rems [dict get $opts -ignore_rems] + set opt_substitutionmap [dict get $opts -substitutionmap] + set opt_crlf_lf_replacements [dict get $opts -crlf_lf_replacements] + # -- --- --- --- --- --- --- + + # #### load file #### + ##set raw_filedata [fcat -translation binary $filepath] + # - as we may need to look at data beyond a ctrl-z (\x1A) section + set fd [open $filepath r] + fconfigure $fd -translation binary + set raw_filedata [read $fd] + close $fd + # ################### + + + set objFile [fileline::textinfo new $raw_filedata] + + if {$opt_ignore_rems} { + #! todo + error "-ignore_rems unimplemented" + if 0 { + #todo - rebuild a raw_filedata value from the resultant lines + #review. @REM can appear after other commands and an ampersand for example. + # - we are interested in stripping lines with leading REMs + # - need to work out if a REM line with dos line-continuation should + + set data "" + set skipped_rems 0 + foreach ln [split $filedata \n] { + set ln [string trim $ln] + if {[string match @REM* $ln] || [string match REM* $ln] || [string match @rem* $ln] || [string match rem* $ln]} { + #ignore + incr skipped_rems + } else { + append data $ln \n ;#!! + } + } + puts stderr "Skipped $skipped_rems rem lines" + set dsize [string length $data] + } + } else { + set dsize [string length $raw_filedata] + } + + puts stdout "Examining [$objFile chunklen] bytes of file $filepath for cmd script issues." + set le_info [$objFile chunk_le_counts 0 end] + set lf_count [dict get $le_info lf] + set crlf_count [dict get $le_info crlf] + set unterminated_count [dict get $le_info unterminated] + set total_count [expr {$lf_count + $crlf_count + $unterminated_count}] + puts stdout "lines ending in lf : $lf_count" + puts stdout "lines ending in crlf : $crlf_count" + puts stdout "unterminated lines : $unterminated_count" ;#commonly 1 for trailing data at end of file. More than one is likely to be an error - or perhaps a policy plugin with different concept of lines? + puts stdout "crlf + lf + unterminated: $total_count" + puts stdout "line count : [$objFile linecount]" + if {$total_count != [$objFile linecount]} { + puts stdout "[a+ yellow bold]WARNING: Linecount mismatch with line-endings - seems fishy[a]" + } + if {$unterminated_count > 1} { + puts stdout "[a+ yellow bold]WARNING: More than one unterminated line reported - seems fishy[a]" + } + puts "Checking line based labels and 512 byte boundaries from call sites for possible labels and code execution points." + set line_count [$objFile linecount] + set callid 0 ;#id for callsite and objects created + set file_offset 0 + set error_labels [list] + set warning_labels [list] + set call_labels_found [dict create] + set target_labels_found [dict create] + set possible_target_labels_found [dict create] + set warning_target_labels_found [dict create] + for {set callingline_index 0} {$callingline_index < $line_count} {incr callingline_index} { + set callingline_info [$objFile lineinfo $callingline_index] + set callingline_payload [dict get $callingline_info payload] + set callingline_len [dict get $callingline_info linelen] + set callingline_num [expr {$callingline_index + 1}] + + set callposn -1 + set trimln [string trim $callingline_payload] + if {[string match "rem *" $trimln] || [string match "@rem *" $trimln] || [string match "REM *" $trimln] || [string match "@REM *" $trimln]} { + #ignore things that look like a call that are beind a REM + } else { + + #todo - better callsite analysis. There can be data between @GOTO or @CALL and : other than just whitespace! + + #todo - allow analysis of colon-less call. May need to check list of internal commands - but what about external ones? + #foreach search_regex [list {(.*\s+|^)(@*call\s*:*)(\S.*)} {(.*\s+|^)(@*CALL\s*:*)(\S.*)} {(.*\s+|^)(@*goto\s*:*)(\S.*)} {(.*\s+|^)(@*GOTO\s*:*)(\S.*)}] {} + foreach search_regex [list {(.*\s+|^)(@*call\s*:)(\S.*)} {(.*\s+|^)(@*CALL\s*:)(\S.*)} {(.*\s+|^)(@*goto\s*:)(\S.*)} {(.*\s*|.*\s+|^)(@*GOTO\s*:)(\S.*)} {(.*\|\|.*)(@*GOTO\s*:)(\S.*)}] { + if {[regexp $search_regex $callingline_payload _m precall call labelplus]} { + #todo further checks to see if it's actually a batch script line + # - - - - work out what cmd.exe considers start of 512B boundaries when scanning from a callsite + #callposn affected by newlines? + #set callposn [expr {$file_offset + [string length $callingline_payload]}] ;#take callposn as end of line .. review - multiline statements? + set callposn [expr {$file_offset + $callingline_len}] + + #Note there are anomalies around target labels in bracketed sections such as IF blocks + #this is bad practice - as it can lead to unbalanced braces - but batch files can still work under cmd.exe with them in some cases + #e.g unbalanced trailing bracket may be ignored. + #A working script with target-labels in braces can fail due to boundary issues we don't detect (callsite for boundary counting may need to be at end of entire multiline if block??) + #For now - just make sure punk templates don't do this - but it would be nice to be able to detect. + + #set callposn $file_offset + #set callposn [expr {$file_offset + [string length $precall]}] + # - - - - + break + } + } + set callsite_labelfound 0 ;#until proven + if {$callposn != -1} { + set callposn_lineindex [lindex [$objFile chunkrange_to_linerange $callposn $callposn] 0] + #the line represented by callposn may actually be beyond the calling_line_index + set labelinfo [batchlib::get_callsite_label $labelplus] + if {[dict get $labelinfo labelfound]} { + set callsite_labelfound 1 + set label [dict get $labelinfo label] + set call_label_record [list label $label line $callingline_num] + dict lappend call_labels_found $label $call_label_record + } else { + puts stderr "[a+ yellow bold]WARNING - apparent callsite $callposn but couldn't verify label[a]" + puts stderr "Line:\n$trimln" + } + } + + #todo - multiple calls on one line. - also - determine what cmd considers the starting point for forward scanning when call is in a structure such as an if statement. + if {$callsite_labelfound} { + puts stdout "[a+ bold cyan]CALLSITE on line $callingline_num ending at byte $callposn[a]" + set callsummary [string range "${call}${labelplus}" 0 100] + if {[string length $callsummary] < [string length ${call}${labelplus}]} { + puts stdout " CALLSITE: $callsummary (truncated to 100 bytes)" + } else { + puts stdout " CALLSITE: '${call}${labelplus}'" + } + puts stdout " [a+ cyan]FULLINE: $callingline_payload[a]" + + + + ################################## + #set labelpluswords [regexp -inline -all {\S+} $labelplus] ;#don't assume labelplus can be treated as Tcl list - use regexp to split + #NOTE it is invalid to assume label always terminated by space - pair of % characters (variable substitution) can contain a space without terminating label + #set word1 [lindex $labelpluswords 0] + + + ################################## + + + set labelsize [string length $label] + #scan forward for labels at boundaries + set forward_chunk [$objFile chunk $callposn end] + set forward_chunk_base $callposn ;#name for clarity + + incr callid + set callvar "call-${callid}_fromline-${callingline_num}" + upvar 0 $callvar objForwardScan + set objForwardScan [fileline::textinfo new $forward_chunk] + + + + ################################################################################################################################## + #Forward scan 1 - check at normal line boundaries - and see if collides with a chunk boundary - and if the label is obscured or ok + set dsize [$objForwardScan chunklen] + set num_boundaries [expr {$dsize / 512} ] + puts "scanning $dsize forward bytes in file starting at $forward_chunk_base for label '$label' - num_boundaries: $num_boundaries" + set total_offset $file_offset + set found_forward_label 0 + foreach scanlineinfo [$objForwardScan lineinfolist 0 end] { + set scanline_start [dict get $scanlineinfo start] + set scanline_bytes [dict get $scanlineinfo linelen] + set scanline [dict get $scanlineinfo payload] + + set line_start_global [expr {$forward_chunk_base + $scanline_start}] + set line_index_global [lindex [$objFile chunkrange_to_linerange $line_start_global $line_start_global] 0] + set line_num_global [expr {$line_index_global + 1}] + + set trimscanline [string trim $scanline] + + set found_targetlabel_at_line 0 ;# until disproven + if {[string first : $scanline] >= 0} { + set labelinfo [batchlib::get_target_label_from_line $scanline] + if {[dict get $labelinfo labelfound] && [dict get $labelinfo label] eq $label} { + #add to target_labels_found record below + set scan_target_label_record [list label $label line $line_num_global] + set found_targetlabel_at_line 1 + } + } + + if {$found_targetlabel_at_line} { + set scan_target_label_same_line_seen false + if {[dict exists $target_labels_found $label]} { + set thislabel_records [dict get $target_labels_found $label] + foreach previous $thislabel_records { + if {[dict get $previous line] eq $line_num_global} { + set scan_target_label_same_line_seen true + } + } + } + incr found_forward_label + if {!$scan_target_label_same_line_seen} { + set label_posn_in_line [string first : $scanline] + set labelposn [expr {$scanline_start + $label_posn_in_line}] + #we only really care about exactly landing on a boundary or else the next 512 byte boundaries above the labelposn + if {($labelposn % 512) == 0} { + set ubound [expr {($labelposn / 512) * 512}] + } else { + set ubound [expr {(($labelposn / 512)+1) * 512}] + } + set lbound [expr {$ubound - $labelsize}] + if {($labelposn >= $lbound) && ($labelposn <= $ubound)} { + dict set scan_target_label_record error linestart_and_call_offset_bytes + lappend error_labels [list label $label linestart_and_call_offset_bytes $labelposn callsite [list call ${call}${labelplus} call_linenum $callingline_num] bad_target_line $line_num_global] + puts stdout "[a+ bold red]ERROR: file line: $line_num_global target-label $trimscanline at line-beginning and with byte offset from callsite: $labelposn offset in file: $line_start_global[a]" + puts stdout "[a+ bold red] This target-label appears to span the 512byte boundary at byte $ubound[a] [a+ yellow bold]from callsite.[a]" + puts [$objForwardScan chunk_boundary_display [dict get $scanlineinfo start] [dict get $scanlineinfo end] 512 -linebase $callposn_lineindex+1 -limit 1] ;#+1 on callposn_linindex to do editor-style linenums + } else { + dict set scan_target_label_record ok 1 + puts stdout "[a+ bold green]OK: file line: $line_num_global target-label $trimscanline at line-beginning and with byte offset from callsite: $labelposn offset in file: $line_start_global[a]" + } + dict lappend target_labels_found $label $scan_target_label_record + } else { + puts stdout "OK - seen label $label on $line_num_global before" + } + } + incr total_offset $scanline_bytes + } + ################################################################################################################################## + + + #todo + #forward scan 2 - check any boundaries missed above because the label isn't at the begining of a line + #these are potentially hidden labels that could activate without requiring the label be at the beginning of a line + #check boundary spans relative to start of this objForwardScan chunk + + #adjust boundary-search by resetting counter each time crlf encountered + set forward_lines [$objForwardScan chunkrange_to_lineinfolist 0 end] + set boundary_positions [list 0] + set scanner_offset 0 + set scanner_position 0 + foreach forwardbline_info $forward_lines { + #review - do we need to check the payload in case we have configured the textinfo object to split the file only on lf - (not true by default) + set forwardbline_len [dict get $forwardbline_info linelen] + set forwardbline_spaninfo [fileline::range_spans_chunk_boundaries [expr {$scanner_position + $scanner_offset}] [expr {$scanner_position + $scanner_offset + $forwardbline_len}] 512] + set forwardbline_boundaries [dict get $forwardbline_spaninfo boundaries] + + foreach b $forwardbline_boundaries { + set relb [expr $b + $scanner_offset] + if {$relb <= [dict get $forwardbline_info end]} { + lappend boundary_positions $relb + } else { + #leave it for the next line - as we may need to adjust offset anyway + break + } + } + if {[dict get $forwardbline_info le] eq "crlf"} { + set scanner_offset [expr {[dict get $forwardbline_info end] - [lindex $boundary_positions end]}] ;#reset on crlf + #puts "+++++ set scanner_offset $scanner_offset" + } + set scanner_position [dict get $forwardbline_info end] + } + set boundary_positions [lsearch -all -not -inline $boundary_positions 0] + if {[llength $boundary_positions]} { + puts stdout "[a+ blue bold]----> [llength $forward_lines] forward lines, boundaries (possibly with offsets) to check $boundary_positions[a]" + } else { + puts stdout "[a+ blue bold]----> [llength $forward_lines] forward lines, No boundaries to check (generally expected for files with crlf line endings and no extremely long lines)[a]" + } + + + if {[llength $boundary_positions]} { + puts stdout "line $callingline_num scan from call label $label ending at position $callposn. Next Callsite-relative boundary [lindex $boundary_positions 0]" + + for {set i 0} {$i < [llength $boundary_positions]} {incr i} { + set b [lindex $boundary_positions $i] + if {$i < [llength $boundary_positions]-1} { + set nextb [lindex $boundary_positions $i+1] + set top $nextb + } else { + set top end + } + + set forwardbline_infolist [$objForwardScan chunkrange_to_lineinfolist $b $top -show_truncated 1] + set forwardbline_info [lindex $forwardbline_infolist 0] + if {[dict get $forwardbline_info is_truncated]} { + set payload_from_boundary [dict get $forwardbline_info truncated] + } else { + set payload_from_boundary [dict get $forwardbline_info payload] + } + set forwardbline_len [dict get $forwardbline_info linelen] + set forwardbline_index [dict get $forwardbline_info lineindex] + set forwardbline_start [dict get $forwardbline_info start] + set forwardbline_start_global [expr {$forward_chunk_base + $forwardbline_start}] + set forwardbline_index_global [lindex [$objFile chunkrange_to_linerange $forwardbline_start_global $forwardbline_start_global] 0] + set forwardbline_num_global [expr {$forwardbline_index_global + 1}] + + set found_targetlabel_at_boundary 0 + if {[string first : $payload_from_boundary] >= 0} { + #puts stdout "Possible label at boundary $b - testing" + set labelinfo [batchlib::get_target_label_from_line $payload_from_boundary] + if {[dict get $labelinfo labelfound] && [dict get $labelinfo label] eq $label} { + incr found_forward_label + set found_targetlabel_at_boundary 1 + } elseif {[dict get $labelinfo labelfound]} { + set unsearched_label [dict get $labelinfo label] + puts stderr "[a+ cyan]Line $forwardbline_num_global: Found an item that cmd may interpret as a target label because of its location at a boundary $b - but it doesn't seem to be the one we are looking for. Looking for '$label' Found: '[dict get $labelinfo label]'[a]" + puts stderr "[a+ yellow]Warning - if the label '$unsearched_label' on line $forwardbline_num_global isn't meant to be a target - it may be safest to make sure batch script isn't using CALL or GOTO with target :$unsearched_label" + puts stdout "linedata:\n" + #puts stdout "'$payload_from_boundary'" + puts [$objForwardScan chunk_boundary_display [dict get $forwardbline_info start] [dict get $forwardbline_info end] 0 -boundaries $b -linebase $callposn_lineindex+1 -limit 1] + + #dubious value to check call_labels_found - as we didn't run through and find all call labels first! + if {$unsearched_label in [dict keys $call_labels_found]} { + set boundary_target_label_record [list label $unsearched_label line $forwardbline_num_global error found_via_boundary_check_on_a_different_call_label] + dict lappend warning_target_labels_found $unsearched_label $boundary_target_label_record + } else { + set possible_target_label_record [list label $unsearched_label line $forwardbline_num_global] + dict lappend possible_target_labels_found $unsearched_label $possible_target_label_record + } + } else { + set note "" + if {[dict exists $labelinfo note]} { + set note [dict get $labelinfo note] + } + if {$note ne "prefix_fail"} { + puts stdout "no label detected at boundary $b - probably ok. Note from target-label scanner: $note" + } + } + if {$found_targetlabel_at_boundary} { + set target_label_record [list label $label line $forwardbline_num_global error call_offset_bytes] + dict lappend target_labels_found $label $target_label_record + set note "possibly unreliable or dangerous target-label at line $forwardbline_num_global may execute line [expr {$forwardbline_num_global +1}].\n" + append note "Target label not at line start but was found by scanning 512byte chunks from callsite with count resets at any crlf encountered\n" + append note "Adjust spacing between line $callingline_num and $forwardbline_num_global to avoid the 512 boundary - and re-test for other boundary problems" + lappend error_labels [list label $label call_offset_bytes $b callsite [list call ${call}${labelplus} call_linenum $callingline_num] note $note] + puts stdout "[a+ bold red]ERROR: line $forwardbline_num_global target-label [dict get $labelinfo rawlabel] found at boundary and with byte offset from callsite: $b [a]" + puts stdout "[a+ bold red] This target-label appears to fall at or just after the 512byte boundary at byte $b[a] [a+ yellow bold]from callsite.[a]" + puts stdout "[a+ bold yellow]Code may execute at line [expr {$forwardbline_num_global + 1}] (or at next 512Byte boundary in some circumstances)[a]" + puts stdout "[a+ bold yellow]Recommend adjusting spacing between line $callingline_num and $forwardbline_num_global[a]" + puts stdout [$objForwardScan chunk_boundary_display [dict get $forwardbline_info start] [dict get $forwardbline_info end] 0 -boundaries $b -linebase $callposn_lineindex+1 -limit 1] + } + #if found any label - peek at next boundary + if {[dict get $labelinfo labelfound] && $i+1 < [llength $boundary_positions]} { + set next_lineinfolist [$objForwardScan chunkrange_to_lineinfolist $nextb end -show_truncated 1] + set next_lineinfo [lindex $next_lineinfolist 0] + puts "peek next boundary data - line [expr {$forwardbline_num_global + 1}]:" + #if {[dict get $next_lineinfo is_truncated]} { + # puts [dict get $next_lineinfo truncated] + #} else { + # puts [dict get $next_lineinfo payload] + #} + puts [$objForwardScan chunk_boundary_display [dict get $next_lineinfo start] [dict get $next_lineinfo end] 0 -boundaries $nextb -linebase $callposn_lineindex+1 -limit 1] + } + } + } + } + $objForwardScan destroy + + #scan behind for labels at boundaries - using offset from start of file + #we do a backward scan even if a forward label has been found, so that we can warn of duplicate labels. + + set prior_start 0 + set prior_end $callingline_index ;#only scan from file start to call-site + + set pline_begin 0 + set found_backward_label 0 + set p_linenum 0 + for {set pidx $prior_start} {$pidx <= $prior_end} {incr pidx} { + set plineinfo [$objFile lineinfo $pidx] + set pline [dict get $plineinfo payload] + incr p_linenum + set pline_bytes [dict get $plineinfo linelen] ;#includes lf or crlf ending bytes + set pline_start $pline_begin + if {$pline_start != [dict get $plineinfo start]} { + error "checkfile error: line $p_linenum - calculated start $pline_start not equal to stored start [dict get $plineinfo start]" + } + set pline_end [expr {$pline_begin + $pline_bytes -1}] + if {$pline_end != [dict get $plineinfo end]} { + error "checkfile error: line $p_linenum - calculated end $pline_end not equal to stored end [dict get $plineinfo end]" + } + + + set trimpline [string trim $pline] + #todo - process leading part of line before : + #e.g the following are valid (leading # is not part of the examples) + # ====== : label + # also + #%=== == : label + # also + #%= ,,,, ;;; = : label + + #these token delimiters (; , = 0x0B ox0C 0xFF ) + #can also occur after the colon e.g + #: ;label + + #the following is a valid target for @GOTO :#something + #: ;#something + + #It is possible for closing bracket ) to also be invisible if there is no open ( active + #This only seems to work for a single ) at beggining of the line multiple ) even separated by spaces or ; etc seem to stop the target being found. + #The lone unbalanced ) can act like a comment in other contexts - and can appear multiple times, but only if first ) on the line is followed by a delimiter + #Essentially all characters following the first ) are ignored - but if the first is something like )) then cmd tries to interpret that as a command and fails + # e.g + #) ignored + #);)))) ignored + #)) causes error as cmd tries to run "))" as a command. + #This is a reason why *target* labels shouldn't appear in bracketed blocks - as code jumps to a point where ( ) will be unbalanced + + #target labels are literal with regards to % ie not subject to % expansion - but ^ must still be processed + if {[string first : $pline] >= 0} { + #space (and some other chars) allowed between colon and label at target - (but not at callsite) + set labelinfo [batchlib::get_target_label_from_line $pline] + if {[dict get $labelinfo labelfound] && [dict get $labelinfo label] eq $label} { + set target_label_record [list label $label line $p_linenum] + puts stdout "$labelinfo" + incr found_backward_label + set prior_label_posn_in_line [string first : $pline] + set prior_label_posn [expr {$pline_begin + $prior_label_posn_in_line}] + if {($prior_label_posn % 512) == 0} { + set p_ubound [expr {($prior_label_posn / 512) * 512}] + } else { + set p_ubound [expr {(($prior_label_posn /512) +1) * 512}] + } + set p_lbound [expr {$p_ubound - $labelsize}] + if {($prior_label_posn >= $p_lbound) && ($prior_label_posn <= $p_ubound)} { + dict set target_label_record error linestart_and_overlap + lappend error_labels [list label $label linestart_and_overlap $prior_label_posn callsite [list call ${call}${labelplus} call_linenum $callingline_num]] + puts stdout "[a+ bold red]ERROR: target-label '$trimpline' at line $p_linenum and offset from file start: $prior_label_posn line start: $pline_begin[a]" + puts stdout "[a+ bold red] This target-label appears to span the 512byte boundary at byte $p_ubound[a] [a+ yellow bold]from file start[a]" + puts [$objFile chunk_boundary_display [dict get $plineinfo start] [dict get $plineinfo end] 512 -linebase 1 -limit 1] + } else { + dict set target_label_record ok 1 + puts stdout "[a+ bold green]OK: file line: $p_linenum target-label '$trimpline' before call from line $callingline_num. Target is at offset from file start: $prior_label_posn line start: $pline_begin[a]" + } + dict lappend call_labels_found $label $target_label_record + } + #else - label we weren't searching for - even if at file boundary it should be picked up when actually searched? review + } + set spaninfo [fileline::range_spans_chunk_boundaries $pline_start $pline_end 512] + if {[dict get $spaninfo is_span]} { + #puts stdout "boundary spanning line $p_linenum byte range $pline_start -> $pline_end [a+ bold purple]$spaninfo[a]" + #check boundaries within the line + set boundaries [dict get $spaninfo boundaries] + foreach b $boundaries { + if {$b == 0} { + #skip - beginning of line already handled (review?) + continue + } + #overlap test is just a warning - we have a label-like thing overlapping the boundary + #todo - take account of fact that target label can be ": labelname" - so using just labelsize won't detect all overlaps + #The label could even be at the end of a long line that appears at first to be a comment e.g something like + # : whatever : sneakylabel + # or + #@REM ============================================================================================================================================================ : sneakylabel + + #The fact that it overlaps - means it's probably not being found with lf line-endings - and only the label :whatever should be found with crlf endings + #- but we won't always catch that something's fishy + #review + set overlaptail [string range $pline [expr {$b - $labelsize}] [expr {($b + $labelsize) -1}]] ;#subtracting labelsize gives earliest possible overlap + if {[string match "*:$label *" $overlaptail] } { + lappend warning_labels [list label $label warning label_spanning callsite [list call ${call}${labelplus} call_linenum $callingline_num]] + puts stdout "[a+ bold yellow] WARNING: possible label $label spans boundary $b from start of file" + } + + set pline_tail [string range $pline $b end] + + if {[string first : $pline_tail] >= 0} { + set labelinfo [batchlib::get_target_label_from_line $pline_tail] + set labelfound 0 + if {[dict get $labelinfo labelfound] && [dict get $labelinfo label] eq $label} { + set labelfound 1 + } elseif {[dict get $labelinfo labelfound]} { + puts stdout "Note: detected target label [dict get $labelinfo label] at file offset $b at boundary with no preceeding newline - but it's not the one we're currently scanning for" + } + if {$labelfound} { + set label_found_name [dict get $labelinfo label] + incr found_backward_label + + lappend error_labels [list label $label_found_name file_offset_bytes $b note "label at boundary but no preceeding newline - cmd may interpret as label and execute following line or code at next boundary" callsite [list call ${call}${labelplus} call_linenum $callingline_num]] + + puts stdout "[a+ bold red]ERROR: *possible* label '$label_found_name' at line $p_linenum and offset from file start: $b line start: $pline_begin[a]" + puts stdout "[a+ bold red] This label with no preceeding newline appears to span the 512byte boundary at byte $b[a] [a+ yellow bold]from file start[a]" + puts stdout "[a+ bold red] cmd.exe may find this label - but it probably shouldn't be relied upon[a]" + puts stdout "[a+ bold yellow] label starting at $b : $pline_tail[a]" + + set target_label_record [list label $label_found_name line $p_linenum] + if {$label_found_name in [dict keys $call_labels_found]} { + dict set target_label_record error "called_label_at_file_offset_boundary" + dict lappend target_labels_found $label_found_name $target_label_record + } else { + #review - we need to get better at finding all calls! + dict set target_label_record error "uncalled_label_at_file_offset_boundary" + dict lappend possible_target_labels_found $label_found_name $target_label_record + } + + + set tail_start $b + set tail_end [expr {$b + [string length $pline_tail]}] + set tail_spaninfo [fileline::range_spans_chunk_boundaries $tail_start $tail_end 512] + if {[dict get $tail_spaninfo is_span]} { + set tail_boundaries [dict get $tail_spaninfo boundaries] + set extra_tail_boundaries [lsearch -all -inline -not $tail_boundaries $b] + if {[llength $extra_tail_boundaries]} { + puts "Line $p_linenum also spans additional boundaries: $extra_tail_boundaries" + set next_boundary [lindex $extra_tail_boundaries 0] + #boundary doesn't reset if no crlf - we are still within the line - so can calc from line beginning + set next_boundary_data [string range $pline [expr {$pline_begin + $next_boundary}] end] + puts "Line $p_linenum data at next boundary: [a+ yellow bold]$next_boundary_data[a]" + puts [$objFile chunk_boundary_display [dict get $plineinfo start] [dict get $plineinfo end] 0 -boundaries $next_boundary -linebase 1 -limit 1] + + puts "[a+ yellow bold]NOTE: cmd may attempt to treat this data as code[a]" + } + } else { + if {$pidx+1 < [$objFile linecount]} { + set nextlineinfo [$objFile lineinfo $pidx+1] + set nextpayload [dict get $nextlineinfo payload] + puts "Line $p_linenum + 1 has data: [a+ yellow bold]$nextpayload[a]" + puts "[a+ yellow bold]NOTE: cmd may attempt to treat this data as code[a]" + } else { + #EOF reached + } + } + } + } + + } + + } + incr pline_begin $pline_bytes + } + + if {$found_forward_label == 0} { + if {[string toupper $label] eq "EOF"} { + #EOF/eof label is special - it doesn't have to exist - but if it does - it probably shouldn't be spanning a boundary + puts stdout "[a+ bold green]OK: label :$label doesn't exist - but it's usually not meant to. callsite: [list call ${call}${labelplus} call_linenum $callingline_num] [a]" + } else { + if {$found_backward_label == 0} { + lappend warning_labels [list label $label warning label_not_found callsite [list call ${call}${labelplus} call_linenum $callingline_num]] + puts stdout "[a+ bold yellow]WARNING: label :$label not found (in forward or backward scan)[a]" + } + } + } + if {($found_forward_label + $found_backward_label) > 1} { + #puts "target_labels_found: $target_labels_found" + dict for {targetkey targethits} $target_labels_found { + set targetlines [list] + foreach record $targethits { + lappend targetlines [dict get $record line] + } + set remaining [list] + set previous "" ; + foreach lnum [lsort -integer -increasing $targetlines] { + if {$previous eq ""} { + lappend remaining $lnum + } else { + if {$lnum-1 == $previous} { + puts stdout "[a+ green bold]OK[a] - target-label $targetkey appears on immediately adjacent lines $previous and $lnum - assuming it is a boundary-avoidance tactic rather than an inadvertent duplicate" + set remaining [lrange $remaining 0 end-1];#retain latest - we will allow a run of targets on subsequent lines + } + lappend remaining $lnum + } + set previous [lindex $remaining end] + } + if {[llength $remaining] > 1} { + lappend warning_labels [list label $label warning multiple_target_labels_found callsite [list call ${call}${labelplus} call_linenum $callingline_num]] + puts stdout "[a+ bold yellow]WARNING: label :$label seems to appear multiple times[a]" + } + } + } + } + } + incr file_offset $callingline_len ;#including per-line stored line-ending + } + if {[dict size $possible_target_labels_found] > 0} { + #puts stdout "Possibly bogus target-labels: [dict keys $possible_target_labels_found]" + set bogus_summary [list] + dict for {pb pbrecords} $possible_target_labels_found { + if {$pb in [dict keys $call_labels_found]} { + puts stdout "[a+ yellow bold]Warning - target for label $pb was found with a record as being possibly bogus. record: $pbrecords [a]" + puts stdout "[a+ yellow bold]Consider moving this target-label and re-checking[a]" + puts stdout "[a+ yellow bold]It may be a call label line that was found by boundary scanning - which shouldn't really happen[a]" + puts stdout "Call record [dict get $call_labels_found $pb]" + lappend warning_labels [list label $pb warning possibly_bogus_target list_of_target_hits $pbrecords] + } + set blines [list] + foreach rec $pbrecords { + lappend blines [dict get $rec line] + } + lappend bogus_summary [list label $pb found_on_lines $blines] + } + puts stdout "[a+ cyan]Possibly bogus target-labels: $bogus_summary[a]" + puts stdout "These are usually nothing to be concerned about. Some will almost always turn up in a polyglot script that contains batch script." + puts stdout "If some of the label names appear to contain newlines, or are prefixes of or exact matches with legitimate labels - you might consider adjusting the boundary spacing with whitespace or comments to get a different result." + } + set result "" + if {[llength $warning_labels]} { + append result "WARNING:" \n + append result "The following labels had warnings" \n + foreach w $warning_labels { + append result " [a+ bold yellow]$w[a]" \n + } + } + if {[llength $error_labels]} { + append result "ERROR: label location errors found" \n + append result "The following labels appear to span 512 Byte boundaries or occur on boundaries without a preceding newline and are likely to cause batch script errors" \n + append result "For labels spanning boundaries the label is likely to be missed by the batch interpreter" \n + append result "For labels occuring at boundaries but not at the beginning of a line, the label may be interpreted as a label when not expected, and the interpreter may run code on next line or next boundary" \n + append result "Try adding comments and/or spacing between the call site at the call_lineum indicated and the label and then re-test in case there are further boundary collisions" \n + foreach err $error_labels { + append result " [a+ bold red]$err[a]" \n + } + } + if {[dict size $warning_target_labels_found] > 0} { + puts stdout "target-labels with minor warnings: [dict keys $warning_target_labels_found]" + } + append result "call-labels-found: [dict keys $call_labels_found]" \n + append result "target-labels-found: [dict keys $target_labels_found]" \n + if {![llength $warning_labels] && ![llength $error_labels]} { + puts stderr \n + puts stderr "[a+ green bold]OK No warnings or errors considered major enough to return in result.[a]" + } + return $result + } + #specific filepath to just wrap one script at the xxx-pre-launch-suprocess site #scriptset name to substiture multiple scriptset.xxx files at the default locations - or as specified in scriptset.wrapconf proc multishell {filepath_or_scriptset args} { set defaults [dict create\ -askme 1\ -outputfolder "\uFFFF"\ -template "\uFFFF"\ + -returnextra 0\ ] set known_opts [dict keys $defaults] dict for {k v} $args { @@ -126,6 +778,7 @@ namespace eval punk::mix::commandset::scriptwrap { set opt_askme [dict get $opts -askme] set opt_template [dict get $opts -template] set opt_outputfolder [dict get $opts -outputfolder] + set opt_returnextra [dict get $opts -returnextra] # -- --- --- --- --- --- --- --- --- --- --- --- @@ -141,13 +794,14 @@ namespace eval punk::mix::commandset::scriptwrap { set specified_path [file join $startdir $filepath_or_scriptset] } set ext [string trim [file extension $filepath_or_scriptset] .] - set allowed_extensions [list wrapconfig tcl ps1 sh bash] + set allowed_extensions [list wrapconfig tcl ps1 sh bash pl] + set extension_langs [list tcl tcl ps1 powershell sh sh bash bash pl perl] #set allowed_extensions [list tcl] set found_script 0 if {[file exists $specified_path]} { set found_script 1 } else { - foreach e $allowed_extensions { + foreach e [concat $allowed_extensions [string toupper $allowed_extensions]] { if {[file exists $filepath_or_scriptset.$e]} { set found_script 1 break @@ -328,7 +982,15 @@ namespace eval punk::mix::commandset::scriptwrap { } set output_file [file join $output_folder $scriptset.$output_extension] if {[file exists $output_file]} { - error "wrap_in_multishell: target file $output_file already exists.. aborting" + set fdexisting [open $output_file r] + fconfigure $fdexisting -translation binary + set existing_file_data [read $fdexisting] + close $fdexisting + set objFile_existing [fileline::textinfo new $existing_file_data] + puts stdout "wrap_in_multishell: target file $output_file already exists. File size: [$objFile_existing chunklen] Line count: [$objFile_existing linecount]" + + $objFile_existing destroy + error "aborting.." } @@ -356,7 +1018,7 @@ namespace eval punk::mix::commandset::scriptwrap { lappend list_input_files $scriptroot/$scriptset.$ext } - #todo - split template at each etc marker and build a dict of parts + #todo - split template at each etc marker and build a dict of parts #hack - process one input @@ -374,7 +1036,9 @@ namespace eval punk::mix::commandset::scriptwrap { puts stdout $ln } puts stdout "-----------------------------------------------\n" - puts stdout "Target for above data is '$output_file'" + puts stdout "Target for above script data is '$output_file'" + set lang [dict get $extension_langs [string tolower $ext]] + puts stdout "Language of script being wrapped is $lang" if {$opt_askme} { set answer [util::askuser "Does this look correct? Y|N"] if {[string tolower $answer] ne "y"} { @@ -389,9 +1053,9 @@ namespace eval punk::mix::commandset::scriptwrap { set existing_payload [list] foreach ln $template_lines { - if {[string match "#*" $ln]} { + if {[string match "#<$lang-pre-launch-subprocess>*" $ln]} { set start_idx $line_idx - } elseif {[string match "#*" $ln]} { + } elseif {[string match "#*" $ln]} { set end_idx $line_idx break } elseif {$start_idx > 0} { @@ -404,20 +1068,20 @@ namespace eval punk::mix::commandset::scriptwrap { incr line_idx } if {($start_idx == 0) || ($end_idx == 0)} { - error "wrap_in_multishell was unable to find payload area in template marked with # and # on separate lines" + error "wrap_in_multishell was unable to find payload area in template marked with #<$lang-pre-launch-subprocess> and # on separate lines" } set existing_string [join $existing_payload \n] if {[string length [string trim $existing_string]]} { - puts stdout "EXISTING PAYLOAD!!" + puts stdout "EXISTING <$lang-pre-launch-subprocess> PAYLOAD!!" puts stdout "-----------------------------------------------\n" puts stdout $existing_string puts stdout "-----------------------------------------------\n" - error "wrap_in_multishell found existing payload.. aborting." + error "wrap_in_multishell found existing payload for language $lang ... aborting." #todo - allow overwrite only in files outside of punkshell distribution? if 0 { - puts stderr "Found existing payload.. overwrite?" + puts stderr "Found existing $lang payload.. overwrite?" if {$opt_askme} { - set answer [util::askuser "Are you sure you want to replace the tcl payload shown above? Y|N"] + set answer [util::askuser "Are you sure you want to replace the $lang payload shown above? Y|N"] if {[string tolower $answer] ne "y"} { puts stderr "mix new aborting due to user response '$answer' (required Y|y to proceed) use -askme 0 to avoid prompts." return @@ -436,78 +1100,80 @@ namespace eval punk::mix::commandset::scriptwrap { puts -nonewline $fdtarget $newscript close $fdtarget puts stdout "Wrote script file at $output_file" - + set check_result [checkfile $output_file] + set with_errors "" + set with_warnings "" + set call_labels [list] + set target_labels [list] + set errorlist [list] + set warninglist [list] + if {$check_result ne ""} { + puts stdout $check_result + set check_lines [split $check_result \n] + foreach cl $check_lines { + set trimcl [string trim $cl] + if {[string match "ERROR:*" $trimcl]} { + set with_errors "[a+ bold red]with errors[a]" + lappend errorlist $trimcl + } + if {[string match "WARNING:*" $trimcl]} { + set with_warnings "[a+ bold yellow] with warnings[a]" + lappend errorlist $trimcl + } + if {[string match "call-labels-found:*" $trimcl]} { + set call_labels [string trim [string range $trimcl [string length "call-labels-found:"] end]] + } + if {[string match "target-labels-found:*" $trimcl]} { + set target_labels [string trim [string range $trimcl [string length "target-labels-found:"] end]] + } + } + } else { + puts stderr "Expected output from checkfile - but got none" + } #even though chmod might exist on windows - we will leave permissions alone if {$::tcl_platform(platform) ne "windows"} { catch {exec chmod +x $output_file} } - puts stdout "-done-" - return $output_file + puts stdout "-done- $with_errors $with_warnings" + if {$opt_returnextra} { + set result [list filename $output_file batch_call_labels $call_labels batch_target_labels $target_labels] + if {[llength $warninglist]} { + dict set result warnings $warninglist + } + if {[llength $errorlist]} { + dict set result errors $errorlist + } + } else { + set result [list filename $output_file] + } + + return $result } namespace eval lib { - #get_wrapper_folders - # scriptpath - file or folder - # It represents the base point from which to search for /wrapper folders either directly above the scriptpath or in the containing project if any - # The cwd will also be searched for /wrapper folder and project - but with lower precedence in the resultset (later in list) - proc get_wrapper_folders {{scriptpath ""}} { - set wrapper_folders [list] - if {$scriptpath ne ""} { - if {[file type $scriptpath] eq "file"} { - set searchbase [file dirname $scriptpath] - } else { - set searchbase $scriptpath - } - if {[file isdirectory [file join $searchbase wrappers]]} { - lappend wrapper_folders [file join $searchbase wrappers] - } - set pathinfo [punk::repo::find_repos $searchbase] - set scriptpath_projectroot [dict get $pathinfo closest] - if {$scriptpath_projectroot ne ""} { - set fld [file join $scriptpath_projectroot src/scriptapps/wrappers] - if {[file isdirectory $fld]} { - if {$fld ni $wrapper_folders} { - lappend wrapper_folders $fld - } - } - } - } - set searchbase [pwd] - set fld [file join $searchbase wrappers] - if {[file isdirectory $fld]} { - if {$fld ni $wrapper_folders} { - lappend wrapper_folders $fld - } - } - set pathinfo [punk::repo::find_repos $searchbase] - set pwd_projectroot [dict get $pathinfo closest] - if {$pwd_projectroot ne ""} { - set fld [file join $pwd_projectroot src/scriptapps/wrappers] - if {[file isdirectory $fld]} { - if {$fld ni $wrapper_folders} { - lappend wrapper_folders $fld - } - } - } - set template_base_dict [punk::mix::base::lib::get_template_basefolders] - set tpldirs [list] - dict for {tdir tsourceinfo} $template_base_dict { - if {[file exists $tdir/utility/scriptappwrappers]} { - lappend tpldirs $tdir - } - } - foreach tpldir $tpldirs { - set fld [file join $tpldir utility scriptappwrappers] - if {[file isdirectory $fld]} { - if {$fld ni $wrapper_folders} { - lappend wrapper_folders $fld - } + proc get_wrapper_folders {args} { + set opts [dict get [punk::get_leading_opts_and_values { + -scriptpath "" + } $args -maxvalues 0] opts] + # -- --- --- --- --- --- --- --- --- + set opt_scriptpath [dict get $opts -scriptpath] + # -- --- --- --- --- --- --- --- --- + + set wrapper_template_bases [list] + set tbasedict [punk::mix::base::lib::get_template_basefolders $opt_scriptpath] + dict for {tbase folderinfo} $tbasedict { + set wrapf [file join $tbase utility/scriptappwrappers] + if {[file isdirectory $wrapf]} { + lappend wrapper_template_bases [list basefolder $wrapf sourceinfo $folderinfo] } } - return $wrapper_folders + return $wrapper_template_bases } + + + proc _scriptapp_tag_from_line {line} { set result [list istag 0 raw ""] ;#default assumption. All #---- @@ -656,6 +1322,238 @@ namespace eval punk::mix::commandset::scriptwrap { } + namespace eval batchlib { + # + #see also: https://www.dostips.com/forum/viewtopic.php?t=3803 'Rules for label names vs GOTO and CALL + # review - we may need different get_callsite_label functions? + + proc get_callsite_label {labelstart} { + #labelstart is the character immediately following the colon (which is optional at callsite) - a label such as ::label doesn't seem valid at call or target sites + #e.g for @goto %= possible comment=% :mylabe%%l etc + #we would expect to be passed only "mylabe%%1 etc" + #It is up to the caller to determine where a callsite label begins. + #note that: + #@REM ----- + #@goto ^ + #:label + #@REM----- + # is a valid callsite - but doesn't appear to be found by the label scanner as it's own target label even though :label is on it's own line from non-batch perspective + # so the caller will have to do some batch-style line processing to find all call sites + #Also, for the following 2 lines + #@REM ^ + #:label + # the label will be found - yet if the :label was a command such as @GOTO - it would not be run as a callsite + + + #a quick'n'dirty fix for some ways various escapes are handled within labels at callsite. + #There seem to be very different rules for labels at target site - presumably because they are not part of a command + # Mostly it seems target labels are more literal with regards to % chars - but ^ are processed the same way at target label + #some rules.. + #callsite labels can't have space between : and label - but target labels can + #label terminated by =,: even if prefixed by ^ and even if in squotes or dquotes + #squotes and dquotes otherwise pass through as part of label + #may resolve variables within the label - but characters from variable value can terminate. + #as we don't have access to the variable values - we should normalize %varname% to empty string at callsite - but perhaps emit warning somewhere + # The target labels don't seem to + #a single % resolves to empty - depending. (starts invar processing - and decides if it was a var depending on whether it was closed?) + #sequences of % don't begin a var - number of % in labelname = number of %s divided by 2 and rounded down. ie 1->0 2->1 3-> 1 4->2 5->2 6->3 etc + #spaces in % wrapped var names don't terminate label + #spaces aren't escaped by ^ or quoting + #sequences of ^ seem to follow same counting rule as % + #e.g @goto :la%path%bel where path begins with C:\Program Files.. becomes label :laC + + if {[string index $labelstart 0] in [list : " " \t = {;}]} { + #return everything as tail - nothing was consumed + return [list labelfound 0 note "invalid first character for callsite label" tail $labelstart] + } + + #The due to whitespace and most chars except : and % being alowed inside vars - it seems the best first step + # -------------- start % handling % + set inputchars [split $labelstart ""] + set percentrun 0 ;#0|1 because we use invar-toggling rather than running total of number of percents in a sequence + set invar 0 + set labelout "" + set varsfound [list] + set varterminals [list :] + set labelterminals [list + , {;} = " " \t] + set varname "" + set caretseq 0 + set inputconsumed 0 + foreach c $inputchars { + if {!$invar} { + if {$c eq "%"} { + set caretseq 0 + set lookahead [lrange $inputchars $inputconsumed+1 end] + if {"%" in $lookahead} { + set invar 1 + incr percentrun + } else { + incr percentrun + } + } elseif {$c eq "^"} { + if {$caretseq} { + set caretseq 0 + append labelout "^" ;#for every pair encountered in direct sequence - second gets included in label + } else { + set caretseq 1 + } + } else { + set caretseq 0 + if {$percentrun && ($c in [list 0 1 2 3 4 5 6 7 8 9])} { + #subst %i with value - here we have no way of getting that - so use blank - user will get warning that target not found + set percentrun 0 + } else { + append labelout [string repeat % [expr {$percentrun / 2}]] + set percentrun 0 + if {$c in $labelterminals} { + break + } + append labelout $c + } + } + } else { + #in var - don't do anything with carets(?) + if {$c eq "%" && $percentrun == 1} { + #double percent - rather than just an empty var - emit one % + append labelout % + set invar 0 + set percentrun 0 + } elseif {$c eq "%"} { + #presume percentrun is 0 + set invar 0 + lappend varsfound $varname; set varname "" + } elseif {$c in $varterminals} { + set invar 0 + lappend varsfound $varname; set varname "" + } else { + if {$percentrun && ($c in [list 0 1 2 3 4 5 6 7 8 9])} { + #review - seems to terminate var - and substitute? + #this branch untested - case where we have %i and further % - what if it was %1var% ? does %1 get substituted ? or %1var% - test + set invar 0 + append varname $c + } else { + append varname $c + } + set percentrun 0 + } + } + incr inputconsumed + } + # -------------- end % handling % + set tail [string range $labelstart $inputconsumed end] + #caret -- etc + if {$labelout eq ""} { + set resultdict [dict create labelfound 0] + if {[llength $varsfound]} { + dict set resultdict vars $varsfound + dict set resultdict note "empty label but vars exist - may be legit" + } else { + dict set resultdict note "empty label - no vars" + } + dict set resultdict tail $tail + return $resultdict + } + + return [list labelfound 1 label $labelout tail $tail] + } + proc get_target_label_from_line {labelline} { + #scan a whole line - or a 'line' starting at some chunk boundary we found for a label + #caller should resolve any trailing caret and subsequent line and include them in the call + #note that we may be scanning all sorts of things in a polyglot file - but we're interested in seeing if cmd.exe might interpret it as a label + #target labels don't have %var% processing - they will be literal + set firstcolon [string first : $labelline] + if {$firstcolon == -1} { + return [list labelfound 0 note "no_colon"] + } + set prefixpart [string range $labelline 0 $firstcolon-1] + set targetpart [string range $labelline $firstcolon+1 end] + + set prefixok 1;#default assumption + set invisible_prefix_chars [list {;} , = " " \t] + set prefixchars [split $prefixpart ""] + # % and ^ in the prefix - whether doubled etc or not - will stop label being found + #ANY first char seems to be allowed in prefixpart (it won't be colon, because we already split on that) + #perhaps this is done by cmd.exe to reduce off-by-one errors?? weird... + # but it does allow labels to be found in certain # tcl/bashsh comment lines, which could be both dangerous and ...useful. + #start prefix check at char 1 instead of 0 + foreach pchar [lrange $prefixchars 1 end] { + if {$pchar ni $invisible_prefix_chars} { + set prefixok 0 + break + } + } + if {!$prefixok} { + return [list labelfound 0 note "prefix_fail"] + } + + #no problems before colon - now see if targetpart can be interpreted as a label + #we again have some potential invisible chars before label begins. + set charindex [expr {$firstcolon +1}] ;#track position so we can return index of where we believe label begins + set targetchars [split $targetpart ""] + set inlabel 0 + set labelposn -1 + # --- + set inlabel_terminals [list : + " " \t \r \n] ;# , ; = don't seem to terminate a target label, but do terminate a calling label + # + and whitespace terminate caller and target + # --- + # consider: + #@goto :14^ + # :14^ + #caller is searching for label "14" but won't match - presumably target scanner has escaped the trailing space + set label "" + set rawlabel "" + set caretseq 0 ;# 0|1 + foreach tchar $targetchars { + if {$tchar in [list + :]} { + break + } + if {!$inlabel} { + if {$tchar ni $invisible_prefix_chars} { + #beginning of target label + set labelposn $charindex + set inlabel 1 + append rawlabel $tchar + if {$tchar eq "^"} { + set caretseq 1 + } else { + append label $tchar + } + } + } else { + if {$tchar in $inlabel_terminals} { + #caret stops them from terminating + if {$caretseq} { + set caretseq 0 + append label $tchar + append rawlabel $tchar + } else { + break + } + } else { + append rawlabel $tchar + if {$tchar eq "^"} { + if {$caretseq} { + set caretseq 0 + append label "^" ;#for every pair encountered in direct sequence - second gets included in label + } else { + set caretseq 1 + } + } else { + set caretseq 0 + append label $tchar ;#for target labels - all including %var% is directly part of the label target + } + } + } + incr charindex + } + if {$labelposn == -1} { + return [list labelfound 0 note "no_label_found_after_colon"] + } + + #return rawlabel so we can see it as it appears in the data - as opposed to how it is interpreted as a label by cmd.exe + return [list labelfound 1 label $label rawlabel $rawlabel] + } + } } diff --git a/src/bootsupport/modules/punk/mix/templates-0.1.0.tm b/src/bootsupport/modules/punk/mix/templates-0.1.0.tm index 46065bda..5521ad8a 100644 --- a/src/bootsupport/modules/punk/mix/templates-0.1.0.tm +++ b/src/bootsupport/modules/punk/mix/templates-0.1.0.tm @@ -26,19 +26,28 @@ namespace eval punk::mix::templates { variable pkg punk::mix::templates variable cap_provider - #punk::cap::register_package punk::mix::templates [list\ - # {punk.templates {relpath ../templates}}\ - #] - namespace eval capsystem { if {[info commands capprovider.registration] eq ""} { punk::cap::class::interface_capprovider.registration create capprovider.registration oo::objdefine capprovider.registration { method get_declarations {} { set decls [list] - lappend decls [list punk.templates {relpath ../templates}] - lappend decls [list punk.templates {relpath ../templates2}] - lappend decls [list punk.test {something blah}] + lappend decls [list punk.templates {path templates pathtype adhoc vendor _project}] ;#todo - split out to a different provider package? + + lappend decls [list punk.templates {path templates pathtype module vendor punk}] + #only punk::templates is allowed to register a _multivendor path - review + #other punk.template providers should use module, absolute, currentproject and shellproject pathtypes only + lappend decls [list punk.templates {path src/decktemplates pathtype currentproject_multivendor vendor punk}] + lappend decls [list punk.templates {path decktemplates pathtype shellproject_multivendor vendor punk}] + + + #we need a way to ensure we don't pull updates from a remote repo into a local project that is actually the same project ? review! + #need flags as to whether/how provider allows template updates that are out of sync with the provider pkg version + #perhaps a separate .txt file (alongside buildversion and description txt files) that has some package require statements (we can't put them in the template itself as the filled template may have nothing to do with the punk.templates provider) + lappend decls [list punk.templates {path src/decktemplates/vendor/punk pathtype currentproject vendor punk allowupdates 0 repo "https://www.gitea1.intx.com.au/jn/punkshell" reposubdir "src/decktemplates/vendor/punk"}] + lappend decls [list punk.isbogus {provider punk::mix::templates something blah}] ;#some capability for which there is no handler to validate - therefore no warning will result. + #review - we should report unhandled caps somewhere, or provide a mechanism to detect/report. + #we don't want to warn at the time this provider is loaded - as handler may legitimately be loaded later. return $decls } } @@ -62,6 +71,7 @@ namespace eval punk::mix::templates { #provider api # -- --- #none - declarations only + #todo - template folder install/update/status methods? } diff --git a/src/modules/punk/mix/templates/layouts/project/src/_vfscommon/lib/common_vfs_libs b/src/bootsupport/modules/punk/mix/templates/layout_refs/project@vendor+punk+project-0.1.ref similarity index 100% rename from src/modules/punk/mix/templates/layouts/project/src/_vfscommon/lib/common_vfs_libs rename to src/bootsupport/modules/punk/mix/templates/layout_refs/project@vendor+punk+project-0.1.ref diff --git a/src/bootsupport/modules/punk/mix/templates/modules/modulename_buildversion.txt b/src/bootsupport/modules/punk/mix/templates/modules/modulename_buildversion.txt new file mode 100644 index 00000000..53815fbd --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/modules/modulename_buildversion.txt @@ -0,0 +1,3 @@ +%Major.Minor.Level% +#First line must be a semantic version number +#all other lines are ignored. diff --git a/src/bootsupport/modules/punk/mix/templates/modules/modulename_description.txt b/src/bootsupport/modules/punk/mix/templates/modules/modulename_description.txt new file mode 100644 index 00000000..ddb209af --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/modules/modulename_description.txt @@ -0,0 +1,10 @@ +Identifier: %package% +Version: %version% +Title: %title% +Creator: %name% <%email%> +Description: %description% +Rights: BSD +URL: %url% +Available: +Architecture: tcl +Subject: diff --git a/src/bootsupport/modules/punk/mix/templates/utility/a b/tcltest.bat b/src/bootsupport/modules/punk/mix/templates/utility/a b/tcltest.bat new file mode 100644 index 00000000..4f798a83 --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/utility/a b/tcltest.bat @@ -0,0 +1,7 @@ +::lindex tcl;#\ +@call tclsh "%~dp0%~n0.bat" %* & goto :eof +# --- --- --- --- --- --- --- --- --- --- --- --- ---begin Tcl +puts stdout "script: [info script]" +puts stdout "argv: $::argc" +puts stdout "args: '$::argv'" + diff --git a/src/modules/punk/mix/templates/layouts/project/src/scriptapps/wrappers/sample_punk-multishell.cmd b/src/bootsupport/modules/punk/mix/templates/utility/multishell.cmd similarity index 100% rename from src/modules/punk/mix/templates/layouts/project/src/scriptapps/wrappers/sample_punk-multishell.cmd rename to src/bootsupport/modules/punk/mix/templates/utility/multishell.cmd diff --git a/src/bootsupport/modules/punk/mix/templates/utility/multishell.ps1 b/src/bootsupport/modules/punk/mix/templates/utility/multishell.ps1 new file mode 100644 index 00000000..c2905c97 --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/utility/multishell.ps1 @@ -0,0 +1,256 @@ +set -- "$@" "a=[list shebangless punk MULTISHELL tclsh sh bash cmd pwsh powershell;proc Hide s {proc $s args {}}; Hide :;rename set s2;Hide set;s2 1 list]"; set -- : "$@"; $1 = @' +: heredoc1 - hide from powershell (close sqote for unix shells) ' \ +: << 'HEREDOC1B_HIDE_FROM_BASH_AND_SH' +: .bat/.cmd launch section, leading colon hides from cmd, trailing slash hides next line from tcl \ +: "[Hide @ECHO; Hide ); Hide (;Hide echo]#not necessary but can help avoid errs in testing" +: Continuation char at end of this line and rem with curly-braces used to exlude Tcl from the whole cmd block \ +@REM { +@REM DO NOT MODIFY FIRST LINE OF THIS SCRIPT. shebang #! line is not required and will reduce functionality. +@REM Even comment lines can be part of the functionality of this script - modify with care. +@REM Change the value of nextshell in the next line if desired, and code within payload sections as appropriate. +@SET "nextshell=pwsh" +@REM nextshell set to pwsh,sh,bash or tclsh +@REM @ECHO nextshell is %nextshell% +@SET "validshells=pwsh,sh,bash,tclsh" +@CALL SET keyRemoved=%%validshells:%nextshell%=%% +@REM Note that 'powershell' e.g v5 is just a fallback for when pwsh is not available +@REM ## ### ### ### ### ### ### ### ### ### ### ### ### ### +@REM -- cmd/batch file section (ignored on unix) +@REM -- This section intended only to launch the next shell +@REM -- Avoid customising this if possible. cmd/batch script is probably the least expressive language. +@REM -- custom windows payloads should be in powershell,tclsh or sh/bash code sections +@REM ## ### ### ### ### ### ### ### ### ### ### ### ### ### +@SETLOCAL EnableExtensions EnableDelayedExpansion +@SET "winpath=%~dp0" +@SET "fname=%~nx0" +@REM @ECHO fname %fname% +@REM @ECHO winpath %winpath% +@IF %nextshell%==pwsh ( + CALL pwsh -nop -c set-executionpolicy -Scope CurrentUser RemoteSigned + COPY "%~dp0%~n0.cmd" "%~dp0%~n0.ps1" >NUL + REM test availability of preferred option of powershell7+ pwsh + CALL pwsh -nop -nol -c write-host "statusmessage: pwsh-found" >NUL + SET pwshtest_exitcode=!errorlevel! + REM ECHO pwshtest_exitcode !pwshtest_exitcode! + IF !pwshtest_exitcode!==0 CALL pwsh -nop -nol "%~dp0%~n0.ps1" %* & SET task_exitcode=!errorlevel! + REM fallback to powershell if pwsh failed + IF NOT !pwshtest_exitcode!==0 ( + REM CALL powershell -nop -nol -c write-host powershell-found + CALL powershell -nop -nol -file "%~dp0%~n0.ps1" %* + SET task_exitcode=!errorlevel! + ) +) ELSE ( + IF %nextshell%==bash ( + CALL :getWslPath %winpath% wslpath + REM ECHO wslfullpath "!wslpath!%fname%" + CALL %nextshell% "!wslpath!%fname%" %* & SET task_exitcode=!errorlevel! + ) ELSE ( + REM probably tclsh or sh + IF NOT "x%keyRemoved%"=="x%validshells%" ( + REM sh uses /c/ instead of /mnt/c - at least if using msys. Todo, review what is the norm on windows with and without msys2,cygwin,wsl + REM and what logic if any may be needed. For now sh with /c/xxx seems to work the same as sh with c:/xxx + CALL %nextshell% "%~dp0%fname%" %* & SET task_exitcode=!errorlevel! + ) ELSE ( + ECHO %fname% has invalid nextshell value %nextshell% valid options are %validshells% + SET task_exitcode=66 + GOTO :exit + ) + ) +) +@GOTO :endlib +:getWslPath +@SETLOCAL + @SET "_path=%~p1" + @SET "name=%~nx1" + @SET "drive=%~d1" + @SET "rtrn=%~2" + @SET "result=/mnt/%drive:~0,1%%_path:\=/%%name%" +@ENDLOCAL & ( + @if "%~2" neq "" ( + SET "%rtrn%=%result%" + ) ELSE ( + ECHO %result% + ) +) +@GOTO :eof +:endlib + +: \ +@REM @SET taskexit_code=!errorlevel! & goto :exit +@GOTO :exit +# } +# rem call %nextshell% "%~dp0%~n0.cmd" %* +# -*- tcl -*- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- tcl script section +# -- This is a punk multishell file +# -- Primary payload target is Tcl, with sh,bash,powershell as helpers +# -- but it may equally be used with any of these being the primary script. +# -- It is tuned to run when called as a batch file, a tcl script a sh/bash script or a pwsh/powershell script +# -- i.e it is a polyglot file. +# -- The specific layout including some lines that appear just as comments is quite sensitive to change. +# -- It can be called on unix or windows platforms with or without the interpreter being specified on the commandline. +# -- e.g ./filename.polypunk.cmd in sh or bash +# -- e.g tclsh filename.cmd +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +rename set ""; rename s2 set; set k {-- "$@" "a}; if {[info exists ::env($k)]} {unset ::env($k)} ;# tidyup +Hide :exit;Hide {<#};Hide '@ +namespace eval ::punk::multishell { + set last_script_root [file dirname [file normalize ${argv0}/__]] + set last_script [file dirname [file normalize [info script]/__]] + if {[info exists argv0] && + $last_script eq $last_script_root + } { + set ::punk::multishell::is_main($last_script) 1 ;#run as executable/script - likely desirable to launch application and return an exitcode + } else { + set ::punk::multishell::is_main($last_script) 0 ;#sourced - likely to be being used as a library - no launch, no exit. Can use return. + } + if {"::punk::multishell::is_main" ni [info commands ::punk::multishell::is_main]} { + proc ::punk::multishell::is_main {{script_name {}}} { + if {$script_name eq ""} { + set script_name [file dirname [file normalize [info script]/--]] + } + return [set ::punk::multishell::is_main($script_name)] + } + } +} +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin Tcl Payload +#puts "script : [info script]" +#puts "argcount : $::argc" +#puts "argvalues: $::argv" +#puts "argv0 : $::argv0" +# -- --- --- --- --- --- --- --- --- --- --- --- + + + +# + + + +# -- --- --- --- --- --- --- --- --- --- --- --- +# -- Best practice is to always return or exit above, or just by leaving the below defaults in place. +# -- If the multishell script is modified to have Tcl below the Tcl Payload section, +# -- then Tcl bracket balancing needs to be carefully managed in the shell and powershell sections below. +# -- Only the # in front of the two relevant if statements below needs to be removed to enable Tcl below +# -- but the sh/bash 'then' and 'fi' would also need to be uncommented. +# -- This facility left in place for experiments on whether configuration payloads etc can be appended +# -- to tail of file - possibly binary with ctrl-z char - but utility is dependent on which other interpreters/shells +# -- can be made to ignore/cope with such data. +if {[::punk::multishell::is_main]} { + exit 0 +} else { + return +} +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end Tcl Payload +# end hide from unix shells \ +HEREDOC1B_HIDE_FROM_BASH_AND_SH +# sh/bash \ +shift && set -- "${@:1:$#-1}" +#------------------------------------------------------ +# -- This if block only needed if Tcl didn't exit or return above. +if false==false # else { + then + : +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- sh/bash script section +# -- leave as is if all that is required is launching the Tcl payload" +# -- +# -- Note that sh/bash script isn't called when running a .bat/.cmd from cmd.exe on windows by default +# -- adjust @call line above ... to something like @call sh ... @call bash .. or @call env sh ... etc as appropriate +# -- if sh/bash scripting needs to run on windows too. +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin sh Payload +#printf "start of bash or sh code" + +# + +# -- --- --- --- --- --- --- --- +exitcode=0 ;#default assumption +#-- sh/bash launches Tcl here instead of shebang line at top +#-- use exec to use exitcode (if any) directly from the tcl script +#exec /usr/bin/env tclsh "$0" "$@" +#-- alternative - can run sh/bash script after the tcl call. +/usr/bin/env tclsh "$0" "$@" +exitcode=$? +#echo "tcl exitcode: ${exitcode}" +#-- override exitcode example +#exit 66 +# -- --- --- --- --- --- --- --- + +# + + +#printf "sh/bash done \n" +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end sh Payload +#------------------------------------------------------ +fi +exit ${exitcode} +# end hide sh/bash block from Tcl +# This comment with closing brace should stay in place whether if commented or not } +#------------------------------------------------------ +# begin hide powershell-block from Tcl - only needed if Tcl didn't exit or return above +if 0 { +: end heredoc1 - end hide from powershell \ +'@ +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- powershell/pwsh section +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +function GetScriptName { $myInvocation.ScriptName } +$scriptname = getScriptName +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin powershell Payload +#"Timestamp : {0,10:yyyy-MM-dd HH:mm:ss}" -f $(Get-Date) | write-host +#"Script Name : {0}" -f $scriptname | write-host +#"Powershell Version: {0}" -f $PSVersionTable.PSVersion.Major | write-host +#"powershell args : {0}" -f ($args -join ", ") | write-host +# -- --- --- --- + +# + + +# -- --- --- --- --- --- --- --- +tclsh $scriptname $args +# -- --- --- --- --- --- --- --- + + +# + +# unbal } + +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end powershell Payload +#"powershell reporting exitcode: {0}" -f $LASTEXITCODE | write-host +Exit $LASTEXITCODE +# heredoc2 for powershell to ignore block below +$1 = @' +' +: end hide powershell-block from Tcl \ +# This comment with closing brace should stay in place whether 'if' commented or not } +: cmd exit label - return exitcode +:exit +: \ +@REM @ECHO exitcode: !task_exitcode! +: \ +@EXIT /B !task_exitcode! +# cmd has exited +: end heredoc2 \ +'@ +<# +# id:tailblock0 +# -- powershell multiline comment +#> +<# +# id:tailblock1 +# + +# +# -- unreachable by tcl directly if ctrl-z character is in the section above. (but file can be read and split on \x1A) +# -- Potential for zip and/or base64 contents, but we can't stop pwsh parser from slurping in the data +# -- so for example a plain text tar archive could cause problems depending on the content. +# -- final line in file must be the powershell multiline comment terminator or other data it can handle. +# -- e.g plain # comment lines will work too +# -- (for example a powershell digital signature is a # commented block of data at the end of the file) +#> + + diff --git a/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/basic/multishell-old.cmd b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/basic/multishell-old.cmd new file mode 100644 index 00000000..1cb9e0ef --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/basic/multishell-old.cmd @@ -0,0 +1,270 @@ +set -- "$@" "a=[list shebangless punk MULTISHELL tclsh sh bash cmd pwsh powershell;proc Hide s {proc $s args {}}; Hide :;rename set s2;Hide set;s2 1 list]"; set -- : "$@"; $1 = @' +: heredoc1 - hide from powershell (close sqote for unix shells) ' \ +: << 'HEREDOC1B_HIDE_FROM_BASH_AND_SH' +: .bat/.cmd launch section, leading colon hides from cmd, trailing slash hides next line from tcl \ +: "[Hide @ECHO; Hide ); Hide (;Hide echo]#not necessary but can help avoid errs in testing" +: Continuation char at end of this line and rem with curly-braces used to exlude Tcl from the whole cmd block \ +@REM { +@REM DO NOT MODIFY FIRST LINE OF THIS SCRIPT. shebang #! line is not required and will reduce functionality. +@REM Even comment lines can be part of the functionality of this script - modify with care. +@REM Change the value of nextshell in the next line if desired, and code within payload sections as appropriate. +@SET "nextshell=pwsh" +@REM nextshell set to pwsh,sh,bash or tclsh +@REM @ECHO nextshell is %nextshell% +@SET "validshells=pwsh,sh,bash,tclsh" +@CALL SET keyRemoved=%%validshells:%nextshell%=%% +@REM Note that 'powershell' e.g v5 is just a fallback for when pwsh is not available +@REM ## ### ### ### ### ### ### ### ### ### ### ### ### ### +@REM -- cmd/batch file section (ignored on unix) +@REM -- This section intended only to launch the next shell +@REM -- Avoid customising this if possible. cmd/batch script is probably the least expressive language. +@REM -- custom windows payloads should be in powershell,tclsh or sh/bash code sections +@REM ## ### ### ### ### ### ### ### ### ### ### ### ### ### +@SETLOCAL EnableExtensions EnableDelayedExpansion +@SET "winpath=%~dp0" +@SET "fname=%~nx0" +@REM @ECHO fname %fname% +@REM @ECHO winpath %winpath% +@IF %nextshell%==pwsh ( + CALL pwsh -nop -c set-executionpolicy -Scope CurrentUser RemoteSigned + COPY "%~dp0%~n0.cmd" "%~dp0%~n0.ps1" >NUL + REM test availability of preferred option of powershell7+ pwsh + CALL pwsh -nop -nol -c write-host "statusmessage: pwsh-found" >NUL + SET pwshtest_exitcode=!errorlevel! + REM ECHO pwshtest_exitcode !pwshtest_exitcode! + IF !pwshtest_exitcode!==0 CALL pwsh -nop -nol "%~dp0%~n0.ps1" %* & SET task_exitcode=!errorlevel! + REM fallback to powershell if pwsh failed + IF NOT !pwshtest_exitcode!==0 ( + REM CALL powershell -nop -nol -c write-host powershell-found + CALL powershell -nop -nol -file "%~dp0%~n0.ps1" %* + SET task_exitcode=!errorlevel! + ) +) ELSE ( + IF %nextshell%==bash ( + CALL :getWslPath %winpath% wslpath + REM ECHO wslfullpath "!wslpath!%fname%" + CALL %nextshell% "!wslpath!%fname%" %* & SET task_exitcode=!errorlevel! + ) ELSE ( + REM probably tclsh or sh + IF NOT "x%keyRemoved%"=="x%validshells%" ( + REM sh uses /c/ instead of /mnt/c - at least if using msys. Todo, review what is the norm on windows with and without msys2,cygwin,wsl + REM and what logic if any may be needed. For now sh with /c/xxx seems to work the same as sh with c:/xxx + CALL %nextshell% "%~dp0%fname%" %* & SET task_exitcode=!errorlevel! + ) ELSE ( + ECHO %fname% has invalid nextshell value %nextshell% valid options are %validshells% + SET task_exitcode=66 + GOTO :exit + ) + ) +) +@GOTO :endlib +:getWslPath +@SETLOCAL + @SET "_path=%~p1" + @SET "name=%~nx1" + @SET "drive=%~d1" + @SET "rtrn=%~2" + @SET "result=/mnt/%drive:~0,1%%_path:\=/%%name%" +@ENDLOCAL & ( + @if "%~2" neq "" ( + SET "%rtrn%=%result%" + ) ELSE ( + ECHO %result% + ) +) +@GOTO :eof +:endlib + +: \ +@REM @SET taskexit_code=!errorlevel! & goto :exit +@GOTO :exit +# } +# rem call %nextshell% "%~dp0%~n0.cmd" %* +# -*- tcl -*- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- tcl script section +# -- This is a punk multishell file +# -- Primary payload target is Tcl, with sh,bash,powershell as helpers +# -- but it may equally be used with any of these being the primary script. +# -- It is tuned to run when called as a batch file, a tcl script a sh/bash script or a pwsh/powershell script +# -- i.e it is a polyglot file. +# -- The specific layout including some lines that appear just as comments is quite sensitive to change. +# -- It can be called on unix or windows platforms with or without the interpreter being specified on the commandline. +# -- e.g ./filename.polypunk.cmd in sh or bash +# -- e.g tclsh filename.cmd +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +rename set ""; rename s2 set; set k {-- "$@" "a}; if {[info exists ::env($k)]} {unset ::env($k)} ;# tidyup +Hide :exit;Hide {<#};Hide '@ +namespace eval ::punk::multishell { + set last_script_root [file dirname [file normalize ${argv0}/__]] + set last_script [file dirname [file normalize [info script]/__]] + if {[info exists argv0] && + $last_script eq $last_script_root + } { + set ::punk::multishell::is_main($last_script) 1 ;#run as executable/script - likely desirable to launch application and return an exitcode + } else { + set ::punk::multishell::is_main($last_script) 0 ;#sourced - likely to be being used as a library - no launch, no exit. Can use return. + } + if {"::punk::multishell::is_main" ni [info commands ::punk::multishell::is_main]} { + proc ::punk::multishell::is_main {{script_name {}}} { + if {$script_name eq ""} { + set script_name [file dirname [file normalize [info script]/--]] + } + if {![info exists ::punk::multishell::is_main($script_name)]} { + #e.g a .dll or something else unanticipated + puts stderr "Warning punk::multishell didn't recognize info script result: $script_name - will treat as if sourced and return instead of exiting" + puts stderr "Info: script_root: [file dirname [file normalize ${argv0}/__]]" + return 0 + } + return [set ::punk::multishell::is_main($script_name)] + } + } +} +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin Tcl Payload +#puts "script : [info script]" +#puts "argcount : $::argc" +#puts "argvalues: $::argv" +#puts "argv0 : $::argv0" +# -- --- --- --- --- --- --- --- --- --- --- --- + + +# +# + + + +# -- --- --- --- --- --- --- --- --- --- --- --- +# -- Best practice is to always return or exit above, or just by leaving the below defaults in place. +# -- If the multishell script is modified to have Tcl below the Tcl Payload section, +# -- then Tcl bracket balancing needs to be carefully managed in the shell and powershell sections below. +# -- Only the # in front of the two relevant if statements below needs to be removed to enable Tcl below +# -- but the sh/bash 'then' and 'fi' would also need to be uncommented. +# -- This facility left in place for experiments on whether configuration payloads etc can be appended +# -- to tail of file - possibly binary with ctrl-z char - but utility is dependent on which other interpreters/shells +# -- can be made to ignore/cope with such data. +if {[::punk::multishell::is_main]} { + exit 0 +} else { + return +} +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end Tcl Payload +# end hide from unix shells \ +HEREDOC1B_HIDE_FROM_BASH_AND_SH +# sh/bash \ +shift && set -- "${@:1:$#-1}" +#------------------------------------------------------ +# -- This if block only needed if Tcl didn't exit or return above. +if false==false # else { + then + : +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- sh/bash script section +# -- leave as is if all that is required is launching the Tcl payload" +# -- +# -- Note that sh/bash script isn't called when running a .bat/.cmd from cmd.exe on windows by default +# -- adjust @call line above ... to something like @call sh ... @call bash .. or @call env sh ... etc as appropriate +# -- if sh/bash scripting needs to run on windows too. +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin sh Payload +#printf "start of bash or sh code" + +# +# + +# -- --- --- --- --- --- --- --- +# +exitcode=0 ;#default assumption +#-- sh/bash launches Tcl here instead of shebang line at top +#-- use exec to use exitcode (if any) directly from the tcl script +#exec /usr/bin/env tclsh "$0" "$@" +#-- alternative - can run sh/bash script after the tcl call. +/usr/bin/env tclsh "$0" "$@" +exitcode=$? +#echo "tcl exitcode: ${exitcode}" +#-- override exitcode example +#exit 66 +# +# -- --- --- --- --- --- --- --- + +# +# + + +#printf "sh/bash done \n" +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end sh Payload +#------------------------------------------------------ +fi +exit ${exitcode} +# end hide sh/bash block from Tcl +# This comment with closing brace should stay in place whether if commented or not } +#------------------------------------------------------ +# begin hide powershell-block from Tcl - only needed if Tcl didn't exit or return above +if 0 { +: end heredoc1 - end hide from powershell \ +'@ +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- powershell/pwsh section +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +function GetScriptName { $myInvocation.ScriptName } +$scriptname = getScriptName +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin powershell Payload +#"Timestamp : {0,10:yyyy-MM-dd HH:mm:ss}" -f $(Get-Date) | write-host +#"Script Name : {0}" -f $scriptname | write-host +#"Powershell Version: {0}" -f $PSVersionTable.PSVersion.Major | write-host +#"powershell args : {0}" -f ($args -join ", ") | write-host +# -- --- --- --- + +# +# + + +# -- --- --- --- --- --- --- --- +# +tclsh $scriptname $args +# +# -- --- --- --- --- --- --- --- + + +# +# + +# unbal } + +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end powershell Payload +#"powershell reporting exitcode: {0}" -f $LASTEXITCODE | write-host +Exit $LASTEXITCODE +# heredoc2 for powershell to ignore block below +$1 = @' +' +: end hide powershell-block from Tcl \ +# This comment with closing brace should stay in place whether 'if' commented or not } +: cmd exit label - return exitcode +:exit +: \ +@REM @ECHO exitcode: !task_exitcode! +: \ +@EXIT /B !task_exitcode! +# cmd has exited +: end heredoc2 \ +'@ +<# +# id:tailblock0 +# -- powershell multiline comment +#> +<# +# id:tailblock1 +# + +# +# -- unreachable by tcl directly if ctrl-z character is in the section above. (but file can be read and split on \x1A) +# -- Potential for zip and/or base64 contents, but we can't stop pwsh parser from slurping in the data +# -- so for example a plain text tar archive could cause problems depending on the content. +# -- final line in file must be the powershell multiline comment terminator or other data it can handle. +# -- e.g plain # comment lines will work too +# -- (for example a powershell digital signature is a # commented block of data at the end of the file) +#> + + diff --git a/src/modules/punk/mix/templates/layouts/project/src/scriptapps/wrappers/sample_punk-shellbat.bat b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/basic/shellbat.bat similarity index 100% rename from src/modules/punk/mix/templates/layouts/project/src/scriptapps/wrappers/sample_punk-shellbat.bat rename to src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/basic/shellbat.bat diff --git a/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/multishell.cmd b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/multishell.cmd new file mode 100644 index 00000000..fff93dcc --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/multishell.cmd @@ -0,0 +1,661 @@ +: "punk MULTISHELL - shebangless polyglot for Tcl Perl sh bash cmd pwsh powershell" + "[rename set s;proc Hide x {proc $x args {}};Hide :]" + "\$(function : {<#pwsh#>})" + "perlhide" + qw^ +set -- "$@" "a=[Hide <#;Hide set;s 1 list]"; set -- : "$@";$1 = @' +: heredoc1 - hide from powershell using @ and squote above. close sqote for unix shells + ' \ +: .bat/.cmd launch section, leading colon hides from cmd, trailing slash hides next line from tcl + \ +: "[Hide @GOTO; Hide =begin; Hide @REM] #not necessary but can help avoid errs in testing" + +: << 'HEREDOC1B_HIDE_FROM_BASH_AND_SH' +: STRONG SUGGESTION: DO NOT MODIFY FIRST LINE OF THIS SCRIPT - except for first double quoted section. +: shebang line is not required on unix or windows and will reduce functionality and/or portability. +: Even comment lines can be part of the functionality of this script (both on unix and windows) - modify with care. +@GOTO :skip_perl_pod_start ^; +=begin excludeperl +: skip_perl_pod_start +: Continuation char at end of this line and rem with curly-braces used to exlude Tcl from the whole cmd block \ +: { +@REM ############################################################################################################################ +@REM THIS IS A POLYGLOT SCRIPT - supporting payloads in Tcl, bash, sh and/or powershelll (powershell.exe or pwsh.exe) +@REM It should remain portable between unix-like OSes & windows if the proper structure is maintained. +@REM ############################################################################################################################ +@REM On windows, change the value of nextshell to one of the listed 2 digit values if desired, and add code within payload sections for tcl,sh,bash,powershell as appropriate. +@REM This wrapper can be edited manually (carefully!) - or sh,bash,tcl,powershell scripts can be wrapped using the Tcl-based punkshell system +@REM e.g from within a running punkshell: deck scriptwrap.multishell -outputfolder +@REM On unix-like systems, call with sh, bash or tclsh. (powershell untested on unix - and requires wscript if security elevation is used) +@REM Due to lack of shebang (#! line) Unix-like systems will probably (hopefully) default to sh if the script is called without an interpreter - but it may depend on the shell in use when called. +@REM If you find yourself really wanting/needing to add a shebang line - do so on the basis that the script will exist on unix-like systems only. +@SETLOCAL EnableExtensions EnableDelayedExpansion +@SET "validshells= ^(10^) 'pwsh' ^(11^) 'sh' (^12^) 'bash' (^13^) 'tclsh' (^14^) 'perl'" +@SET "shells[10]=pwsh" +@SET "shells[11]=sh" +@set "shells[12]=bash" +@SET "shells[13]=tclsh" +@SET "shells[14]=perl" +: +@SET "nextshell=13" +: +@rem asadmin is for automatic elevation to administrator. Separate window will be created (seems unavoidable with current elevation mechanism) and user will still get security prompt (probably reasonable). +: +@SET "asadmin=0" +: +@REM nextshell set to index for validshells .eg 10 for pwsh +@REM @ECHO nextshell is %nextshell% +@SET "selected=!shells[%nextshell%]!" +@REM @ECHO selected %selected% +@CALL SET "keyRemoved=%%validshells:'!selected!'=%%" +@REM @ECHO keyremoved %keyRemoved% +@REM Note that 'powershell' e.g v5 is just a fallback for when pwsh is not available +@REM ## ### ### ### ### ### ### ### ### ### ### ### ### ### +@REM -- cmd/batch file section (ignored on unix but should be left in place) +@REM -- This section intended mainly to launch the next shell (and to escalate privileges if necessary) +@REM -- Avoid customising this if you are not familiar with batch scripting. cmd/batch script can be useful, but is probably the least expressive language and most error prone. +@REM -- For example - as this file needs to use unix-style lf line-endings - the label scanner is susceptible to the 512Byte boundary issue: https://www.dostips.com/forum/viewtopic.php?t=8988#p58888 +@REM -- This label issue can be triggered/abused in files with crlf line endings too - but it is less likely to happen accidentaly. +@REm -- See also: https://stackoverflow.com/questions/4094699/how-does-the-windows-command-interpreter-cmd-exe-parse-scripts/4095133#4095133 +@REM ############################################################################################################################ +@REM -- Due to this issue -seemingly trivial edits of the batch file section can break the script! (for Windows anyway) +@REM -- Even something as simple as adding or removing an @REM +@REM -- From within punkshell - use: +@REM -- deck scriptwrap.checkfile +@REM -- to check your templates or final wrapped scripts for byte boundary issues +@REM -- It will report any labels that are on boundaries +@REM -- This is why the nextshell value above is a 2 digit key instead of a string - so that editing the value doesn't change the byte offsets. +@REM -- Editing your sh,bash,tcl,pwsh payloads is much less likely to cause an issue. There is the possibility of the final batch :exit_multishell label spanning a boundary - so testing using deck scriptwrap.checkfile is still recommended. +@REM -- Alternatively, as you should do anyway - test the final script on windows +@REM -- Aside from adding comments/whitespace to tweak the location of labels - you can try duplicating the label (e.g just add the label on a line above) but this is not guaranteed to work in all situations. +@REM -- '@REM' is a safer comment mechanism than a leading colon - which is used sparingly here. +@REM -- A colon anywhere in the script that happens to land on a 512 Byte boundary (from file start or from a callsite) could be misinterpreted as a label +@REM -- It is unknown what versions of cmd interpreters behave this way - and deck scriptwrap.checkfile doesn't check all such boundaries. +@REm -- For this reason, batch labels should be chosen to be relatively unlikely to collide with other strings in the file, and simple names such as :exit or :end should probably be avoided +@REM ############################################################################################################################ +@REM -- custom windows payloads should be in powershell,tclsh (or sh/bash if available) code sections +@REM ## ### ### ### ### ### ### ### ### ### ### ### ### ### +@SET "winpath=%~dp0" +@SET "fname=%~nx0" +@REM @ECHO fname %fname% +@REM @ECHO winpath %winpath% +@REM @ECHO commandlineascalled %0 +@REM @ECHO commandlineresolved %~f0 +@CALL :getNormalizedScriptTail nftail +@REM @ECHO normalizedscripttail %nftail% +@CALL :getFileTail %0 clinetail +@REM @ECHO clinetail %clinetail% +@CALL :stringToUpper %~nx0 capscripttail +@REM @ECHO capscriptname: %capscripttail% + +@IF "%nftail%"=="%capscripttail%" ( + @ECHO forcing asadmin=1 due to file name on filesystem being uppercase + @SET "asadmin=1" +) else ( + @CALL :stringToUpper %clinetail% capcmdlinetail + @REM @ECHO capcmdlinetail !capcmdlinetail! + IF "%clinetail%"=="!capcmdlinetail!" ( + @ECHO forcing asadmin=1 due to cmdline scriptname in uppercase + @set "asadmin=1" + ) +) +@SET "vbsGetPrivileges=%temp%\punk_bat_elevate_%fname%.vbs" +@SET arglist=%* +@IF "%1"=="PUNK-ELEVATED" ( + GOTO :gotPrivileges +) +@IF !asadmin!==1 ( + net file 1>NUL 2>NUL + @IF '!errorlevel!'=='0' ( GOTO :gotPrivileges ) else ( GOTO :getPrivileges ) +) +@GOTO skip_privileges +:getPrivileges +@IF '%1'=='PUNK-ELEVATED' (echo PUNK-ELEVATED & shift /1 & goto :gotPrivileges ) +@ECHO Set UAC = CreateObject^("Shell.Application"^) > "%vbsGetPrivileges%" +@ECHO args = "PUNK-ELEVATED " >> "%vbsGetPrivileges%" +@ECHO For Each strArg in WScript.Arguments >> "%vbsGetPrivileges%" +@ECHO args = args ^& strArg ^& " " >> "%vbsGetPrivileges%" +@ECHO Next >> "%vbsGetPrivileges%" +@ECHO UAC.ShellExecute "%~dp0%~n0%~x0", args, "", "runas", 1 >> "%vbsGetPrivileges%" +@ECHO Launching script in new windows due to administrator elevation +@"%SystemRoot%\System32\WScript.exe" "%vbsGetPrivileges%" %* +@EXIT /B + +:gotPrivileges +@REM setlocal & pushd . +@PUSHD . +@cd /d %~dp0 +@IF "%1"=="PUNK-ELEVATED" ( + @DEL "%vbsGetPrivileges%" 1>nul 2>nul + @SET arglist=%arglist:~14% +) + +:skip_privileges +@SET need_ps1=0 +@REM we want the ps1 to exist even if the nextshell isn't powershell +@if not exist "%~dp0%~n0.ps1" ( + @SET need_ps1=1 +) ELSE ( + fc "%~dp0%~n0%~x0" "%~dp0%~n0.ps1" >nul || goto different + @REM @ECHO "files same" + @SET need_ps1=0 +) +@GOTO :pscontinue +:different +@REM @ECHO "files differ" +@SET need_ps1=1 +:pscontinue +@IF !need_ps1!==1 ( + COPY "%~dp0%~n0%~x0" "%~dp0%~n0.ps1" >NUL +) +@REM avoid using CALL to launch pwsh,tclsh etc - it will intercept some args such as /? +@IF "!shells[%nextshell%]!"=="pwsh" ( + REM pws vs powershell hasn't been tested because we didn't need to copy cmd to ps1 this time + REM test availability of preferred option of powershell7+ pwsh + pwsh -nop -nol -c set-executionpolicy -Scope Process Unrestricted; write-host "statusmessage: pwsh-found" >NUL + SET pwshtest_exitcode=!errorlevel! + REM ECHO pwshtest_exitcode !pwshtest_exitcode! + REM fallback to powershell if pwsh failed + IF !pwshtest_exitcode!==0 ( + pwsh -nop -nol -c set-executionpolicy -Scope Process Unrestricted; "%~dp0%~n0.ps1" %arglist% + SET task_exitcode=!errorlevel! + ) ELSE ( + REM CALL powershell -nop -nol -c write-host powershell-found + REM powershell -nop -nol -file "%~dp0%~n0.ps1" %* + powershell -nop -nol -c set-executionpolicy -Scope Process Unrestricted; %~dp0%~n0.ps1" %arglist% + SET task_exitcode=!errorlevel! + ) +) ELSE ( + IF "!shells[%nextshell%]!"=="bash" ( + CALL :getWslPath %winpath% wslpath + REM ECHO wslfullpath "!wslpath!%fname%" + !shells[%nextshell%]! "!wslpath!%fname%" %arglist% + SET task_exitcode=!errorlevel! + ) ELSE ( + REM probably tclsh or sh + IF NOT "x%keyRemoved%"=="x%validshells%" ( + REM sh on windows uses /c/ instead of /mnt/c - at least if using msys. Todo, review what is the norm on windows with and without msys2,cygwin,wsl + REM and what logic if any may be needed. For now sh with /c/xxx seems to work the same as sh with c:/xxx + !shells[%nextshell%]! "%~dp0%fname%" %arglist% + SET task_exitcode=!errorlevel! + ) ELSE ( + ECHO %fname% has invalid nextshell value ^(%nextshell%^) !shells[%nextshell%]! valid options are %validshells% + SET task_exitcode=66 + @REM boundary padding + GOTO :exit_multishell + ) + ) +) +@REM batch file library functions +@REM boundary padding +@GOTO :endlib + +:getWslPath +@SETLOCAL + @SET "_path=%~p1" + @SET "name=%~nx1" + @SET "drive=%~d1" + @SET "rtrn=%~2" + @SET "result=/mnt/%drive:~0,1%%_path:\=/%%name%" +@ENDLOCAL & ( + @if "%~2" neq "" ( + SET "%rtrn%=%result%" + ) ELSE ( + ECHO %result% + ) +) +@EXIT /B + +:getFileTail +@REM return tail of file without any normalization e.g c:/punkshell/bin/Punk.cmd returns Punk.cmd even if file is punk.cmd +@REM we can't use things such as %~nx1 as it can change capitalisation +@REM This function is designed explicitly to preserve capitalisation +@REM accepts full paths with either / or \ as delimiters - or +@SETLOCAL + @SET "rtrn=%~2" + @SET "arg=%~1" + @REM @SET "result=%_arg:*/=%" + @REM @SET "result=%~1" + @SET LF=^ + + + : The above 2 empty lines are important. Don't remove + @CALL :stringContains "!arg!" "\" hasBackSlash + @IF "!hasBackslash!"=="true" ( + @for %%A in ("!LF!") do @( + @FOR /F %%B in ("!arg:\=%%~A!") do @set "result=%%B" + ) + ) ELSE ( + @CALL :stringContains "!arg!" "/" hasForwardSlash + @IF "!hasForwardSlash!"=="true" ( + @FOR %%A in ("!LF!") do @( + @FOR /F %%B in ("!arg:/=%%~A!") do @set "result=%%B" + ) + ) ELSE ( + @set "result=%arg%" + ) + ) +@ENDLOCAL & ( + @if "%~2" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO %result% + ) +) +@EXIT /B +@REM boundary padding +@REM boundary padding +:getNormalizedScriptTail +@SETLOCAL + @SET "result=%~nx0" + @SET "rtrn=%~1" +@ENDLOCAL & ( + @IF "%~1" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO %result% + ) +) +@EXIT /B + +:getNormalizedFileTailFromPath +@REM warn via echo, and do not set return variable if path not found +@REM note that %~nx1 does not preserve case of provided path - hence the name 'normalized' +@REM boundary padding +@REM boundary padding +@REM boundary padding +@REM boundary padding +@SETLOCAL + @CALL :stringContains %~1 "\" hasBackSlash + @CALL :stringContains %~1 "/" hasForwardSlash + @IF "%hasBackslash%-%hasForwardslash%"=="false-false" ( + @SET "P=%cd%%~1" + @CALL :getNormalizedFileTailFromPath "!P!" ftail2 + @SET "result=!ftail2!" + ) else ( + @IF EXIST "%~1" ( + @SET "result=%~nx1" + ) else ( + @ECHO error getNormalizedFileTailFromPath file not found: %~1 + @EXIT /B 1 + ) + ) + @SET "rtrn=%~2" +@ENDLOCAL & ( + @IF "%~2" neq "" ( + SET "%rtrn%=%result%" + ) ELSE ( + @ECHO getNormalizedFileTailFromPath %1 result: %result% + ) +) +@EXIT /B + +:stringContains +@REM usage: @CALL:stringContains string needle returnvarname +@SETLOCAL + @SET "rtrn=%~3" + @SET "string=%~1" + @SET "needle=%~2" + @IF "!string:%needle%=!"=="!string!" @( + @SET "result=false" + ) ELSE ( + @SET "result=true" + ) +@ENDLOCAL & ( + @IF "%~3" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO stringContains %string% %needle% result: %result% + ) +) +@EXIT /B + +:stringToUpper +@SETLOCAL + @SET "rtrn=%~2" + @SET "string=%~1" + @SET "capstring=%~1" + @FOR %%A in (A B C D E F G H I J K L M N O P Q R S T U V W X Y Z) DO @( + @SET "capstring=!capstring:%%A=%%A!" + ) + @SET "result=!capstring!" +@ENDLOCAL & ( + @IF "%~2" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO stringToUpper %string% result: %result% + ) +) +@EXIT /B + +:isNumeric +@SETLOCAL + @SET "notnumeric="&FOR /F "delims=0123456789" %%i in ("%1") do set "notnumeric=%%i" + @IF defined notnumeric ( + @SET "result=false" + ) else ( + @SET "result=true" + ) + @SET "rtrn=%~2" +@ENDLOCAL & ( + @IF "%~2" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO %result% + ) +) +@EXIT /B + +:endlib +: \ +@REM @SET taskexit_code=!errorlevel! & goto :exit_multishell +@GOTO :exit_multishell +# } +# -*- tcl -*- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- tcl script section +# -- This is a punk multishell file +# -- Primary payload target is Tcl, with sh,bash,powershell as helpers +# -- but it may equally be used with any of these being the primary script. +# -- It is tuned to run when called as a batch file, a tcl script a sh/bash script or a pwsh/powershell script +# -- i.e it is a polyglot file. +# -- The specific layout including some lines that appear just as comments is quite sensitive to change. +# -- It can be called on unix or windows platforms with or without the interpreter being specified on the commandline. +# -- e.g ./filename.polypunk.cmd in sh or bash +# -- e.g tclsh filename.cmd +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +rename set ""; rename s set; set k {-- "$@" "a}; if {[info exists ::env($k)]} {unset ::env($k)} ;# tidyup and restore +Hide :exit_multishell;Hide {<#};Hide '@ +namespace eval ::punk::multishell { + set last_script_root [file dirname [file normalize ${argv0}/__]] + set last_script [file dirname [file normalize [info script]/__]] + if {[info exists argv0] && + $last_script eq $last_script_root + } { + set ::punk::multishell::is_main($last_script) 1 ;#run as executable/script - likely desirable to launch application and return an exitcode + } else { + set ::punk::multishell::is_main($last_script) 0 ;#sourced - likely to be being used as a library - no launch, no exit. Can use return. + } + if {"::punk::multishell::is_main" ni [info commands ::punk::multishell::is_main]} { + proc ::punk::multishell::is_main {{script_name {}}} { + if {$script_name eq ""} { + set script_name [file dirname [file normalize [info script]/--]] + } + if {![info exists ::punk::multishell::is_main($script_name)]} { + #e.g a .dll or something else unanticipated + puts stderr "Warning punk::multishell didn't recognize info script result: $script_name - will treat as if sourced and return instead of exiting" + puts stderr "Info: script_root: [file dirname [file normalize ${argv0}/__]]" + return 0 + } + return [set ::punk::multishell::is_main($script_name)] + } + } +} +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin Tcl Payload +#puts "script : [info script]" +#puts "argcount : $::argc" +#puts "argvalues: $::argv" +#puts "argv0 : $::argv0" +# -- --- --- --- --- --- --- --- --- --- --- --- + + +# +# + +# +# + + +# +# + + +# -- --- --- --- --- --- --- --- --- --- --- --- +# -- Best practice is to always return or exit above, or just by leaving the below defaults in place. +# -- If the multishell script is modified to have Tcl below the Tcl Payload section, +# -- then Tcl bracket balancing needs to be carefully managed in the shell and powershell sections below. +# -- Only the # in front of the two relevant if statements below needs to be removed to enable Tcl below +# -- but the sh/bash 'then' and 'fi' would also need to be uncommented. +# -- This facility left in place for experiments on whether configuration payloads etc can be appended +# -- to tail of file - possibly binary with ctrl-z char - but utility is dependent on which other interpreters/shells +# -- can be made to ignore/cope with such data. +if {[::punk::multishell::is_main]} { + exit 0 +} else { + return +} +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end Tcl Payload +# end hide from unix shells \ +HEREDOC1B_HIDE_FROM_BASH_AND_SH +# sh/bash \ +shift && set -- "${@:1:$#-1}" +#------------------------------------------------------ +# -- This if block only needed if Tcl didn't exit or return above. +if false==false # else { + then + : # +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- sh/bash script section +# -- leave as is if all that is required is launching the Tcl payload" +# -- +# -- Note that sh/bash script isn't called when running a .bat/.cmd from cmd.exe on windows by default +# -- adjust the %nextshell% value above +# -- if sh/bash scripting needs to run on windows too. +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin sh Payload +exitcode=0 +#printf "start of bash or sh code" + +# +# + +# -- --- --- --- --- --- --- --- +# +#-- sh/bash launches Tcl here instead of shebang line at top +#-- use exec to use exitcode (if any) directly from the tcl script +#exec /usr/bin/env tclsh "$0" "$@" +#-- alternative - can run sh/bash script after the tcl call. +/usr/bin/env tclsh "$0" "$@" +exitcode=$? +#echo "sh/bash reporting tcl exitcode: ${exitcode}" +#-- override exitcode example +#exit 66 +# +# -- --- --- --- --- --- --- --- + +# +# + + +#printf "sh/bash done \n" +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end sh Payload +#------------------------------------------------------ +fi +exit ${exitcode} +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- Perl script section +# -- leave the script below as is, if all that is required is launching the Tcl payload" +# -- +# -- Note that perl script isn't called by default when simply running this script by name +# -- adjust the nextshell value at the top of the script to point to perl +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +=cut +#!/user/bin/perl +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin perl Payload +my $exit_code = 0; +#use ExtUtils::Installed; +#my $installed = ExtUtils::Installed->new(); +#my @modules = $installed->modules(); +#print "Modules:\n"; +#foreach my $m (@modules) { +# print "$m\n"; +#} +# -- --- --- + + + +my $scriptname = $0; +print "perl $scriptname\n"; +my $i =1; +foreach my $a(@ARGV) { + print "Arg # $i: $a\n"; +} + +# +# + + + +# -- --- --- --- --- --- --- --- +# +$exit_code=system("tclsh", $scriptname, @ARGV); +#print "perl reporting tcl exitcode: $exit_code"; +# +# -- --- --- --- --- --- --- --- + +# +# + + +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end perl Payload +exit $exit_code; +__END__ + +# end hide sh/bash/perl block from Tcl +# This comment with closing brace should stay in place whether if commented or not } +#------------------------------------------------------ +# begin hide powershell-block from Tcl - only needed if Tcl didn't exit or return above +if 0 { +: end heredoc1 - end hide from powershell \ +'@ +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- powershell/pwsh section +# -- Do not edit if current file is the .ps1 +# -- Edit the corresponding .cmd and it will autocopy +# -- unbalanced braces { } here *even in comments* will cause problems if there was no Tcl exit or return above +# -- custom script should generally go below the begin_powershell_payload line +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +function GetScriptName { $myInvocation.ScriptName } +$scriptname = GetScriptName +function GetDynamicParamDictionary { + [CmdletBinding()] + param( + [Parameter(ValueFromPipeline=$true, Mandatory=$true)] + [string] $CommandName + ) + + begin { + # Get a list of params that should be ignored (they're common to all advanced functions) + $CommonParameterNames = [System.Runtime.Serialization.FormatterServices]::GetUninitializedObject([type] [System.Management.Automation.Internal.CommonParameters]) | + Get-Member -MemberType Properties | + Select-Object -ExpandProperty Name + } + + process { + # Create the dictionary that this scriptblock will return: + $DynParamDictionary = New-Object System.Management.Automation.RuntimeDefinedParameterDictionary + + # Convert to object array and get rid of Common params: + (Get-Command $CommandName | select -exp Parameters).GetEnumerator() | + Where-Object { $CommonParameterNames -notcontains $_.Key } | + ForEach-Object { + $DynamicParameter = New-Object System.Management.Automation.RuntimeDefinedParameter ( + $_.Key, + $_.Value.ParameterType, + $_.Value.Attributes + ) + $DynParamDictionary.Add($_.Key, $DynamicParameter) + } + + # Return the dynamic parameters + return $DynParamDictionary + } +} +# GetDynamicParamDictionary +# - This can make it easier to share a single set of param definitions between functions +# - sample usage +#function ParameterDefinitions { +# param( +# [Parameter(Mandatory)][string] $myargument +# ) +#} +#function psmain { +# [CmdletBinding()] +# param() +# dynamicparam { GetDynamicParamDictionary ParameterDefinitions } +# process { +# #called once with $PSBoundParameters dictionary +# #can be used to validate arguments, or set a simpler variable name for access +# switch ($PSBoundParameters.keys) { +# 'myargumentname' { +# Set-Variable -Name $_ -Value $PSBoundParameters."$_" +# } +# #... +# } +# foreach ($boundparam in $PSBoundParameters.GetEnumerator()) { +# #... +# } +# } +# end { +# #Main function logic +# Write-Host "myargumentname value is: $myargumentname" +# #myotherfunction @PSBoundParameters +# } +#} +#psmain @args +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin powershell Payload +#"Timestamp : {0,10:yyyy-MM-dd HH:mm:ss}" -f $(Get-Date) | write-host +#"Script Name : {0}" -f $scriptname | write-host +#"Powershell Version: {0}" -f $PSVersionTable.PSVersion.Major | write-host +#"powershell args : {0}" -f ($args -join ", ") | write-host +# -- --- --- --- + +# +# + + +# -- --- --- --- --- --- --- --- +# +tclsh $scriptname $args +#"powershell reporting exitcode: {0}" -f $LASTEXITCODE | write-host +# +# -- --- --- --- --- --- --- --- + + +# +# + +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end powershell Payload +Exit $LASTEXITCODE +# heredoc2 for powershell to ignore block below +$1 = @' +' +: comment end hide powershell-block from Tcl \ +# This comment with closing brace should stay in place whether 'if' commented or not } +: multishell doubled-up cmd exit label - return exitcode +:exit_multishell +:exit_multishell +: \ +@REM @ECHO exitcode: !task_exitcode! +: \ +@IF "%1"=="PUNK-ELEVATED" (echo. & @cmd /k echo elevated prompt: type exit to quit) +: \ +@EXIT /B !task_exitcode! +# cmd has exited +: comment end heredoc2 \ +'@ +<# +# id:tailblock0 +# -- powershell multiline comment +#> +<# +no script engine should try to run me +# id:tailblock1 +# + +# +# -- unreachable by tcl directly if ctrl-z character is in the section above. (but file can be read and split on \x1A) +# -- Potential for zip and/or base64 contents, but we can't stop pwsh parser from slurping in the data +# -- so for example a plain text tar archive could cause problems depending on the content. +# -- final line in file must be the powershell multiline comment terminator or other data it can handle. +# -- e.g plain # comment lines will work too +# -- (for example a powershell digital signature is a # commented block of data at the end of the file) +#> + + diff --git a/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/multishell1.cmd b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/multishell1.cmd new file mode 100644 index 00000000..17fe4c15 --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/multishell1.cmd @@ -0,0 +1,524 @@ +: "[rename set s;proc Hide x {proc $x args {}};Hide :]" "\$(function : {<#pwsh#>})" ^ +set -- "$@" "a=[list shebangless punk MULTISHELL tclsh sh bash cmd pwsh powershell;proc Hide x {proc $x args {}}; Hide <#;Hide set;s 1 list]"; set -- : "$@";$1 = @' +: heredoc1 - hide from powershell using @ and squote above. (close sqote for unix shells) ' \ +: .bat/.cmd launch section, leading colon hides from cmd, trailing slash hides next line from tcl \ +: "[Hide @ECHO; Hide ); Hide (;Hide echo; Hide @REM]#not necessary but can help avoid errs in testing" +: << 'HEREDOC1B_HIDE_FROM_BASH_AND_SH' +: Continuation char at end of this line and rem with curly-braces used to exlude Tcl from the whole cmd block \ +: { +: STRONG SUGGESTION: DO NOT MODIFY FIRST LINE OF THIS SCRIPT. shebang #! line is not required on unix or windows and will reduce functionality and/or portability. +: Even comment lines can be part of the functionality of this script (both on unix and windows) - modify with care. +@REM ############################################################################################################################ +@REM THIS IS A POLYGLOT SCRIPT - supporting payloads in Tcl, bash, sh and/or powershelll (powershell.exe or pwsh.exe) +@REM It should remain portable between unix-like OSes & windows if the proper structure is maintained. +@REM ############################################################################################################################ +@REM On windows, change the value of nextshell to one of the listed 2 digit values if desired, and add code within payload sections for tcl,sh,bash,powershell as appropriate. +@REM This wrapper can be edited manually (carefully!) - or sh,bash,tcl,powershell scripts can be wrapped using the Tcl-based punkshell system +@REM e.g from within a running punkshell: pmix scriptwrap.multishell -outputfolder +@REM On unix-like systems, call with sh, bash or tclsh. (powershell untested on unix - and requires wscript if security elevation is used) +@REM Due to lack of shebang (#! line) Unix-like systems will probably (hopefully) default to sh if the script is called without an interpreter - but it may depend on the shell in use when called. +@REM If you find yourself really wanting/needing to add a shebang line - do so on the basis that the script will exist on unix-like systems only. +@SETLOCAL EnableExtensions EnableDelayedExpansion +@SET "validshells= ^(10^) 'pwsh' ^(11^) 'sh' (^12^) 'bash' (^13^) 'tclsh'" +@SET "shells[10]=pwsh" +@SET "shells[11]=sh" +@set "shells[12]=bash" +@SET "shells[13]=tclsh" +: +@SET "nextshell=13" +: +@rem asadmin is for automatic elevation to administrator. Separate window will be created (seems unavoidable with current elevation mechanism) and user will still get security prompt (probably reasonable). +: +@SET "asadmin=0" +: +@REM nextshell set to index for validshells .eg 10 for pwsh +@REM @ECHO nextshell is %nextshell% +@SET "selected=!shells[%nextshell%]!" +@REM @ECHO selected %selected% +@CALL SET "keyRemoved=%%validshells:'!selected!'=%%" +@REM @ECHO keyremoved %keyRemoved% +@REM Note that 'powershell' e.g v5 is just a fallback for when pwsh is not available +@REM ## ### ### ### ### ### ### ### ### ### ### ### ### ### +@REM -- cmd/batch file section (ignored on unix but should be left in place) +@REM -- This section intended mainly to launch the next shell (and to escalate privileges if necessary) +@REM -- Avoid customising this if you are not familiar with batch scripting. cmd/batch script can be useful, but is probably the least expressive language and most error prone. +@REM -- For example - as this file needs to use unix-style lf line-endings - the label scanner is susceptible to the 512Byte boundary issue: https://www.dostips.com/forum/viewtopic.php?t=8988#p58888 +@REM -- This label issue can be triggered/abused in files with crlf line endings too - but it is less likely to happen accidentaly. +@REm -- See also: https://stackoverflow.com/questions/4094699/how-does-the-windows-command-interpreter-cmd-exe-parse-scripts/4095133#4095133 +@REM ############################################################################################################################ +@REM -- Due to this issue -seemingly trivial edits of the batch file section can break the script! (for Windows anyway) +@REM -- Even something as simple as adding or removing an @REM +@REM -- From within punkshell - use: +@REM -- pmix scriptwrap.checkfile +@REM -- to check your templates or final wrapped scripts for byte boundary issues +@REM -- It will report any labels that are on boundaries +@REM -- This is why the nextshell value above is a 2 digit key instead of a string - so that editing the value doesn't change the byte offsets. +@REM -- Editing your sh,bash,tcl,pwsh payloads is much less likely to cause an issue. There is the possibility of the final batch :exit_multishell label spanning a boundary - so testing using pmix scriptwrap.checkfile is still recommended. +@REM -- Alternatively, as you should do anyway - test the final script on windows +@REM -- Aside from adding comments/whitespace to tweak the location of labels - you can try duplicating the label (e.g just add the label on a line above) but this is not guaranteed to work in all situations. +@REM -- '@REM' is a safer comment mechanism than a leading colon - which is used sparingly here. +@REM -- A colon anywhere in the script that happens to land on a 512 Byte boundary (from file start or from a callsite) could be misinterpreted as a label +@REM -- It is unknown what versions of cmd interpreters behave this way - and pmix scriptwrap.checkfile doesn't check all such boundaries. +@REm -- For this reason, batch labels should be chosen to be relatively unlikely to collide with other strings in the file, and simple names such as :exit or :end should probably be avoided +@REM ############################################################################################################################ +@REM -- custom windows payloads should be in powershell,tclsh (or sh/bash if available) code sections +@REM ## ### ### ### ### ### ### ### ### ### ### ### ### ### +@SET "winpath=%~dp0" +@SET "fname=%~nx0" +@REM @ECHO fname %fname% +@REM @ECHO winpath %winpath% +@REM @ECHO commandlineascalled %0 +@REM @ECHO commandlineresolved %~f0 +@CALL :getNormalizedScriptTail nftail +@REM @ECHO normalizedscripttail %nftail% +@CALL :getFileTail %0 clinetail +@REM @ECHO clinetail %clinetail% +@CALL :stringToUpper %~nx0 capscripttail +@REM @ECHO capscriptname: %capscripttail% + +@IF "%nftail%"=="%capscripttail%" ( + @ECHO forcing asadmin=1 due to file name on filesystem being uppercase + @SET "asadmin=1" +) else ( + @CALL :stringToUpper %clinetail% capcmdlinetail + @REM @ECHO capcmdlinetail !capcmdlinetail! + IF "%clinetail%"=="!capcmdlinetail!" ( + @ECHO forcing asadmin=1 due to cmdline scriptname in uppercase + @set "asadmin=1" + ) +) +@SET "vbsGetPrivileges=%temp%\punk_bat_elevate_%fname%.vbs" +@SET arglist=%* +@IF "%1"=="PUNK-ELEVATED" ( + GOTO :gotPrivileges +) +@IF !asadmin!==1 ( + net file 1>NUL 2>NUL + @IF '!errorlevel!'=='0' ( GOTO :gotPrivileges ) else ( GOTO :getPrivileges ) +) +@GOTO skip_privileges +:getPrivileges +@IF '%1'=='PUNK-ELEVATED' (echo PUNK-ELEVATED & shift /1 & goto :gotPrivileges ) +@ECHO Set UAC = CreateObject^("Shell.Application"^) > "%vbsGetPrivileges%" +@ECHO args = "PUNK-ELEVATED " >> "%vbsGetPrivileges%" +@ECHO For Each strArg in WScript.Arguments >> "%vbsGetPrivileges%" +@ECHO args = args ^& strArg ^& " " >> "%vbsGetPrivileges%" +@ECHO Next >> "%vbsGetPrivileges%" +@ECHO UAC.ShellExecute "%~dp0%~n0.cmd", args, "", "runas", 1 >> "%vbsGetPrivileges%" +@ECHO Launching script in new windows due to administrator elevation +@"%SystemRoot%\System32\WScript.exe" "%vbsGetPrivileges%" %* +@EXIT /B + +:gotPrivileges +@REM setlocal & pushd . +@PUSHD . +@cd /d %~dp0 +@IF "%1"=="PUNK-ELEVATED" ( + @DEL "%vbsGetPrivileges%" 1>nul 2>nul + @SET arglist=%arglist:~14% +) + +:skip_privileges +@SET need_ps1=0 +@REM we want the ps1 to exist even if the nextshell isn't powershell +@if not exist "%~dp0%~n0.ps1" ( + @SET need_ps1=1 +) ELSE ( + fc "%~dp0%~n0.cmd" "%~dp0%~n0.ps1" >nul || goto different + @REM @ECHO "files same" + @SET need_ps1=0 +) +@GOTO :pscontinue +:different +@REM @ECHO "files differ" +@SET need_ps1=1 +:pscontinue +@IF !need_ps1!==1 ( + COPY "%~dp0%~n0.cmd" "%~dp0%~n0.ps1" >NUL +) +@REM avoid using CALL to launch pwsh,tclsh etc - it will intercept some args such as /? +@IF "!shells[%nextshell%]!"=="pwsh" ( + REM pws vs powershell hasn't been tested because we didn't need to copy cmd to ps1 this time + REM test availability of preferred option of powershell7+ pwsh + pwsh -nop -nol -c set-executionpolicy -Scope Process Unrestricted; write-host "statusmessage: pwsh-found" >NUL + SET pwshtest_exitcode=!errorlevel! + REM ECHO pwshtest_exitcode !pwshtest_exitcode! + REM fallback to powershell if pwsh failed + IF !pwshtest_exitcode!==0 ( + pwsh -nop -nol -c set-executionpolicy -Scope Process Unrestricted; "%~dp0%~n0.ps1" %arglist% & SET task_exitcode=!errorlevel! + ) ELSE ( + REM CALL powershell -nop -nol -c write-host powershell-found + REM powershell -nop -nol -file "%~dp0%~n0.ps1" %* + powershell -nop -nol -c set-executionpolicy -Scope Process Unrestricted; %~dp0%~n0.ps1" %arglist% + SET task_exitcode=!errorlevel! + ) +) ELSE ( + IF "!shells[%nextshell%]!"=="bash" ( + CALL :getWslPath %winpath% wslpath + REM ECHO wslfullpath "!wslpath!%fname%" + !shells[%nextshell%]! "!wslpath!%fname%" %arglist% & SET task_exitcode=!errorlevel! + ) ELSE ( + REM probably tclsh or sh + IF NOT "x%keyRemoved%"=="x%validshells%" ( + REM sh on windows uses /c/ instead of /mnt/c - at least if using msys. Todo, review what is the norm on windows with and without msys2,cygwin,wsl + REM and what logic if any may be needed. For now sh with /c/xxx seems to work the same as sh with c:/xxx + !shells[%nextshell%]! "%~dp0%fname%" %arglist% & SET task_exitcode=!errorlevel! + ) ELSE ( + ECHO %fname% has invalid nextshell value ^(%nextshell%^) !shells[%nextshell%]! valid options are %validshells% + SET task_exitcode=66 + GOTO :exit_multishell + ) + ) +) +@REM batch file library functions +@GOTO :endlib + +:getWslPath +@SETLOCAL + @SET "_path=%~p1" + @SET "name=%~nx1" + @SET "drive=%~d1" + @SET "rtrn=%~2" + @SET "result=/mnt/%drive:~0,1%%_path:\=/%%name%" +@ENDLOCAL & ( + @if "%~2" neq "" ( + SET "%rtrn%=%result%" + ) ELSE ( + ECHO %result% + ) +) +@EXIT /B + +:getFileTail +@REM return tail of file without any normalization e.g c:/punkshell/bin/Punk.cmd returns Punk.cmd even if file is punk.cmd +@REM we can't use things such as %~nx1 as it can change capitalisation +@REM This function is designed explicitly to preserve capitalisation +@REM accepts full paths with either / or \ as delimiters - or +@SETLOCAL + @SET "rtrn=%~2" + @SET "arg=%~1" + @REM @SET "result=%_arg:*/=%" + @REM @SET "result=%~1" + @SET LF=^ + + + : The above 2 empty lines are important. Don't remove + @CALL :stringContains "!arg!" "\" hasBackSlash + @IF "!hasBackslash!"=="true" ( + @for %%A in ("!LF!") do @( + @FOR /F %%B in ("!arg:\=%%~A!") do @set "result=%%B" + ) + ) ELSE ( + @CALL :stringContains "!arg!" "/" hasForwardSlash + @IF "!hasForwardSlash!"=="true" ( + @FOR %%A in ("!LF!") do @( + @FOR /F %%B in ("!arg:/=%%~A!") do @set "result=%%B" + ) + ) ELSE ( + @set "result=%arg%" + ) + ) +@ENDLOCAL & ( + @if "%~2" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO %result% + ) +) +@EXIT /B +@REM boundary padding +:getNormalizedScriptTail +@SETLOCAL + @SET "result=%~nx0" + @SET "rtrn=%~1" +@ENDLOCAL & ( + @IF "%~1" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO %result% + ) +) +@EXIT /B + +:getNormalizedFileTailFromPath +@REM warn via echo, and do not set return variable if path not found +@REM note that %~nx1 does not preserve case of provided path - hence the name 'normalized' +@REM boundary padding +@REM boundary padding +@SETLOCAL + @CALL :stringContains %~1 "\" hasBackSlash + @CALL :stringContains %~1 "/" hasForwardSlash + @IF "%hasBackslash%-%hasForwardslash%"=="false-false" ( + @SET "P=%cd%%~1" + @CALL :getNormalizedFileTailFromPath "!P!" ftail2 + @SET "result=!ftail2!" + ) else ( + @IF EXIST "%~1" ( + @SET "result=%~nx1" + ) else ( + @ECHO error getNormalizedFileTailFromPath file not found: %~1 + @EXIT /B 1 + ) + ) + @SET "rtrn=%~2" +@ENDLOCAL & ( + @IF "%~2" neq "" ( + SET "%rtrn%=%result%" + ) ELSE ( + @ECHO getNormalizedFileTailFromPath %1 result: %result% + ) +) +@EXIT /B + +:stringContains +@REM usage: @CALL:stringContains string needle returnvarname +@SETLOCAL + @SET "rtrn=%~3" + @SET "string=%~1" + @SET "needle=%~2" + @IF "!string:%needle%=!"=="!string!" @( + @SET "result=false" + ) ELSE ( + @SET "result=true" + ) +@ENDLOCAL & ( + @IF "%~3" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO stringContains %string% %needle% result: %result% + ) +) +@EXIT /B + +:stringToUpper +@SETLOCAL + @SET "rtrn=%~2" + @SET "string=%~1" + @SET "capstring=%~1" + @FOR %%A in (A B C D E F G H I J K L M N O P Q R S T U V W X Y Z) DO @( + @SET "capstring=!capstring:%%A=%%A!" + ) + @SET "result=!capstring!" +@ENDLOCAL & ( + @IF "%~2" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO stringToUpper %string% result: %result% + ) +) +@EXIT /B + +:isNumeric +@SETLOCAL + @SET "notnumeric="&FOR /F "delims=0123456789" %%i in ("%1") do set "notnumeric=%%i" + @IF defined notnumeric ( + @SET "result=false" + ) else ( + @SET "result=true" + ) + @SET "rtrn=%~2" +@ENDLOCAL & ( + @IF "%~2" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO %result% + ) +) +@EXIT /B + +:endlib +: \ +@REM @SET taskexit_code=!errorlevel! & goto :exit_multishell +@GOTO :exit_multishell +# } +# -*- tcl -*- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- tcl script section +# -- This is a punk multishell file +# -- Primary payload target is Tcl, with sh,bash,powershell as helpers +# -- but it may equally be used with any of these being the primary script. +# -- It is tuned to run when called as a batch file, a tcl script a sh/bash script or a pwsh/powershell script +# -- i.e it is a polyglot file. +# -- The specific layout including some lines that appear just as comments is quite sensitive to change. +# -- It can be called on unix or windows platforms with or without the interpreter being specified on the commandline. +# -- e.g ./filename.polypunk.cmd in sh or bash +# -- e.g tclsh filename.cmd +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +rename set ""; rename s set; set k {-- "$@" "a}; if {[info exists ::env($k)]} {unset ::env($k)} ;# tidyup and restore +Hide :exit_multishell;Hide {<#};Hide '@ +namespace eval ::punk::multishell { + set last_script_root [file dirname [file normalize ${argv0}/__]] + set last_script [file dirname [file normalize [info script]/__]] + if {[info exists argv0] && + $last_script eq $last_script_root + } { + set ::punk::multishell::is_main($last_script) 1 ;#run as executable/script - likely desirable to launch application and return an exitcode + } else { + set ::punk::multishell::is_main($last_script) 0 ;#sourced - likely to be being used as a library - no launch, no exit. Can use return. + } + if {"::punk::multishell::is_main" ni [info commands ::punk::multishell::is_main]} { + proc ::punk::multishell::is_main {{script_name {}}} { + if {$script_name eq ""} { + set script_name [file dirname [file normalize [info script]/--]] + } + if {![info exists ::punk::multishell::is_main($script_name)]} { + #e.g a .dll or something else unanticipated + puts stderr "Warning punk::multishell didn't recognize info script result: $script_name - will treat as if sourced and return instead of exiting" + puts stderr "Info: script_root: [file dirname [file normalize ${argv0}/__]]" + return 0 + } + return [set ::punk::multishell::is_main($script_name)] + } + } +} +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin Tcl Payload +#puts "script : [info script]" +#puts "argcount : $::argc" +#puts "argvalues: $::argv" +#puts "argv0 : $::argv0" +# -- --- --- --- --- --- --- --- --- --- --- --- + + +# +# + + + +# -- --- --- --- --- --- --- --- --- --- --- --- +# -- Best practice is to always return or exit above, or just by leaving the below defaults in place. +# -- If the multishell script is modified to have Tcl below the Tcl Payload section, +# -- then Tcl bracket balancing needs to be carefully managed in the shell and powershell sections below. +# -- Only the # in front of the two relevant if statements below needs to be removed to enable Tcl below +# -- but the sh/bash 'then' and 'fi' would also need to be uncommented. +# -- This facility left in place for experiments on whether configuration payloads etc can be appended +# -- to tail of file - possibly binary with ctrl-z char - but utility is dependent on which other interpreters/shells +# -- can be made to ignore/cope with such data. +if {[::punk::multishell::is_main]} { + exit 0 +} else { + return +} +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end Tcl Payload +# end hide from unix shells \ +HEREDOC1B_HIDE_FROM_BASH_AND_SH +# sh/bash \ +shift && set -- "${@:1:$#-1}" +#------------------------------------------------------ +# -- This if block only needed if Tcl didn't exit or return above. +if false==false # else { + then + : # +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- sh/bash script section +# -- leave as is if all that is required is launching the Tcl payload" +# -- +# -- Note that sh/bash script isn't called when running a .bat/.cmd from cmd.exe on windows by default +# -- adjust @call line above ... to something like @call sh ... @call bash .. or @call env sh ... etc as appropriate +# -- if sh/bash scripting needs to run on windows too. +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin sh Payload +#printf "start of bash or sh code" + +# +# + +# -- --- --- --- --- --- --- --- +# +exitcode=0 ;#default assumption +#-- sh/bash launches Tcl here instead of shebang line at top +#-- use exec to use exitcode (if any) directly from the tcl script +#exec /usr/bin/env tclsh "$0" "$@" +#-- alternative - can run sh/bash script after the tcl call. +/usr/bin/env tclsh "$0" "$@" +exitcode=$? +#echo "tcl exitcode: ${exitcode}" +#-- override exitcode example +#exit 66 +# +# -- --- --- --- --- --- --- --- + +# +# + + +#printf "sh/bash done \n" +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end sh Payload +#------------------------------------------------------ +fi +exit ${exitcode} +# end hide sh/bash block from Tcl +# This comment with closing brace should stay in place whether if commented or not } +#------------------------------------------------------ +# begin hide powershell-block from Tcl - only needed if Tcl didn't exit or return above +if 0 { +: end heredoc1 - end hide from powershell \ +'@ +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- powershell/pwsh section +# -- Do not edit if current file is the .ps1 +# -- Edit the corresponding .cmd and it will autocopy +# -- unbalanced braces { } here *even in comments* will cause problems if there was no Tcl exit or return above +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +function GetScriptName { $myInvocation.ScriptName } +$scriptname = getScriptName +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin powershell Payload +#"Timestamp : {0,10:yyyy-MM-dd HH:mm:ss}" -f $(Get-Date) | write-host +#"Script Name : {0}" -f $scriptname | write-host +#"Powershell Version: {0}" -f $PSVersionTable.PSVersion.Major | write-host +#"powershell args : {0}" -f ($args -join ", ") | write-host +# -- --- --- --- + +# +# + + +# -- --- --- --- --- --- --- --- +# +tclsh $scriptname $args +# +# -- --- --- --- --- --- --- --- + + +# +# + +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end powershell Payload +#"powershell reporting exitcode: {0}" -f $LASTEXITCODE | write-host +Exit $LASTEXITCODE +# heredoc2 for powershell to ignore block below +$1 = @' +' +: comment end hide powershell-block from Tcl \ +# This comment with closing brace should stay in place whether 'if' commented or not } +: multishell doubled-up cmd exit label - return exitcode +:exit_multishell +:exit_multishell +: \ +@REM @ECHO exitcode: !task_exitcode! +: \ +@IF "%1"=="PUNK-ELEVATED" (echo. & @cmd /k echo elevated prompt: type exit to quit) +: \ +@EXIT /B !task_exitcode! +# cmd has exited +: comment end heredoc2 \ +'@ +<# +# id:tailblock0 +# -- powershell multiline comment +#> +<# +# id:tailblock1 +# + +# +# -- unreachable by tcl directly if ctrl-z character is in the section above. (but file can be read and split on \x1A) +# -- Potential for zip and/or base64 contents, but we can't stop pwsh parser from slurping in the data +# -- so for example a plain text tar archive could cause problems depending on the content. +# -- final line in file must be the powershell multiline comment terminator or other data it can handle. +# -- e.g plain # comment lines will work too +# -- (for example a powershell digital signature is a # commented block of data at the end of the file) +#> + + diff --git a/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/punk-multishell-old.cmd b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/punk-multishell-old.cmd new file mode 100644 index 00000000..1cb9e0ef --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/punk-multishell-old.cmd @@ -0,0 +1,270 @@ +set -- "$@" "a=[list shebangless punk MULTISHELL tclsh sh bash cmd pwsh powershell;proc Hide s {proc $s args {}}; Hide :;rename set s2;Hide set;s2 1 list]"; set -- : "$@"; $1 = @' +: heredoc1 - hide from powershell (close sqote for unix shells) ' \ +: << 'HEREDOC1B_HIDE_FROM_BASH_AND_SH' +: .bat/.cmd launch section, leading colon hides from cmd, trailing slash hides next line from tcl \ +: "[Hide @ECHO; Hide ); Hide (;Hide echo]#not necessary but can help avoid errs in testing" +: Continuation char at end of this line and rem with curly-braces used to exlude Tcl from the whole cmd block \ +@REM { +@REM DO NOT MODIFY FIRST LINE OF THIS SCRIPT. shebang #! line is not required and will reduce functionality. +@REM Even comment lines can be part of the functionality of this script - modify with care. +@REM Change the value of nextshell in the next line if desired, and code within payload sections as appropriate. +@SET "nextshell=pwsh" +@REM nextshell set to pwsh,sh,bash or tclsh +@REM @ECHO nextshell is %nextshell% +@SET "validshells=pwsh,sh,bash,tclsh" +@CALL SET keyRemoved=%%validshells:%nextshell%=%% +@REM Note that 'powershell' e.g v5 is just a fallback for when pwsh is not available +@REM ## ### ### ### ### ### ### ### ### ### ### ### ### ### +@REM -- cmd/batch file section (ignored on unix) +@REM -- This section intended only to launch the next shell +@REM -- Avoid customising this if possible. cmd/batch script is probably the least expressive language. +@REM -- custom windows payloads should be in powershell,tclsh or sh/bash code sections +@REM ## ### ### ### ### ### ### ### ### ### ### ### ### ### +@SETLOCAL EnableExtensions EnableDelayedExpansion +@SET "winpath=%~dp0" +@SET "fname=%~nx0" +@REM @ECHO fname %fname% +@REM @ECHO winpath %winpath% +@IF %nextshell%==pwsh ( + CALL pwsh -nop -c set-executionpolicy -Scope CurrentUser RemoteSigned + COPY "%~dp0%~n0.cmd" "%~dp0%~n0.ps1" >NUL + REM test availability of preferred option of powershell7+ pwsh + CALL pwsh -nop -nol -c write-host "statusmessage: pwsh-found" >NUL + SET pwshtest_exitcode=!errorlevel! + REM ECHO pwshtest_exitcode !pwshtest_exitcode! + IF !pwshtest_exitcode!==0 CALL pwsh -nop -nol "%~dp0%~n0.ps1" %* & SET task_exitcode=!errorlevel! + REM fallback to powershell if pwsh failed + IF NOT !pwshtest_exitcode!==0 ( + REM CALL powershell -nop -nol -c write-host powershell-found + CALL powershell -nop -nol -file "%~dp0%~n0.ps1" %* + SET task_exitcode=!errorlevel! + ) +) ELSE ( + IF %nextshell%==bash ( + CALL :getWslPath %winpath% wslpath + REM ECHO wslfullpath "!wslpath!%fname%" + CALL %nextshell% "!wslpath!%fname%" %* & SET task_exitcode=!errorlevel! + ) ELSE ( + REM probably tclsh or sh + IF NOT "x%keyRemoved%"=="x%validshells%" ( + REM sh uses /c/ instead of /mnt/c - at least if using msys. Todo, review what is the norm on windows with and without msys2,cygwin,wsl + REM and what logic if any may be needed. For now sh with /c/xxx seems to work the same as sh with c:/xxx + CALL %nextshell% "%~dp0%fname%" %* & SET task_exitcode=!errorlevel! + ) ELSE ( + ECHO %fname% has invalid nextshell value %nextshell% valid options are %validshells% + SET task_exitcode=66 + GOTO :exit + ) + ) +) +@GOTO :endlib +:getWslPath +@SETLOCAL + @SET "_path=%~p1" + @SET "name=%~nx1" + @SET "drive=%~d1" + @SET "rtrn=%~2" + @SET "result=/mnt/%drive:~0,1%%_path:\=/%%name%" +@ENDLOCAL & ( + @if "%~2" neq "" ( + SET "%rtrn%=%result%" + ) ELSE ( + ECHO %result% + ) +) +@GOTO :eof +:endlib + +: \ +@REM @SET taskexit_code=!errorlevel! & goto :exit +@GOTO :exit +# } +# rem call %nextshell% "%~dp0%~n0.cmd" %* +# -*- tcl -*- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- tcl script section +# -- This is a punk multishell file +# -- Primary payload target is Tcl, with sh,bash,powershell as helpers +# -- but it may equally be used with any of these being the primary script. +# -- It is tuned to run when called as a batch file, a tcl script a sh/bash script or a pwsh/powershell script +# -- i.e it is a polyglot file. +# -- The specific layout including some lines that appear just as comments is quite sensitive to change. +# -- It can be called on unix or windows platforms with or without the interpreter being specified on the commandline. +# -- e.g ./filename.polypunk.cmd in sh or bash +# -- e.g tclsh filename.cmd +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +rename set ""; rename s2 set; set k {-- "$@" "a}; if {[info exists ::env($k)]} {unset ::env($k)} ;# tidyup +Hide :exit;Hide {<#};Hide '@ +namespace eval ::punk::multishell { + set last_script_root [file dirname [file normalize ${argv0}/__]] + set last_script [file dirname [file normalize [info script]/__]] + if {[info exists argv0] && + $last_script eq $last_script_root + } { + set ::punk::multishell::is_main($last_script) 1 ;#run as executable/script - likely desirable to launch application and return an exitcode + } else { + set ::punk::multishell::is_main($last_script) 0 ;#sourced - likely to be being used as a library - no launch, no exit. Can use return. + } + if {"::punk::multishell::is_main" ni [info commands ::punk::multishell::is_main]} { + proc ::punk::multishell::is_main {{script_name {}}} { + if {$script_name eq ""} { + set script_name [file dirname [file normalize [info script]/--]] + } + if {![info exists ::punk::multishell::is_main($script_name)]} { + #e.g a .dll or something else unanticipated + puts stderr "Warning punk::multishell didn't recognize info script result: $script_name - will treat as if sourced and return instead of exiting" + puts stderr "Info: script_root: [file dirname [file normalize ${argv0}/__]]" + return 0 + } + return [set ::punk::multishell::is_main($script_name)] + } + } +} +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin Tcl Payload +#puts "script : [info script]" +#puts "argcount : $::argc" +#puts "argvalues: $::argv" +#puts "argv0 : $::argv0" +# -- --- --- --- --- --- --- --- --- --- --- --- + + +# +# + + + +# -- --- --- --- --- --- --- --- --- --- --- --- +# -- Best practice is to always return or exit above, or just by leaving the below defaults in place. +# -- If the multishell script is modified to have Tcl below the Tcl Payload section, +# -- then Tcl bracket balancing needs to be carefully managed in the shell and powershell sections below. +# -- Only the # in front of the two relevant if statements below needs to be removed to enable Tcl below +# -- but the sh/bash 'then' and 'fi' would also need to be uncommented. +# -- This facility left in place for experiments on whether configuration payloads etc can be appended +# -- to tail of file - possibly binary with ctrl-z char - but utility is dependent on which other interpreters/shells +# -- can be made to ignore/cope with such data. +if {[::punk::multishell::is_main]} { + exit 0 +} else { + return +} +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end Tcl Payload +# end hide from unix shells \ +HEREDOC1B_HIDE_FROM_BASH_AND_SH +# sh/bash \ +shift && set -- "${@:1:$#-1}" +#------------------------------------------------------ +# -- This if block only needed if Tcl didn't exit or return above. +if false==false # else { + then + : +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- sh/bash script section +# -- leave as is if all that is required is launching the Tcl payload" +# -- +# -- Note that sh/bash script isn't called when running a .bat/.cmd from cmd.exe on windows by default +# -- adjust @call line above ... to something like @call sh ... @call bash .. or @call env sh ... etc as appropriate +# -- if sh/bash scripting needs to run on windows too. +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin sh Payload +#printf "start of bash or sh code" + +# +# + +# -- --- --- --- --- --- --- --- +# +exitcode=0 ;#default assumption +#-- sh/bash launches Tcl here instead of shebang line at top +#-- use exec to use exitcode (if any) directly from the tcl script +#exec /usr/bin/env tclsh "$0" "$@" +#-- alternative - can run sh/bash script after the tcl call. +/usr/bin/env tclsh "$0" "$@" +exitcode=$? +#echo "tcl exitcode: ${exitcode}" +#-- override exitcode example +#exit 66 +# +# -- --- --- --- --- --- --- --- + +# +# + + +#printf "sh/bash done \n" +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end sh Payload +#------------------------------------------------------ +fi +exit ${exitcode} +# end hide sh/bash block from Tcl +# This comment with closing brace should stay in place whether if commented or not } +#------------------------------------------------------ +# begin hide powershell-block from Tcl - only needed if Tcl didn't exit or return above +if 0 { +: end heredoc1 - end hide from powershell \ +'@ +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- powershell/pwsh section +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +function GetScriptName { $myInvocation.ScriptName } +$scriptname = getScriptName +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin powershell Payload +#"Timestamp : {0,10:yyyy-MM-dd HH:mm:ss}" -f $(Get-Date) | write-host +#"Script Name : {0}" -f $scriptname | write-host +#"Powershell Version: {0}" -f $PSVersionTable.PSVersion.Major | write-host +#"powershell args : {0}" -f ($args -join ", ") | write-host +# -- --- --- --- + +# +# + + +# -- --- --- --- --- --- --- --- +# +tclsh $scriptname $args +# +# -- --- --- --- --- --- --- --- + + +# +# + +# unbal } + +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end powershell Payload +#"powershell reporting exitcode: {0}" -f $LASTEXITCODE | write-host +Exit $LASTEXITCODE +# heredoc2 for powershell to ignore block below +$1 = @' +' +: end hide powershell-block from Tcl \ +# This comment with closing brace should stay in place whether 'if' commented or not } +: cmd exit label - return exitcode +:exit +: \ +@REM @ECHO exitcode: !task_exitcode! +: \ +@EXIT /B !task_exitcode! +# cmd has exited +: end heredoc2 \ +'@ +<# +# id:tailblock0 +# -- powershell multiline comment +#> +<# +# id:tailblock1 +# + +# +# -- unreachable by tcl directly if ctrl-z character is in the section above. (but file can be read and split on \x1A) +# -- Potential for zip and/or base64 contents, but we can't stop pwsh parser from slurping in the data +# -- so for example a plain text tar archive could cause problems depending on the content. +# -- final line in file must be the powershell multiline comment terminator or other data it can handle. +# -- e.g plain # comment lines will work too +# -- (for example a powershell digital signature is a # commented block of data at the end of the file) +#> + + diff --git a/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/punk-multishell.cmd b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/punk-multishell.cmd new file mode 100644 index 00000000..0e6b9ff9 --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/punk-multishell.cmd @@ -0,0 +1,661 @@ +: "punk MULTISHELL - shebangless polyglot for Tcl Perl sh bash cmd pwsh powershell" + "[rename set s;proc Hide x {proc $x args {}};Hide :]" + "\$(function : {<#pwsh#>})" + "perlhide" + qw^ +set -- "$@" "a=[Hide <#;Hide set;s 1 list]"; set -- : "$@";$1 = @' +: heredoc1 - hide from powershell using @ and squote above. close sqote for unix shells + ' \ +: .bat/.cmd launch section, leading colon hides from cmd, trailing slash hides next line from tcl + \ +: "[Hide @GOTO; Hide =begin; Hide @REM] #not necessary but can help avoid errs in testing" + +: << 'HEREDOC1B_HIDE_FROM_BASH_AND_SH' +: STRONG SUGGESTION: DO NOT MODIFY FIRST LINE OF THIS SCRIPT - except for first double quoted section. +: shebang line is not required on unix or windows and will reduce functionality and/or portability. +: Even comment lines can be part of the functionality of this script (both on unix and windows) - modify with care. +@GOTO :skip_perl_pod_start ^; +=begin excludeperl +: skip_perl_pod_start +: Continuation char at end of this line and rem with curly-braces used to exlude Tcl from the whole cmd block \ +: { +@REM ############################################################################################################################ +@REM THIS IS A POLYGLOT SCRIPT - supporting payloads in Tcl, bash, sh and/or powershelll (powershell.exe or pwsh.exe) +@REM It should remain portable between unix-like OSes & windows if the proper structure is maintained. +@REM ############################################################################################################################ +@REM On windows, change the value of nextshell to one of the listed 2 digit values if desired, and add code within payload sections for tcl,sh,bash,powershell as appropriate. +@REM This wrapper can be edited manually (carefully!) - or sh,bash,tcl,powershell scripts can be wrapped using the Tcl-based punkshell system +@REM e.g from within a running punkshell: pmix scriptwrap.multishell -outputfolder +@REM On unix-like systems, call with sh, bash or tclsh. (powershell untested on unix - and requires wscript if security elevation is used) +@REM Due to lack of shebang (#! line) Unix-like systems will probably (hopefully) default to sh if the script is called without an interpreter - but it may depend on the shell in use when called. +@REM If you find yourself really wanting/needing to add a shebang line - do so on the basis that the script will exist on unix-like systems only. +@SETLOCAL EnableExtensions EnableDelayedExpansion +@SET "validshells= ^(10^) 'pwsh' ^(11^) 'sh' (^12^) 'bash' (^13^) 'tclsh' (^14^) 'perl'" +@SET "shells[10]=pwsh" +@SET "shells[11]=sh" +@set "shells[12]=bash" +@SET "shells[13]=tclsh" +@SET "shells[14]=perl" +: +@SET "nextshell=13" +: +@rem asadmin is for automatic elevation to administrator. Separate window will be created (seems unavoidable with current elevation mechanism) and user will still get security prompt (probably reasonable). +: +@SET "asadmin=0" +: +@REM nextshell set to index for validshells .eg 10 for pwsh +@REM @ECHO nextshell is %nextshell% +@SET "selected=!shells[%nextshell%]!" +@REM @ECHO selected %selected% +@CALL SET "keyRemoved=%%validshells:'!selected!'=%%" +@REM @ECHO keyremoved %keyRemoved% +@REM Note that 'powershell' e.g v5 is just a fallback for when pwsh is not available +@REM ## ### ### ### ### ### ### ### ### ### ### ### ### ### +@REM -- cmd/batch file section (ignored on unix but should be left in place) +@REM -- This section intended mainly to launch the next shell (and to escalate privileges if necessary) +@REM -- Avoid customising this if you are not familiar with batch scripting. cmd/batch script can be useful, but is probably the least expressive language and most error prone. +@REM -- For example - as this file needs to use unix-style lf line-endings - the label scanner is susceptible to the 512Byte boundary issue: https://www.dostips.com/forum/viewtopic.php?t=8988#p58888 +@REM -- This label issue can be triggered/abused in files with crlf line endings too - but it is less likely to happen accidentaly. +@REm -- See also: https://stackoverflow.com/questions/4094699/how-does-the-windows-command-interpreter-cmd-exe-parse-scripts/4095133#4095133 +@REM ############################################################################################################################ +@REM -- Due to this issue -seemingly trivial edits of the batch file section can break the script! (for Windows anyway) +@REM -- Even something as simple as adding or removing an @REM +@REM -- From within punkshell - use: +@REM -- pmix scriptwrap.checkfile +@REM -- to check your templates or final wrapped scripts for byte boundary issues +@REM -- It will report any labels that are on boundaries +@REM -- This is why the nextshell value above is a 2 digit key instead of a string - so that editing the value doesn't change the byte offsets. +@REM -- Editing your sh,bash,tcl,pwsh payloads is much less likely to cause an issue. There is the possibility of the final batch :exit_multishell label spanning a boundary - so testing using pmix scriptwrap.checkfile is still recommended. +@REM -- Alternatively, as you should do anyway - test the final script on windows +@REM -- Aside from adding comments/whitespace to tweak the location of labels - you can try duplicating the label (e.g just add the label on a line above) but this is not guaranteed to work in all situations. +@REM -- '@REM' is a safer comment mechanism than a leading colon - which is used sparingly here. +@REM -- A colon anywhere in the script that happens to land on a 512 Byte boundary (from file start or from a callsite) could be misinterpreted as a label +@REM -- It is unknown what versions of cmd interpreters behave this way - and pmix scriptwrap.checkfile doesn't check all such boundaries. +@REm -- For this reason, batch labels should be chosen to be relatively unlikely to collide with other strings in the file, and simple names such as :exit or :end should probably be avoided +@REM ############################################################################################################################ +@REM -- custom windows payloads should be in powershell,tclsh (or sh/bash if available) code sections +@REM ## ### ### ### ### ### ### ### ### ### ### ### ### ### +@SET "winpath=%~dp0" +@SET "fname=%~nx0" +@REM @ECHO fname %fname% +@REM @ECHO winpath %winpath% +@REM @ECHO commandlineascalled %0 +@REM @ECHO commandlineresolved %~f0 +@CALL :getNormalizedScriptTail nftail +@REM @ECHO normalizedscripttail %nftail% +@CALL :getFileTail %0 clinetail +@REM @ECHO clinetail %clinetail% +@CALL :stringToUpper %~nx0 capscripttail +@REM @ECHO capscriptname: %capscripttail% + +@IF "%nftail%"=="%capscripttail%" ( + @ECHO forcing asadmin=1 due to file name on filesystem being uppercase + @SET "asadmin=1" +) else ( + @CALL :stringToUpper %clinetail% capcmdlinetail + @REM @ECHO capcmdlinetail !capcmdlinetail! + IF "%clinetail%"=="!capcmdlinetail!" ( + @ECHO forcing asadmin=1 due to cmdline scriptname in uppercase + @set "asadmin=1" + ) +) +@SET "vbsGetPrivileges=%temp%\punk_bat_elevate_%fname%.vbs" +@SET arglist=%* +@IF "%1"=="PUNK-ELEVATED" ( + GOTO :gotPrivileges +) +@IF !asadmin!==1 ( + net file 1>NUL 2>NUL + @IF '!errorlevel!'=='0' ( GOTO :gotPrivileges ) else ( GOTO :getPrivileges ) +) +@GOTO skip_privileges +:getPrivileges +@IF '%1'=='PUNK-ELEVATED' (echo PUNK-ELEVATED & shift /1 & goto :gotPrivileges ) +@ECHO Set UAC = CreateObject^("Shell.Application"^) > "%vbsGetPrivileges%" +@ECHO args = "PUNK-ELEVATED " >> "%vbsGetPrivileges%" +@ECHO For Each strArg in WScript.Arguments >> "%vbsGetPrivileges%" +@ECHO args = args ^& strArg ^& " " >> "%vbsGetPrivileges%" +@ECHO Next >> "%vbsGetPrivileges%" +@ECHO UAC.ShellExecute "%~dp0%~n0%~x0", args, "", "runas", 1 >> "%vbsGetPrivileges%" +@ECHO Launching script in new windows due to administrator elevation +@"%SystemRoot%\System32\WScript.exe" "%vbsGetPrivileges%" %* +@EXIT /B + +:gotPrivileges +@REM setlocal & pushd . +@PUSHD . +@cd /d %~dp0 +@IF "%1"=="PUNK-ELEVATED" ( + @DEL "%vbsGetPrivileges%" 1>nul 2>nul + @SET arglist=%arglist:~14% +) + +:skip_privileges +@SET need_ps1=0 +@REM we want the ps1 to exist even if the nextshell isn't powershell +@if not exist "%~dp0%~n0.ps1" ( + @SET need_ps1=1 +) ELSE ( + fc "%~dp0%~n0%~x0" "%~dp0%~n0.ps1" >nul || goto different + @REM @ECHO "files same" + @SET need_ps1=0 +) +@GOTO :pscontinue +:different +@REM @ECHO "files differ" +@SET need_ps1=1 +:pscontinue +@IF !need_ps1!==1 ( + COPY "%~dp0%~n0%~x0" "%~dp0%~n0.ps1" >NUL +) +@REM avoid using CALL to launch pwsh,tclsh etc - it will intercept some args such as /? +@IF "!shells[%nextshell%]!"=="pwsh" ( + REM pws vs powershell hasn't been tested because we didn't need to copy cmd to ps1 this time + REM test availability of preferred option of powershell7+ pwsh + pwsh -nop -nol -c set-executionpolicy -Scope Process Unrestricted; write-host "statusmessage: pwsh-found" >NUL + SET pwshtest_exitcode=!errorlevel! + REM ECHO pwshtest_exitcode !pwshtest_exitcode! + REM fallback to powershell if pwsh failed + IF !pwshtest_exitcode!==0 ( + pwsh -nop -nol -c set-executionpolicy -Scope Process Unrestricted; "%~dp0%~n0.ps1" %arglist% + SET task_exitcode=!errorlevel! + ) ELSE ( + REM CALL powershell -nop -nol -c write-host powershell-found + REM powershell -nop -nol -file "%~dp0%~n0.ps1" %* + powershell -nop -nol -c set-executionpolicy -Scope Process Unrestricted; %~dp0%~n0.ps1" %arglist% + SET task_exitcode=!errorlevel! + ) +) ELSE ( + IF "!shells[%nextshell%]!"=="bash" ( + CALL :getWslPath %winpath% wslpath + REM ECHO wslfullpath "!wslpath!%fname%" + !shells[%nextshell%]! "!wslpath!%fname%" %arglist% + SET task_exitcode=!errorlevel! + ) ELSE ( + REM probably tclsh or sh + IF NOT "x%keyRemoved%"=="x%validshells%" ( + REM sh on windows uses /c/ instead of /mnt/c - at least if using msys. Todo, review what is the norm on windows with and without msys2,cygwin,wsl + REM and what logic if any may be needed. For now sh with /c/xxx seems to work the same as sh with c:/xxx + !shells[%nextshell%]! "%~dp0%fname%" %arglist% + SET task_exitcode=!errorlevel! + ) ELSE ( + ECHO %fname% has invalid nextshell value ^(%nextshell%^) !shells[%nextshell%]! valid options are %validshells% + SET task_exitcode=66 + @REM boundary padding + GOTO :exit_multishell + ) + ) +) +@REM batch file library functions +@REM boundary padding +@GOTO :endlib + +:getWslPath +@SETLOCAL + @SET "_path=%~p1" + @SET "name=%~nx1" + @SET "drive=%~d1" + @SET "rtrn=%~2" + @SET "result=/mnt/%drive:~0,1%%_path:\=/%%name%" +@ENDLOCAL & ( + @if "%~2" neq "" ( + SET "%rtrn%=%result%" + ) ELSE ( + ECHO %result% + ) +) +@EXIT /B + +:getFileTail +@REM return tail of file without any normalization e.g c:/punkshell/bin/Punk.cmd returns Punk.cmd even if file is punk.cmd +@REM we can't use things such as %~nx1 as it can change capitalisation +@REM This function is designed explicitly to preserve capitalisation +@REM accepts full paths with either / or \ as delimiters - or +@SETLOCAL + @SET "rtrn=%~2" + @SET "arg=%~1" + @REM @SET "result=%_arg:*/=%" + @REM @SET "result=%~1" + @SET LF=^ + + + : The above 2 empty lines are important. Don't remove + @CALL :stringContains "!arg!" "\" hasBackSlash + @IF "!hasBackslash!"=="true" ( + @for %%A in ("!LF!") do @( + @FOR /F %%B in ("!arg:\=%%~A!") do @set "result=%%B" + ) + ) ELSE ( + @CALL :stringContains "!arg!" "/" hasForwardSlash + @IF "!hasForwardSlash!"=="true" ( + @FOR %%A in ("!LF!") do @( + @FOR /F %%B in ("!arg:/=%%~A!") do @set "result=%%B" + ) + ) ELSE ( + @set "result=%arg%" + ) + ) +@ENDLOCAL & ( + @if "%~2" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO %result% + ) +) +@EXIT /B +@REM boundary padding +@REM boundary padding +:getNormalizedScriptTail +@SETLOCAL + @SET "result=%~nx0" + @SET "rtrn=%~1" +@ENDLOCAL & ( + @IF "%~1" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO %result% + ) +) +@EXIT /B + +:getNormalizedFileTailFromPath +@REM warn via echo, and do not set return variable if path not found +@REM note that %~nx1 does not preserve case of provided path - hence the name 'normalized' +@REM boundary padding +@REM boundary padding +@REM boundary padding +@REM boundary padding +@SETLOCAL + @CALL :stringContains %~1 "\" hasBackSlash + @CALL :stringContains %~1 "/" hasForwardSlash + @IF "%hasBackslash%-%hasForwardslash%"=="false-false" ( + @SET "P=%cd%%~1" + @CALL :getNormalizedFileTailFromPath "!P!" ftail2 + @SET "result=!ftail2!" + ) else ( + @IF EXIST "%~1" ( + @SET "result=%~nx1" + ) else ( + @ECHO error getNormalizedFileTailFromPath file not found: %~1 + @EXIT /B 1 + ) + ) + @SET "rtrn=%~2" +@ENDLOCAL & ( + @IF "%~2" neq "" ( + SET "%rtrn%=%result%" + ) ELSE ( + @ECHO getNormalizedFileTailFromPath %1 result: %result% + ) +) +@EXIT /B + +:stringContains +@REM usage: @CALL:stringContains string needle returnvarname +@SETLOCAL + @SET "rtrn=%~3" + @SET "string=%~1" + @SET "needle=%~2" + @IF "!string:%needle%=!"=="!string!" @( + @SET "result=false" + ) ELSE ( + @SET "result=true" + ) +@ENDLOCAL & ( + @IF "%~3" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO stringContains %string% %needle% result: %result% + ) +) +@EXIT /B + +:stringToUpper +@SETLOCAL + @SET "rtrn=%~2" + @SET "string=%~1" + @SET "capstring=%~1" + @FOR %%A in (A B C D E F G H I J K L M N O P Q R S T U V W X Y Z) DO @( + @SET "capstring=!capstring:%%A=%%A!" + ) + @SET "result=!capstring!" +@ENDLOCAL & ( + @IF "%~2" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO stringToUpper %string% result: %result% + ) +) +@EXIT /B + +:isNumeric +@SETLOCAL + @SET "notnumeric="&FOR /F "delims=0123456789" %%i in ("%1") do set "notnumeric=%%i" + @IF defined notnumeric ( + @SET "result=false" + ) else ( + @SET "result=true" + ) + @SET "rtrn=%~2" +@ENDLOCAL & ( + @IF "%~2" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO %result% + ) +) +@EXIT /B + +:endlib +: \ +@REM @SET taskexit_code=!errorlevel! & goto :exit_multishell +@GOTO :exit_multishell +# } +# -*- tcl -*- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- tcl script section +# -- This is a punk multishell file +# -- Primary payload target is Tcl, with sh,bash,powershell as helpers +# -- but it may equally be used with any of these being the primary script. +# -- It is tuned to run when called as a batch file, a tcl script a sh/bash script or a pwsh/powershell script +# -- i.e it is a polyglot file. +# -- The specific layout including some lines that appear just as comments is quite sensitive to change. +# -- It can be called on unix or windows platforms with or without the interpreter being specified on the commandline. +# -- e.g ./filename.polypunk.cmd in sh or bash +# -- e.g tclsh filename.cmd +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +rename set ""; rename s set; set k {-- "$@" "a}; if {[info exists ::env($k)]} {unset ::env($k)} ;# tidyup and restore +Hide :exit_multishell;Hide {<#};Hide '@ +namespace eval ::punk::multishell { + set last_script_root [file dirname [file normalize ${argv0}/__]] + set last_script [file dirname [file normalize [info script]/__]] + if {[info exists argv0] && + $last_script eq $last_script_root + } { + set ::punk::multishell::is_main($last_script) 1 ;#run as executable/script - likely desirable to launch application and return an exitcode + } else { + set ::punk::multishell::is_main($last_script) 0 ;#sourced - likely to be being used as a library - no launch, no exit. Can use return. + } + if {"::punk::multishell::is_main" ni [info commands ::punk::multishell::is_main]} { + proc ::punk::multishell::is_main {{script_name {}}} { + if {$script_name eq ""} { + set script_name [file dirname [file normalize [info script]/--]] + } + if {![info exists ::punk::multishell::is_main($script_name)]} { + #e.g a .dll or something else unanticipated + puts stderr "Warning punk::multishell didn't recognize info script result: $script_name - will treat as if sourced and return instead of exiting" + puts stderr "Info: script_root: [file dirname [file normalize ${argv0}/__]]" + return 0 + } + return [set ::punk::multishell::is_main($script_name)] + } + } +} +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin Tcl Payload +#puts "script : [info script]" +#puts "argcount : $::argc" +#puts "argvalues: $::argv" +#puts "argv0 : $::argv0" +# -- --- --- --- --- --- --- --- --- --- --- --- + + +# +# + +# +# + + +# +# + + +# -- --- --- --- --- --- --- --- --- --- --- --- +# -- Best practice is to always return or exit above, or just by leaving the below defaults in place. +# -- If the multishell script is modified to have Tcl below the Tcl Payload section, +# -- then Tcl bracket balancing needs to be carefully managed in the shell and powershell sections below. +# -- Only the # in front of the two relevant if statements below needs to be removed to enable Tcl below +# -- but the sh/bash 'then' and 'fi' would also need to be uncommented. +# -- This facility left in place for experiments on whether configuration payloads etc can be appended +# -- to tail of file - possibly binary with ctrl-z char - but utility is dependent on which other interpreters/shells +# -- can be made to ignore/cope with such data. +if {[::punk::multishell::is_main]} { + exit 0 +} else { + return +} +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end Tcl Payload +# end hide from unix shells \ +HEREDOC1B_HIDE_FROM_BASH_AND_SH +# sh/bash \ +shift && set -- "${@:1:$#-1}" +#------------------------------------------------------ +# -- This if block only needed if Tcl didn't exit or return above. +if false==false # else { + then + : # +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- sh/bash script section +# -- leave as is if all that is required is launching the Tcl payload" +# -- +# -- Note that sh/bash script isn't called when running a .bat/.cmd from cmd.exe on windows by default +# -- adjust the %nextshell% value above +# -- if sh/bash scripting needs to run on windows too. +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin sh Payload +exitcode=0 +#printf "start of bash or sh code" + +# +# + +# -- --- --- --- --- --- --- --- +# +#-- sh/bash launches Tcl here instead of shebang line at top +#-- use exec to use exitcode (if any) directly from the tcl script +#exec /usr/bin/env tclsh "$0" "$@" +#-- alternative - can run sh/bash script after the tcl call. +/usr/bin/env tclsh "$0" "$@" +exitcode=$? +#echo "sh/bash reporting tcl exitcode: ${exitcode}" +#-- override exitcode example +#exit 66 +# +# -- --- --- --- --- --- --- --- + +# +# + + +#printf "sh/bash done \n" +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end sh Payload +#------------------------------------------------------ +fi +exit ${exitcode} +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- Perl script section +# -- leave the script below as is, if all that is required is launching the Tcl payload" +# -- +# -- Note that perl script isn't called by default when simply running this script by name +# -- adjust the nextshell value at the top of the script to point to perl +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +=cut +#!/user/bin/perl +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin perl Payload +my $exit_code = 0; +#use ExtUtils::Installed; +#my $installed = ExtUtils::Installed->new(); +#my @modules = $installed->modules(); +#print "Modules:\n"; +#foreach my $m (@modules) { +# print "$m\n"; +#} +# -- --- --- + + + +my $scriptname = $0; +print "perl $scriptname\n"; +my $i =1; +foreach my $a(@ARGV) { + print "Arg # $i: $a\n"; +} + +# +# + + + +# -- --- --- --- --- --- --- --- +# +$exit_code=system("tclsh", $scriptname, @ARGV); +#print "perl reporting tcl exitcode: $exit_code"; +# +# -- --- --- --- --- --- --- --- + +# +# + + +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end perl Payload +exit $exit_code; +__END__ + +# end hide sh/bash/perl block from Tcl +# This comment with closing brace should stay in place whether if commented or not } +#------------------------------------------------------ +# begin hide powershell-block from Tcl - only needed if Tcl didn't exit or return above +if 0 { +: end heredoc1 - end hide from powershell \ +'@ +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- powershell/pwsh section +# -- Do not edit if current file is the .ps1 +# -- Edit the corresponding .cmd and it will autocopy +# -- unbalanced braces { } here *even in comments* will cause problems if there was no Tcl exit or return above +# -- custom script should generally go below the begin_powershell_payload line +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +function GetScriptName { $myInvocation.ScriptName } +$scriptname = GetScriptName +function GetDynamicParamDictionary { + [CmdletBinding()] + param( + [Parameter(ValueFromPipeline=$true, Mandatory=$true)] + [string] $CommandName + ) + + begin { + # Get a list of params that should be ignored (they're common to all advanced functions) + $CommonParameterNames = [System.Runtime.Serialization.FormatterServices]::GetUninitializedObject([type] [System.Management.Automation.Internal.CommonParameters]) | + Get-Member -MemberType Properties | + Select-Object -ExpandProperty Name + } + + process { + # Create the dictionary that this scriptblock will return: + $DynParamDictionary = New-Object System.Management.Automation.RuntimeDefinedParameterDictionary + + # Convert to object array and get rid of Common params: + (Get-Command $CommandName | select -exp Parameters).GetEnumerator() | + Where-Object { $CommonParameterNames -notcontains $_.Key } | + ForEach-Object { + $DynamicParameter = New-Object System.Management.Automation.RuntimeDefinedParameter ( + $_.Key, + $_.Value.ParameterType, + $_.Value.Attributes + ) + $DynParamDictionary.Add($_.Key, $DynamicParameter) + } + + # Return the dynamic parameters + return $DynParamDictionary + } +} +# GetDynamicParamDictionary +# - This can make it easier to share a single set of param definitions between functions +# - sample usage +#function ParameterDefinitions { +# param( +# [Parameter(Mandatory)][string] $myargument +# ) +#} +#function psmain { +# [CmdletBinding()] +# param() +# dynamicparam { GetDynamicParamDictionary ParameterDefinitions } +# process { +# #called once with $PSBoundParameters dictionary +# #can be used to validate arguments, or set a simpler variable name for access +# switch ($PSBoundParameters.keys) { +# 'myargumentname' { +# Set-Variable -Name $_ -Value $PSBoundParameters."$_" +# } +# #... +# } +# foreach ($boundparam in $PSBoundParameters.GetEnumerator()) { +# #... +# } +# } +# end { +# #Main function logic +# Write-Host "myargumentname value is: $myargumentname" +# #myotherfunction @PSBoundParameters +# } +#} +#psmain @args +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin powershell Payload +#"Timestamp : {0,10:yyyy-MM-dd HH:mm:ss}" -f $(Get-Date) | write-host +#"Script Name : {0}" -f $scriptname | write-host +#"Powershell Version: {0}" -f $PSVersionTable.PSVersion.Major | write-host +#"powershell args : {0}" -f ($args -join ", ") | write-host +# -- --- --- --- + +# +# + + +# -- --- --- --- --- --- --- --- +# +tclsh $scriptname $args +#"powershell reporting exitcode: {0}" -f $LASTEXITCODE | write-host +# +# -- --- --- --- --- --- --- --- + + +# +# + +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end powershell Payload +Exit $LASTEXITCODE +# heredoc2 for powershell to ignore block below +$1 = @' +' +: comment end hide powershell-block from Tcl \ +# This comment with closing brace should stay in place whether 'if' commented or not } +: multishell doubled-up cmd exit label - return exitcode +:exit_multishell +:exit_multishell +: \ +@REM @ECHO exitcode: !task_exitcode! +: \ +@IF "%1"=="PUNK-ELEVATED" (echo. & @cmd /k echo elevated prompt: type exit to quit) +: \ +@EXIT /B !task_exitcode! +# cmd has exited +: comment end heredoc2 \ +'@ +<# +# id:tailblock0 +# -- powershell multiline comment +#> +<# +no script engine should try to run me +# id:tailblock1 +# + +# +# -- unreachable by tcl directly if ctrl-z character is in the section above. (but file can be read and split on \x1A) +# -- Potential for zip and/or base64 contents, but we can't stop pwsh parser from slurping in the data +# -- so for example a plain text tar archive could cause problems depending on the content. +# -- final line in file must be the powershell multiline comment terminator or other data it can handle. +# -- e.g plain # comment lines will work too +# -- (for example a powershell digital signature is a # commented block of data at the end of the file) +#> + + diff --git a/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/punk-multishell1.cmd b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/punk-multishell1.cmd new file mode 100644 index 00000000..17fe4c15 --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/punk-multishell1.cmd @@ -0,0 +1,524 @@ +: "[rename set s;proc Hide x {proc $x args {}};Hide :]" "\$(function : {<#pwsh#>})" ^ +set -- "$@" "a=[list shebangless punk MULTISHELL tclsh sh bash cmd pwsh powershell;proc Hide x {proc $x args {}}; Hide <#;Hide set;s 1 list]"; set -- : "$@";$1 = @' +: heredoc1 - hide from powershell using @ and squote above. (close sqote for unix shells) ' \ +: .bat/.cmd launch section, leading colon hides from cmd, trailing slash hides next line from tcl \ +: "[Hide @ECHO; Hide ); Hide (;Hide echo; Hide @REM]#not necessary but can help avoid errs in testing" +: << 'HEREDOC1B_HIDE_FROM_BASH_AND_SH' +: Continuation char at end of this line and rem with curly-braces used to exlude Tcl from the whole cmd block \ +: { +: STRONG SUGGESTION: DO NOT MODIFY FIRST LINE OF THIS SCRIPT. shebang #! line is not required on unix or windows and will reduce functionality and/or portability. +: Even comment lines can be part of the functionality of this script (both on unix and windows) - modify with care. +@REM ############################################################################################################################ +@REM THIS IS A POLYGLOT SCRIPT - supporting payloads in Tcl, bash, sh and/or powershelll (powershell.exe or pwsh.exe) +@REM It should remain portable between unix-like OSes & windows if the proper structure is maintained. +@REM ############################################################################################################################ +@REM On windows, change the value of nextshell to one of the listed 2 digit values if desired, and add code within payload sections for tcl,sh,bash,powershell as appropriate. +@REM This wrapper can be edited manually (carefully!) - or sh,bash,tcl,powershell scripts can be wrapped using the Tcl-based punkshell system +@REM e.g from within a running punkshell: pmix scriptwrap.multishell -outputfolder +@REM On unix-like systems, call with sh, bash or tclsh. (powershell untested on unix - and requires wscript if security elevation is used) +@REM Due to lack of shebang (#! line) Unix-like systems will probably (hopefully) default to sh if the script is called without an interpreter - but it may depend on the shell in use when called. +@REM If you find yourself really wanting/needing to add a shebang line - do so on the basis that the script will exist on unix-like systems only. +@SETLOCAL EnableExtensions EnableDelayedExpansion +@SET "validshells= ^(10^) 'pwsh' ^(11^) 'sh' (^12^) 'bash' (^13^) 'tclsh'" +@SET "shells[10]=pwsh" +@SET "shells[11]=sh" +@set "shells[12]=bash" +@SET "shells[13]=tclsh" +: +@SET "nextshell=13" +: +@rem asadmin is for automatic elevation to administrator. Separate window will be created (seems unavoidable with current elevation mechanism) and user will still get security prompt (probably reasonable). +: +@SET "asadmin=0" +: +@REM nextshell set to index for validshells .eg 10 for pwsh +@REM @ECHO nextshell is %nextshell% +@SET "selected=!shells[%nextshell%]!" +@REM @ECHO selected %selected% +@CALL SET "keyRemoved=%%validshells:'!selected!'=%%" +@REM @ECHO keyremoved %keyRemoved% +@REM Note that 'powershell' e.g v5 is just a fallback for when pwsh is not available +@REM ## ### ### ### ### ### ### ### ### ### ### ### ### ### +@REM -- cmd/batch file section (ignored on unix but should be left in place) +@REM -- This section intended mainly to launch the next shell (and to escalate privileges if necessary) +@REM -- Avoid customising this if you are not familiar with batch scripting. cmd/batch script can be useful, but is probably the least expressive language and most error prone. +@REM -- For example - as this file needs to use unix-style lf line-endings - the label scanner is susceptible to the 512Byte boundary issue: https://www.dostips.com/forum/viewtopic.php?t=8988#p58888 +@REM -- This label issue can be triggered/abused in files with crlf line endings too - but it is less likely to happen accidentaly. +@REm -- See also: https://stackoverflow.com/questions/4094699/how-does-the-windows-command-interpreter-cmd-exe-parse-scripts/4095133#4095133 +@REM ############################################################################################################################ +@REM -- Due to this issue -seemingly trivial edits of the batch file section can break the script! (for Windows anyway) +@REM -- Even something as simple as adding or removing an @REM +@REM -- From within punkshell - use: +@REM -- pmix scriptwrap.checkfile +@REM -- to check your templates or final wrapped scripts for byte boundary issues +@REM -- It will report any labels that are on boundaries +@REM -- This is why the nextshell value above is a 2 digit key instead of a string - so that editing the value doesn't change the byte offsets. +@REM -- Editing your sh,bash,tcl,pwsh payloads is much less likely to cause an issue. There is the possibility of the final batch :exit_multishell label spanning a boundary - so testing using pmix scriptwrap.checkfile is still recommended. +@REM -- Alternatively, as you should do anyway - test the final script on windows +@REM -- Aside from adding comments/whitespace to tweak the location of labels - you can try duplicating the label (e.g just add the label on a line above) but this is not guaranteed to work in all situations. +@REM -- '@REM' is a safer comment mechanism than a leading colon - which is used sparingly here. +@REM -- A colon anywhere in the script that happens to land on a 512 Byte boundary (from file start or from a callsite) could be misinterpreted as a label +@REM -- It is unknown what versions of cmd interpreters behave this way - and pmix scriptwrap.checkfile doesn't check all such boundaries. +@REm -- For this reason, batch labels should be chosen to be relatively unlikely to collide with other strings in the file, and simple names such as :exit or :end should probably be avoided +@REM ############################################################################################################################ +@REM -- custom windows payloads should be in powershell,tclsh (or sh/bash if available) code sections +@REM ## ### ### ### ### ### ### ### ### ### ### ### ### ### +@SET "winpath=%~dp0" +@SET "fname=%~nx0" +@REM @ECHO fname %fname% +@REM @ECHO winpath %winpath% +@REM @ECHO commandlineascalled %0 +@REM @ECHO commandlineresolved %~f0 +@CALL :getNormalizedScriptTail nftail +@REM @ECHO normalizedscripttail %nftail% +@CALL :getFileTail %0 clinetail +@REM @ECHO clinetail %clinetail% +@CALL :stringToUpper %~nx0 capscripttail +@REM @ECHO capscriptname: %capscripttail% + +@IF "%nftail%"=="%capscripttail%" ( + @ECHO forcing asadmin=1 due to file name on filesystem being uppercase + @SET "asadmin=1" +) else ( + @CALL :stringToUpper %clinetail% capcmdlinetail + @REM @ECHO capcmdlinetail !capcmdlinetail! + IF "%clinetail%"=="!capcmdlinetail!" ( + @ECHO forcing asadmin=1 due to cmdline scriptname in uppercase + @set "asadmin=1" + ) +) +@SET "vbsGetPrivileges=%temp%\punk_bat_elevate_%fname%.vbs" +@SET arglist=%* +@IF "%1"=="PUNK-ELEVATED" ( + GOTO :gotPrivileges +) +@IF !asadmin!==1 ( + net file 1>NUL 2>NUL + @IF '!errorlevel!'=='0' ( GOTO :gotPrivileges ) else ( GOTO :getPrivileges ) +) +@GOTO skip_privileges +:getPrivileges +@IF '%1'=='PUNK-ELEVATED' (echo PUNK-ELEVATED & shift /1 & goto :gotPrivileges ) +@ECHO Set UAC = CreateObject^("Shell.Application"^) > "%vbsGetPrivileges%" +@ECHO args = "PUNK-ELEVATED " >> "%vbsGetPrivileges%" +@ECHO For Each strArg in WScript.Arguments >> "%vbsGetPrivileges%" +@ECHO args = args ^& strArg ^& " " >> "%vbsGetPrivileges%" +@ECHO Next >> "%vbsGetPrivileges%" +@ECHO UAC.ShellExecute "%~dp0%~n0.cmd", args, "", "runas", 1 >> "%vbsGetPrivileges%" +@ECHO Launching script in new windows due to administrator elevation +@"%SystemRoot%\System32\WScript.exe" "%vbsGetPrivileges%" %* +@EXIT /B + +:gotPrivileges +@REM setlocal & pushd . +@PUSHD . +@cd /d %~dp0 +@IF "%1"=="PUNK-ELEVATED" ( + @DEL "%vbsGetPrivileges%" 1>nul 2>nul + @SET arglist=%arglist:~14% +) + +:skip_privileges +@SET need_ps1=0 +@REM we want the ps1 to exist even if the nextshell isn't powershell +@if not exist "%~dp0%~n0.ps1" ( + @SET need_ps1=1 +) ELSE ( + fc "%~dp0%~n0.cmd" "%~dp0%~n0.ps1" >nul || goto different + @REM @ECHO "files same" + @SET need_ps1=0 +) +@GOTO :pscontinue +:different +@REM @ECHO "files differ" +@SET need_ps1=1 +:pscontinue +@IF !need_ps1!==1 ( + COPY "%~dp0%~n0.cmd" "%~dp0%~n0.ps1" >NUL +) +@REM avoid using CALL to launch pwsh,tclsh etc - it will intercept some args such as /? +@IF "!shells[%nextshell%]!"=="pwsh" ( + REM pws vs powershell hasn't been tested because we didn't need to copy cmd to ps1 this time + REM test availability of preferred option of powershell7+ pwsh + pwsh -nop -nol -c set-executionpolicy -Scope Process Unrestricted; write-host "statusmessage: pwsh-found" >NUL + SET pwshtest_exitcode=!errorlevel! + REM ECHO pwshtest_exitcode !pwshtest_exitcode! + REM fallback to powershell if pwsh failed + IF !pwshtest_exitcode!==0 ( + pwsh -nop -nol -c set-executionpolicy -Scope Process Unrestricted; "%~dp0%~n0.ps1" %arglist% & SET task_exitcode=!errorlevel! + ) ELSE ( + REM CALL powershell -nop -nol -c write-host powershell-found + REM powershell -nop -nol -file "%~dp0%~n0.ps1" %* + powershell -nop -nol -c set-executionpolicy -Scope Process Unrestricted; %~dp0%~n0.ps1" %arglist% + SET task_exitcode=!errorlevel! + ) +) ELSE ( + IF "!shells[%nextshell%]!"=="bash" ( + CALL :getWslPath %winpath% wslpath + REM ECHO wslfullpath "!wslpath!%fname%" + !shells[%nextshell%]! "!wslpath!%fname%" %arglist% & SET task_exitcode=!errorlevel! + ) ELSE ( + REM probably tclsh or sh + IF NOT "x%keyRemoved%"=="x%validshells%" ( + REM sh on windows uses /c/ instead of /mnt/c - at least if using msys. Todo, review what is the norm on windows with and without msys2,cygwin,wsl + REM and what logic if any may be needed. For now sh with /c/xxx seems to work the same as sh with c:/xxx + !shells[%nextshell%]! "%~dp0%fname%" %arglist% & SET task_exitcode=!errorlevel! + ) ELSE ( + ECHO %fname% has invalid nextshell value ^(%nextshell%^) !shells[%nextshell%]! valid options are %validshells% + SET task_exitcode=66 + GOTO :exit_multishell + ) + ) +) +@REM batch file library functions +@GOTO :endlib + +:getWslPath +@SETLOCAL + @SET "_path=%~p1" + @SET "name=%~nx1" + @SET "drive=%~d1" + @SET "rtrn=%~2" + @SET "result=/mnt/%drive:~0,1%%_path:\=/%%name%" +@ENDLOCAL & ( + @if "%~2" neq "" ( + SET "%rtrn%=%result%" + ) ELSE ( + ECHO %result% + ) +) +@EXIT /B + +:getFileTail +@REM return tail of file without any normalization e.g c:/punkshell/bin/Punk.cmd returns Punk.cmd even if file is punk.cmd +@REM we can't use things such as %~nx1 as it can change capitalisation +@REM This function is designed explicitly to preserve capitalisation +@REM accepts full paths with either / or \ as delimiters - or +@SETLOCAL + @SET "rtrn=%~2" + @SET "arg=%~1" + @REM @SET "result=%_arg:*/=%" + @REM @SET "result=%~1" + @SET LF=^ + + + : The above 2 empty lines are important. Don't remove + @CALL :stringContains "!arg!" "\" hasBackSlash + @IF "!hasBackslash!"=="true" ( + @for %%A in ("!LF!") do @( + @FOR /F %%B in ("!arg:\=%%~A!") do @set "result=%%B" + ) + ) ELSE ( + @CALL :stringContains "!arg!" "/" hasForwardSlash + @IF "!hasForwardSlash!"=="true" ( + @FOR %%A in ("!LF!") do @( + @FOR /F %%B in ("!arg:/=%%~A!") do @set "result=%%B" + ) + ) ELSE ( + @set "result=%arg%" + ) + ) +@ENDLOCAL & ( + @if "%~2" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO %result% + ) +) +@EXIT /B +@REM boundary padding +:getNormalizedScriptTail +@SETLOCAL + @SET "result=%~nx0" + @SET "rtrn=%~1" +@ENDLOCAL & ( + @IF "%~1" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO %result% + ) +) +@EXIT /B + +:getNormalizedFileTailFromPath +@REM warn via echo, and do not set return variable if path not found +@REM note that %~nx1 does not preserve case of provided path - hence the name 'normalized' +@REM boundary padding +@REM boundary padding +@SETLOCAL + @CALL :stringContains %~1 "\" hasBackSlash + @CALL :stringContains %~1 "/" hasForwardSlash + @IF "%hasBackslash%-%hasForwardslash%"=="false-false" ( + @SET "P=%cd%%~1" + @CALL :getNormalizedFileTailFromPath "!P!" ftail2 + @SET "result=!ftail2!" + ) else ( + @IF EXIST "%~1" ( + @SET "result=%~nx1" + ) else ( + @ECHO error getNormalizedFileTailFromPath file not found: %~1 + @EXIT /B 1 + ) + ) + @SET "rtrn=%~2" +@ENDLOCAL & ( + @IF "%~2" neq "" ( + SET "%rtrn%=%result%" + ) ELSE ( + @ECHO getNormalizedFileTailFromPath %1 result: %result% + ) +) +@EXIT /B + +:stringContains +@REM usage: @CALL:stringContains string needle returnvarname +@SETLOCAL + @SET "rtrn=%~3" + @SET "string=%~1" + @SET "needle=%~2" + @IF "!string:%needle%=!"=="!string!" @( + @SET "result=false" + ) ELSE ( + @SET "result=true" + ) +@ENDLOCAL & ( + @IF "%~3" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO stringContains %string% %needle% result: %result% + ) +) +@EXIT /B + +:stringToUpper +@SETLOCAL + @SET "rtrn=%~2" + @SET "string=%~1" + @SET "capstring=%~1" + @FOR %%A in (A B C D E F G H I J K L M N O P Q R S T U V W X Y Z) DO @( + @SET "capstring=!capstring:%%A=%%A!" + ) + @SET "result=!capstring!" +@ENDLOCAL & ( + @IF "%~2" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO stringToUpper %string% result: %result% + ) +) +@EXIT /B + +:isNumeric +@SETLOCAL + @SET "notnumeric="&FOR /F "delims=0123456789" %%i in ("%1") do set "notnumeric=%%i" + @IF defined notnumeric ( + @SET "result=false" + ) else ( + @SET "result=true" + ) + @SET "rtrn=%~2" +@ENDLOCAL & ( + @IF "%~2" neq "" ( + @SET "%rtrn%=%result%" + ) ELSE ( + @ECHO %result% + ) +) +@EXIT /B + +:endlib +: \ +@REM @SET taskexit_code=!errorlevel! & goto :exit_multishell +@GOTO :exit_multishell +# } +# -*- tcl -*- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- tcl script section +# -- This is a punk multishell file +# -- Primary payload target is Tcl, with sh,bash,powershell as helpers +# -- but it may equally be used with any of these being the primary script. +# -- It is tuned to run when called as a batch file, a tcl script a sh/bash script or a pwsh/powershell script +# -- i.e it is a polyglot file. +# -- The specific layout including some lines that appear just as comments is quite sensitive to change. +# -- It can be called on unix or windows platforms with or without the interpreter being specified on the commandline. +# -- e.g ./filename.polypunk.cmd in sh or bash +# -- e.g tclsh filename.cmd +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +rename set ""; rename s set; set k {-- "$@" "a}; if {[info exists ::env($k)]} {unset ::env($k)} ;# tidyup and restore +Hide :exit_multishell;Hide {<#};Hide '@ +namespace eval ::punk::multishell { + set last_script_root [file dirname [file normalize ${argv0}/__]] + set last_script [file dirname [file normalize [info script]/__]] + if {[info exists argv0] && + $last_script eq $last_script_root + } { + set ::punk::multishell::is_main($last_script) 1 ;#run as executable/script - likely desirable to launch application and return an exitcode + } else { + set ::punk::multishell::is_main($last_script) 0 ;#sourced - likely to be being used as a library - no launch, no exit. Can use return. + } + if {"::punk::multishell::is_main" ni [info commands ::punk::multishell::is_main]} { + proc ::punk::multishell::is_main {{script_name {}}} { + if {$script_name eq ""} { + set script_name [file dirname [file normalize [info script]/--]] + } + if {![info exists ::punk::multishell::is_main($script_name)]} { + #e.g a .dll or something else unanticipated + puts stderr "Warning punk::multishell didn't recognize info script result: $script_name - will treat as if sourced and return instead of exiting" + puts stderr "Info: script_root: [file dirname [file normalize ${argv0}/__]]" + return 0 + } + return [set ::punk::multishell::is_main($script_name)] + } + } +} +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin Tcl Payload +#puts "script : [info script]" +#puts "argcount : $::argc" +#puts "argvalues: $::argv" +#puts "argv0 : $::argv0" +# -- --- --- --- --- --- --- --- --- --- --- --- + + +# +# + + + +# -- --- --- --- --- --- --- --- --- --- --- --- +# -- Best practice is to always return or exit above, or just by leaving the below defaults in place. +# -- If the multishell script is modified to have Tcl below the Tcl Payload section, +# -- then Tcl bracket balancing needs to be carefully managed in the shell and powershell sections below. +# -- Only the # in front of the two relevant if statements below needs to be removed to enable Tcl below +# -- but the sh/bash 'then' and 'fi' would also need to be uncommented. +# -- This facility left in place for experiments on whether configuration payloads etc can be appended +# -- to tail of file - possibly binary with ctrl-z char - but utility is dependent on which other interpreters/shells +# -- can be made to ignore/cope with such data. +if {[::punk::multishell::is_main]} { + exit 0 +} else { + return +} +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end Tcl Payload +# end hide from unix shells \ +HEREDOC1B_HIDE_FROM_BASH_AND_SH +# sh/bash \ +shift && set -- "${@:1:$#-1}" +#------------------------------------------------------ +# -- This if block only needed if Tcl didn't exit or return above. +if false==false # else { + then + : # +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- sh/bash script section +# -- leave as is if all that is required is launching the Tcl payload" +# -- +# -- Note that sh/bash script isn't called when running a .bat/.cmd from cmd.exe on windows by default +# -- adjust @call line above ... to something like @call sh ... @call bash .. or @call env sh ... etc as appropriate +# -- if sh/bash scripting needs to run on windows too. +# -- +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin sh Payload +#printf "start of bash or sh code" + +# +# + +# -- --- --- --- --- --- --- --- +# +exitcode=0 ;#default assumption +#-- sh/bash launches Tcl here instead of shebang line at top +#-- use exec to use exitcode (if any) directly from the tcl script +#exec /usr/bin/env tclsh "$0" "$@" +#-- alternative - can run sh/bash script after the tcl call. +/usr/bin/env tclsh "$0" "$@" +exitcode=$? +#echo "tcl exitcode: ${exitcode}" +#-- override exitcode example +#exit 66 +# +# -- --- --- --- --- --- --- --- + +# +# + + +#printf "sh/bash done \n" +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end sh Payload +#------------------------------------------------------ +fi +exit ${exitcode} +# end hide sh/bash block from Tcl +# This comment with closing brace should stay in place whether if commented or not } +#------------------------------------------------------ +# begin hide powershell-block from Tcl - only needed if Tcl didn't exit or return above +if 0 { +: end heredoc1 - end hide from powershell \ +'@ +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +# -- powershell/pwsh section +# -- Do not edit if current file is the .ps1 +# -- Edit the corresponding .cmd and it will autocopy +# -- unbalanced braces { } here *even in comments* will cause problems if there was no Tcl exit or return above +# ## ### ### ### ### ### ### ### ### ### ### ### ### ### +function GetScriptName { $myInvocation.ScriptName } +$scriptname = getScriptName +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---begin powershell Payload +#"Timestamp : {0,10:yyyy-MM-dd HH:mm:ss}" -f $(Get-Date) | write-host +#"Script Name : {0}" -f $scriptname | write-host +#"Powershell Version: {0}" -f $PSVersionTable.PSVersion.Major | write-host +#"powershell args : {0}" -f ($args -join ", ") | write-host +# -- --- --- --- + +# +# + + +# -- --- --- --- --- --- --- --- +# +tclsh $scriptname $args +# +# -- --- --- --- --- --- --- --- + + +# +# + +# -- --- --- --- --- --- --- --- --- --- --- --- --- ---end powershell Payload +#"powershell reporting exitcode: {0}" -f $LASTEXITCODE | write-host +Exit $LASTEXITCODE +# heredoc2 for powershell to ignore block below +$1 = @' +' +: comment end hide powershell-block from Tcl \ +# This comment with closing brace should stay in place whether 'if' commented or not } +: multishell doubled-up cmd exit label - return exitcode +:exit_multishell +:exit_multishell +: \ +@REM @ECHO exitcode: !task_exitcode! +: \ +@IF "%1"=="PUNK-ELEVATED" (echo. & @cmd /k echo elevated prompt: type exit to quit) +: \ +@EXIT /B !task_exitcode! +# cmd has exited +: comment end heredoc2 \ +'@ +<# +# id:tailblock0 +# -- powershell multiline comment +#> +<# +# id:tailblock1 +# + +# +# -- unreachable by tcl directly if ctrl-z character is in the section above. (but file can be read and split on \x1A) +# -- Potential for zip and/or base64 contents, but we can't stop pwsh parser from slurping in the data +# -- so for example a plain text tar archive could cause problems depending on the content. +# -- final line in file must be the powershell multiline comment terminator or other data it can handle. +# -- e.g plain # comment lines will work too +# -- (for example a powershell digital signature is a # commented block of data at the end of the file) +#> + + diff --git a/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/punk-shellbat.bat b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/punk-shellbat.bat new file mode 100644 index 00000000..aa9039a9 --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/utility/scriptappwrappers/punk-shellbat.bat @@ -0,0 +1,112 @@ +: "[proc : args {}]" ;# *tcl shellbat - call with sh,bash,tclsh on any platform, or with cmd on windows. +: <<'HIDE_FROM_BASH_AND_SH' +: ;# leading colon hides from .bat, trailing slash hides next line from tcl \ +@call tclsh "%~dp0%~n0.bat" %* +: ;#\ +@set taskexitcode=%errorlevel% & goto :exit +# -*- tcl -*- +# ################################################################################################# +# This is a tcl shellbat file +# It is tuned to run when called as a batch file, a tcl script, an sh script or a bash script, +# so the specific layout and characters used are quite sensitive to change. +# It can be called on unix or windows platforms with or without the interpreter being specified on the commandline. +# e.g ./filename.sh.bat in sh or bash or powershell +# e.g filename.sh or filename.sh.bat at windows command prompt +# e.g tclsh filename.sh.bat | sh filename.sh.bat | bash filename.sh.bat +# In all cases an arbitrary number of arguments are accepted +# To avoid the initial commandline on stdout when calling as a batch file on windows, use: +# cmd /Q /c filename.sh.bat +# (because we cannot use @if to silence it, as this isn't understood by tcl,sh or bash) +# ################################################################################################# +#fconfigure stdout -translation crlf +# --- --- --- --- --- --- --- --- --- --- --- --- ---begin Tcl Payload +#puts "script : [info script]" +#puts "argcount : $::argc" +#puts "argvalues: $::argv" + + +# +# + +# --- --- --- --- --- --- --- --- --- --- --- --- --- +# only exit if needed. see exitcode notes at bottom of file and exit there for consistency across invocation methods +# --- --- --- --- --- --- --- --- --- --- --- --- ---end Tcl Payload +#-- +#-- bash/sh code follows. +#-- protect from tcl using line continuation char on the previous comment for each line, like so: \ +printf "etc" +#-- or alternatively place sh/bash script within the false==false block +#-- whilst being careful to balance braces {} +#-- For more complex needs you should call out to external scripts +#-- +#-- END marker for hide_from_bash_and_sh\ +HIDE_FROM_BASH_AND_SH + +#--------------------------------------------------------- +#-- This if statement hides(mostly) a sh/bash code block from Tcl +if false==false # else { +then +: +#--------------------------------------------------------- + #-- leave as is if all that's required is launching the Tcl payload" + #-- + #-- Note that sh/bash script isn't called when running a .bat from cmd.exe on windows by default + #-- adjust line 4: @call tclsh ... to something like @call sh ... @call bash .. or @call env sh ... etc as appropriate + #-- if sh/bash scripting needs to run on windows too. + #-- + #printf "start of bash or sh code" + + # + # + + + #-- sh/bash launches Tcl here instead of shebang line at top + # + #-- use exec to use exitcode (if any) directly from the tcl script + exec /usr/bin/env tclsh "$0" "$@" + # + + #-- alternative - if sh/bash script required to run after the tcl call. + #/usr/bin/env tclsh "$0" "$@" + #tcl_exitcode=$? + #echo "tcl_exitcode: ${tcl_exitcode}" + + # + # + + #-- override exitcode example + #exit 66 + + #printf "No need for trailing slashes for sh/bash code here\n" +#--------------------------------------------------------- +fi +# closing brace for Tcl } +#--------------------------------------------------------- + +#-- tcl and shell script now both active + +#-- comment for line sample 1 with trailing continuation slash \ +#printf "tcl-invisible sh/bash line sample 1 \n" + +#-- comment for line sample 2 with trailing continuation slash \ +#printf "tcl-invisible sh/bash line sample 2 \n" + + +#-- Consistent exitcode from sh,bash,tclsh or cmd +#-- Call exit in tcl (or sh/bash) code only if explicitly required, otherwise leave this commented out. +#-- (script might be more widely useable without explicit exit. e.g in tcl: set ::argc 1; set ::argv "val"; source filename.sh.bat ) +#-- exit line unprotected by trailing slash will work for tcl and/or sh/bash +#exit 0 +#exit 42 + + + +#-- make sure sh/bash/tcl all skip over .bat style exit \ +: <<'shell_end' +#-- .bat exit with exitcode from tcl process \ +:exit +: ;# \ +@exit /B %taskexitcode% +# .bat has exited \ +shell_end + diff --git a/src/bootsupport/modules/punk/mix/templates/utility/shellbat.txt b/src/bootsupport/modules/punk/mix/templates/utility/shellbat.txt new file mode 100644 index 00000000..25c7d1d8 --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/utility/shellbat.txt @@ -0,0 +1,104 @@ +: "[proc : args {}]" ;# *tcl shellbat - call with sh,bash,tclsh on any platform, or with cmd on windows. +: <<'HIDE_FROM_BASH_AND_SH' +: ;# leading colon hides from .bat, trailing slash hides next line from tcl \ +@call tclsh "%~dp0%~n0.bat" %* +: ;#\ +@set taskexitcode=%errorlevel% & goto :exit +# -*- tcl -*- +# ################################################################################################# +# This is a tcl shellbat file +# It is tuned to run when called as a batch file, a tcl script, an sh script or a bash script, +# so the specific layout and characters used are quite sensitive to change. +# It can be called on unix or windows platforms with or without the interpreter being specified on the commandline. +# e.g ./filename.sh.bat in sh or bash or powershell +# e.g filename.sh or filename.sh.bat at windows command prompt +# e.g tclsh filename.sh.bat | sh filename.sh.bat | bash filename.sh.bat +# In all cases an arbitrary number of arguments are accepted +# To avoid the initial commandline on stdout when calling as a batch file on windows, use: +# cmd /Q /c filename.sh.bat +# (because we cannot use @if to silence it, as this isn't understood by tcl,sh or bash) +# ################################################################################################# +#fconfigure stdout -translation crlf +# --- --- --- --- --- --- --- --- --- --- --- --- ---begin Tcl Payload +#puts "script : [info script]" +#puts "argcount : $::argc" +#puts "argvalues: $::argv" + + +# + +# --- --- --- --- --- --- --- --- --- --- --- --- --- +# only exit if needed. see exitcode notes at bottom of file and exit there for consistency across invocation methods +# --- --- --- --- --- --- --- --- --- --- --- --- ---end Tcl Payload +#-- +#-- bash/sh code follows. +#-- protect from tcl using line continuation char on the previous comment for each line, like so: \ +printf "etc" +#-- or alternatively place sh/bash script within the false==false block +#-- whilst being careful to balance braces {} +#-- For more complex needs you should call out to external scripts +#-- +#-- END marker for hide_from_bash_and_sh\ +HIDE_FROM_BASH_AND_SH + +#--------------------------------------------------------- +#-- This if statement hides(mostly) a sh/bash code block from Tcl +if false==false # else { +then +: +#--------------------------------------------------------- + #-- leave as is if all that's required is launching the Tcl payload" + #-- + #-- Note that sh/bash script isn't called when running a .bat from cmd.exe on windows by default + #-- adjust line 4: @call tclsh ... to something like @call sh ... @call bash .. or @call env sh ... etc as appropriate + #-- if sh/bash scripting needs to run on windows too. + #-- + #printf "start of bash or sh code" + + + #-- sh/bash launches Tcl here instead of shebang line at top + + #-- use exec to use exitcode (if any) directly from the tcl script + exec /usr/bin/env tclsh "$0" "$@" + + #-- alternative - if sh/bash script required to run after the tcl call. + #/usr/bin/env tclsh "$0" "$@" + #tcl_exitcode=$? + #echo "tcl_exitcode: ${tcl_exitcode}" + + #-- override exitcode example + #exit 66 + + #printf "No need for trailing slashes for sh/bash code here\n" +#--------------------------------------------------------- +fi +# closing brace for Tcl } +#--------------------------------------------------------- + +#-- tcl and shell script now both active + +#-- comment for line sample 1 with trailing continuation slash \ +#printf "tcl-invisible sh/bash line sample 1 \n" + +#-- comment for line sample 2 with trailing continuation slash \ +#printf "tcl-invisible sh/bash line sample 2 \n" + + +#-- Consistent exitcode from sh,bash,tclsh or cmd +#-- Call exit in tcl (or sh/bash) code only if explicitly required, otherwise leave this commented out. +#-- (script might be more widely useable without explicit exit. e.g in tcl: set ::argc 1; set ::argv "val"; source filename.sh.bat ) +#-- exit line unprotected by trailing slash will work for tcl and/or sh/bash +#exit 0 +#exit 42 + + + +#-- make sure sh/bash/tcl all skip over .bat style exit \ +: <<'shell_end' +#-- .bat exit with exitcode from tcl process \ +:exit +: ;# \ +@exit /B %taskexitcode% +# .bat has exited \ +shell_end + diff --git a/src/bootsupport/modules/punk/mix/templates/utility/shellbat_v1.txt b/src/bootsupport/modules/punk/mix/templates/utility/shellbat_v1.txt new file mode 100644 index 00000000..e504ee01 --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/utility/shellbat_v1.txt @@ -0,0 +1,106 @@ +if (true=="shellbat") #;#\ +: <<'HIDE_FROM_BASH_AND_SH' +::lindex tcl;# leading colons hide from .bat, trailing slash hides next line from tcl \ +@call tclsh "%~dp0%~n0.bat" %* +::lindex tcl;#\ +@set taskexitcode=%errorlevel% & goto :exit +# -*- tcl -*- +# ################################################################################################# +# This is a tcl shellbat file +# It is tuned to run when called as a batch file, a tcl script, an sh script or a bash script, +# so the specific layout and characters used are quite sensitive to change. +# It can be called on unix or windows platforms with or without the interpreter being specified on the commandline. +# e.g ./filename.sh.bat in sh or bash or powershell +# e.g filename.sh or filename.sh.bat at windows command prompt +# e.g tclsh filename.sh.bat | sh filename.sh.bat | bash filename.sh.bat +# In all cases an arbitrary number of arguments are accepted +# To avoid the initial commandline on stdout when calling as a batch file on windows, use: +# cmd /Q /c filename.sh.bat +# (because we cannot use @if to silence it, as this isn't understood by tcl,sh or bash) +# ################################################################################################# +#fconfigure stdout -translation crlf +# --- --- --- --- --- --- --- --- --- --- --- --- ---begin Tcl Payload +#puts "script : [info script]" +#puts "argcount : $::argc" +#puts "argvalues: $::argv" + +# + +# --- --- --- --- --- --- --- --- --- --- --- --- --- +# only exit if needed. see exitcode notes at bottom of file and exit there for consistency across invocation methods +# --- --- --- --- --- --- --- --- --- --- --- --- ---end Tcl Payload +#-- +#-- bash/sh code follows. +#-- protect from tcl using line continuation char on the previous comment for each line, like so: \ +printf "etc" +#-- or alternatively place sh/bash script within the false==false block +#-- whilst being careful to balance braces {} +#-- For more complex needs you should call out to external scripts +#-- +#-- END marker for hide_from_bash_and_sh\ +HIDE_FROM_BASH_AND_SH +#\ +then + +#--------------------------------------------------------- +if false==false # else { +then +: +#--------------------------------------------------------- + #-- leave as is if all that's required is launching the Tcl payload" + #-- + #-- Note that sh/bash script isn't called when running a .bat from cmd.exe on windows by default + #-- adjust line 4: @call tclsh ... to something like @call sh ... @call bash .. or @call env sh ... etc as appropriate + #-- if sh/bash scripting needs to run on windows too. + #-- + #printf "start of bash or sh code" + + + #-- sh/bash launches Tcl here instead of shebang line at top + + #-- use exec to use exitcode (if any) directly from the tcl script + exec /usr/bin/env tclsh "$0" "$@" + + #-- alternative - if sh/bash script required to run after the tcl call. + #/usr/bin/env tclsh "$0" "$@" + #tcl_exitcode=$? + #echo "tcl_exitcode: ${tcl_exitcode}" + + #-- override exitcode example + #exit 66 + + #printf "No need for trailing slashes for sh/bash code here\n" +#--------------------------------------------------------- +fi +# } +#--------------------------------------------------------- + +#-- comment for line sample 1 with trailing continuation slash \ +#printf "tcl-invisible sh/bash line sample 1 \n" + +#-- comment for line sample 2 with trailing continuation slash \ +#printf "tcl-invisible sh/bash line sample 2 \n" + + +#-- Consistent exitcode from sh,bash,tclsh or cmd +#-- Call exit in tcl (or sh/bash) code only if explicitly required, otherwise leave this commented out. +#-- (script might be more widely useable without explicit exit. e.g in tcl: set ::argc 1; set ::argv "val"; source filename.sh.bat ) +#-- exit line unprotected by trailing slash will work for tcl and/or sh/bash +#exit 0 +#exit 42 + + +#--------------------------------------------------------- +#-- end if true==shellbat on very first line\ +fi +#--------------------------------------------------------- + +#-- make sure sh/bash/tcl all skip over .bat style exit \ +: <<'shell_end' +#-- .bat exit with exitcode from tcl process \ +:exit +::lindex tcl;#\ +@exit /B %taskexitcode% +#\ +shell_end + diff --git a/src/bootsupport/modules/punk/mix/templates/utility/tclbatheader.txt b/src/bootsupport/modules/punk/mix/templates/utility/tclbatheader.txt new file mode 100644 index 00000000..b2e0367f --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/utility/tclbatheader.txt @@ -0,0 +1,3 @@ +::lindex tcl;#\ +@call tclsh "%~dp0%~n0.bat" %* & goto :eof +# --- --- --- --- --- --- --- --- --- --- --- --- ---begin Tcl diff --git a/src/bootsupport/modules/punk/mix/templates/utility/tclbattest.bat b/src/bootsupport/modules/punk/mix/templates/utility/tclbattest.bat new file mode 100644 index 00000000..396aea56 --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/utility/tclbattest.bat @@ -0,0 +1,8 @@ +::lindex tcl;#\ +@call tclsh "%~dp0%~n0.bat" %* & goto :eof +# --- --- --- --- --- --- --- --- --- --- --- --- ---begin Tcl +puts stdout "exe: [info nameof]" +puts stdout "scr: [info script]" +puts stdout "argc: $::argc" +puts stdout "argv: '$::argv'" + diff --git a/src/bootsupport/modules/punk/mix/templates/utility/tclbattest2.bat b/src/bootsupport/modules/punk/mix/templates/utility/tclbattest2.bat new file mode 100644 index 00000000..fbf2fcd0 --- /dev/null +++ b/src/bootsupport/modules/punk/mix/templates/utility/tclbattest2.bat @@ -0,0 +1,19 @@ +::set - { +@goto start +# -- tcl bat +:start +@echo off +set script=%0 +echo %* +if exist %script%.bat set script=%script%.bat +tclsh %script% %* +goto end of BAT file +};unset - ;# --- --- --- --- --- --- --- --- --- --- --- --- ---begin Tcl + +puts stdout "exe: [info nameof]" +puts stdout "scr: [info script]" +puts stdout "argc: $::argc" +puts stdout "argv: '$::argv'" + +# -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---end Tcl\ +:end of BAT file diff --git a/src/bootsupport/modules/punk/mix/util-0.1.0.tm b/src/bootsupport/modules/punk/mix/util-0.1.0.tm index 5622bc02..44c01721 100644 --- a/src/bootsupport/modules/punk/mix/util-0.1.0.tm +++ b/src/bootsupport/modules/punk/mix/util-0.1.0.tm @@ -35,13 +35,10 @@ namespace eval punk::mix::util { namespace export * - + #NOTE fileutil::cat seems to silently ignore options if passed at end instead of before file! proc fcat {args} { variable has_winpath - if {$::tcl_platform(platform) ne "windows"} { - return [fileutil::cat {*}$args] - } set knownopts [list -eofchar -translation -encoding --] set last_opt 0 @@ -73,7 +70,21 @@ namespace eval punk::mix::util { if {![llength $paths]} { error "Unable to find file in the supplied arguments: $args. Ensure options are all -opt val pairs and that file name(s) follow" } + #puts stderr "opts: $opts paths: $paths" + + #let's proceed, but warn the user if an apparent option is in paths + foreach opt [list -encoding -eofchar -translation] { + if {$opt in $paths} { + puts stderr "fcat WARNING: apparent option $opt found after file argument(s) (expected them before filenames). Passing to fileutil::cat anyway - but for at least some versions, these options may be ignored. commandline 'fcat $args'" + } + } + + if {$::tcl_platform(platform) ne "windows"} { + return [fileutil::cat {*}$args] + } + + set finalpaths [list] foreach p $paths { if {$has_winpath && [punk::winpath::illegalname_test $p]} { diff --git a/src/bootsupport/modules/punk/ns-0.1.0.tm b/src/bootsupport/modules/punk/ns-0.1.0.tm index 156d51d1..c9bbc3d3 100644 --- a/src/bootsupport/modules/punk/ns-0.1.0.tm +++ b/src/bootsupport/modules/punk/ns-0.1.0.tm @@ -17,7 +17,7 @@ # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ ## Requirements ##e.g package require frobz - +package require punk::args namespace eval ::punk_dynamic::ns { @@ -356,8 +356,8 @@ namespace eval punk::ns { } - return [list_as_lines [lsort $result]] - #.= lsort $result |> list_as_lines + return [list_as_lines -- [lsort $result]] + #.= lsort $result |> list_as_lines -- } proc nsglob_as_re {glob} { @@ -1601,23 +1601,33 @@ namespace eval punk::ns { } interp alias "" use "" punk::ns::pkguse - proc nsimport_noclobber {pattern {ns ""}} { - set source_ns [namespace qualifiers $pattern] + proc nsimport_noclobber {args} { + set argspecs { + -targetnamespace -default "" -optional 1 + -prefix -default "" -optional 1 + sourcepattern -type string -optional 0 + } + lassign [punk::args::opts_values $argspecs $args -minvalues 1 -maxvalues 1] _o opts _v values + set sourcepattern [dict get $values sourcepattern] + + set source_ns [namespace qualifiers $sourcepattern] if {![namespace exists $source_ns]} { error "nsimport_noclobber error namespace $source_ns not found" } - if {$ns eq ""} { - set ns [uplevel 1 {namespace current}] - } elseif {![string match ::* $ns]} { - set nscaller [uplevel 1 {namespace current}] - set ns [punk::nsjoin $nscaller $ns] + set target_ns [dict get $opts -targetnamespace] + set nscaller [uplevel 1 {namespace current}] + if {$target_ns eq ""} { + set target_ns $nscaller + } elseif {![string match ::* $target_ns]} { + set target_ns [punk::nsjoin $nscaller $target_ns] } + set a_export_patterns [namespace eval $source_ns {namespace export}] - set a_commands [info commands $pattern] + set a_commands [info commands $sourcepattern] set a_tails [lmap v $a_commands {namespace tail $v}] set a_exported_tails [list] - foreach pattern $a_export_patterns { - set matches [lsearch -all -inline $a_tails $pattern] + foreach epattern $a_export_patterns { + set matches [lsearch -all -inline $a_tails $epattern] foreach m $matches { if {$m ni $a_exported_tails} { lappend a_exported_tails $m @@ -1626,7 +1636,7 @@ namespace eval punk::ns { } set imported_commands [list] foreach e $a_exported_tails { - set imported [namespace eval $ns [string map [list $e $source_ns] { + set imported [namespace eval $target_ns [string map [list $e $source_ns] { set cmd "" if {![catch {namespace import ::}]} { set cmd diff --git a/src/bootsupport/modules/punk/path-0.1.0.tm b/src/bootsupport/modules/punk/path-0.1.0.tm index f877417a..2f1ee6c5 100644 --- a/src/bootsupport/modules/punk/path-0.1.0.tm +++ b/src/bootsupport/modules/punk/path-0.1.0.tm @@ -198,29 +198,47 @@ namespace eval punk::path { #todo - implement treefiles which acts like dirfiles but allows path globbing in the same way as punk::ns::ns/ #then review if treefiles can replace dirfiles or if both should exist (dirfiles can have literal glob chars in path segments - but that is a rare usecase) - proc treefilenames {basepath tailglob args} { + proc treefilenames {args} { #*** !doctools - #[call [fun treefilenames] [arg basepath] [arg tailglob] [opt {option value...}]] - #basic (glob based) list of filenames matching tailglob - recursive - #no natsorting - so order is dependent on filesystem - set defaults [dict create\ - -call-depth-internal 0\ - -antiglob_paths {}\ - ] - set opts [dict merge $defaults $args] + #[call [fun treefilenames] [opt {option value...}] [opt {globpattern...}]] + #[para]basic (glob based) list of filenames matching each pattern in tailglobs - recursive + #[para] options: + #[para] [opt -dir] + #[para] defaults to [lb]pwd[rb] - base path for tree to search + #[para] [opt -antiglob_paths] + #[para] list of path patterns to exclude - may include * and ** path segments e.g /usr/** + #[para]no natsorting - so order is dependent on filesystem + + lassign [punk::get_leading_opts_and_values { + -directory "\uFFFF" + -call-depth-internal 0 + -antiglob_paths {} + } $args] _o opts _v tailglobs + + # -- --- --- --- --- --- --- set opt_antiglob_paths [dict get $opts -antiglob_paths] set CALLDEPTH [dict get $opts -call-depth-internal] + # -- --- --- --- --- --- --- + set opt_dir [dict get $opts -directory] + if {$opt_dir eq "\uFFFF"} { + set opt_dir [pwd] + } + # -- --- --- --- --- --- --- set files [list] if {$CALLDEPTH == 0} { - if {![file isdirectory $basepath]} { + if {![file isdirectory $opt_dir]} { return [list] } + set opts [dict merge $opts [list -directory $opt_dir]] + if {![llength $tailglobs]} { + lappend tailglobs * + } } set skip 0 foreach anti $opt_antiglob_paths { - if {[globmatchpath $anti $basepath]} { + if {[globmatchpath $anti $opt_dir]} { set skip 1 break } @@ -230,9 +248,9 @@ namespace eval punk::path { } #todo - account for vfs where matched path could appear to be a directory but is mounted so could be a desired match? - set dirfiles [glob -nocomplain -dir $basepath -type f $tailglob] + set dirfiles [lsort [glob -nocomplain -dir $opt_dir -type f {*}$tailglobs]] lappend files {*}$dirfiles - set dirdirs [glob -nocomplain -dir $basepath -type d *] + set dirdirs [glob -nocomplain -dir $opt_dir -type d *] foreach dir $dirdirs { set skip 0 foreach anti $opt_antiglob_paths { @@ -244,8 +262,8 @@ namespace eval punk::path { if {$skip} { continue } - set nextargs [dict merge $args [list -call-depth-internal [incr CALLDEPTH]]] - lappend files {*}[treefilenames $dir $tailglob {*}$nextargs] + set nextopts [dict merge $opts [list -directory $dir -call-depth-internal [incr CALLDEPTH]]] + lappend files {*}[treefilenames {*}$nextopts {*}$tailglobs] } return $files } @@ -270,14 +288,30 @@ namespace eval punk::path { #[item] #[para] Notes: #[para] Both paths must be the same type - ie both absolute or both relative - #[para] Case sensitive. ie relative /etc /etC + #[para] Case sensitive. ie punk::path::relative /etc /etC # will return ../etC #[para] On windows, the drive-letter component (only) is not case sensitive - #[para] ie relative c:/etc C:/etc returns . - #[para] but relative c:/etc C:/Etc returns ../Etc + #[example_begin] + # P% punk::path::relative c:/etc C:/etc + # - . + #[example_end] + #[para] The part following the driveletter is case sensitive so in the following cases it recognises the driveletter matches but not the tail + #[example_begin] + # P% punk::path::relative c:/etc C:/Etc + # - ../Etc + #[example_end] #[para] On windows, if the paths are absolute and specifiy different volumes, only the location will be returned. - # ie relative c:/etc d:/etc/blah - # returns d:/etc/blah + #[example_begin] + # P% punk::path::relative c:/etc d:/etc/blah + # - d:/etc/blah + #[example_end] + #[para] Unix-like examples: + #[example_begin] + # P% punk::path::relative /usr/local/etc/ /usr/local/etc/somewhere/below + # - somewhere/below + # P% punk::path::relative /usr/local/etc/somewhere /usr/local/lib/here + # - ../../lib/here + #[example_end] #[list_end] #see also kettle diff --git a/src/bootsupport/modules/punk/repo-0.1.1.tm b/src/bootsupport/modules/punk/repo-0.1.1.tm index d2aaf9fc..1a8ac6ec 100644 --- a/src/bootsupport/modules/punk/repo-0.1.1.tm +++ b/src/bootsupport/modules/punk/repo-0.1.1.tm @@ -779,10 +779,10 @@ namespace eval punk::repo { package require overtype set title1 "Path" - set widest1 [tcl::mathfunc::max {*}[lmap v [concat [list $title1] $col1items] {punk::strlen $v}]] + set widest1 [tcl::mathfunc::max {*}[lmap v [concat [list $title1] $col1items] {lib::strlen $v}]] set col1 [string repeat " " $widest1] set title2 "Repo-type(s)" - set widest2 [tcl::mathfunc::max {*}[lmap v [concat [list $title2] $col2items] {punk::strlen $v}]] + set widest2 [tcl::mathfunc::max {*}[lmap v [concat [list $title2] $col2items] {string length $v}]] set col2 [string repeat " " $widest2] set tablewidth [expr {$widest1 + 1 + $widest2}] @@ -827,19 +827,61 @@ namespace eval punk::repo { } } proc fossil_get_repository_folder_for_project {projectname args} { - - set defaults [list -parentfolder \uFFFF -extrachoice \uFFFF] + set defaults [list\ + -parentfolder \uFFFF\ + -extrachoices \uFFFF\ + -askpath 0\ + -ansi \uFFFF\ + -ansi_prompt \uFFFF\ + -ansi_warning \uFFFF\ + ] + if {[llength $args] % 2 != 0} { + error "fossil_get_repository_folder requires args to be option-value pairs. Received '$args'" + } + dict for {k v} $args { + if {$k ni [dict keys $defaults]} { + error "fossil_get_repository_folder unrecognised option $k. Known options: [dict keys $defaults]" + } + } set opts [dict merge $defaults $args] - + # -- --- --- --- --- --- set opt_parentfolder [dict get $opts -parentfolder] if {$opt_parentfolder eq "\uFFFF"} { set opt_parentfolder [pwd] } - set opt_extrachoice [dict get $opts -extrachoice] - set extrachoice "" - if {$opt_extrachoice ne "\uFFFF"} { - set extrachoice $opt_extrachoice + # -- --- --- --- --- --- + set opt_extrachoices [dict get $opts -extrachoices] + set extrachoices [list] + if {$opt_extrachoices ne "\uFFFF"} { + set extrachoices $opt_extrachoices + } + # -- --- --- --- --- --- + set opt_askpath [dict get $opts -askpath] + # -- --- --- --- --- --- + set opt_ansi [dict get $opts -ansi] + set opt_ansi_prompt [dict get $opts -ansi_prompt] + set opt_ansi_warning [dict get $opts -ansi_warning] + if {$opt_ansi eq "\uFFFF"} { + set opt_ansi 1 + } + if {$opt_ansi} { + if {$opt_ansi_prompt eq "\uFFFF"} { + set ansiprompt [a+ green bold] + } else { + set ansiprompt [$opt_ansi_prompt] + } + if {$opt_ansi_warning eq "\uFFFF"} { + set ansiwarn [a+ red bold] + } else { + set ansiwarn [$opt_ansi_warning] + } + set ansireset [a] + } else { + set ansiprompt "" + set ansiwarn "" + set ansireset "" } + # -- --- --- --- --- --- set startdir $opt_parentfolder @@ -849,25 +891,8 @@ namespace eval punk::repo { return } - set fossilinfo [exec {*}$fossil_prog info] ;#will give us the necessary config-db info whether in a project folder or not - set matching_lines [punk::repo::grep {config-db:*} $fossilinfo] - if {[llength $matching_lines] != 1} { - puts stderr "Unable to find config-db info from fossil. Check your fossil installation." - puts stderr "Fossil output was:" - puts stderr "-------------" - puts stderr "$fossilinfo" - puts stderr "-------------" - puts stderr "config-db info:" - puts stderr "$matching_lines" - return - } - set trimmedline [string trim [lindex $matching_lines 0]] - set firstcolon [string first : $trimmedline] - set config_db_path [string trim [string range $trimmedline $firstcolon+1 end]] - if {![file exists $config_db_path]} { - puts stderr "Unable to verify fossil global configuration info at path: $config_db_path" - return - } + set config_db_path [fossil_get_configdb] + set config_db_folder [file dirname $config_db_path] #NOTE: we could use fossil all info to detect all locations of .fossil files - but there may be many that are specific to projects if the user wasn't in the habit of using a default location @@ -914,10 +939,11 @@ namespace eval punk::repo { if {$tp ni $testpaths} { lappend testpaths $tp } - if {[string length $extrachoice]} { - set tp $extrachoice - if {$tp ni $testpaths} { - lappend testpaths $tp + if {[llength $extrachoices]} { + foreach tp $extrachoices { + if {$tp ni $testpaths} { + lappend testpaths $tp + } } } @@ -956,7 +982,7 @@ namespace eval punk::repo { set existingfossils "( no existing .fossil files found )" } if {"$projectname.fossil" in $existing_fossils} { - set conflict "CONFLICT - $projectname.fossil already exists in this folder" + set conflict "${ansiwarn}CONFLICT - $projectname.fossil already exists in this folder${ansireset}" } else { set conflict "" } @@ -975,9 +1001,9 @@ namespace eval punk::repo { set menu_message "" if {[llength $choice_folders]} { - append menu_message "Select the number of the folder to use to store the .fossil repository file" \n + append menu_message "${ansiprompt}Select the number of the folder to use to store the .fossil repository file${ansireset}" \n } else { - append menu_message "--- NO suitable writable folders or locations found for .fossil file. Consider setting FOSSIL_HOME environment variable and check that folders are writable.--" \n + append menu_message "${ansiwarn}--- NO suitable writable folders or locations found for .fossil file. Consider setting FOSSIL_HOME environment variable and check that folders are writable.--${ansireset}" \n } set conflicted_options [list] @@ -1003,7 +1029,7 @@ namespace eval punk::repo { if {[llength $readonly_repo_folder_locations]} { append menu_message "--------------------------------------------------" \n foreach readonly $readonly_repo_folder_locations { - append menu_message " $readonly" \n + append menu_message "${ansiwarn} $readonly${ansireset}" \n } append menu_message "--------------------------------------------------" \n } @@ -1011,11 +1037,11 @@ namespace eval punk::repo { #see if we can reasonably use the only available option and not bug the user #Todo - option to always prompt? #we will not auto-select if there is even one conflicted_option - as that seems like something you should know about - if {![llength $conflicted_options] && ([llength $choice_folders] == 1)} { + if {![llength $conflicted_options] && ([llength $choice_folders] == 1) && !$opt_askpath} { set repo_folder_choice [lindex $choice_folders 0] set repository_folder [dict get $repo_folder_choice folder] } else { - if {[llength $choice_folders]} { + if {[llength $choice_folders] || $opt_askpath} { puts stdout $menu_message set max [llength $choice_folders] if {$max == 1} { @@ -1023,24 +1049,73 @@ namespace eval punk::repo { } else { set rangemsg "a number from 1 to $max" } - set answer [askuser "Enter $rangemsg to select location. (or N to abort)"] - if {![string is integer -strict $answer]} { - puts stderr "Aborting" - return + set menuprompt "${ansiprompt}Enter $rangemsg to select location. (or N to abort)${ansireset}" + if {$opt_askpath} { + set askpathprompt "${ansiprompt}Enter the word: path followed by an absolute path to a folder if you would like to manually enter a folder${ansireset}" + append menuprompt \n $askpathprompt } + set answer [askuser $menuprompt] + if {$opt_askpath && [string match "path*" [string tolower $answer]]} { + set is_done 0 + set repository_folder [string trim [string range $answer 4 end]] + while {!$is_done} { + + if {![file isdirectory $repository_folder]} { + puts stderr "${ansiwarn}Sorry - unable to find entered location '$repository_folder'${ansireset}" + if {[file isdirectory [file dirname $repository_folder]]} { + set answer [askuser "${ansiprompt}Do you want to create this folder? Type just the word mkdir to create it, or N for no${ansireset}"] + if {[string equal mkdir [string tolower $answer]]} { + if {[catch {file mkdir $repository_folder} errM]} { + puts stderr "Failed to create folder $repository_folder. Error $errM" + } + } + } else { + puts stderr "${ansiwarn}Not offering to create directory because parent folder not found${ansireset}" + } + } - set index [expr {int($answer) -1}] - if {$index >= 0 && $index <= $max-1} { - set repo_folder_choice [lindex $choice_folders $index] - set repository_folder [dict get $repo_folder_choice folder] - puts stdout "Selected fossil location $repository_folder" + if {![file isdirectory $repository_folder]} { + set answer [askuser "${ansiprompt}Try again? (Y|N)${ansireset}"] + if {[string match y* [string tolower $answer]]} { + set answer [askuser $askpathprompt] + if {[string match "path*" [string tolower $answer]]} { + set repository_folder [string trim [string range $answer 4 end]] + } else { + puts stderr "Answer didn't begin with the word path" + set is_done 1 + } + } else { + set is_done 1 + } + } else { + set is_done 1 + } + } + if {$is_done && ![file isdirectory $repository_folder]} { + puts stderr "Aborting" + return + } } else { - puts stderr " No menu number matched - aborting." - return + if {![string is integer -strict $answer]} { + puts stderr "Aborting" + return + } + set index [expr {int($answer) -1}] + if {$index >= 0 && $index <= $max-1} { + set repo_folder_choice [lindex $choice_folders $index] + set repository_folder [dict get $repo_folder_choice folder] + puts stdout "Selected fossil location $repository_folder" + } else { + puts stderr " No menu number matched - aborting." + return + } } + + + } else { puts stdout $menu_message - set answer [askuser "Hit enter to exit"] + set answer [askuser "${ansiprompt}Hit enter to exit${ansireset}"] return } } @@ -1053,26 +1128,26 @@ namespace eval punk::repo { if {$path eq {}} { set path [pwd] } # ::kettle::path::revision.git do_in_path $path { - try { - #git describe will error with 'No names found' if repo has no tags - #set v [::exec {*}[auto_execok git] describe] - set v [::exec {*}[auto_execok git] rev-parse HEAD] ;# consider 'git rev-parse --short HEAD' - } on error {e o} { - set v [lindex [split [dict get $o -errorinfo] \n] 0] - } + try { + #git describe will error with 'No names found' if repo has no tags + #set v [::exec {*}[auto_execok git] describe] + set v [::exec {*}[auto_execok git] rev-parse HEAD] ;# consider 'git rev-parse --short HEAD' + } on error {e o} { + set v [lindex [split [dict get $o -errorinfo] \n] 0] + } } return [string trim $v] } proc git_remote {{path {{}}}} { if {$path eq {}} { set path [pwd] } do_in_path $path { - try { - #git describe will error with 'No names found' if repo has no tags - #set v [::exec {*}[auto_execok git] describe] - set v [::exec {*}[auto_execok git] -remote -v] ;# consider 'git rev-parse --short HEAD' - } on error {e o} { - set v [lindex [split [dict get $o -errorinfo] \n] 0] - } + try { + #git describe will error with 'No names found' if repo has no tags + #set v [::exec {*}[auto_execok git] describe] + set v [::exec {*}[auto_execok git] -remote -v] ;# consider 'git rev-parse --short HEAD' + } on error {e o} { + set v [lindex [split [dict get $o -errorinfo] \n] 0] + } } return [string trim $v] } @@ -1104,6 +1179,148 @@ namespace eval punk::repo { return Unknown } } + proc fossil_get_configdb {{path {}}} { + #fossil info will *usually* give us the necessary config-db info whether in a project folder or not but.. + #a) It's expensive to shell-out and call it + #b) it won't give us a result if we are in a checkout folder which has had its repository moved + #this fairly extensive mechanism is designed to find it even if the environment has some weird goings-on regarding the filesystem/environment variables + #This is unlikely to be necessary in most scenarios, where the location is related to the user's home directory + + #attempt 1 - environment vars and well-known locations + #This is first because it's faster - but hopefully it's aligned with how fossil does it + + if {"windows" eq $::tcl_platform(platform)} { + foreach varname [list FOSSIL_HOME LOCALAPPDATA APPDATA USERPROFILES] { + if {[info exists ::env($varname)]} { + set testfile [file join $::env($varname) _fossil] + if {[file exists $testfile]} { + return $testfile + } + } + } + if {[info exists ::env(HOMEDRIVE)] && [info exists ::env(HOMEPATH)]} { + set testfile $::env(HOMEDRIVE)$::env(HOMEPATH)\\_fossil" + if {[file exists $testfile]} { + return $testfile + } + } + } else { + foreach varname [list FOSSIL_HOME HOME ] { + if {[info exists ::env($varname)]} { + set testfile [file join $::env($varname) .fossil] + if {[file exists $testfile]} { + return $testfile + } + } + } + if {[info exists ::env(XDG_CONFIG_HOME)]} { + set testfile [file join $::env(XDG_CONFIG_HOME) fossil.db] + if {[file exists $testfile]} { + return $testfile + } + set testfile [file join $::env(XDG_CONFIG_HOME) .config fossil.db] + if {[file exists $testfile]} { + return $testfile + } + } + if {[info exists ::env(HOME)]} { + set testfile [file join $::env(HOME) .config fossil.db] + if {[file exists $testfile]} { + return $testfile + } + } + } + + + set original_cwd [pwd] + #attempt2 - let fossil do it for us - hopefully based on current folder + if {$path eq {}} {set path [pwd]} + set fossilcmd [auto_execok fossil] + if {![llength $fossilcmd]} { + set fossil_ok 0 + } else { + set fossil_ok 1 + } + try { + while {$fossil_ok} { + cd $path + if {[catch {exec {*}$fossilcmd info} fossilinfo]} { + #a detached repo above us can result in an error from fossil info + set next [file dirname $path] + } else { + set matching_lines [punk::repo::grep {config-db:*} $fossilinfo] + if {[llength $matching_lines] == 1} { + set trimmedline [string trim [lindex $matching_lines 0]] + set firstcolon [string first : $trimmedline] + set config_db_path [string trim [string range $trimmedline $firstcolon+1 end]] + if {[file exists $config_db_path]} { + return $config_db_path + } + } + set next [file dirname $path] + } + + if {$next eq $path || $next eq ""} { + break + } + set path $next + } + } on error {errmsg options} { + #puts "errmsg:$errmgs options:$options" + } finally { + cd $original_cwd + } + + #attempt 3 - getting desperate.. find other repos, determine their checkouts and run fossil in them to get a result + if {$fossil_ok} { + #It should be extremely rare to need to resort to sqlite on the databases to find other potential repo paths + #Conceivably only on some weird VFS or where some other filesystem strangeness is going on with our original path - or if the root volume itself is a broken fossil checkout + #Examining the other repos gives us a chance at discovering some other filesystem/paths where things may not be broken + if {![catch {package require sqlite3} errPackage]} { + #use fossil all ls and sqlite + if {[catch {exec {*}$fossilcmd all ls} repolines]} { + error "fossil_get_configdb cannot find repositories" + } else { + set repolines [string map [list \r\n \n] $repolines] + set repolist [split $repolines \n] + set dbcmd "fossil_get_configdb_tempdb" + foreach repodb $repolist { + catch {rename $dbcmd ""} + if {[file exists $repodb]} { + if {![catch {sqlite3 $dbcmd $repodb}]} { + set ckoutrecords [$dbcmd eval {select name from config where name like 'ckout:%'}] + catch {$dbcmd close} + foreach ck $ckoutrecords { + set ckfolder [string range $ck 6 end] + #puts stdout "ckfolder $ckfolder" + if {[file isdirectory $ckfolder]} { + set result "" + do_in_path $ckfolder { + if {![catch {exec {*}$fossilcmd info} fossilinfo]} { + set matching_lines [punk::repo::grep {config-db:*} $fossilinfo] + if {[llength $matching_lines] == 1} { + set trimmedline [string trim [lindex $matching_lines 0]] + set firstcolon [string first : $trimmedline] + set config_db_path [string trim [string range $trimmedline $firstcolon+1 end]] + if {[file exists $config_db_path]} { + set result $config_db_path + } + } + } + } + if {$result ne ""} { + return $result + } + } + } + } + } + } + } + } + } + error "fossil_get_configdb exhausted search options" + } #------------------------------------ #temporarily cd to workpath to run script - return to correct path even on failure @@ -1256,7 +1473,19 @@ namespace eval punk::repo { interp alias {} gconf {} git config --global -l } +namespace eval punk::repo::lib { + #----------------------------------------------------------------------------------- + #strlen is important for testing issues with string representationa and shimmering. + #This specific implementation with append (as at 2023-09) is designed to ensure the original str representation isn't changed + #It may need to be reviewed with different Tcl versions in case the append empty string is 'optimised/tuned' in some way that affects the behaviour + #The use of this function instead of string length can make a difference in certain circumstances with 'path' object representations + proc strlen {str} { + append str2 $str {} + string length $str2 + } + #----------------------------------------------------------------------------------- +} # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ ## Ready diff --git a/src/bootsupport/modules/punkcheck-0.1.0.tm b/src/bootsupport/modules/punkcheck-0.1.0.tm index 4582ac50..c58fe244 100644 --- a/src/bootsupport/modules/punkcheck-0.1.0.tm +++ b/src/bootsupport/modules/punkcheck-0.1.0.tm @@ -1,5 +1,5 @@ # -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt +# Maintenance Instruction: leave the 999999.xxx.x as is and use 'deck make' or src/make.tcl to update from -buildversion.txt # # Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. # Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. @@ -36,6 +36,7 @@ namespace eval punkcheck { uuid\ start_installer_event installfile_* + #antiglob_dir & antiglob_file entries match the pattern at any level - should not contain path separators variable default_antiglob_dir_core [list "#*" "_aside" ".git" ".fossil*"] variable default_antiglob_file_core "" proc uuid {} { @@ -1155,15 +1156,15 @@ namespace eval punkcheck { -installer punkcheck::install\ ] - set opts [dict merge $defaults $args] if {([llength $args] %2) != 0} { error "punkcheck::install requires option-style arguments to be in pairs. Received args: $args" } - foreach k [dict keys $args] { + foreach {k -} $args { if {$k ni [dict keys $defaults]} { error "punkcheck::install unrecognised option '$k' known options: '[dict keys $defaults]'" } } + set opts [dict merge $defaults $args] #The choice to recurse using the original values of srcdir & tgtdir, and passing the subpath down as a list in -subdirlist seems an odd one. #(as opposed to a more 'standard' mechanism of adjusting srcdir & tgtdir as we move down the tree) @@ -1194,6 +1195,12 @@ namespace eval punkcheck { } # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- set opt_antiglob_file [dict get $opts -antiglob_file] + #validate no path seps + foreach af $opt_antiglob_file { + if {[llength [file split $af]] > 1} { + error "punkcheck::install received invalid -antiglob_file entry '$af'. -antiglob_file entries are meant to match to a file name at any level so cannot contain path separators" + } + } # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- set opt_antiglob_dir_core [dict get $opts -antiglob_dir_core] if {$opt_antiglob_dir_core eq "\uFFFF"} { @@ -1202,8 +1209,15 @@ namespace eval punkcheck { } # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- set opt_antiglob_dir [dict get $opts -antiglob_dir] + #validate no path seps + foreach ad $opt_antiglob_dir { + if {[llength [file split $ad]] > 1} { + error "punkcheck::install received invalid -antiglob_dir entry '$ad'. -antiglob_dir entries are meant to match to a directory name at any level so cannot contain path separators" + } + } # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- set opt_antiglob_paths [dict get $opts -antiglob_paths] ;#todo - combine with config file in source tree .punkcheckpublish (?) + #antiglob_paths will usually contain file separators - and may contain glob patterns within each segment set antiglob_paths_matched [list] # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- set known_whats [list no-targets newer-targets older-targets all-targets installedsourcechanged-targets synced-targets] @@ -1586,6 +1600,21 @@ namespace eval punkcheck { continue } + set relative_source_path [file join $relative_source_dir $d] + set is_antipath 0 + foreach antipath $opt_antiglob_paths { + #puts "testing folder - globmatchpath $antipath vs $relative_source_path" + if {[punk::path::globmatchpath $antipath $relative_source_path]} { + lappend antiglob_paths_matched [file join $current_source_dir $d] + puts stdout "SKIPPING FOLDER $relative_source_path due to antiglob_path-match: $antipath " + set is_antipath 1 + break + } + } + if {$is_antipath} { + continue + } + if {![file exists $current_target_dir/$d]} { file mkdir $current_target_dir/$d diff --git a/src/bootsupport/modules/smtp-1.5.1.tm b/src/bootsupport/modules/smtp-1.5.1.tm new file mode 100644 index 00000000..f0d93f7b --- /dev/null +++ b/src/bootsupport/modules/smtp-1.5.1.tm @@ -0,0 +1,1508 @@ +# smtp.tcl - SMTP client +# +# Copyright (c) 1999-2000 Marshall T. Rose +# Copyright (c) 2003-2006 Pat Thoyts +# +# See the file "license.terms" for information on usage and redistribution +# of this file, and for a DISCLAIMER OF ALL WARRANTIES. +# + +package require Tcl 8.3 +package require mime 1.4.1 + +catch { + package require SASL 1.0; # tcllib 1.8 + package require SASL::NTLM 1.0; # tcllib 1.8 +} + +# +# state variables: +# +# sd: socket to server +# afterID: afterID associated with ::smtp::timer +# options: array of user-supplied options +# readable: semaphore for vwait +# addrs: number of recipients negotiated +# error: error during read +# line: response read from server +# crP: just put a \r in the data +# nlP: just put a \n in the data +# size: number of octets sent in DATA +# + +namespace eval ::smtp { + variable trf 1 + variable smtp + array set smtp { uid 0 } + + namespace export sendmessage +} + +if {[catch {package require Trf 2.0}]} { + # Trf is not available, but we can live without it as long as the + # transform and unstack procs are defined. + + # Warning! + # This is a fragile emulation of the more general calling sequence + # that appears to work with this code here. + + proc transform {args} { + upvar state mystate + set mystate(size) 1 + } + proc unstack {channel} { + # do nothing + return + } + set ::smtp::trf 0 +} + + +# ::smtp::sendmessage -- +# +# Sends a mime object (containing a message) to some recipients +# +# Arguments: +# part The MIME object containing the message to send +# args A list of arguments specifying various options for sending the +# message: +# -atleastone A boolean specifying whether or not to send the +# message at all if any of the recipients are +# invalid. A value of false (as defined by +# ::smtp::boolean) means that ALL recipients must be +# valid in order to send the message. A value of +# true means that as long as at least one recipient +# is valid, the message will be sent. +# -debug A boolean specifying whether or not debugging is +# on. If debugging is enabled, status messages are +# printed to stderr while trying to send mail. +# -queue A boolean specifying whether or not the message +# being sent should be queued for later delivery. +# -header A single RFC 822 header key and value (as a list), +# used to specify to whom to send the message +# (To, Cc, Bcc), the "From", etc. +# -originator The originator of the message (equivalent to +# specifying a From header). +# -recipients A string containing recipient e-mail addresses. +# NOTE: This option overrides any recipient addresses +# specified with -header. +# -servers A list of mail servers that could process the +# request. +# -ports A list of SMTP ports to use for each SMTP server +# specified +# -client The string to use as our host name for EHLO or HELO +# This defaults to 'localhost' or [info hostname] +# -maxsecs Maximum number of seconds to allow the SMTP server +# to accept the message. If not specified, the default +# is 120 seconds. +# -usetls A boolean flag. If the server supports it and we +# have the package, use TLS to secure the connection. +# -tlspolicy A command to call if the TLS negotiation fails for +# some reason. Return 'insecure' to continue with +# normal SMTP or 'secure' to close the connection and +# try another server. +# -tlsimport after a succesfull socket command, import tls on +# channel - used for native smtps negotiation +# -username These are needed if your SMTP server requires +# -password authentication. +# +# Results: +# Message is sent. On success, return "". On failure, throw an +# exception with an error code and error message. + +proc ::smtp::sendmessage {part args} { + global errorCode errorInfo + + # Here are the meanings of the following boolean variables: + # aloP -- value of -atleastone option above. + # debugP -- value of -debug option above. + # origP -- 1 if -originator option was specified, 0 otherwise. + # queueP -- value of -queue option above. + + set aloP 0 + set debugP 0 + set origP 0 + set queueP 0 + set maxsecs 120 + set originator "" + set recipients "" + set servers [list localhost] + set client "" ;# default is set after options processing + set ports [list 25] + set tlsP 1 + set tlspolicy {} + set tlsimport 0 + set username {} + set password {} + + array set header "" + + # lowerL will contain the list of header keys (converted to lower case) + # specified with various -header options. mixedL is the mixed-case version + # of the list. + set lowerL "" + set mixedL "" + + # Parse options (args). + + if {[expr {[llength $args]%2}]} { + # Some option didn't get a value. + error "Each option must have a value! Invalid option list: $args" + } + + foreach {option value} $args { + switch -- $option { + -atleastone {set aloP [boolean $value]} + -debug {set debugP [boolean $value]} + -queue {set queueP [boolean $value]} + -usetls {set tlsP [boolean $value]} + -tlspolicy {set tlspolicy $value} + -tlsimport {set tlsimport [boolean $value]} + -maxsecs {set maxsecs [expr {$value < 0 ? 0 : $value}]} + -header { + if {[llength $value] != 2} { + error "-header expects a key and a value, not $value" + } + set mixed [lindex $value 0] + set lower [string tolower $mixed] + set disallowedHdrList \ + [list content-type \ + content-transfer-encoding \ + content-md5 \ + mime-version] + if {[lsearch -exact $disallowedHdrList $lower] > -1} { + error "Content-Type, Content-Transfer-Encoding,\ + Content-MD5, and MIME-Version cannot be user-specified." + } + if {[lsearch -exact $lowerL $lower] < 0} { + lappend lowerL $lower + lappend mixedL $mixed + } + + lappend header($lower) [lindex $value 1] + } + + -originator { + set originator $value + if {$originator == ""} { + set origP 1 + } + } + + -recipients { + set recipients $value + } + + -servers { + set servers $value + } + + -client { + set client $value + } + + -ports { + set ports $value + } + + -username { set username $value } + -password { set password $value } + + default { + error "unknown option $option" + } + } + } + + if {[lsearch -glob $lowerL resent-*] >= 0} { + set prefixL resent- + set prefixM Resent- + } else { + set prefixL "" + set prefixM "" + } + + # Set a bunch of variables whose value will be the real header to be used + # in the outbound message (with proper case and prefix). + + foreach mixed {From Sender To cc Dcc Bcc Date Message-ID} { + set lower [string tolower $mixed] + # FRINK: nocheck + set ${lower}L $prefixL$lower + # FRINK: nocheck + set ${lower}M $prefixM$mixed + } + + if {$origP} { + # -originator was specified with "", so SMTP sender should be marked "". + set sender "" + } else { + # -originator was specified with a value, OR -originator wasn't + # specified at all. + + # If no -originator was provided, get the originator from the "From" + # header. If there was no "From" header get it from the username + # executing the script. + + set who "-originator" + if {$originator == ""} { + if {![info exists header($fromL)]} { + set originator $::tcl_platform(user) + } else { + set originator [join $header($fromL) ,] + + # Indicate that we're using the From header for the originator. + + set who $fromM + } + } + + # If there's no "From" header, create a From header with the value + # of -originator as the value. + + if {[lsearch -exact $lowerL $fromL] < 0} { + lappend lowerL $fromL + lappend mixedL $fromM + lappend header($fromL) $originator + } + + # ::mime::parseaddress returns a list whose elements are huge key-value + # lists with info about the addresses. In this case, we only want one + # originator, so we want the length of the main list to be 1. + + set addrs [::mime::parseaddress $originator] + if {[llength $addrs] > 1} { + error "too many mailboxes in $who: $originator" + } + array set aprops {error "invalid address \"$from\""} + array set aprops [lindex $addrs 0] + if {$aprops(error) != ""} { + error "error in $who: $aprops(error)" + } + + # sender = validated originator or the value of the From header. + + set sender $aprops(address) + + # If no Sender header has been specified and From is different from + # originator, then set the sender header to the From. Otherwise, don't + # specify a Sender header. + set from [join $header($fromL) ,] + if {[lsearch -exact $lowerL $senderL] < 0 && \ + [string compare $originator $from]} { + if {[info exists aprops]} { + unset aprops + } + array set aprops {error "invalid address \"$from\""} + array set aprops [lindex [::mime::parseaddress $from] 0] + if {$aprops(error) != ""} { + error "error in $fromM: $aprops(error)" + } + if {[string compare $aprops(address) $sender]} { + lappend lowerL $senderL + lappend mixedL $senderM + lappend header($senderL) $aprops(address) + } + } + } + + # We're done parsing the arguments. + + if {$recipients != ""} { + set who -recipients + } elseif {![info exists header($toL)]} { + error "need -header \"$toM ...\"" + } else { + set recipients [join $header($toL) ,] + # Add Cc values to recipients list + set who $toM + if {[info exists header($ccL)]} { + append recipients ,[join $header($ccL) ,] + append who /$ccM + } + + set dccInd [lsearch -exact $lowerL $dccL] + if {$dccInd >= 0} { + # Add Dcc values to recipients list, and get rid of Dcc header + # since we don't want to output that. + append recipients ,[join $header($dccL) ,] + append who /$dccM + + unset header($dccL) + set lowerL [lreplace $lowerL $dccInd $dccInd] + set mixedL [lreplace $mixedL $dccInd $dccInd] + } + } + + set brecipients "" + set bccInd [lsearch -exact $lowerL $bccL] + if {$bccInd >= 0} { + set bccP 1 + + # Build valid bcc list and remove bcc element of header array (so that + # bcc info won't be sent with mail). + foreach addr [::mime::parseaddress [join $header($bccL) ,]] { + if {[info exists aprops]} { + unset aprops + } + array set aprops {error "invalid address \"$from\""} + array set aprops $addr + if {$aprops(error) != ""} { + error "error in $bccM: $aprops(error)" + } + lappend brecipients $aprops(address) + } + + unset header($bccL) + set lowerL [lreplace $lowerL $bccInd $bccInd] + set mixedL [lreplace $mixedL $bccInd $bccInd] + } else { + set bccP 0 + } + + # If there are no To headers, add "" to bcc list. WHY?? + if {[lsearch -exact $lowerL $toL] < 0} { + lappend lowerL $bccL + lappend mixedL $bccM + lappend header($bccL) "" + } + + # Construct valid recipients list from recipients list. + + set vrecipients "" + foreach addr [::mime::parseaddress $recipients] { + if {[info exists aprops]} { + unset aprops + } + array set aprops {error "invalid address \"$from\""} + array set aprops $addr + if {$aprops(error) != ""} { + error "error in $who: $aprops(error)" + } + lappend vrecipients $aprops(address) + } + + # If there's no date header, get the date from the mime message. Same for + # the message-id. + + if {([lsearch -exact $lowerL $dateL] < 0) \ + && ([catch {::mime::getheader $part $dateL}])} { + lappend lowerL $dateL + lappend mixedL $dateM + lappend header($dateL) [::mime::parsedatetime -now proper] + } + + if {([lsearch -exact $lowerL ${message-idL}] < 0) \ + && ([catch {::mime::getheader $part ${message-idL}}])} { + lappend lowerL ${message-idL} + lappend mixedL ${message-idM} + lappend header(${message-idL}) [::mime::uniqueID] + + } + + set origheaders {} + set orignames [join [lmap name [::mime::getheader $part -names] { + list [string tolower $name] $name + }]] + + # Take all the headers defined earlier and add them to the MIME message. + foreach lower $lowerL mixed $mixedL { + foreach value $header($lower) { + if {![dict exists $origheaders $lower]} { + if {![catch {::mime::getheader $part $lower} cres]} { + dict set origheaderx $lower $cres + } + } + ::mime::setheader $part $mixed $value -mode append + } + } + + if {[string length $client] < 1} { + if {![string compare $servers localhost]} { + set client localhost + } else { + set client [info hostname] + } + } + + # Create smtp token, which essentially means begin talking to the SMTP + # server. + set token [initialize -debug $debugP -client $client \ + -maxsecs $maxsecs -usetls $tlsP \ + -multiple $bccP -queue $queueP \ + -servers $servers -ports $ports \ + -tlspolicy $tlspolicy -tlsimport $tlsimport \ + -username $username -password $password] + + if {![string match "::smtp::*" $token]} { + # An error occurred and $token contains the error info + array set respArr $token + return -code error $respArr(diagnostic) + } + + set code [catch { sendmessageaux $token $part \ + $sender $vrecipients $aloP } \ + cres copts] + + # Send the message to bcc recipients as a MIME attachment. + + if {($code == 0) && ($bccP)} { + set inner [::mime::initialize -canonical message/rfc822 \ + -header [list Content-Description \ + "Original Message"] \ + -parts [list $part]] + + set subject "\[$bccM\]" + if {[info exists header(subject)]} { + append subject " " [lindex $header(subject) 0] + } + + set outer [::mime::initialize \ + -canonical multipart/digest \ + -header [list From $originator] \ + -header [list Bcc ""] \ + -header [list Date \ + [::mime::parsedatetime -now proper]] \ + -header [list Subject $subject] \ + -header [list Message-ID [::mime::uniqueID]] \ + -header [list Content-Description \ + "Blind Carbon Copy"] \ + -parts [list $inner]] + + + set code [catch { sendmessageaux $token $outer \ + $sender $brecipients \ + $aloP } cres2 copts2] + if {$code == 0} { + append cres $cres2 + } + + catch { ::mime::finalize $inner -subordinates none } + catch { ::mime::finalize $outer -subordinates none } + } + + # Determine if there was any error in prior operations and set errorcodes + # and error messages appropriately. + + switch -- $code { + 0 { + set status orderly + } + default { + set status abort + } + } + + # Destroy SMTP token 'cause we're done with it. + catch { finalize $token -close $status } + + # Restore provided MIME object to original state (without the SMTP + # headers). To avoid an incorect attempt to set a read-only header like + # "Content-Type', the only original headers that were saved were those that + # were later modified. + foreach {key value} $origheaders { + mime::setheader $part $key {} -mode delete + ::mime::setheader $part [dict get orignames $key] $value -mode append + } + + return -options $copts $cres +} + +# ::smtp::sendmessageaux -- +# +# Sends a mime object (containing a message) to some recipients using an +# existing SMTP token. +# +# Arguments: +# token SMTP token that has an open connection to the SMTP server. +# part The MIME object containing the message to send. +# originator The e-mail address of the entity sending the message, +# usually the From clause. +# recipients List of e-mail addresses to whom message will be sent. +# aloP Boolean "atleastone" setting; see the -atleastone option +# in ::smtp::sendmessage for details. +# +# Results: +# Message is sent. On success, return "". On failure, throw an +# exception with an error code and error message. + +proc ::smtp::sendmessageaux {token part originator recipients aloP} { + global errorCode errorInfo + + winit $token $part $originator + + set goodP 0 + set badP 0 + set oops "" + foreach recipient $recipients { + set code [catch { waddr $token $recipient } result] + set ecode $errorCode + set einfo $errorInfo + + switch -- $code { + 0 { + incr goodP + } + + 7 { + incr badP + + array set response $result + lappend oops [list $recipient $response(code) \ + $response(diagnostic)] + } + + default { + return -code $code -errorinfo $einfo -errorcode $ecode $result + } + } + } + + if {($goodP) && ((!$badP) || ($aloP))} { + wtext $token $part + } else { + catch { talk $token 300 RSET } + } + + return $oops +} + +# ::smtp::initialize -- +# +# Create an SMTP token and open a connection to the SMTP server. +# +# Arguments: +# args A list of arguments specifying various options for sending the +# message: +# -debug A boolean specifying whether or not debugging is +# on. If debugging is enabled, status messages are +# printed to stderr while trying to send mail. +# -client Either localhost or the name of the local host. +# -multiple Multiple messages will be sent using this token. +# -queue A boolean specifying whether or not the message +# being sent should be queued for later delivery. +# -servers A list of mail servers that could process the +# request. +# -ports A list of ports on mail servers that could process +# the request (one port per server-- defaults to 25). +# -usetls A boolean to indicate we will use TLS if possible. +# -tlspolicy Command called if TLS setup fails. +# -tlsimport after a succesfull socket command, import tls on +# channel - used for native smtps negotiation +# -username These provide the authentication information +# -password to be used if needed by the SMTP server. +# +# Results: +# On success, return an smtp token. On failure, throw +# an exception with an error code and error message. + +proc ::smtp::initialize {args} { + global errorCode errorInfo + + variable smtp + + set token [namespace current]::[incr smtp(uid)] + # FRINK: nocheck + variable $token + upvar 0 $token state + + array set state [list afterID "" options "" readable 0] + array set options [list -debug 0 -client localhost -multiple 1 \ + -maxsecs 120 -queue 0 -servers localhost \ + -ports 25 -usetls 1 -tlspolicy {} \ + -tlsimport 0 \ + -username {} -password {}] + array set options $args + set state(options) [array get options] + + # Iterate through servers until one accepts a connection (and responds + # nicely). + + foreach server $options(-servers) port $options(-ports) { + if {$server == ""} continue + + set state(readable) 0 + if {$port == ""} { set port 25 } + + if {$options(-debug)} { + puts stderr "Trying $server..." + flush stderr + } + + if {[info exists state(sd)]} { + unset state(sd) + } + + if {[set code [catch { + set state(sd) [socket -async $server $port] + if { $options(-tlsimport) } { + package require tls + tls::import $state(sd) + } + fconfigure $state(sd) -blocking off -translation binary + fileevent $state(sd) readable [list ::smtp::readable $token] + } result]]} { + set ecode $errorCode + set einfo $errorInfo + + catch { close $state(sd) } + continue + } + + if {[set code [catch { hear $token 600 } result]]} { + array set response [list code 400 diagnostic $result] + } else { + array set response $result + } + set ecode $errorCode + set einfo $errorInfo + switch -- $response(code) { + 220 { + } + + 421 - default { + # 421 - Temporary problem on server + catch {close $state(sd)} + continue + } + } + + set r [initialize_ehlo $token] + if {$r != {}} { + return $r + } + } + + # None of the servers accepted our connection, so close everything up and + # return an error. + finalize $token -close drop + + return -code $code -errorinfo $einfo -errorcode $ecode $result +} + +# If we cannot load the tls package, ignore the error +# Result value is a Tcl return code, not a bool. +# 0 == OK +proc ::smtp::load_tls {} { + set r [catch {package require tls}] + if {$r} {set ::errorInfo ""} + return $r +} + +proc ::smtp::initialize_ehlo {token} { + global errorCode errorInfo + upvar einfo einfo + upvar ecode ecode + upvar code code + + # FRINK: nocheck + variable $token + upvar 0 $token state + array set options $state(options) + + # Try enhanced SMTP first. + + if {[set code [catch {smtp::talk $token 300 "EHLO $options(-client)"} \ + result]]} { + array set response [list code 400 diagnostic $result args ""] + } else { + array set response $result + } + set ecode $errorCode + set einfo $errorInfo + if {(500 <= $response(code)) && ($response(code) <= 599)} { + if {[set code [catch { talk $token 300 \ + "HELO $options(-client)" } \ + result]]} { + array set response [list code 400 diagnostic $result args ""] + } else { + array set response $result + } + set ecode $errorCode + set einfo $errorInfo + } + + if {$response(code) == 250} { + # Successful response to HELO or EHLO command, so set up queuing + # and whatnot and return the token. + + set state(esmtp) $response(args) + + if {(!$options(-multiple)) \ + && ([lsearch $response(args) ONEX] >= 0)} { + catch {smtp::talk $token 300 ONEX} + } + if {($options(-queue)) \ + && ([lsearch $response(args) XQUE] >= 0)} { + catch {smtp::talk $token 300 QUED} + } + + # Support STARTTLS extension. + # The state(tls) item is used to see if we have already tried this. + if {($options(-usetls)) && ![info exists state(tls)] \ + && (([lsearch $response(args) STARTTLS] >= 0) + || ([lsearch $response(args) TLS] >= 0))} { + if {[load_tls] == 0} { + set state(tls) 0 + if {![catch {smtp::talk $token 300 STARTTLS} resp]} { + array set starttls $resp + if {$starttls(code) == 220} { + fileevent $state(sd) readable {} + catch { + ::tls::import $state(sd) + catch {::tls::handshake $state(sd)} msg + set state(tls) 1 + } + fileevent $state(sd) readable \ + [list ::smtp::readable $token] + return [initialize_ehlo $token] + } else { + # Call a TLS client policy proc here + # returns secure - close and try another server. + # returns insecure - continue on current socket + set policy insecure + if {$options(-tlspolicy) != {}} { + catch { + eval $options(-tlspolicy) \ + [list $starttls(code)] \ + [list $starttls(diagnostic)] + } policy + } + if {$policy != "insecure"} { + set code error + set ecode $starttls(code) + set einfo $starttls(diagnostic) + catch {close $state(sd)} + return {} + } + } + } + } + } + + # If we have not already tried and the server supports it and we + # have a username -- lets try to authenticate. + # + if {![info exists state(auth)] + && [llength [package provide SASL]] != 0 + && [set andx [lsearch -glob $response(args) "AUTH*"]] >= 0 + && [string length $options(-username)] > 0 } { + + # May be AUTH mech or AUTH=mech + # We want to use the strongest mechanism that has been offered + # and that we support. If we cannot find a mechanism that + # succeeds, we will go ahead and try to carry on unauthenticated. + # This may still work else we'll get an unauthorised error later. + + set mechs [string range [lindex $response(args) $andx] 5 end] + foreach mech [SASL::mechanisms] { + if {[lsearch -exact $mechs $mech] == -1} { continue } + if {[catch { + Authenticate $token $mech + } msg]} { + if {$options(-debug)} { + puts stderr "AUTH $mech failed: $msg " + flush stderr + } + } + if {[info exists state(auth)] && $state(auth)} { + if {$state(auth) == 1} { + break + } else { + # After successful AUTH we are supposed to redo + # our connection for mechanisms that setup a new + # security layer -- these should set state(auth) + # greater than 1 + fileevent $state(sd) readable \ + [list ::smtp::readable $token] + return [initialize_ehlo $token] + } + } + } + } + + return $token + } else { + # Bad response; close the connection and hope the next server + # is happier. + catch {close $state(sd)} + } + return {} +} + +proc ::smtp::SASLCallback {token context command args} { + upvar #0 $token state + upvar #0 $context ctx + array set options $state(options) + switch -exact -- $command { + login { return "" } + username { return $options(-username) } + password { return $options(-password) } + hostname { return [info host] } + realm { + if {[string equal $ctx(mech) "NTLM"] \ + && [info exists ::env(USERDOMAIN)]} { + return $::env(USERDOMAIN) + } else { + return "" + } + } + default { + return -code error "error: unsupported SASL information requested" + } + } +} + +proc ::smtp::Authenticate {token mechanism} { + upvar 0 $token state + package require base64 + set ctx [SASL::new -mechanism $mechanism \ + -callback [list [namespace origin SASLCallback] $token]] + + set state(auth) 0 + set result [smtp::talk $token 300 "AUTH $mechanism"] + array set response $result + + while {$response(code) == 334} { + # The NTLM initial response is not base64 encoded so handle it. + if {[catch {base64::decode $response(diagnostic)} challenge]} { + set challenge $response(diagnostic) + } + SASL::step $ctx $challenge + set result [smtp::talk $token 300 \ + [base64::encode -maxlen 0 [SASL::response $ctx]]] + array set response $result + } + + if {$response(code) == 235} { + set state(auth) 1 + return $result + } else { + return -code 7 $result + } +} + +# ::smtp::finalize -- +# +# Deletes an SMTP token by closing the connection to the SMTP server, +# cleanup up various state. +# +# Arguments: +# token SMTP token that has an open connection to the SMTP server. +# args Optional arguments, where the only useful option is -close, +# whose valid values are the following: +# orderly Normal successful completion. Close connection and +# clear state variables. +# abort A connection exists to the SMTP server, but it's in +# a weird state and needs to be reset before being +# closed. Then clear state variables. +# drop No connection exists, so we just need to clean up +# state variables. +# +# Results: +# SMTP connection is closed and state variables are cleared. If there's +# an error while attempting to close the connection to the SMTP server, +# throw an exception with the error code and error message. + +proc ::smtp::finalize {token args} { + global errorCode errorInfo + # FRINK: nocheck + variable $token + upvar 0 $token state + + array set options [list -close orderly] + array set options $args + + switch -- $options(-close) { + orderly { + set code [catch { talk $token 120 QUIT } result] + } + + abort { + set code [catch { + talk $token 0 RSET + talk $token 0 QUIT + } result] + } + + drop { + set code 0 + set result "" + } + + default { + error "unknown value for -close $options(-close)" + } + } + set ecode $errorCode + set einfo $errorInfo + + catch { close $state(sd) } + + if {$state(afterID) != ""} { + catch { after cancel $state(afterID) } + } + + foreach name [array names state] { + unset state($name) + } + # FRINK: nocheck + unset $token + + return -code $code -errorinfo $einfo -errorcode $ecode $result +} + +# ::smtp::winit -- +# +# Send originator info to SMTP server. This occurs after HELO/EHLO +# command has completed successfully (in ::smtp::initialize). This function +# is called by ::smtp::sendmessageaux. +# +# Arguments: +# token SMTP token that has an open connection to the SMTP server. +# part MIME token for the message to be sent. May be used for +# handling some SMTP extensions. +# originator The e-mail address of the entity sending the message, +# usually the From clause. +# mode SMTP command specifying the mode of communication. Default +# value is MAIL. +# +# Results: +# Originator info is sent and SMTP server's response is returned. If an +# error occurs, throw an exception. + +proc ::smtp::winit {token part originator {mode MAIL}} { + # FRINK: nocheck + variable $token + upvar 0 $token state + + if {[lsearch -exact [list MAIL SEND SOML SAML] $mode] < 0} { + error "unknown origination mode $mode" + } + + set from "$mode FROM:<$originator>" + + # RFC 1870 - SMTP Service Extension for Message Size Declaration + if {[info exists state(esmtp)] + && [lsearch -glob $state(esmtp) "SIZE*"] != -1} { + catch { + set size [string length [mime::buildmessage $part]] + append from " SIZE=$size" + } + } + + array set response [set result [talk $token 600 $from]] + + if {$response(code) == 250} { + set state(addrs) 0 + return $result + } else { + return -code 7 $result + } +} + +# ::smtp::waddr -- +# +# Send recipient info to SMTP server. This occurs after originator info +# is sent (in ::smtp::winit). This function is called by +# ::smtp::sendmessageaux. +# +# Arguments: +# token SMTP token that has an open connection to the SMTP server. +# recipient One of the recipients to whom the message should be +# delivered. +# +# Results: +# Recipient info is sent and SMTP server's response is returned. If an +# error occurs, throw an exception. + +proc ::smtp::waddr {token recipient} { + # FRINK: nocheck + variable $token + upvar 0 $token state + + set result [talk $token 3600 "RCPT TO:<$recipient>"] + array set response $result + + switch -- $response(code) { + 250 - 251 { + incr state(addrs) + return $result + } + + default { + return -code 7 $result + } + } +} + +# ::smtp::wtext -- +# +# Send message to SMTP server. This occurs after recipient info +# is sent (in ::smtp::winit). This function is called by +# ::smtp::sendmessageaux. +# +# Arguments: +# token SMTP token that has an open connection to the SMTP server. +# part The MIME object containing the message to send. +# +# Results: +# MIME message is sent and SMTP server's response is returned. If an +# error occurs, throw an exception. + +proc ::smtp::wtext {token part} { + # FRINK: nocheck + variable $token + upvar 0 $token state + array set options $state(options) + + set result [talk $token 300 DATA] + array set response $result + if {$response(code) != 354} { + return -code 7 $result + } + + if {[catch { wtextaux $token $part } result]} { + catch { puts -nonewline $state(sd) "\r\n.\r\n" ; flush $state(sd) } + return -code 7 [list code 400 diagnostic $result] + } + + set secs $options(-maxsecs) + + set result [talk $token $secs .] + array set response $result + switch -- $response(code) { + 250 - 251 { + return $result + } + + default { + return -code 7 $result + } + } +} + +# ::smtp::wtextaux -- +# +# Helper function that coordinates writing the MIME message to the socket. +# In particular, it stacks the channel leading to the SMTP server, sets up +# some file events, sends the message, unstacks the channel, resets the +# file events to their original state, and returns. +# +# Arguments: +# token SMTP token that has an open connection to the SMTP server. +# part The MIME object containing the message to send. +# +# Results: +# Message is sent. If anything goes wrong, throw an exception. + +proc ::smtp::wtextaux {token part} { + global errorCode errorInfo + + # FRINK: nocheck + variable $token + upvar 0 $token state + + # Workaround a bug with stacking channels on top of TLS. + # FRINK: nocheck + set trf [set [namespace current]::trf] + if {[info exists state(tls)] && $state(tls)} { + set trf 0 + } + + flush $state(sd) + fileevent $state(sd) readable "" + if {$trf} { + transform -attach $state(sd) -command [list ::smtp::wdata $token] + fconfigure $state(sd) -blocking on + } else { + set state(size) 1 + } + fileevent $state(sd) readable [list ::smtp::readable $token] + + # If trf is not available, get the contents of the message, + # replace all '.'s that start their own line with '..'s, and + # then write the mime body out to the filehandle. Do not forget to + # deal with bare LF's here too (SF bug #499242). + + if {$trf} { + set code [catch { ::mime::copymessage $part $state(sd) } result] + } else { + set code [catch { ::mime::buildmessage $part } result] + if {$code == 0} { + # Detect and transform bare LF's into proper CR/LF + # sequences. + + while {[regsub -all -- {([^\r])\n} $result "\\1\r\n" result]} {} + regsub -all -- {\n\.} $result "\n.." result + + # Fix for bug #827436 - mail data must end with CRLF.CRLF + if {[string compare [string index $result end] "\n"] != 0} { + append result "\r\n" + } + set state(size) [string length $result] + puts -nonewline $state(sd) $result + set result "" + } + } + set ecode $errorCode + set einfo $errorInfo + + flush $state(sd) + fileevent $state(sd) readable "" + if {$trf} { + unstack $state(sd) + fconfigure $state(sd) -blocking off + } + fileevent $state(sd) readable [list ::smtp::readable $token] + + return -code $code -errorinfo $einfo -errorcode $ecode $result +} + +# ::smtp::wdata -- +# +# This is the custom transform using Trf to do CR/LF translation. If Trf +# is not installed on the system, then this function never gets called and +# no translation occurs. +# +# Arguments: +# token SMTP token that has an open connection to the SMTP server. +# command Trf provided command for manipulating socket data. +# buffer Data to be converted. +# +# Results: +# buffer is translated, and state(size) is set. If Trf is not installed +# on the system, the transform proc defined at the top of this file sets +# state(size) to 1. state(size) is used later to determine a timeout +# value. + +proc ::smtp::wdata {token command buffer} { + # FRINK: nocheck + variable $token + upvar 0 $token state + + switch -- $command { + create/write - + clear/write - + delete/write { + set state(crP) 0 + set state(nlP) 1 + set state(size) 0 + } + + write { + set result "" + + foreach c [split $buffer ""] { + switch -- $c { + "." { + if {$state(nlP)} { + append result . + } + set state(crP) 0 + set state(nlP) 0 + } + + "\r" { + set state(crP) 1 + set state(nlP) 0 + } + + "\n" { + if {!$state(crP)} { + append result "\r" + } + set state(crP) 0 + set state(nlP) 1 + } + + default { + set state(crP) 0 + set state(nlP) 0 + } + } + + append result $c + } + + incr state(size) [string length $result] + return $result + } + + flush/write { + set result "" + + if {!$state(nlP)} { + if {!$state(crP)} { + append result "\r" + } + append result "\n" + } + + incr state(size) [string length $result] + return $result + } + + create/read - + delete/read { + # Bugfix for [#539952] + } + + query/ratio { + # Indicator for unseekable channel, + # for versions of Trf which ask for + # this. + return {0 0} + } + query/maxRead { + # No limits on reading bytes from the channel below, for + # versions of Trf which ask for this information + return -1 + } + + default { + # Silently pass all unknown commands. + #error "Unknown command \"$command\"" + } + } + + return "" +} + +# ::smtp::talk -- +# +# Sends an SMTP command to a server +# +# Arguments: +# token SMTP token that has an open connection to the SMTP server. +# secs Timeout after which command should be aborted. +# command Command to send to SMTP server. +# +# Results: +# command is sent and response is returned. If anything goes wrong, throw +# an exception. + +proc ::smtp::talk {token secs command} { + # FRINK: nocheck + variable $token + upvar 0 $token state + + array set options $state(options) + + if {$options(-debug)} { + puts stderr "--> $command (wait upto $secs seconds)" + flush stderr + } + + if {[catch { puts -nonewline $state(sd) "$command\r\n" + flush $state(sd) } result]} { + return [list code 400 diagnostic $result] + } + + if {$secs == 0} { + return "" + } + + return [hear $token $secs] +} + +# ::smtp::hear -- +# +# Listens for SMTP server's response to some prior command. +# +# Arguments: +# token SMTP token that has an open connection to the SMTP server. +# secs Timeout after which we should stop waiting for a response. +# +# Results: +# Response is returned. + +proc ::smtp::hear {token secs} { + # FRINK: nocheck + variable $token + upvar 0 $token state + + array set options $state(options) + + array set response [list args ""] + + set firstP 1 + while {1} { + if {$secs >= 0} { + ## SF [ 836442 ] timeout with large data + ## correction, aotto 031105 - + if {$secs > 600} {set secs 600} + set state(afterID) [after [expr {$secs*1000}] \ + [list ::smtp::timer $token]] + } + + if {!$state(readable)} { + vwait ${token}(readable) + } + + # Wait until socket is readable. + if {$state(readable) != -1} { + catch { after cancel $state(afterID) } + set state(afterID) "" + } + + if {$state(readable) < 0} { + array set response [list code 400 diagnostic $state(error)] + break + } + set state(readable) 0 + + if {$options(-debug)} { + puts stderr "<-- $state(line)" + flush stderr + } + + if {[string length $state(line)] < 3} { + array set response \ + [list code 500 \ + diagnostic "response too short: $state(line)"] + break + } + + if {$firstP} { + set firstP 0 + + if {[scan [string range $state(line) 0 2] %d response(code)] \ + != 1} { + array set response \ + [list code 500 \ + diagnostic "unrecognizable code: $state(line)"] + break + } + + set response(diagnostic) \ + [string trim [string range $state(line) 4 end]] + } else { + lappend response(args) \ + [string trim [string range $state(line) 4 end]] + } + + # When status message line ends in -, it means the message is complete. + + if {[string compare [string index $state(line) 3] -]} { + break + } + } + + return [array get response] +} + +# ::smtp::readable -- +# +# Reads a line of data from SMTP server when the socket is readable. This +# is the callback of "fileevent readable". +# +# Arguments: +# token SMTP token that has an open connection to the SMTP server. +# +# Results: +# state(line) contains the line of data and state(readable) is reset. +# state(readable) gets the following values: +# -3 if there's a premature eof, +# -2 if reading from socket fails. +# 1 if reading from socket was successful + +proc ::smtp::readable {token} { + # FRINK: nocheck + variable $token + upvar 0 $token state + + if {[catch { array set options $state(options) }]} { + return + } + + set state(line) "" + if {[catch { gets $state(sd) state(line) } result]} { + set state(readable) -2 + set state(error) $result + } elseif {$result == -1} { + if {[eof $state(sd)]} { + set state(readable) -3 + set state(error) "premature end-of-file from server" + } + } else { + # If the line ends in \r, remove the \r. + if {![string compare [string index $state(line) end] "\r"]} { + set state(line) [string range $state(line) 0 end-1] + } + set state(readable) 1 + } + + if {$state(readable) < 0} { + if {$options(-debug)} { + puts stderr " ... $state(error) ..." + flush stderr + } + + catch { fileevent $state(sd) readable "" } + } +} + +# ::smtp::timer -- +# +# Handles timeout condition on any communication with the SMTP server. +# +# Arguments: +# token SMTP token that has an open connection to the SMTP server. +# +# Results: +# Sets state(readable) to -1 and state(error) to an error message. + +proc ::smtp::timer {token} { + # FRINK: nocheck + variable $token + upvar 0 $token state + + array set options $state(options) + + set state(afterID) "" + set state(readable) -1 + set state(error) "read from server timed out" + + if {$options(-debug)} { + puts stderr " ... $state(error) ..." + flush stderr + } +} + +# ::smtp::boolean -- +# +# Helper function for unifying boolean values to 1 and 0. +# +# Arguments: +# value Some kind of value that represents true or false (i.e. 0, 1, +# false, true, no, yes, off, on). +# +# Results: +# Return 1 if the value is true, 0 if false. If the input value is not +# one of the above, throw an exception. + +proc ::smtp::boolean {value} { + switch -- [string tolower $value] { + 0 - false - no - off { + return 0 + } + + 1 - true - yes - on { + return 1 + } + + default { + error "unknown boolean value: $value" + } + } +} + +# ------------------------------------------------------------------------- + +package provide smtp 1.5.1 + +# ------------------------------------------------------------------------- +# Local variables: +# indent-tabs-mode: nil +# End: diff --git a/src/build.tcl b/src/build.tcl index ee541f5b..734ccb87 100644 --- a/src/build.tcl +++ b/src/build.tcl @@ -1,6 +1,6 @@ #!/bin/sh # -*- tcl -*- \ # 'build.tcl' name as required by kettle -# Can be run directly - but also using `pmix Kettle ...` or `pmix KettleShell ...`\ +# Can be run directly - but also using `deck Kettle ...` or `deck KettleShell ...`\ exec ./kettle -f "$0" "${1+$@}" kettle doc diff --git a/src/modules/punk/mix/templates/layouts/project/src/buildsuites/samplesuite1/tcl/empty_project_source.txt b/src/decktemplates/custom/_project/layout_refs/@custom+_project+punk.shell-0.1.ref similarity index 100% rename from src/modules/punk/mix/templates/layouts/project/src/buildsuites/samplesuite1/tcl/empty_project_source.txt rename to src/decktemplates/custom/_project/layout_refs/@custom+_project+punk.shell-0.1.ref diff --git a/src/modules/punk/mix/templates/layouts/project/src/buildsuites/samplesuite1/tk/empty_project_source.txt b/src/decktemplates/custom/_project/layout_refs/test1@vendor+punk+sample-0.1.ref similarity index 100% rename from src/modules/punk/mix/templates/layouts/project/src/buildsuites/samplesuite1/tk/empty_project_source.txt rename to src/decktemplates/custom/_project/layout_refs/test1@vendor+punk+sample-0.1.ref diff --git a/src/decktemplates/vendor/_project/layout_refs/@vendor+_project+punk.shell-0.1.ref b/src/decktemplates/vendor/_project/layout_refs/@vendor+_project+punk.shell-0.1.ref new file mode 100644 index 00000000..e69de29b diff --git a/src/decktemplates/vendor/_project/layout_refs/test2@vendor+punk+sample-0.1.ref b/src/decktemplates/vendor/_project/layout_refs/test2@vendor+punk+sample-0.1.ref new file mode 100644 index 00000000..e69de29b diff --git a/src/decktemplates/vendor/punk/layout_refs/@vendor+punk+basic.ref b/src/decktemplates/vendor/punk/layout_refs/@vendor+punk+basic.ref new file mode 100644 index 00000000..e69de29b diff --git a/src/decktemplates/vendor/punk/layout_refs/@vendor+punk+minimal.ref b/src/decktemplates/vendor/punk/layout_refs/@vendor+punk+minimal.ref new file mode 100644 index 00000000..e69de29b diff --git a/src/decktemplates/vendor/punk/layout_refs/@vendor+punk+sample-0.1.ref b/src/decktemplates/vendor/punk/layout_refs/@vendor+punk+sample-0.1.ref new file mode 100644 index 00000000..e69de29b diff --git a/src/decktemplates/vendor/punk/layout_refs/project@vendor+punk+project-0.1.ref b/src/decktemplates/vendor/punk/layout_refs/project@vendor+punk+project-0.1.ref new file mode 100644 index 00000000..e69de29b diff --git a/src/decktemplates/vendor/punk/modules/template_module-0.0.2.tm b/src/decktemplates/vendor/punk/modules/template_module-0.0.2.tm index bde17848..f8f01847 100644 --- a/src/decktemplates/vendor/punk/modules/template_module-0.0.2.tm +++ b/src/decktemplates/vendor/punk/modules/template_module-0.0.2.tm @@ -108,10 +108,15 @@ namespace eval %pkg% { - #proc sample1 {p1 args} { + #proc sample1 {p1 n args} { # #*** !doctools - # #[call [fun sample1] [arg p1] [opt {option value...}]] + # #[call [fun sample1] [arg p1] [arg n] [opt {option value...}]] # #[para]Description of sample1 + # #[para] Arguments: + # # [list_begin arguments] + # # [arg_def tring p1] A description of string argument p1. + # # [arg_def integer n] A description of integer argument n. + # # [list_end] # return "ok" #} diff --git a/src/make.tcl b/src/make.tcl index c01c9775..10d8e7ed 100644 --- a/src/make.tcl +++ b/src/make.tcl @@ -339,28 +339,28 @@ if {$::punkmake::command eq "bootsupport"} { } } - foreach folder $bootsupport_module_folders { - #explicitly ignore punk/mix/templates folder even if specified in config. - #punk/mix/templates contains modules including punk/mix/templates itself - the actual templates aren't needed for the bootsupport system, - # as make.tcl shouldn't be building new projects from the one being made. - #review. - #should we be autodetecting such recursive folder structures - (or is the bootsupport copying in need of a rethink?) - if {[string trim $folder /] eq "punk/mix/templates"} { - puts stderr "IGNORING punk/mix/templates - not needed/desirable in bootsupport" - continue - } - set src [file join $projectroot/modules $folder] - if {![file isdirectory $src]} { - puts stderr "bootsupport folder not found: $src" - continue - } - set tgt [file join $targetroot $folder] - file mkdir $tgt + if {[llength $bootsupport_module_folders] % 2 != 0} { + #todo - change include_modules.config structure to be line based? we have no way of verifying paired entries because we accept a flat list + puts stderr "WARNING - Skipping bootsupport_module_folders - list should be a list of base subpath pairs" + } else { + foreach {base subfolder} $bootsupport_module_folders { + #user should be careful not to include recursive/cyclic structures e.g module that has a folder which contains other modules from this project + #It will probably work somewhat.. but may make updates confusing.. or worse - start making deeper and deeper copies + set src [file join $projectroot $base $subfolder] + if {![file isdirectory $src]} { + puts stderr "bootsupport folder not found: $src" + continue + } + + #subfolder is the common relative path - so don't include the base in the target path + set tgt [file join $targetroot $subfolder] + file mkdir $tgt - puts stdout "BOOTSUPPORT non_tm_files $src - copying to $tgt (if source file changed)" - set overwrite "installedsourcechanged-targets" - set resultdict [punkcheck::install_non_tm_files $src $tgt -installer make.tcl -overwrite $overwrite -punkcheck_folder $projectroot/src/bootsupport] - puts stdout [punkcheck::summarize_install_resultdict $resultdict] + puts stdout "BOOTSUPPORT non_tm_files $src - copying to $tgt (if source file changed)" + set overwrite "installedsourcechanged-targets" + set resultdict [punkcheck::install_non_tm_files $src $tgt -installer make.tcl -overwrite $overwrite -punkcheck_folder $projectroot/src/bootsupport] + puts stdout [punkcheck::summarize_install_resultdict $resultdict] + } } } @@ -370,13 +370,13 @@ if {$::punkmake::command eq "bootsupport"} { #/modules/punk/mix/templates/layouts only applies if the project has it's own copy of the punk/mix modules. Generally this should only apply to the punkshell project itself. set layout_bases [list\ - $sourcefolder/mixtemplates/layouts\ - $sourcefolder/modules/punk/mix/templates/layouts\ + $sourcefolder/project_layouts/custom/_project\ ] foreach project_layout_base $layout_bases { if {[file exists $project_layout_base]} { set project_layouts [glob -nocomplain -dir $project_layout_base -type d -tail *] foreach layoutname $project_layouts { + #don't auto-create src/bootsupport - just update it if it exists if {[file exists [file join $project_layout_base $layoutname/src/bootsupport]]} { set antipaths [list\ README.md\ @@ -451,61 +451,67 @@ if {[file exists $sourcefolder/vendormodules]} { #e.g The default project layout is mainly folder structure and readme files - but has some scripts developed under the main src that we want to sync #src to src/modules/punk/mix/templates/layouts/project/src -set layout_update_list [list\ +set old_layout_update_list [list\ [list project $sourcefolder/modules/punk/mix/templates]\ [list basic $sourcefolder/mixtemplates]\ ] +set layout_bases [list\ + $sourcefolder/project_layouts/custom/_project\ + ] -foreach layoutinfo $layout_update_list { - lassign $layoutinfo layout templatebase - if {![file exists $templatebase]} { +foreach layoutbase $layout_bases { + if {![file exists $layoutbase]} { continue } - set config [dict create\ - -make-step sync_templates\ - ] - #---------- - set tpl_installer [punkcheck::installtrack new make.tcl $templatebase/.punkcheck] - $tpl_installer set_source_target $sourcefolder $templatebase - set tpl_event [$tpl_installer start_event $config] - #---------- - set pairs [list] - set pairs [list\ - [list $sourcefolder/build.tcl $templatebase/layouts/$layout/src/build.tcl]\ - [list $sourcefolder/make.tcl $templatebase/layouts/$layout/src/make.tcl]\ - ] - - foreach filepair $pairs { - lassign $filepair srcfile tgtfile - file mkdir [file dirname $tgtfile] + set project_layouts [glob -nocomplain -dir $layoutbase -type d -tail *] + foreach layoutname $project_layouts { + set config [dict create\ + -make-step sync_layouts\ + ] #---------- - $tpl_event targetset_init INSTALL $tgtfile - $tpl_event targetset_addsource $srcfile + set tpl_installer [punkcheck::installtrack new make.tcl $layoutbase/.punkcheck] + $tpl_installer set_source_target $sourcefolder $layoutbase + set tpl_event [$tpl_installer start_event $config] #---------- - if {\ - [llength [dict get [$tpl_event targetset_source_changes] changed]]\ - || [llength [$tpl_event get_targets_exist]] < [llength [$tpl_event get_targets]]\ - } { - $tpl_event targetset_started - # -- --- --- --- --- --- - puts stdout "punk module templates: Copying from $srcfile to $tgtfile" - if {[catch { - file copy -force $srcfile $tgtfile - } errM]} { - $tpl_event targetset_end FAILED -note "copy failed with err: $errM" + set pairs [list] + set pairs [list\ + [list $sourcefolder/build.tcl $layoutbase/$layoutname/src/build.tcl]\ + [list $sourcefolder/make.tcl $layoutbase/$layoutname/src/make.tcl]\ + ] + + foreach filepair $pairs { + lassign $filepair srcfile tgtfile + + file mkdir [file dirname $tgtfile] + #---------- + $tpl_event targetset_init INSTALL $tgtfile + $tpl_event targetset_addsource $srcfile + #---------- + if {\ + [llength [dict get [$tpl_event targetset_source_changes] changed]]\ + || [llength [$tpl_event get_targets_exist]] < [llength [$tpl_event get_targets]]\ + } { + $tpl_event targetset_started + # -- --- --- --- --- --- + puts stdout "PROJECT LAYOUT update - layoutname: $layoutname Copying from $srcfile to $tgtfile" + if {[catch { + file copy -force $srcfile $tgtfile + } errM]} { + $tpl_event targetset_end FAILED -note "layout:$layoutname copy failed with err: $errM" + } else { + $tpl_event targetset_end OK -note "layout:$layoutname" + } + # -- --- --- --- --- --- } else { - $tpl_event targetset_end OK -note "test" + puts stderr "." + $tpl_event targetset_end SKIPPED } - # -- --- --- --- --- --- - } else { - puts stderr "." - $tpl_event targetset_end SKIPPED } - } - $tpl_event end - $tpl_event destroy - $tpl_installer destroy + $tpl_event end + $tpl_event destroy + $tpl_installer destroy + } } ######################################################## @@ -658,7 +664,7 @@ if {[file exists $mapfile]} { set vfs_folders [glob -nocomplain -dir $sourcefolder -types d -tail *.vfs] #add any extra .vfs folders found in runtime/mapvfs.config file (e.g myotherruntimes/something.vfs) -foreach vfs [dict keys $vfs_runtime_map] { +dict for {vfs -} $vfs_runtime_map { if {$vfs ni $vfs_folders} { lappend vfs_folders $vfs } diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/README.md b/src/mixtemplates/layouts/basic/src/bootsupport/README.md deleted file mode 100644 index f1f51421..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/README.md +++ /dev/null @@ -1 +0,0 @@ -bootsupport libs and modules \ No newline at end of file diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/cksum-1.1.4.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/cksum-1.1.4.tm deleted file mode 100644 index 0fb17981..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/cksum-1.1.4.tm +++ /dev/null @@ -1,200 +0,0 @@ -# cksum.tcl - Copyright (C) 2002 Pat Thoyts -# -# Provides a Tcl only implementation of the unix cksum(1) command. This is -# similar to the sum(1) command but the algorithm is better defined and -# standardized across multiple platforms by POSIX 1003.2/D11.2 -# -# This command has been verified against the cksum command from the GNU -# textutils package version 2.0 -# -# ------------------------------------------------------------------------- -# See the file "license.terms" for information on usage and redistribution -# of this file, and for a DISCLAIMER OF ALL WARRANTIES. -# ------------------------------------------------------------------------- - -package require Tcl 8.5-; # tcl minimum version - -namespace eval ::crc { - namespace export cksum - - variable cksum_tbl [list 0x0 \ - 0x04C11DB7 0x09823B6E 0x0D4326D9 0x130476DC 0x17C56B6B \ - 0x1A864DB2 0x1E475005 0x2608EDB8 0x22C9F00F 0x2F8AD6D6 \ - 0x2B4BCB61 0x350C9B64 0x31CD86D3 0x3C8EA00A 0x384FBDBD \ - 0x4C11DB70 0x48D0C6C7 0x4593E01E 0x4152FDA9 0x5F15ADAC \ - 0x5BD4B01B 0x569796C2 0x52568B75 0x6A1936C8 0x6ED82B7F \ - 0x639B0DA6 0x675A1011 0x791D4014 0x7DDC5DA3 0x709F7B7A \ - 0x745E66CD 0x9823B6E0 0x9CE2AB57 0x91A18D8E 0x95609039 \ - 0x8B27C03C 0x8FE6DD8B 0x82A5FB52 0x8664E6E5 0xBE2B5B58 \ - 0xBAEA46EF 0xB7A96036 0xB3687D81 0xAD2F2D84 0xA9EE3033 \ - 0xA4AD16EA 0xA06C0B5D 0xD4326D90 0xD0F37027 0xDDB056FE \ - 0xD9714B49 0xC7361B4C 0xC3F706FB 0xCEB42022 0xCA753D95 \ - 0xF23A8028 0xF6FB9D9F 0xFBB8BB46 0xFF79A6F1 0xE13EF6F4 \ - 0xE5FFEB43 0xE8BCCD9A 0xEC7DD02D 0x34867077 0x30476DC0 \ - 0x3D044B19 0x39C556AE 0x278206AB 0x23431B1C 0x2E003DC5 \ - 0x2AC12072 0x128E9DCF 0x164F8078 0x1B0CA6A1 0x1FCDBB16 \ - 0x018AEB13 0x054BF6A4 0x0808D07D 0x0CC9CDCA 0x7897AB07 \ - 0x7C56B6B0 0x71159069 0x75D48DDE 0x6B93DDDB 0x6F52C06C \ - 0x6211E6B5 0x66D0FB02 0x5E9F46BF 0x5A5E5B08 0x571D7DD1 \ - 0x53DC6066 0x4D9B3063 0x495A2DD4 0x44190B0D 0x40D816BA \ - 0xACA5C697 0xA864DB20 0xA527FDF9 0xA1E6E04E 0xBFA1B04B \ - 0xBB60ADFC 0xB6238B25 0xB2E29692 0x8AAD2B2F 0x8E6C3698 \ - 0x832F1041 0x87EE0DF6 0x99A95DF3 0x9D684044 0x902B669D \ - 0x94EA7B2A 0xE0B41DE7 0xE4750050 0xE9362689 0xEDF73B3E \ - 0xF3B06B3B 0xF771768C 0xFA325055 0xFEF34DE2 0xC6BCF05F \ - 0xC27DEDE8 0xCF3ECB31 0xCBFFD686 0xD5B88683 0xD1799B34 \ - 0xDC3ABDED 0xD8FBA05A 0x690CE0EE 0x6DCDFD59 0x608EDB80 \ - 0x644FC637 0x7A089632 0x7EC98B85 0x738AAD5C 0x774BB0EB \ - 0x4F040D56 0x4BC510E1 0x46863638 0x42472B8F 0x5C007B8A \ - 0x58C1663D 0x558240E4 0x51435D53 0x251D3B9E 0x21DC2629 \ - 0x2C9F00F0 0x285E1D47 0x36194D42 0x32D850F5 0x3F9B762C \ - 0x3B5A6B9B 0x0315D626 0x07D4CB91 0x0A97ED48 0x0E56F0FF \ - 0x1011A0FA 0x14D0BD4D 0x19939B94 0x1D528623 0xF12F560E \ - 0xF5EE4BB9 0xF8AD6D60 0xFC6C70D7 0xE22B20D2 0xE6EA3D65 \ - 0xEBA91BBC 0xEF68060B 0xD727BBB6 0xD3E6A601 0xDEA580D8 \ - 0xDA649D6F 0xC423CD6A 0xC0E2D0DD 0xCDA1F604 0xC960EBB3 \ - 0xBD3E8D7E 0xB9FF90C9 0xB4BCB610 0xB07DABA7 0xAE3AFBA2 \ - 0xAAFBE615 0xA7B8C0CC 0xA379DD7B 0x9B3660C6 0x9FF77D71 \ - 0x92B45BA8 0x9675461F 0x8832161A 0x8CF30BAD 0x81B02D74 \ - 0x857130C3 0x5D8A9099 0x594B8D2E 0x5408ABF7 0x50C9B640 \ - 0x4E8EE645 0x4A4FFBF2 0x470CDD2B 0x43CDC09C 0x7B827D21 \ - 0x7F436096 0x7200464F 0x76C15BF8 0x68860BFD 0x6C47164A \ - 0x61043093 0x65C52D24 0x119B4BE9 0x155A565E 0x18197087 \ - 0x1CD86D30 0x029F3D35 0x065E2082 0x0B1D065B 0x0FDC1BEC \ - 0x3793A651 0x3352BBE6 0x3E119D3F 0x3AD08088 0x2497D08D \ - 0x2056CD3A 0x2D15EBE3 0x29D4F654 0xC5A92679 0xC1683BCE \ - 0xCC2B1D17 0xC8EA00A0 0xD6AD50A5 0xD26C4D12 0xDF2F6BCB \ - 0xDBEE767C 0xE3A1CBC1 0xE760D676 0xEA23F0AF 0xEEE2ED18 \ - 0xF0A5BD1D 0xF464A0AA 0xF9278673 0xFDE69BC4 0x89B8FD09 \ - 0x8D79E0BE 0x803AC667 0x84FBDBD0 0x9ABC8BD5 0x9E7D9662 \ - 0x933EB0BB 0x97FFAD0C 0xAFB010B1 0xAB710D06 0xA6322BDF \ - 0xA2F33668 0xBCB4666D 0xB8757BDA 0xB5365D03 0xB1F740B4 ] - - variable uid - if {![info exists uid]} {set uid 0} -} - -# crc::CksumInit -- -# -# Create and initialize a cksum context. This is cleaned up when we -# call CksumFinal to obtain the result. -# -proc ::crc::CksumInit {} { - variable uid - set token [namespace current]::[incr uid] - upvar #0 $token state - array set state {t 0 l 0} - return $token -} - -proc ::crc::CksumUpdate {token data} { - variable cksum_tbl - upvar #0 $token state - set t $state(t) - binary scan $data c* r - foreach {n} $r { - set index [expr { (($t >> 24) ^ ($n & 0xFF)) & 0xFF }] - # Since the introduction of built-in bigInt support with Tcl - # 8.5, bit-shifting $t to the left no longer overflows, - # keeping it 32 bits long. The value grows bigger and bigger - # instead - a severe hit on performance. For this reason we - # do a bitwise AND against 0xFFFFFFFF at each step to keep the - # value within limits. - set t [expr {0xFFFFFFFF & (($t << 8) ^ [lindex $cksum_tbl $index])}] - incr state(l) - } - set state(t) $t - return -} - -proc ::crc::CksumFinal {token} { - variable cksum_tbl - upvar #0 $token state - set t $state(t) - for {set i $state(l)} {$i > 0} {set i [expr {$i>>8}]} { - set index [expr {(($t >> 24) ^ $i) & 0xFF}] - set t [expr {0xFFFFFFFF & (($t << 8) ^ [lindex $cksum_tbl $index])}] - } - unset state - return [expr {~$t & 0xFFFFFFFF}] -} - -# crc::Pop -- -# -# Pop the nth element off a list. Used in options processing. -# -proc ::crc::Pop {varname {nth 0}} { - upvar $varname args - set r [lindex $args $nth] - set args [lreplace $args $nth $nth] - return $r -} - -# Description: -# Provide a Tcl equivalent of the unix cksum(1) command. -# Options: -# -filename name - return a checksum for the specified file. -# -format string - return the checksum using this format string. -# -chunksize size - set the chunking read size -# -proc ::crc::cksum {args} { - array set opts [list -filename {} -channel {} -chunksize 4096 \ - -format %u -command {}] - while {[string match -* [set option [lindex $args 0]]]} { - switch -glob -- $option { - -file* { set opts(-filename) [Pop args 1] } - -chan* { set opts(-channel) [Pop args 1] } - -chunk* { set opts(-chunksize) [Pop args 1] } - -for* { set opts(-format) [Pop args 1] } - -command { set opts(-command) [Pop args 1] } - default { - if {[llength $args] == 1} { break } - if {[string compare $option "--"] == 0} { Pop args ; break } - set err [join [lsort [array names opts -*]] ", "] - return -code error "bad option \"option\": must be $err" - } - } - Pop args - } - - if {$opts(-filename) != {}} { - set opts(-channel) [open $opts(-filename) r] - fconfigure $opts(-channel) -translation binary - } - - if {$opts(-channel) == {}} { - - if {[llength $args] != 1} { - return -code error "wrong # args: should be\ - cksum ?-format string?\ - -channel chan | -filename file | string" - } - set tok [CksumInit] - CksumUpdate $tok [lindex $args 0] - set r [CksumFinal $tok] - - } else { - - set tok [CksumInit] - while {![eof $opts(-channel)]} { - CksumUpdate $tok [read $opts(-channel) $opts(-chunksize)] - } - set r [CksumFinal $tok] - - if {$opts(-filename) != {}} { - close $opts(-channel) - } - } - - return [format $opts(-format) $r] -} - -# ------------------------------------------------------------------------- - -package provide cksum 1.1.4 - -# ------------------------------------------------------------------------- -# Local variables: -# mode: tcl -# indent-tabs-mode: nil -# End: diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/cmdline-1.5.2.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/cmdline-1.5.2.tm deleted file mode 100644 index 4e5e1df9..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/cmdline-1.5.2.tm +++ /dev/null @@ -1,933 +0,0 @@ -# cmdline.tcl -- -# -# This package provides a utility for parsing command line -# arguments that are processed by our various applications. -# It also includes a utility routine to determine the -# application name for use in command line errors. -# -# Copyright (c) 1998-2000 by Ajuba Solutions. -# Copyright (c) 2001-2015 by Andreas Kupries . -# Copyright (c) 2003 by David N. Welton -# See the file "license.terms" for information on usage and redistribution -# of this file, and for a DISCLAIMER OF ALL WARRANTIES. - -package require Tcl 8.5- -package provide cmdline 1.5.2 - -namespace eval ::cmdline { - namespace export getArgv0 getopt getKnownOpt getfiles getoptions \ - getKnownOptions usage -} - -# ::cmdline::getopt -- -# -# The cmdline::getopt works in a fashion like the standard -# C based getopt function. Given an option string and a -# pointer to an array or args this command will process the -# first argument and return info on how to proceed. -# -# Arguments: -# argvVar Name of the argv list that you -# want to process. If options are found the -# arg list is modified and the processed arguments -# are removed from the start of the list. -# optstring A list of command options that the application -# will accept. If the option ends in ".arg" the -# getopt routine will use the next argument as -# an argument to the option. Otherwise the option -# is a boolean that is set to 1 if present. -# optVar The variable pointed to by optVar -# contains the option that was found (without the -# leading '-' and without the .arg extension). -# valVar Upon success, the variable pointed to by valVar -# contains the value for the specified option. -# This value comes from the command line for .arg -# options, otherwise the value is 1. -# If getopt fails, the valVar is filled with an -# error message. -# -# Results: -# The getopt function returns 1 if an option was found, 0 if no more -# options were found, and -1 if an error occurred. - -proc ::cmdline::getopt {argvVar optstring optVar valVar} { - upvar 1 $argvVar argsList - upvar 1 $optVar option - upvar 1 $valVar value - - set result [getKnownOpt argsList $optstring option value] - - if {$result < 0} { - # Collapse unknown-option error into any-other-error result. - set result -1 - } - return $result -} - -# ::cmdline::getKnownOpt -- -# -# The cmdline::getKnownOpt works in a fashion like the standard -# C based getopt function. Given an option string and a -# pointer to an array or args this command will process the -# first argument and return info on how to proceed. -# -# Arguments: -# argvVar Name of the argv list that you -# want to process. If options are found the -# arg list is modified and the processed arguments -# are removed from the start of the list. Note that -# unknown options and the args that follow them are -# left in this list. -# optstring A list of command options that the application -# will accept. If the option ends in ".arg" the -# getopt routine will use the next argument as -# an argument to the option. Otherwise the option -# is a boolean that is set to 1 if present. -# optVar The variable pointed to by optVar -# contains the option that was found (without the -# leading '-' and without the .arg extension). -# valVar Upon success, the variable pointed to by valVar -# contains the value for the specified option. -# This value comes from the command line for .arg -# options, otherwise the value is 1. -# If getopt fails, the valVar is filled with an -# error message. -# -# Results: -# The getKnownOpt function returns 1 if an option was found, -# 0 if no more options were found, -1 if an unknown option was -# encountered, and -2 if any other error occurred. - -proc ::cmdline::getKnownOpt {argvVar optstring optVar valVar} { - upvar 1 $argvVar argsList - upvar 1 $optVar option - upvar 1 $valVar value - - # default settings for a normal return - set value "" - set option "" - set result 0 - - # check if we're past the end of the args list - if {[llength $argsList] != 0} { - - # if we got -- or an option that doesn't begin with -, return (skipping - # the --). otherwise process the option arg. - switch -glob -- [set arg [lindex $argsList 0]] { - "--" { - set argsList [lrange $argsList 1 end] - } - "--*" - - "-*" { - set option [string range $arg 1 end] - if {[string equal [string range $option 0 0] "-"]} { - set option [string range $arg 2 end] - } - - # support for format: [-]-option=value - set idx [string first "=" $option 1] - if {$idx != -1} { - set _val [string range $option [expr {$idx+1}] end] - set option [string range $option 0 [expr {$idx-1}]] - } - - if {[lsearch -exact $optstring $option] != -1} { - # Booleans are set to 1 when present - set value 1 - set result 1 - set argsList [lrange $argsList 1 end] - } elseif {[lsearch -exact $optstring "$option.arg"] != -1} { - set result 1 - set argsList [lrange $argsList 1 end] - - if {[info exists _val]} { - set value $_val - } elseif {[llength $argsList]} { - set value [lindex $argsList 0] - set argsList [lrange $argsList 1 end] - } else { - set value "Option \"$option\" requires an argument" - set result -2 - } - } else { - # Unknown option. - set value "Illegal option \"-$option\"" - set result -1 - } - } - default { - # Skip ahead - } - } - } - - return $result -} - -# ::cmdline::getoptions -- -# -# Process a set of command line options, filling in defaults -# for those not specified. This also generates an error message -# that lists the allowed flags if an incorrect flag is specified. -# -# Arguments: -# argvVar The name of the argument list, typically argv. -# We remove all known options and their args from it. -# In other words, after the call to this command the -# referenced variable contains only the non-options, -# and unknown options. -# optlist A list-of-lists where each element specifies an option -# in the form: -# (where flag takes no argument) -# flag comment -# -# (or where flag takes an argument) -# flag default comment -# -# If flag ends in ".arg" then the value is taken from the -# command line. Otherwise it is a boolean and appears in -# the result if present on the command line. If flag ends -# in ".secret", it will not be displayed in the usage. -# usage Text to include in the usage display. Defaults to -# "options:" -# -# Results -# Name value pairs suitable for using with array set. -# A modified `argvVar`. - -proc ::cmdline::getoptions {argvVar optlist {usage options:}} { - upvar 1 $argvVar argv - - set opts [GetOptionDefaults $optlist result] - - set argc [llength $argv] - while {[set err [getopt argv $opts opt arg]]} { - if {$err < 0} { - set result(?) "" - break - } - set result($opt) $arg - } - if {[info exist result(?)] || [info exists result(help)]} { - Error [usage $optlist $usage] USAGE - } - return [array get result] -} - -# ::cmdline::getKnownOptions -- -# -# Process a set of command line options, filling in defaults -# for those not specified. This ignores unknown flags, but generates -# an error message that lists the correct usage if a known option -# is used incorrectly. -# -# Arguments: -# argvVar The name of the argument list, typically argv. This -# We remove all known options and their args from it. -# In other words, after the call to this command the -# referenced variable contains only the non-options, -# and unknown options. -# optlist A list-of-lists where each element specifies an option -# in the form: -# flag default comment -# If flag ends in ".arg" then the value is taken from the -# command line. Otherwise it is a boolean and appears in -# the result if present on the command line. If flag ends -# in ".secret", it will not be displayed in the usage. -# usage Text to include in the usage display. Defaults to -# "options:" -# -# Results -# Name value pairs suitable for using with array set. -# A modified `argvVar`. - -proc ::cmdline::getKnownOptions {argvVar optlist {usage options:}} { - upvar 1 $argvVar argv - - set opts [GetOptionDefaults $optlist result] - - # As we encounter them, keep the unknown options and their - # arguments in this list. Before we return from this procedure, - # we'll prepend these args to the argList so that the application - # doesn't lose them. - - set unknownOptions [list] - - set argc [llength $argv] - while {[set err [getKnownOpt argv $opts opt arg]]} { - if {$err == -1} { - # Unknown option. - - # Skip over any non-option items that follow it. - # For now, add them to the list of unknownOptions. - lappend unknownOptions [lindex $argv 0] - set argv [lrange $argv 1 end] - while {([llength $argv] != 0) \ - && ![string match "-*" [lindex $argv 0]]} { - lappend unknownOptions [lindex $argv 0] - set argv [lrange $argv 1 end] - } - } elseif {$err == -2} { - set result(?) "" - break - } else { - set result($opt) $arg - } - } - - # Before returning, prepend the any unknown args back onto the - # argList so that the application doesn't lose them. - set argv [concat $unknownOptions $argv] - - if {[info exist result(?)] || [info exists result(help)]} { - Error [usage $optlist $usage] USAGE - } - return [array get result] -} - -# ::cmdline::GetOptionDefaults -- -# -# This internal procedure processes the option list (that was passed to -# the getopt or getKnownOpt procedure). The defaultArray gets an index -# for each option in the option list, the value of which is the option's -# default value. -# -# Arguments: -# optlist A list-of-lists where each element specifies an option -# in the form: -# flag default comment -# If flag ends in ".arg" then the value is taken from the -# command line. Otherwise it is a boolean and appears in -# the result if present on the command line. If flag ends -# in ".secret", it will not be displayed in the usage. -# defaultArrayVar The name of the array in which to put argument defaults. -# -# Results -# Name value pairs suitable for using with array set. - -proc ::cmdline::GetOptionDefaults {optlist defaultArrayVar} { - upvar 1 $defaultArrayVar result - - set opts {? help} - foreach opt $optlist { - set name [lindex $opt 0] - if {[regsub -- {\.secret$} $name {} name] == 1} { - # Need to hide this from the usage display and getopt - } - lappend opts $name - if {[regsub -- {\.arg$} $name {} name] == 1} { - - # Set defaults for those that take values. - - set default [lindex $opt 1] - set result($name) $default - } else { - # The default for booleans is false - set result($name) 0 - } - } - return $opts -} - -# ::cmdline::usage -- -# -# Generate an error message that lists the allowed flags. -# -# Arguments: -# optlist As for cmdline::getoptions -# usage Text to include in the usage display. Defaults to -# "options:" -# -# Results -# A formatted usage message - -proc ::cmdline::usage {optlist {usage {options:}}} { - set str "[getArgv0] $usage\n" - set longest 20 - set lines {} - foreach opt [concat $optlist \ - {{- "Forcibly stop option processing"} {help "Print this message"} {? "Print this message"}}] { - set name "-[lindex $opt 0]" - if {[regsub -- {\.secret$} $name {} name] == 1} { - # Hidden option - continue - } - if {[regsub -- {\.arg$} $name {} name] == 1} { - append name " value" - set desc "[lindex $opt 2] <[lindex $opt 1]>" - } else { - set desc "[lindex $opt 1]" - } - set n [string length $name] - if {$n > $longest} { set longest $n } - # max not available before 8.5 - set longest [expr {max($longest, )}] - lappend lines $name $desc - } - foreach {name desc} $lines { - append str "[string trimright [format " %-*s %s" $longest $name $desc]]\n" - } - - return $str -} - -# ::cmdline::getfiles -- -# -# Given a list of file arguments from the command line, compute -# the set of valid files. On windows, file globbing is performed -# on each argument. On Unix, only file existence is tested. If -# a file argument produces no valid files, a warning is optionally -# generated. -# -# This code also uses the full path for each file. If not -# given it prepends [pwd] to the filename. This ensures that -# these files will never conflict with files in our zip file. -# -# Arguments: -# patterns The file patterns specified by the user. -# quiet If this flag is set, no warnings will be generated. -# -# Results: -# Returns the list of files that match the input patterns. - -proc ::cmdline::getfiles {patterns quiet} { - set result {} - if {$::tcl_platform(platform) == "windows"} { - foreach pattern $patterns { - set pat [file join $pattern] - set files [glob -nocomplain -- $pat] - if {$files == {}} { - if {! $quiet} { - puts stdout "warning: no files match \"$pattern\"" - } - } else { - foreach file $files { - lappend result $file - } - } - } - } else { - set result $patterns - } - set files {} - foreach file $result { - # Make file an absolute path so that we will never conflict - # with files that might be contained in our zip file. - set fullPath [file join [pwd] $file] - - if {[file isfile $fullPath]} { - lappend files $fullPath - } elseif {! $quiet} { - puts stdout "warning: no files match \"$file\"" - } - } - return $files -} - -# ::cmdline::getArgv0 -- -# -# This command returns the "sanitized" version of argv0. It will strip -# off the leading path and remove the ".bin" extensions that our apps -# use because they must be wrapped by a shell script. -# -# Arguments: -# None. -# -# Results: -# The application name that can be used in error messages. - -proc ::cmdline::getArgv0 {} { - global argv0 - - set name [file tail $argv0] - return [file rootname $name] -} - -## -# ### ### ### ######### ######### ######### -## -# Now the typed versions of the above commands. -## -# ### ### ### ######### ######### ######### -## - -# typedCmdline.tcl -- -# -# This package provides a utility for parsing typed command -# line arguments that may be processed by various applications. -# -# Copyright (c) 2000 by Ross Palmer Mohn. -# See the file "license.terms" for information on usage and redistribution -# of this file, and for a DISCLAIMER OF ALL WARRANTIES. -# -# RCS: @(#) $Id: cmdline.tcl,v 1.28 2011/02/23 17:41:52 andreas_kupries Exp $ - -namespace eval ::cmdline { - namespace export typedGetopt typedGetoptions typedUsage - - # variable cmdline::charclasses -- - # - # Create regexp list of allowable character classes - # from "string is" error message. - # - # Results: - # String of character class names separated by "|" characters. - - variable charclasses - #checker exclude badKey - catch {string is . .} charclasses - variable dummy - regexp -- {must be (.+)$} $charclasses dummy charclasses - regsub -all -- {, (or )?} $charclasses {|} charclasses - unset dummy -} - -# ::cmdline::typedGetopt -- -# -# The cmdline::typedGetopt works in a fashion like the standard -# C based getopt function. Given an option string and a -# pointer to a list of args this command will process the -# first argument and return info on how to proceed. In addition, -# you may specify a type for the argument to each option. -# -# Arguments: -# argvVar Name of the argv list that you want to process. -# If options are found, the arg list is modified -# and the processed arguments are removed from the -# start of the list. -# -# optstring A list of command options that the application -# will accept. If the option ends in ".xxx", where -# xxx is any valid character class to the tcl -# command "string is", then typedGetopt routine will -# use the next argument as a typed argument to the -# option. The argument must match the specified -# character classes (e.g. integer, double, boolean, -# xdigit, etc.). Alternatively, you may specify -# ".arg" for an untyped argument. -# -# optVar Upon success, the variable pointed to by optVar -# contains the option that was found (without the -# leading '-' and without the .xxx extension). If -# typedGetopt fails the variable is set to the empty -# string. SOMETIMES! Different for each -value! -# -# argVar Upon success, the variable pointed to by argVar -# contains the argument for the specified option. -# If typedGetopt fails, the variable is filled with -# an error message. -# -# Argument type syntax: -# Option that takes no argument. -# foo -# -# Option that takes a typeless argument. -# foo.arg -# -# Option that takes a typed argument. Allowable types are all -# valid character classes to the tcl command "string is". -# Currently must be one of alnum, alpha, ascii, control, -# boolean, digit, double, false, graph, integer, lower, print, -# punct, space, true, upper, wordchar, or xdigit. -# foo.double -# -# Option that takes an argument from a list. -# foo.(bar|blat) -# -# Argument quantifier syntax: -# Option that takes an optional argument. -# foo.arg? -# -# Option that takes a list of arguments terminated by "--". -# foo.arg+ -# -# Option that takes an optional list of arguments terminated by "--". -# foo.arg* -# -# Argument quantifiers work on all argument types, so, for -# example, the following is a valid option specification. -# foo.(bar|blat|blah)? -# -# Argument syntax miscellany: -# Options may be specified on the command line using a unique, -# shortened version of the option name. Given that program foo -# has an option list of {bar.alpha blah.arg blat.double}, -# "foo -b fob" returns an error, but "foo -ba fob" -# successfully returns {bar fob} -# -# Results: -# The typedGetopt function returns one of the following: -# 1 a valid option was found -# 0 no more options found to process -# -1 invalid option -# -2 missing argument to a valid option -# -3 argument to a valid option does not match type -# -# Known Bugs: -# When using options which include special glob characters, -# you must use the exact option. Abbreviating it can cause -# an error in the "cmdline::prefixSearch" procedure. - -proc ::cmdline::typedGetopt {argvVar optstring optVar argVar} { - variable charclasses - - upvar $argvVar argsList - - upvar $optVar retvar - upvar $argVar optarg - - # default settings for a normal return - set optarg "" - set retvar "" - set retval 0 - - # check if we're past the end of the args list - if {[llength $argsList] != 0} { - - # if we got -- or an option that doesn't begin with -, return (skipping - # the --). otherwise process the option arg. - switch -glob -- [set arg [lindex $argsList 0]] { - "--" { - set argsList [lrange $argsList 1 end] - } - - "-*" { - # Create list of options without their argument extensions - - set optstr "" - foreach str $optstring { - lappend optstr [file rootname $str] - } - - set _opt [string range $arg 1 end] - - set i [prefixSearch $optstr [file rootname $_opt]] - if {$i != -1} { - set opt [lindex $optstring $i] - - set quantifier "none" - if {[regexp -- {\.[^.]+([?+*])$} $opt dummy quantifier]} { - set opt [string range $opt 0 end-1] - } - - if {[string first . $opt] == -1} { - set retval 1 - set retvar $opt - set argsList [lrange $argsList 1 end] - - } elseif {[regexp -- "\\.(arg|$charclasses)\$" $opt dummy charclass] - || [regexp -- {\.\(([^)]+)\)} $opt dummy charclass]} { - if {[string equal arg $charclass]} { - set type arg - } elseif {[regexp -- "^($charclasses)\$" $charclass]} { - set type class - } else { - set type oneof - } - - set argsList [lrange $argsList 1 end] - set opt [file rootname $opt] - - while {1} { - if {[llength $argsList] == 0 - || [string equal "--" [lindex $argsList 0]]} { - if {[string equal "--" [lindex $argsList 0]]} { - set argsList [lrange $argsList 1 end] - } - - set oneof "" - if {$type == "arg"} { - set charclass an - } elseif {$type == "oneof"} { - set oneof ", one of $charclass" - set charclass an - } - - if {$quantifier == "?"} { - set retval 1 - set retvar $opt - set optarg "" - } elseif {$quantifier == "+"} { - set retvar $opt - if {[llength $optarg] < 1} { - set retval -2 - set optarg "Option requires at least one $charclass argument$oneof -- $opt" - } else { - set retval 1 - } - } elseif {$quantifier == "*"} { - set retval 1 - set retvar $opt - } else { - set optarg "Option requires $charclass argument$oneof -- $opt" - set retvar $opt - set retval -2 - } - set quantifier "" - } elseif {($type == "arg") - || (($type == "oneof") - && [string first "|[lindex $argsList 0]|" "|$charclass|"] != -1) - || (($type == "class") - && [string is $charclass [lindex $argsList 0]])} { - set retval 1 - set retvar $opt - lappend optarg [lindex $argsList 0] - set argsList [lrange $argsList 1 end] - } else { - set oneof "" - if {$type == "arg"} { - set charclass an - } elseif {$type == "oneof"} { - set oneof ", one of $charclass" - set charclass an - } - set optarg "Option requires $charclass argument$oneof -- $opt" - set retvar $opt - set retval -3 - - if {$quantifier == "?"} { - set retval 1 - set optarg "" - } - set quantifier "" - } - if {![regexp -- {[+*]} $quantifier]} { - break; - } - } - } else { - Error \ - "Illegal option type specification: must be one of $charclasses" \ - BAD OPTION TYPE - } - } else { - set optarg "Illegal option -- $_opt" - set retvar $_opt - set retval -1 - } - } - default { - # Skip ahead - } - } - } - - return $retval -} - -# ::cmdline::typedGetoptions -- -# -# Process a set of command line options, filling in defaults -# for those not specified. This also generates an error message -# that lists the allowed options if an incorrect option is -# specified. -# -# Arguments: -# argvVar The name of the argument list, typically argv -# optlist A list-of-lists where each element specifies an option -# in the form: -# -# option default comment -# -# Options formatting is as described for the optstring -# argument of typedGetopt. Default is for optionally -# specifying a default value. Comment is for optionally -# specifying a comment for the usage display. The -# options "--", "-help", and "-?" are automatically included -# in optlist. -# -# Argument syntax miscellany: -# Options formatting and syntax is as described in typedGetopt. -# There are two additional suffixes that may be applied when -# passing options to typedGetoptions. -# -# You may add ".multi" as a suffix to any option. For options -# that take an argument, this means that the option may be used -# more than once on the command line and that each additional -# argument will be appended to a list, which is then returned -# to the application. -# foo.double.multi -# -# If a non-argument option is specified as ".multi", it is -# toggled on and off for each time it is used on the command -# line. -# foo.multi -# -# If an option specification does not contain the ".multi" -# suffix, it is not an error to use an option more than once. -# In this case, the behavior for options with arguments is that -# the last argument is the one that will be returned. For -# options that do not take arguments, using them more than once -# has no additional effect. -# -# Options may also be hidden from the usage display by -# appending the suffix ".secret" to any option specification. -# Please note that the ".secret" suffix must be the last suffix, -# after any argument type specification and ".multi" suffix. -# foo.xdigit.multi.secret -# -# Results -# Name value pairs suitable for using with array set. - -proc ::cmdline::typedGetoptions {argvVar optlist {usage options:}} { - variable charclasses - - upvar 1 $argvVar argv - - set opts {? help} - foreach opt $optlist { - set name [lindex $opt 0] - if {[regsub -- {\.secret$} $name {} name] == 1} { - # Remove this extension before passing to typedGetopt. - } - if {[regsub -- {\.multi$} $name {} name] == 1} { - # Remove this extension before passing to typedGetopt. - - regsub -- {\..*$} $name {} temp - set multi($temp) 1 - } - lappend opts $name - if {[regsub -- "\\.(arg|$charclasses|\\(.+).?\$" $name {} name] == 1} { - # Set defaults for those that take values. - # Booleans are set just by being present, or not - - set dflt [lindex $opt 1] - if {$dflt != {}} { - set defaults($name) $dflt - } - } - } - set argc [llength $argv] - while {[set err [typedGetopt argv $opts opt arg]]} { - if {$err == 1} { - if {[info exists result($opt)] - && [info exists multi($opt)]} { - # Toggle boolean options or append new arguments - - if {$arg == ""} { - unset result($opt) - } else { - set result($opt) "$result($opt) $arg" - } - } else { - set result($opt) "$arg" - } - } elseif {($err == -1) || ($err == -3)} { - Error [typedUsage $optlist $usage] USAGE - } elseif {$err == -2 && ![info exists defaults($opt)]} { - Error [typedUsage $optlist $usage] USAGE - } - } - if {[info exists result(?)] || [info exists result(help)]} { - Error [typedUsage $optlist $usage] USAGE - } - foreach {opt dflt} [array get defaults] { - if {![info exists result($opt)]} { - set result($opt) $dflt - } - } - return [array get result] -} - -# ::cmdline::typedUsage -- -# -# Generate an error message that lists the allowed flags, -# type of argument taken (if any), default value (if any), -# and an optional description. -# -# Arguments: -# optlist As for cmdline::typedGetoptions -# -# Results -# A formatted usage message - -proc ::cmdline::typedUsage {optlist {usage {options:}}} { - variable charclasses - - set str "[getArgv0] $usage\n" - set longest 20 - set lines {} - foreach opt [concat $optlist \ - {{help "Print this message"} {? "Print this message"}}] { - set name "-[lindex $opt 0]" - if {[regsub -- {\.secret$} $name {} name] == 1} { - # Hidden option - continue - } - - if {[regsub -- {\.multi$} $name {} name] == 1} { - # Display something about multiple options - } - - if {[regexp -- "\\.(arg|$charclasses)\$" $name dummy charclass] || - [regexp -- {\.\(([^)]+)\)} $opt dummy charclass] - } { - regsub -- "\\..+\$" $name {} name - append name " $charclass" - set desc [lindex $opt 2] - set default [lindex $opt 1] - if {$default != ""} { - append desc " <$default>" - } - } else { - set desc [lindex $opt 1] - } - lappend accum $name $desc - set n [string length $name] - if {$n > $longest} { set longest $n } - # max not available before 8.5 - set longest [expr {max($longest, [string length $name])}] - } - foreach {name desc} $accum { - append str "[string trimright [format " %-*s %s" $longest $name $desc]]\n" - } - return $str -} - -# ::cmdline::prefixSearch -- -# -# Search a Tcl list for a pattern; searches first for an exact match, -# and if that fails, for a unique prefix that matches the pattern -# (i.e, first "lsearch -exact", then "lsearch -glob $pattern*" -# -# Arguments: -# list list of words -# pattern word to search for -# -# Results: -# Index of found word is returned. If no exact match or -# unique short version is found then -1 is returned. - -proc ::cmdline::prefixSearch {list pattern} { - # Check for an exact match - - if {[set pos [::lsearch -exact $list $pattern]] > -1} { - return $pos - } - - # Check for a unique short version - - set slist [lsort $list] - if {[set pos [::lsearch -glob $slist $pattern*]] > -1} { - # What if there is nothing for the check variable? - - set check [lindex $slist [expr {$pos + 1}]] - if {[string first $pattern $check] != 0} { - return [::lsearch -exact $list [lindex $slist $pos]] - } - } - return -1 -} -# ::cmdline::Error -- -# -# Internal helper to throw errors with a proper error-code attached. -# -# Arguments: -# message text of the error message to throw. -# args additional parts of the error code to use, -# with CMDLINE as basic prefix added by this command. -# -# Results: -# An error is thrown, always. - -proc ::cmdline::Error {message args} { - return -code error -errorcode [linsert $args 0 CMDLINE] $message -} diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/fileutil-1.16.1.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/fileutil-1.16.1.tm deleted file mode 100644 index 6d5c737e..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/fileutil-1.16.1.tm +++ /dev/null @@ -1,2311 +0,0 @@ -# fileutil.tcl -- -# -# Tcl implementations of standard UNIX utilities. -# -# Copyright (c) 1998-2000 by Ajuba Solutions. -# Copyright (c) 2002 by Phil Ehrens (fileType) -# Copyright (c) 2005-2013 by Andreas Kupries -# -# See the file "license.terms" for information on usage and redistribution -# of this file, and for a DISCLAIMER OF ALL WARRANTIES. - -package require Tcl 8.5- -package require cmdline -package provide fileutil 1.16.1 - -namespace eval ::fileutil { - namespace export \ - grep find findByPattern cat touch foreachLine \ - jail stripPwd stripN stripPath tempdir tempfile \ - install fileType writeFile appendToFile \ - insertIntoFile removeFromFile replaceInFile \ - updateInPlace test tempdirReset maketempdir -} - -# ::fileutil::grep -- -# -# Implementation of grep. Adapted from the Tcler's Wiki. -# -# Arguments: -# pattern pattern to search for. -# files list of files to search; if NULL, uses stdin. -# -# Results: -# results list of matches - -proc ::fileutil::grep {pattern {files {}}} { - set result [list] - if {[llength $files] == 0} { - # read from stdin - set lnum 0 - while {[gets stdin line] >= 0} { - incr lnum - if {[regexp -- $pattern $line]} { - lappend result "${lnum}:${line}" - } - } - } else { - foreach filename $files { - set file [open $filename r] - set lnum 0 - while {[gets $file line] >= 0} { - incr lnum - if {[regexp -- $pattern $line]} { - lappend result "${filename}:${lnum}:${line}" - } - } - close $file - } - } - return $result -} - -# ::fileutil::find == - -# Below is the core command, which is portable across Tcl versions and -# platforms. Functionality which is common or platform and/or Tcl -# version dependent, has been factored out/ encapsulated into separate -# (small) commands. Only these commands may have multiple variant -# implementations per the available features of the Tcl core / -# platform. -# -# These commands are -# -# FADD - Add path result, performs filtering. Portable! -# GLOBF - Return files in a directory. Tcl version/platform dependent. -# GLOBD - Return dirs in a directory. Tcl version/platform dependent. -# ACCESS - Check directory for accessibility. Tcl version/platform dependent. - -proc ::fileutil::find {{basedir .} {filtercmd {}}} { - set result {} - set filt [string length $filtercmd] - - if {[file isfile $basedir]} { - # The base is a file, and therefore only possible result, - # modulo filtering. - - FADD $basedir - - } elseif {[file isdirectory $basedir]} { - # For a directory as base we do an iterative recursion through - # the directory hierarchy starting at the base. We use a queue - # (Tcl list) of directories we have to check. We access it by - # index, and stop when we have reached beyond the end of the - # list. This is faster than removing elements from the be- - # ginning of the list, as that entails copying down a possibly - # large list of directories, making it O(n*n). The index is - # faster, O(n), at the expense of memory. Nothing is deleted - # from the list until we have processed all directories in the - # hierarchy. - # - # We scan each directory at least twice. First for files, then - # for directories. The scans may internally make several - # passes (normal vs hidden files). - # - # Looped directory structures due to symbolic links are - # handled by _fully_ normalizing directory paths and checking - # if we encountered the normalized form before. The array - # 'known' is our cache where we record the known normalized - # paths. - - set pending [list $basedir] - set at 0 - array set parent {} - array set norm {} - Enter {} $basedir - - while {$at < [llength $pending]} { - # Get next directory not yet processed. - set current [lindex $pending $at] - incr at - - # Is the directory accessible? Continue if not. - ACCESS $current - - # Files first, then the sub-directories ... - - foreach f [GLOBF $current] { FADD $f } - - foreach f [GLOBD $current] { - # Ignore current and parent directory, this needs - # explicit filtering outside of the filter command. - if { - [string equal [file tail $f] "."] || - [string equal [file tail $f] ".."] - } continue - - # Extend result, modulo filtering. - FADD $f - - # Detection of symlink loops via a portable path - # normalization computing a canonical form of the path - # followed by a check if that canonical form was - # encountered before. If ok, record directory for - # expansion in future iterations. - - Enter $current $f - if {[Cycle $f]} continue - - lappend pending $f - } - } - } else { - return -code error "$basedir does not exist" - } - - return $result -} - -proc ::fileutil::Enter {parent path} { - upvar 1 parent _parent norm _norm - set _parent($path) $parent - set _norm($path) [fullnormalize $path] - return -} - -proc ::fileutil::Cycle {path} { - upvar 1 parent _parent norm _norm - set nform $_norm($path) - set paren $_parent($path) - while {$paren ne {}} { - if {$_norm($paren) eq $nform} { return yes } - set paren $_parent($paren) - } - return no -} - -# Helper command for fileutil::find. Performs the filtering of the -# result per a filter command for the candidates found by the -# traversal core, see above. This is portable. - -proc ::fileutil::FADD {filename} { - upvar 1 result result filt filt filtercmd filtercmd - if {!$filt} { - lappend result $filename - return - } - - set here [pwd] - cd [file dirname $filename] - - if {[uplevel 2 [linsert $filtercmd end [file tail $filename]]]} { - lappend result $filename - } - - cd $here - return -} - -# The next three helper commands for fileutil::find depend strongly on -# the version of Tcl, and partially on the platform. - -# 1. The -directory and -types switches were added to glob in Tcl -# 8.3. This means that we have to emulate them for Tcl 8.2. -# -# 2. In Tcl 8.3 using -types f will return only true files, but not -# links to files. This changed in 8.4+ where links to files are -# returned as well. So for 8.3 we have to handle the links -# separately (-types l) and also filter on our own. -# Note that Windows file links are hard links which are reported by -# -types f, but not -types l, so we can optimize that for the two -# platforms. -# -# Note further that we have to handle broken links on our own. They -# are not returned by glob yet we want them in the output. -# -# 3. In Tcl 8.3 we also have a crashing bug in glob (SIGABRT, "stat on -# a known file") when trying to perform 'glob -types {hidden f}' on -# a directory without e'x'ecute permissions. We code around by -# testing if we can cd into the directory (stat might return enough -# information too (mode), but possibly also not portable). -# -# For Tcl 8.2 and 8.4+ glob simply delivers an empty result -# (-nocomplain), without crashing. For them this command is defined -# so that the bytecode compiler removes it from the bytecode. -# -# This bug made the ACCESS helper necessary. -# We code around the problem by testing if we can cd into the -# directory (stat might return enough information too (mode), but -# possibly also not portable). - -if {[package vsatisfies [package present Tcl] 8.5]} { - # Tcl 8.5+. - # We have to check readability of "current" on our own, glob - # changed to error out instead of returning nothing. - - proc ::fileutil::ACCESS {args} {} - - proc ::fileutil::GLOBF {current} { - if {![file readable $current] || - [BadLink $current]} { - return {} - } - - set res [lsort -unique [concat \ - [glob -nocomplain -directory $current -types f -- *] \ - [glob -nocomplain -directory $current -types {hidden f} -- *]]] - - # Look for broken links (They are reported as neither file nor directory). - foreach l [lsort -unique [concat \ - [glob -nocomplain -directory $current -types l -- *] \ - [glob -nocomplain -directory $current -types {hidden l} -- *]]] { - if {[file isfile $l]} continue - if {[file isdirectory $l]} continue - lappend res $l - } - return [lsort -unique $res] - } - - proc ::fileutil::GLOBD {current} { - if {![file readable $current] || - [BadLink $current]} { - return {} - } - - lsort -unique [concat \ - [glob -nocomplain -directory $current -types d -- *] \ - [glob -nocomplain -directory $current -types {hidden d} -- *]] - } - - proc ::fileutil::BadLink {current} { - if {[file type $current] ne "link"} { return no } - - set dst [file join [file dirname $current] [file readlink $current]] - - if {![file exists $dst] || - ![file readable $dst]} { - return yes - } - - return no - } -} elseif {[package vsatisfies [package present Tcl] 8.4]} { - # Tcl 8.4+. - # (Ad 1) We have -directory, and -types, - # (Ad 2) Links are returned for -types f/d if they refer to files/dirs. - # (Ad 3) No bug to code around - - proc ::fileutil::ACCESS {args} {} - - proc ::fileutil::GLOBF {current} { - set res [lsort -unique [concat \ - [glob -nocomplain -directory $current -types f -- *] \ - [glob -nocomplain -directory $current -types {hidden f} -- *]]] - - # Look for broken links (They are reported as neither file nor directory). - foreach l [lsort -unique [concat \ - [glob -nocomplain -directory $current -types l -- *] \ - [glob -nocomplain -directory $current -types {hidden l} -- *]]] { - if {[file isfile $l]} continue - if {[file isdirectory $l]} continue - lappend res $l - } - return [lsort -unique $res] - } - - proc ::fileutil::GLOBD {current} { - lsort -unique [concat \ - [glob -nocomplain -directory $current -types d -- *] \ - [glob -nocomplain -directory $current -types {hidden d} -- *]] - } - -} elseif {[package vsatisfies [package present Tcl] 8.3]} { - # 8.3. - # (Ad 1) We have -directory, and -types, - # (Ad 2) Links are NOT returned for -types f/d, collect separately. - # No symbolic file links on Windows. - # (Ad 3) Bug to code around. - - proc ::fileutil::ACCESS {current} { - if {[catch { - set h [pwd] ; cd $current ; cd $h - }]} {return -code continue} - return - } - - if {[string equal $::tcl_platform(platform) windows]} { - proc ::fileutil::GLOBF {current} { - concat \ - [glob -nocomplain -directory $current -types f -- *] \ - [glob -nocomplain -directory $current -types {hidden f} -- *]] - } - } else { - proc ::fileutil::GLOBF {current} { - set l [concat \ - [glob -nocomplain -directory $current -types f -- *] \ - [glob -nocomplain -directory $current -types {hidden f} -- *]] - - foreach x [concat \ - [glob -nocomplain -directory $current -types l -- *] \ - [glob -nocomplain -directory $current -types {hidden l} -- *]] { - if {[file isdirectory $x]} continue - # We have now accepted files, links to files, and broken links. - lappend l $x - } - - return $l - } - } - - proc ::fileutil::GLOBD {current} { - set l [concat \ - [glob -nocomplain -directory $current -types d -- *] \ - [glob -nocomplain -directory $current -types {hidden d} -- *]] - - foreach x [concat \ - [glob -nocomplain -directory $current -types l -- *] \ - [glob -nocomplain -directory $current -types {hidden l} -- *]] { - if {![file isdirectory $x]} continue - lappend l $x - } - - return $l - } -} else { - # 8.2. - # (Ad 1,2,3) We do not have -directory, nor -types. Full emulation required. - - proc ::fileutil::ACCESS {args} {} - - if {[string equal $::tcl_platform(platform) windows]} { - # Hidden files cannot be handled by Tcl 8.2 in glob. We have - # to punt. - - proc ::fileutil::GLOBF {current} { - set current \\[join [split $current {}] \\] - set res {} - foreach x [glob -nocomplain -- [file join $current *]] { - if {[file isdirectory $x]} continue - if {[catch {file type $x}]} continue - # We have now accepted files, links to files, and - # broken links. We may also have accepted a directory - # as well, if the current path was inaccessible. This - # however will cause 'file type' to throw an error, - # hence the second check. - lappend res $x - } - return $res - } - - proc ::fileutil::GLOBD {current} { - set current \\[join [split $current {}] \\] - set res {} - foreach x [glob -nocomplain -- [file join $current *]] { - if {![file isdirectory $x]} continue - lappend res $x - } - return $res - } - } else { - # Hidden files on Unix are dot-files. We emulate the switch - # '-types hidden' by using an explicit pattern. - - proc ::fileutil::GLOBF {current} { - set current \\[join [split $current {}] \\] - set res {} - foreach x [glob -nocomplain -- [file join $current *] [file join $current .*]] { - if {[file isdirectory $x]} continue - if {[catch {file type $x}]} continue - # We have now accepted files, links to files, and - # broken links. We may also have accepted a directory - # as well, if the current path was inaccessible. This - # however will cause 'file type' to throw an error, - # hence the second check. - - lappend res $x - } - return $res - } - - proc ::fileutil::GLOBD {current} { - set current \\[join [split $current {}] \\] - set res {} - foreach x [glob -nocomplain -- $current/* [file join $current .*]] { - if {![file isdirectory $x]} continue - lappend res $x - } - return $res - } - } -} - -# ::fileutil::findByPattern -- -# -# Specialization of find. Finds files based on their names, -# which have to match the specified patterns. Options are used -# to specify which type of patterns (regexp-, glob-style) is -# used. -# -# Arguments: -# basedir Directory to start searching from. -# args Options (-glob, -regexp, --) followed by a -# list of patterns to search for. -# -# Results: -# files a list of interesting files. - -proc ::fileutil::findByPattern {basedir args} { - set pos 0 - set cmd ::fileutil::FindGlob - foreach a $args { - incr pos - switch -glob -- $a { - -- {break} - -regexp {set cmd ::fileutil::FindRegexp} - -glob {set cmd ::fileutil::FindGlob} - -* {return -code error "Unknown option $a"} - default {incr pos -1 ; break} - } - } - - set args [lrange $args $pos end] - - if {[llength $args] != 1} { - set pname [lindex [info level 0] 0] - return -code error \ - "wrong#args for \"$pname\", should be\ - \"$pname basedir ?-regexp|-glob? ?--? patterns\"" - } - - set patterns [lindex $args 0] - return [find $basedir [list $cmd $patterns]] -} - - -# ::fileutil::FindRegexp -- -# -# Internal helper. Filter command used by 'findByPattern' -# to match files based on regular expressions. -# -# Arguments: -# patterns List of regular expressions to match against. -# filename Name of the file to match against the patterns. -# Results: -# interesting A boolean flag. Set to true if the file -# matches at least one of the patterns. - -proc ::fileutil::FindRegexp {patterns filename} { - foreach p $patterns { - if {[regexp -- $p $filename]} { - return 1 - } - } - return 0 -} - -# ::fileutil::FindGlob -- -# -# Internal helper. Filter command used by 'findByPattern' -# to match files based on glob expressions. -# -# Arguments: -# patterns List of glob expressions to match against. -# filename Name of the file to match against the patterns. -# Results: -# interesting A boolean flag. Set to true if the file -# matches at least one of the patterns. - -proc ::fileutil::FindGlob {patterns filename} { - foreach p $patterns { - if {[string match $p $filename]} { - return 1 - } - } - return 0 -} - -# ::fileutil::stripPwd -- -# -# If the specified path references is a path in [pwd] (or [pwd] itself) it -# is made relative to [pwd]. Otherwise it is left unchanged. -# In the case of [pwd] itself the result is the string '.'. -# -# Arguments: -# path path to modify -# -# Results: -# path The (possibly) modified path. - -proc ::fileutil::stripPwd {path} { - - # [file split] is used to generate a canonical form for both - # paths, for easy comparison, and also one which is easy to modify - # using list commands. - - set pwd [pwd] - if {[string equal $pwd $path]} { - return "." - } - - set pwd [file split $pwd] - set npath [file split $path] - - if {[string match ${pwd}* $npath]} { - set path [eval [linsert [lrange $npath [llength $pwd] end] 0 file join ]] - } - return $path -} - -# ::fileutil::stripN -- -# -# Removes N elements from the beginning of the path. -# -# Arguments: -# path path to modify -# n number of elements to strip -# -# Results: -# path The modified path - -proc ::fileutil::stripN {path n} { - set path [file split $path] - if {$n >= [llength $path]} { - return {} - } else { - return [eval [linsert [lrange $path $n end] 0 file join]] - } -} - -# ::fileutil::stripPath -- -# -# If the specified path references/is a path in prefix (or prefix itself) it -# is made relative to prefix. Otherwise it is left unchanged. -# In the case of it being prefix itself the result is the string '.'. -# -# Arguments: -# prefix prefix to strip from the path. -# path path to modify -# -# Results: -# path The (possibly) modified path. - -if {[string equal $tcl_platform(platform) windows]} { - - # Windows. While paths are stored with letter-case preserved al - # comparisons have to be done case-insensitive. For reference see - # SF Tcllib Bug 2499641. - - proc ::fileutil::stripPath {prefix path} { - # [file split] is used to generate a canonical form for both - # paths, for easy comparison, and also one which is easy to modify - # using list commands. - - set prefix [file split $prefix] - set npath [file split $path] - - if {[string equal -nocase $prefix $npath]} { - return "." - } - - if {[string match -nocase "${prefix} *" $npath]} { - set path [eval [linsert [lrange $npath [llength $prefix] end] 0 file join ]] - } - return $path - } -} else { - proc ::fileutil::stripPath {prefix path} { - # [file split] is used to generate a canonical form for both - # paths, for easy comparison, and also one which is easy to modify - # using list commands. - - set prefix [file split $prefix] - set npath [file split $path] - - if {[string equal $prefix $npath]} { - return "." - } - - if {[string match "${prefix} *" $npath]} { - set path [eval [linsert [lrange $npath [llength $prefix] end] 0 file join ]] - } - return $path - } -} - -# ::fileutil::jail -- -# -# Ensures that the input path 'filename' stays within the -# directory 'jail'. In this way it prevents user-supplied paths -# from escaping the jail. -# -# Arguments: -# jail The path to the directory the other must -# not escape from. -# filename The path to prevent from escaping. -# -# Results: -# path The (possibly) modified path surely within -# the confines of the jail. - -proc fileutil::jail {jail filename} { - if {![string equal [file pathtype $filename] "relative"]} { - # Although the path to check is absolute (or volumerelative on - # windows) we cannot perform a simple prefix check to see if - # the path is inside the jail or not. We have to normalize - # both path and jail and then we can check. If the path is - # outside we make the original path relative and prefix it - # with the original jail. We do make the jail pseudo-absolute - # by prefixing it with the current working directory for that. - - # Normalized jail. Fully resolved sym links, if any. Our main - # complication is that normalize does not resolve symlinks in the - # last component of the path given to it, so we add a bogus - # component, resolve, and then strip it off again. That is why the - # code is so large and long. - - set njail [eval [list file join] [lrange [file split \ - [Normalize [file join $jail __dummy__]]] 0 end-1]] - - # Normalize filename. Fully resolved sym links, if - # any. S.a. for an explanation of the complication. - - set nfile [eval [list file join] [lrange [file split \ - [Normalize [file join $filename __dummy__]]] 0 end-1]] - - if {[string match ${njail}* $nfile]} { - return $filename - } - - # Outside the jail, put it inside. ... We normalize the input - # path lexically for this, to prevent escapes still lurking in - # the original path. (We cannot use the normalized path, - # symlinks may have bent it out of shape in unrecognizable ways. - - return [eval [linsert [lrange [file split \ - [lexnormalize $filename]] 1 end] 0 file join [pwd] $jail]] - } else { - # The path is relative, consider it as outside - # implicitly. Normalize it lexically! to prevent escapes, then - # put the jail in front, use PWD to ensure absoluteness. - - return [eval [linsert [file split [lexnormalize $filename]] 0 \ - file join [pwd] $jail]] - } -} - - -# ::fileutil::test -- -# -# Simple API to testing various properties of -# a path (read, write, file/dir, existence) -# -# Arguments: -# path path to test -# codes names of the properties to test -# msgvar Name of variable to leave an error -# message in. Optional. -# label Label for error message, optional -# -# Results: -# ok boolean flag, set if the path passes -# all tests. - -namespace eval ::fileutil { - variable test - array set test { - read {readable {Read access is denied}} - write {writable {Write access is denied}} - exec {executable {Is not executable}} - exists {exists {Does not exist}} - file {isfile {Is not a file}} - dir {isdirectory {Is not a directory}} - } -} - -proc ::fileutil::test {path codes {msgvar {}} {label {}}} { - variable test - - if {[string equal $msgvar ""]} { - set msg "" - } else { - upvar 1 $msgvar msg - } - - if {![string equal $label ""]} {append label { }} - - if {![regexp {^(read|write|exec|exists|file|dir)} $codes]} { - # Translate single characters into proper codes - set codes [string map { - r read w write e exists x exec f file d dir - } [split $codes {}]] - } - - foreach c $codes { - foreach {cmd text} $test($c) break - if {![file $cmd $path]} { - set msg "$label\"$path\": $text" - return 0 - } - } - - return 1 -} - -# ::fileutil::cat -- -# -# Tcl implementation of the UNIX "cat" command. Returns the contents -# of the specified files. -# -# Arguments: -# args names of the files to read, interspersed with options -# to set encodings, translations, or eofchar. -# -# Results: -# data data read from the file. - -proc ::fileutil::cat {args} { - # Syntax: (?options? file)+ - # options = -encoding ENC - # | -translation TRA - # | -eofchar ECH - # | -- - - if {![llength $args]} { - # Argument processing stopped with arguments missing. - return -code error \ - "wrong#args: should be\ - [lindex [info level 0] 0] ?-eofchar|-translation|-encoding arg?+ file ..." - } - - # We go through the arguments using foreach and keeping track of - # the index we are at. We do not shift the arguments out to the - # left. That is inherently quadratic, copying everything down. - - set opts {} - set mode maybeopt - set channels {} - - foreach a $args { - if {[string equal $mode optarg]} { - lappend opts $a - set mode maybeopt - continue - } elseif {[string equal $mode maybeopt]} { - if {[string match -* $a]} { - switch -exact -- $a { - -encoding - - -translation - - -eofchar { - lappend opts $a - set mode optarg - continue - } - -- { - set mode file - continue - } - default { - return -code error \ - "Bad option \"$a\",\ - expected one of\ - -encoding, -eofchar,\ - or -translation" - } - } - } - # Not an option, but a file. Change mode and fall through. - set mode file - } - # Process file arguments - - if {[string equal $a -]} { - # Stdin reference is special. - - # Test that the current options are all ok. - # For stdin we have to avoid closing it. - - set old [fconfigure stdin] - set fail [catch { - SetOptions stdin $opts - } msg] ; # {} - SetOptions stdin $old - - if {$fail} { - return -code error $msg - } - - lappend channels [list $a $opts 0] - } else { - if {![file exists $a]} { - return -code error "Cannot read file \"$a\", does not exist" - } elseif {![file isfile $a]} { - return -code error "Cannot read file \"$a\", is not a file" - } elseif {![file readable $a]} { - return -code error "Cannot read file \"$a\", read access is denied" - } - - # Test that the current options are all ok. - set c [open $a r] - set fail [catch { - SetOptions $c $opts - } msg] ; # {} - close $c - if {$fail} { - return -code error $msg - } - - lappend channels [list $a $opts [file size $a]] - } - - # We may have more options and files coming after. - set mode maybeopt - } - - if {![string equal $mode maybeopt]} { - # Argument processing stopped with arguments missing. - return -code error \ - "wrong#args: should be\ - [lindex [info level 0] 0] ?-eofchar|-translation|-encoding arg?+ file ..." - } - - set data "" - foreach c $channels { - foreach {fname opts size} $c break - - if {[string equal $fname -]} { - set old [fconfigure stdin] - SetOptions stdin $opts - append data [read stdin] - SetOptions stdin $old - continue - } - - set c [open $fname r] - SetOptions $c $opts - - if {$size > 0} { - # Used the [file size] command to get the size, which - # preallocates memory, rather than trying to grow it as - # the read progresses. - append data [read $c $size] - } else { - # if the file has zero bytes it is either empty, or - # something where [file size] reports 0 but the file - # actually has data (like the files in the /proc - # filesystem on Linux). - append data [read $c] - } - close $c - } - - return $data -} - -# ::fileutil::writeFile -- -# -# Write the specified data into the named file, -# creating it if necessary. -# -# Arguments: -# options... Options and arguments. -# filename Path to the file to write. -# data The data to write into the file -# -# Results: -# None. - -proc ::fileutil::writeFile {args} { - # Syntax: ?options? file data - # options = -encoding ENC - # | -translation TRA - # | -eofchar ECH - # | -- - - Spec Writable $args opts fname data - - # Now perform the requested operation. - - file mkdir [file dirname $fname] - set c [open $fname w] - SetOptions $c $opts - puts -nonewline $c $data - close $c - return -} - -# ::fileutil::appendToFile -- -# -# Append the specified data at the end of the named file, -# creating it if necessary. -# -# Arguments: -# options... Options and arguments. -# filename Path to the file to extend. -# data The data to extend the file with. -# -# Results: -# None. - -proc ::fileutil::appendToFile {args} { - # Syntax: ?options? file data - # options = -encoding ENC - # | -translation TRA - # | -eofchar ECH - # | -- - - Spec Writable $args opts fname data - - # Now perform the requested operation. - - file mkdir [file dirname $fname] - set c [open $fname a] - SetOptions $c $opts - set at [tell $c] - puts -nonewline $c $data - close $c - return $at -} - -# ::fileutil::insertIntoFile -- -# -# Insert the specified data into the named file, -# creating it if necessary, at the given locaton. -# -# Arguments: -# options... Options and arguments. -# filename Path to the file to extend. -# data The data to extend the file with. -# -# Results: -# None. - -proc ::fileutil::insertIntoFile {args} { - - # Syntax: ?options? file at data - # options = -encoding ENC - # | -translation TRA - # | -eofchar ECH - # | -- - - Spec ReadWritable $args opts fname at data - - set max [file size $fname] - CheckLocation $at $max insertion - - if {[string length $data] == 0} { - # Another degenerate case, inserting nothing. - # Leave the file well enough alone. - return - } - - foreach {c o t} [Open2 $fname $opts] break - - # The degenerate cases of both appending and insertion at the - # beginning of the file allow more optimized implementations of - # the operation. - - if {$at == 0} { - puts -nonewline $o $data - fcopy $c $o - } elseif {$at == $max} { - fcopy $c $o - puts -nonewline $o $data - } else { - fcopy $c $o -size $at - puts -nonewline $o $data - fcopy $c $o - } - - Close2 $fname $t $c $o - return -} - -# ::fileutil::removeFromFile -- -# -# Remove n characters from the named file, -# starting at the given locaton. -# -# Arguments: -# options... Options and arguments. -# filename Path to the file to extend. -# at Location to start the removal from. -# n Number of characters to remove. -# -# Results: -# None. - -proc ::fileutil::removeFromFile {args} { - - # Syntax: ?options? file at n - # options = -encoding ENC - # | -translation TRA - # | -eofchar ECH - # | -- - - Spec ReadWritable $args opts fname at n - - set max [file size $fname] - CheckLocation $at $max removal - CheckLength $n $at $max removal - - if {$n == 0} { - # Another degenerate case, removing nothing. - # Leave the file well enough alone. - return - } - - foreach {c o t} [Open2 $fname $opts] break - - # The degenerate cases of both removal from the beginning or end - # of the file allow more optimized implementations of the - # operation. - - if {$at == 0} { - seek $c $n current - fcopy $c $o - } elseif {($at + $n) == $max} { - fcopy $c $o -size $at - # Nothing further to copy. - } else { - fcopy $c $o -size $at - seek $c $n current - fcopy $c $o - } - - Close2 $fname $t $c $o - return -} - -# ::fileutil::replaceInFile -- -# -# Remove n characters from the named file, -# starting at the given locaton, and replace -# it with the given data. -# -# Arguments: -# options... Options and arguments. -# filename Path to the file to extend. -# at Location to start the removal from. -# n Number of characters to remove. -# data The replacement data. -# -# Results: -# None. - -proc ::fileutil::replaceInFile {args} { - - # Syntax: ?options? file at n data - # options = -encoding ENC - # | -translation TRA - # | -eofchar ECH - # | -- - - Spec ReadWritable $args opts fname at n data - - set max [file size $fname] - CheckLocation $at $max replacement - CheckLength $n $at $max replacement - - if { - ($n == 0) && - ([string length $data] == 0) - } { - # Another degenerate case, replacing nothing with - # nothing. Leave the file well enough alone. - return - } - - foreach {c o t} [Open2 $fname $opts] break - - # Check for degenerate cases and handle them separately, - # i.e. strip the no-op parts out of the general implementation. - - if {$at == 0} { - if {$n == 0} { - # Insertion instead of replacement. - - puts -nonewline $o $data - fcopy $c $o - - } elseif {[string length $data] == 0} { - # Removal instead of replacement. - - seek $c $n current - fcopy $c $o - - } else { - # General replacement at front. - - seek $c $n current - puts -nonewline $o $data - fcopy $c $o - } - } elseif {($at + $n) == $max} { - if {$n == 0} { - # Appending instead of replacement - - fcopy $c $o - puts -nonewline $o $data - - } elseif {[string length $data] == 0} { - # Truncating instead of replacement - - fcopy $c $o -size $at - # Nothing further to copy. - - } else { - # General replacement at end - - fcopy $c $o -size $at - puts -nonewline $o $data - } - } else { - if {$n == 0} { - # General insertion. - - fcopy $c $o -size $at - puts -nonewline $o $data - fcopy $c $o - - } elseif {[string length $data] == 0} { - # General removal. - - fcopy $c $o -size $at - seek $c $n current - fcopy $c $o - - } else { - # General replacement. - - fcopy $c $o -size $at - seek $c $n current - puts -nonewline $o $data - fcopy $c $o - } - } - - Close2 $fname $t $c $o - return -} - -# ::fileutil::updateInPlace -- -# -# Run command prefix on the contents of the -# file and replace them with the result of -# the command. -# -# Arguments: -# options... Options and arguments. -# filename Path to the file to extend. -# cmd Command prefix to run. -# -# Results: -# None. - -proc ::fileutil::updateInPlace {args} { - # Syntax: ?options? file cmd - # options = -encoding ENC - # | -translation TRA - # | -eofchar ECH - # | -- - - Spec ReadWritable $args opts fname cmd - - # readFile/cat inlined ... - - set c [open $fname r] - SetOptions $c $opts - set data [read $c] - close $c - - # Transformation. Abort and do not modify the target file if an - # error was raised during this step. - - lappend cmd $data - set code [catch {uplevel 1 $cmd} res] - if {$code} { - return -code $code $res - } - - # writeFile inlined, with careful preservation of old contents - # until we are sure that the write was ok. - - if {[catch { - file rename -force $fname ${fname}.bak - - set o [open $fname w] - SetOptions $o $opts - puts -nonewline $o $res - close $o - - file delete -force ${fname}.bak - } msg]} { - if {[file exists ${fname}.bak]} { - catch { - file rename -force ${fname}.bak $fname - } - return -code error $msg - } - } - return -} - -proc ::fileutil::Writable {fname mv} { - upvar 1 $mv msg - if {[file exists $fname]} { - if {![file isfile $fname]} { - set msg "Cannot use file \"$fname\", is not a file" - return 0 - } elseif {![file writable $fname]} { - set msg "Cannot use file \"$fname\", write access is denied" - return 0 - } - } - return 1 -} - -proc ::fileutil::ReadWritable {fname mv} { - upvar 1 $mv msg - if {![file exists $fname]} { - set msg "Cannot use file \"$fname\", does not exist" - return 0 - } elseif {![file isfile $fname]} { - set msg "Cannot use file \"$fname\", is not a file" - return 0 - } elseif {![file writable $fname]} { - set msg "Cannot use file \"$fname\", write access is denied" - return 0 - } elseif {![file readable $fname]} { - set msg "Cannot use file \"$fname\", read access is denied" - return 0 - } - return 1 -} - -proc ::fileutil::Spec {check alist ov fv args} { - upvar 1 $ov opts $fv fname - - set n [llength $args] ; # Num more args - incr n ; # Count path as well - - set opts {} - set mode maybeopt - - set at 0 - foreach a $alist { - if {[string equal $mode optarg]} { - lappend opts $a - set mode maybeopt - incr at - continue - } elseif {[string equal $mode maybeopt]} { - if {[string match -* $a]} { - switch -exact -- $a { - -encoding - - -translation - - -eofchar { - lappend opts $a - set mode optarg - incr at - continue - } - -- { - # Stop processing. - incr at - break - } - default { - return -code error \ - "Bad option \"$a\",\ - expected one of\ - -encoding, -eofchar,\ - or -translation" - } - } - } - # Not an option, but a file. - # Stop processing. - break - } - } - - if {([llength $alist] - $at) != $n} { - # Argument processing stopped with arguments missing, or too - # many - return -code error \ - "wrong#args: should be\ - [lindex [info level 1] 0] ?-eofchar|-translation|-encoding arg? file $args" - } - - set fname [lindex $alist $at] - incr at - foreach \ - var $args \ - val [lrange $alist $at end] { - upvar 1 $var A - set A $val - } - - # Check given path ... - - if {![eval [linsert $check end $a msg]]} { - return -code error $msg - } - - return -} - -proc ::fileutil::Open2 {fname opts} { - set c [open $fname r] - set t [tempfile] - set o [open $t w] - - SetOptions $c $opts - SetOptions $o $opts - - return [list $c $o $t] -} - -proc ::fileutil::Close2 {f temp in out} { - close $in - close $out - - file copy -force $f ${f}.bak - file rename -force $temp $f - file delete -force ${f}.bak - return -} - -proc ::fileutil::SetOptions {c opts} { - if {![llength $opts]} return - eval [linsert $opts 0 fconfigure $c] - return -} - -proc ::fileutil::CheckLocation {at max label} { - if {![string is integer -strict $at]} { - return -code error \ - "Expected integer but got \"$at\"" - } elseif {$at < 0} { - return -code error \ - "Bad $label point $at, before start of data" - } elseif {$at > $max} { - return -code error \ - "Bad $label point $at, behind end of data" - } -} - -proc ::fileutil::CheckLength {n at max label} { - if {![string is integer -strict $n]} { - return -code error \ - "Expected integer but got \"$n\"" - } elseif {$n < 0} { - return -code error \ - "Bad $label size $n" - } elseif {($at + $n) > $max} { - return -code error \ - "Bad $label size $n, going behind end of data" - } -} - -# ::fileutil::foreachLine -- -# -# Executes a script for every line in a file. -# -# Arguments: -# var name of the variable to contain the lines -# filename name of the file to read. -# cmd The script to execute. -# -# Results: -# None. - -proc ::fileutil::foreachLine {var filename cmd} { - upvar 1 $var line - set fp [open $filename r] - - # -future- Use try/eval from tcllib/control - catch { - set code 0 - set result {} - set return 0 - while {[gets $fp line] >= 0} { - set code [catch {uplevel 1 $cmd} result options] - if {$code == 2} { - set return 1 - set code [dict get $options -code] - break - } elseif {$code != 0 && $code != 4} { - break - } - } - } - close $fp - - if {$return || $code == 1 || $code > 4} { - return -options $options $result - } - return $result -} - -# ::fileutil::touch -- -# -# Tcl implementation of the UNIX "touch" command. -# -# touch [-a] [-m] [-c] [-r ref_file] [-t time] filename ... -# -# Arguments: -# -a change the access time only, unless -m also specified -# -m change the modification time only, unless -a also specified -# -c silently prevent creating a file if it did not previously exist -# -r ref_file use the ref_file's time instead of the current time -# -t time use the specified time instead of the current time -# ("time" is an integer clock value, like [clock seconds]) -# filename ... the files to modify -# -# Results -# None. -# -# Errors: -# Both of "-r" and "-t" cannot be specified. - -if {[package vsatisfies [package provide Tcl] 8.3]} { - namespace eval ::fileutil { - namespace export touch - } - - proc ::fileutil::touch {args} { - # Don't bother catching errors, just let them propagate up - - set options { - {a "set the atime only"} - {m "set the mtime only"} - {c "do not create non-existant files"} - {r.arg "" "use time from ref_file"} - {t.arg -1 "use specified time"} - } - set usage ": [lindex [info level 0] 0]\ - \[options] filename ...\noptions:" - array set params [::cmdline::getoptions args $options $usage] - - # process -a and -m options - set set_atime [set set_mtime "true"] - if { $params(a) && ! $params(m)} {set set_mtime "false"} - if {! $params(a) && $params(m)} {set set_atime "false"} - - # process -r and -t - set has_t [expr {$params(t) != -1}] - set has_r [expr {[string length $params(r)] > 0}] - if {$has_t && $has_r} { - return -code error "Cannot specify both -r and -t" - } elseif {$has_t} { - set atime [set mtime $params(t)] - } elseif {$has_r} { - file stat $params(r) stat - set atime $stat(atime) - set mtime $stat(mtime) - } else { - set atime [set mtime [clock seconds]] - } - - # do it - foreach filename $args { - if {! [file exists $filename]} { - if {$params(c)} {continue} - close [open $filename w] - } - if {$set_atime} {file atime $filename $atime} - if {$set_mtime} {file mtime $filename $mtime} - } - return - } -} - -# ::fileutil::fileType -- -# -# Do some simple heuristics to determine file type. -# -# -# Arguments: -# filename Name of the file to test. -# -# Results -# type Type of the file. May be a list if multiple tests -# are positive (eg, a file could be both a directory -# and a link). In general, the list proceeds from most -# general (eg, binary) to most specific (eg, gif), so -# the full type for a GIF file would be -# "binary graphic gif" -# -# At present, the following types can be detected: -# -# directory -# empty -# binary -# text -# script -# executable [elf, dos, ne, pe] -# binary graphic [gif, jpeg, png, tiff, bitmap, icns] -# ps, eps, pdf -# html -# xml -# message pgp -# compressed [bzip, gzip, zip, tar] -# audio [mpeg, wave] -# gravity_wave_data_frame -# link -# doctools, doctoc, and docidx documentation files. -# - -proc ::fileutil::fileType {filename} { - ;## existence test - if { ! [ file exists $filename ] } { - set err "file not found: '$filename'" - return -code error $err - } - ;## directory test - if { [ file isdirectory $filename ] } { - set type directory - if { ! [ catch {file readlink $filename} ] } { - lappend type link - } - return $type - } - ;## empty file test - if { ! [ file size $filename ] } { - set type empty - if { ! [ catch {file readlink $filename} ] } { - lappend type link - } - return $type - } - set bin_rx {[\x00-\x08\x0b\x0e-\x1f]} - - if { [ catch { - set fid [ open $filename r ] - fconfigure $fid -translation binary - fconfigure $fid -buffersize 1024 - fconfigure $fid -buffering full - set test [ read $fid 1024 ] - ::close $fid - } err ] } { - catch { ::close $fid } - return -code error "::fileutil::fileType: $err" - } - - if { [ regexp $bin_rx $test ] } { - set type binary - set binary 1 - } else { - set type text - set binary 0 - } - - # SF Tcllib bug [795585]. Allowing whitespace between #! - # and path of script interpreter - - set metakit 0 - - if { [ regexp {^\#\!\s*(\S+)} $test -> terp ] } { - lappend type script $terp - } elseif {([regexp "\\\[manpage_begin " $test] && - !([regexp -- {--- !doctools ---} $test] || [regexp -- "!tcl\.tk//DSL doctools//EN//" $test])) || - ([regexp -- {--- doctools ---} $test] || [regexp -- "tcl\.tk//DSL doctools//EN//" $test])} { - lappend type doctools - } elseif {([regexp "\\\[toc_begin " $test] && - !([regexp -- {--- !doctoc ---} $test] || [regexp -- "!tcl\.tk//DSL doctoc//EN//" $test])) || - ([regexp -- {--- doctoc ---} $test] || [regexp -- "tcl\.tk//DSL doctoc//EN//" $test])} { - lappend type doctoc - } elseif {([regexp "\\\[index_begin " $test] && - !([regexp -- {--- !docidx ---} $test] || [regexp -- "!tcl\.tk//DSL docidx//EN//" $test])) || - ([regexp -- {--- docidx ---} $test] || [regexp -- "tcl\.tk//DSL docidx//EN//" $test])} { - lappend type docidx - } elseif {[regexp -- "tcl\\.tk//DSL diagram//EN//" $test]} { - lappend type tkdiagram - } elseif { $binary && [ regexp {^[\x7F]ELF} $test ] } { - lappend type executable elf - } elseif { $binary && [string match "MZ*" $test] } { - if { [scan [string index $test 24] %c] < 64 } { - lappend type executable dos - } else { - binary scan [string range $test 60 61] s next - set sig [string range $test $next [expr {$next + 1}]] - if { $sig == "NE" || $sig == "PE" } { - lappend type executable [string tolower $sig] - } else { - lappend type executable dos - } - } - } elseif { $binary && [string match "SQLite format 3\x00*" $test] } { - lappend type sqlite3 - - # Check for various sqlite-based application file formats. - set appid [string range $test 68 71] - if {$appid eq "\x0f\x05\x51\x12"} { - lappend type fossil-checkout - } elseif {$appid eq "\x0f\x05\x51\x13"} { - lappend type fossil-global-config - } elseif {$appid eq "\x0f\x05\x51\x11"} { - lappend type fossil-repository - } else { - # encode the appid as hex and append that. - binary scan $appid H8 aid - lappend type A$aid - } - - } elseif { $binary && [string match "BZh91AY\&SY*" $test] } { - lappend type compressed bzip - } elseif { $binary && [string match "\x1f\x8b*" $test] } { - lappend type compressed gzip - } elseif { $binary && [string range $test 257 262] == "ustar\x00" } { - lappend type compressed tar - } elseif { $binary && [string match "\x50\x4b\x03\x04*" $test] } { - lappend type compressed zip - } elseif { $binary && [string match "GIF*" $test] } { - lappend type graphic gif - } elseif { $binary && [string match "icns*" $test] } { - lappend type graphic icns bigendian - } elseif { $binary && [string match "snci*" $test] } { - lappend type graphic icns smallendian - } elseif { $binary && [string match "\x89PNG*" $test] } { - lappend type graphic png - } elseif { $binary && [string match "\xFF\xD8\xFF*" $test] } { - binary scan $test x3H2x2a5 marker txt - if { $marker == "e0" && $txt == "JFIF\x00" } { - lappend type graphic jpeg jfif - } elseif { $marker == "e1" && $txt == "Exif\x00" } { - lappend type graphic jpeg exif - } - } elseif { $binary && [string match "MM\x00\**" $test] } { - lappend type graphic tiff - } elseif { $binary && [string match "BM*" $test] && [string range $test 6 9] == "\x00\x00\x00\x00" } { - lappend type graphic bitmap - } elseif { ! $binary && [string match -nocase "*\*" $test] } { - lappend type html - } elseif {[string match "\%PDF\-*" $test] } { - lappend type pdf - } elseif { [string match "\%\!PS\-*" $test] } { - lappend type ps - if { [string match "* EPSF\-*" $test] } { - lappend type eps - } - } elseif { [string match -nocase "*\<\?xml*" $test] } { - lappend type xml - if { [ regexp -nocase {\<\!DOCTYPE\s+(\S+)} $test -> doctype ] } { - lappend type $doctype - } - } elseif { [string match {*BEGIN PGP MESSAGE*} $test] } { - lappend type message pgp - } elseif { $binary && [string match {IGWD*} $test] } { - lappend type gravity_wave_data_frame - } elseif {[string match "JL\x1a\x00*" $test] && ([file size $filename] >= 27)} { - lappend type metakit smallendian - set metakit 1 - } elseif {[string match "LJ\x1a\x00*" $test] && ([file size $filename] >= 27)} { - lappend type metakit bigendian - set metakit 1 - } elseif { $binary && [string match "RIFF*" $test] && [string range $test 8 11] == "WAVE" } { - lappend type audio wave - } elseif { $binary && [string match "ID3*" $test] } { - lappend type audio mpeg - } elseif { $binary && [binary scan $test S tmp] && [expr {$tmp & 0xFFE0}] == 65504 } { - lappend type audio mpeg - } - - # Additional checks of file contents at the end of the file, - # possibly pointing into the middle too (attached metakit, - # attached zip). - - ## Metakit File format: http://www.equi4.com/metakit/metakit-ff.html - ## Metakit database attached ? ## - - if {!$metakit && ([file size $filename] >= 27)} { - # The offsets in the footer are in always bigendian format - - if { [ catch { - set fid [ open $filename r ] - fconfigure $fid -translation binary - fconfigure $fid -buffersize 1024 - fconfigure $fid -buffering full - seek $fid -16 end - set test [ read $fid 16 ] - ::close $fid - } err ] } { - catch { ::close $fid } - return -code error "::fileutil::fileType: $err" - } - - binary scan $test IIII __ hdroffset __ __ - set hdroffset [expr {[file size $filename] - 16 - $hdroffset}] - - # Further checks iff the offset is actually inside the file. - - if {($hdroffset >= 0) && ($hdroffset < [file size $filename])} { - # Seek to the specified location and try to match a metakit header - # at this location. - - if { [ catch { - set fid [ open $filename r ] - fconfigure $fid -translation binary - fconfigure $fid -buffersize 1024 - fconfigure $fid -buffering full - seek $fid $hdroffset start - set test [ read $fid 16 ] - ::close $fid - } err ] } { - catch { ::close $fid } - return -code error "::fileutil::fileType: $err" - } - - if {[string match "JL\x1a\x00*" $test]} { - lappend type attached metakit smallendian - set metakit 1 - } elseif {[string match "LJ\x1a\x00*" $test]} { - lappend type attached metakit bigendian - set metakit 1 - } - } - } - - ## Zip File Format: http://zziplib.sourceforge.net/zzip-parse.html - ## http://www.pkware.com/products/enterprise/white_papers/appnote.html - - - ;## lastly, is it a link? - if { ! [ catch {file readlink $filename} ] } { - lappend type link - } - return $type -} - -# ::fileutil::tempdir -- -# -# Return the correct directory to use for temporary files. -# Python attempts this sequence, which seems logical: -# -# 1. The directory named by the `TMPDIR' environment variable. -# -# 2. The directory named by the `TEMP' environment variable. -# -# 3. The directory named by the `TMP' environment variable. -# -# 4. A platform-specific location: -# * On Macintosh, the `Temporary Items' folder. -# -# * On Windows, the directories `C:\\TEMP', `C:\\TMP', -# `\\TEMP', and `\\TMP', in that order. -# -# * On all other platforms, the directories `/tmp', -# `/var/tmp', and `/usr/tmp', in that order. -# -# 5. As a last resort, the current working directory. -# -# The code here also does -# -# 0. The directory set by invoking tempdir with an argument. -# If this is present it is used exclusively. -# -# Arguments: -# None. -# -# Side Effects: -# None. -# -# Results: -# The directory for temporary files. - -proc ::fileutil::tempdir {args} { - if {[llength $args] > 1} { - return -code error {wrong#args: should be "::fileutil::tempdir ?path?"} - } elseif {[llength $args] == 1} { - variable tempdir [lindex $args 0] - variable tempdirSet 1 - return - } - return [Normalize [TempDir]] -} - -proc ::fileutil::tempdirReset {} { - variable tempdir {} - variable tempdirSet 0 - return -} - -proc ::fileutil::TempDir {} { - global tcl_platform env - variable tempdir - variable tempdirSet - - set attempdirs [list] - set problems {} - - if {$tempdirSet} { - lappend attempdirs $tempdir - lappend problems {User/Application specified tempdir} - } else { - foreach tmp {TMPDIR TEMP TMP} { - if { [info exists env($tmp)] } { - lappend attempdirs $env($tmp) - } else { - lappend problems "No environment variable $tmp" - } - } - - switch $tcl_platform(platform) { - windows { - lappend attempdirs "C:\\TEMP" "C:\\TMP" "\\TEMP" "\\TMP" - } - macintosh { - lappend attempdirs $env(TRASH_FOLDER) ;# a better place? - } - default { - lappend attempdirs \ - [file join / tmp] \ - [file join / var tmp] \ - [file join / usr tmp] - } - } - - lappend attempdirs [pwd] - } - - foreach tmp $attempdirs { - if { [file isdirectory $tmp] && [file writable $tmp] } { - return $tmp - } elseif { ![file isdirectory $tmp] } { - lappend problems "Not a directory: $tmp" - } else { - lappend problems "Not writable: $tmp" - } - } - - # Fail if nothing worked. - return -code error "Unable to determine a proper directory for temporary files\n[join $problems \n]" -} - -namespace eval ::fileutil { - variable tempdir {} - variable tempdirSet 0 -} - -# ::fileutil::maketempdir -- - -proc ::fileutil::maketempdir {args} { - return [Normalize [MakeTempDir $args]] -} - -proc ::fileutil::MakeTempDir {config} { - # Setup of default configuration. - array set options {} - set options(-suffix) "" - set options(-prefix) "tmp" - set options(-dir) [tempdir] - - # TODO: Check for and reject options not in -suffix, -prefix, -dir - # Merge user configuration, overwrite defaults. - array set options $config - - # See also "tempfile" below. Could be shareable internal configuration. - set chars abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 - set nrand_chars 10 - set maxtries 10 - - for {set i 0} {$i < $maxtries} {incr i} { - # Build up the candidate name. See also "tempfile". - set directory_name $options(-prefix) - for {set j 0} {$j < $nrand_chars} {incr j} { - append directory_name \ - [string index $chars [expr {int(rand() * 62)}]] - } - append directory_name $options(-suffix) - set path [file join $options(-dir) $directory_name] - - # Try to create. Try again if already exists, or trouble - # with creation and setting of perms. - # - # Note: The last looks as if it is able to leave partial - # directories behind (created, trouble with perms). But - # deleting ... Might pull the rug out from somebody else. - - if {[file exists $path]} continue - if {[catch { - file mkdir $path - if {$::tcl_platform(platform) eq "unix"} { - file attributes $path -permissions 0700 - } - }]} continue - - return $path - } - return -code error "Failed to find an unused temporary directory name" -} - -# ::fileutil::tempfile -- -# -# generate a temporary file name suitable for writing to -# the file name will be unique, writable and will be in the -# appropriate system specific temp directory -# Code taken from http://mini.net/tcl/772 attributed to -# Igor Volobouev and anon. -# -# Arguments: -# prefix - a prefix for the filename, p -# Results: -# returns a file name -# - -proc ::fileutil::tempfile {{prefix {}}} { - return [Normalize [TempFile $prefix]] -} - -proc ::fileutil::TempFile {prefix} { - set tmpdir [tempdir] - - set chars "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" - set nrand_chars 10 - set maxtries 10 - set access [list RDWR CREAT EXCL] - set permission 0600 - set channel "" - set checked_dir_writable 0 - - for {set i 0} {$i < $maxtries} {incr i} { - set newname $prefix - for {set j 0} {$j < $nrand_chars} {incr j} { - append newname [string index $chars \ - [expr {int(rand()*62)}]] - } - set newname [file join $tmpdir $newname] - - if {[catch {open $newname $access $permission} channel]} { - if {!$checked_dir_writable} { - set dirname [file dirname $newname] - if {![file writable $dirname]} { - return -code error "Directory $dirname is not writable" - } - set checked_dir_writable 1 - } - } else { - # Success - close $channel - return $newname - } - - } - if {[string compare $channel ""]} { - return -code error "Failed to open a temporary file: $channel" - } else { - return -code error "Failed to find an unused temporary file name" - } -} - -# ::fileutil::install -- -# -# Tcl version of the 'install' command, which copies files from -# one places to another and also optionally sets some attributes -# such as group, owner, and permissions. -# -# Arguments: -# -m Change the file permissions to the specified -# value. Valid arguments are those accepted by -# file attributes -permissions -# -# Results: -# None. - -# TODO - add options for group/owner manipulation. - -proc ::fileutil::install {args} { - set options { - {m.arg "" "Set permission mode"} - } - set usage ": [lindex [info level 0] 0]\ -\[options] source destination \noptions:" - array set params [::cmdline::getoptions args $options $usage] - # Args should now just be the source and destination. - if { [llength $args] < 2 } { - return -code error $usage - } - set src [lindex $args 0] - set dst [lindex $args 1] - file copy -force $src $dst - if { $params(m) != "" } { - set targets [::fileutil::find $dst] - foreach fl $targets { - file attributes $fl -permissions $params(m) - } - } -} - -# ### ### ### ######### ######### ######### - -proc ::fileutil::lexnormalize {sp} { - set spx [file split $sp] - - # Resolution of embedded relative modifiers (., and ..). - - if { - ([lsearch -exact $spx . ] < 0) && - ([lsearch -exact $spx ..] < 0) - } { - # Quick path out if there are no relative modifiers - return $sp - } - - set absolute [expr {![string equal [file pathtype $sp] relative]}] - # A volumerelative path counts as absolute for our purposes. - - set sp $spx - set np {} - set noskip 1 - - while {[llength $sp]} { - set ele [lindex $sp 0] - set sp [lrange $sp 1 end] - set islast [expr {[llength $sp] == 0}] - - if {[string equal $ele ".."]} { - if { - ($absolute && ([llength $np] > 1)) || - (!$absolute && ([llength $np] >= 1)) - } { - # .. : Remove the previous element added to the - # new path, if there actually is enough to remove. - set np [lrange $np 0 end-1] - } - } elseif {[string equal $ele "."]} { - # Ignore .'s, they stay at the current location - continue - } else { - # A regular element. - lappend np $ele - } - } - if {[llength $np] > 0} { - return [eval [linsert $np 0 file join]] - # 8.5: return [file join {*}$np] - } - return {} -} - -# ### ### ### ######### ######### ######### -## Forward compatibility. Some routines require path normalization, -## something we have supported by the builtin 'file' only since Tcl -## 8.4. For versions of Tcl before that, to be supported by the -## module, we implement a normalizer in Tcl itself. Slow, but working. - -if {[package vcompare [package provide Tcl] 8.4] < 0} { - # Pre 8.4. We do not have 'file normalize'. We create an - # approximation for it based on earlier commands. - - # ... Hm. This is lexical normalization. It does not resolve - # symlinks in the path to their origin. - - proc ::fileutil::Normalize {sp} { - set sp [file split $sp] - - # Conversion of the incoming path to absolute. - if {[string equal [file pathtype [lindex $sp 0]] "relative"]} { - set sp [file split [eval [list file join [pwd]] $sp]] - } - - # Resolution of symlink components, and embedded relative - # modifiers (., and ..). - - set np {} - set noskip 1 - while {[llength $sp]} { - set ele [lindex $sp 0] - set sp [lrange $sp 1 end] - set islast [expr {[llength $sp] == 0}] - - if {[string equal $ele ".."]} { - if {[llength $np] > 1} { - # .. : Remove the previous element added to the - # new path, if there actually is enough to remove. - set np [lrange $np 0 end-1] - } - } elseif {[string equal $ele "."]} { - # Ignore .'s, they stay at the current location - continue - } else { - # A regular element. If it is not the last component - # then check if the combination is a symlink, and if - # yes, resolve it. - - lappend np $ele - - if {!$islast && $noskip} { - # The flag 'noskip' is technically not required, - # just 'file exists'. However if a path P does not - # exist, then all longer paths starting with P can - # not exist either, and using the flag to store - # this knowledge then saves us a number of - # unnecessary stat calls. IOW this a performance - # optimization. - - set p [eval file join $np] - set noskip [file exists $p] - if {$noskip} { - if {[string equal link [file type $p]]} { - set dst [file readlink $p] - - # We always push the destination in front of - # the source path (in expanded form). So that - # we handle .., .'s, and symlinks inside of - # this path as well. An absolute path clears - # the result, a relative one just removes the - # last, now resolved component. - - set sp [eval [linsert [file split $dst] 0 linsert $sp 0]] - - if {![string equal relative [file pathtype $dst]]} { - # Absolute|volrelative destination, clear - # result, we have to start over. - set np {} - } else { - # Relative link, just remove the resolved - # component again. - set np [lrange $np 0 end-1] - } - } - } - } - } - } - if {[llength $np] > 0} { - return [eval file join $np] - } - return {} - } -} else { - proc ::fileutil::Normalize {sp} { - file normalize $sp - } -} - -# ::fileutil::relative -- -# -# Taking two _directory_ paths, a base and a destination, computes the path -# of the destination relative to the base. -# -# Arguments: -# base The path to make the destination relative to. -# dst The destination path -# -# Results: -# The path of the destination, relative to the base. - -proc ::fileutil::relative {base dst} { - # Ensure that the link to directory 'dst' is properly done relative to - # the directory 'base'. - - if {![string equal [file pathtype $base] [file pathtype $dst]]} { - return -code error "Unable to compute relation for paths of different pathtypes: [file pathtype $base] vs. [file pathtype $dst], ($base vs. $dst)" - } - - set base [lexnormalize [file join [pwd] $base]] - set dst [lexnormalize [file join [pwd] $dst]] - - set save $dst - set base [file split $base] - set dst [file split $dst] - - while {[string equal [lindex $dst 0] [lindex $base 0]]} { - set dst [lrange $dst 1 end] - set base [lrange $base 1 end] - if {![llength $dst]} {break} - } - - set dstlen [llength $dst] - set baselen [llength $base] - - if {($dstlen == 0) && ($baselen == 0)} { - # Cases: - # (a) base == dst - - set dst . - } else { - # Cases: - # (b) base is: base/sub = sub - # dst is: base = {} - - # (c) base is: base = {} - # dst is: base/sub = sub - - while {$baselen > 0} { - set dst [linsert $dst 0 ..] - incr baselen -1 - } - # 8.5: set dst [file join {*}$dst] - set dst [eval [linsert $dst 0 file join]] - } - - return $dst -} - -# ::fileutil::relativeUrl -- -# -# Taking two _file_ paths, a base and a destination, computes the path -# of the destination relative to the base, from the inside of the base. -# -# This is how a browser resolves relative links in a file, hence the -# url in the command name. -# -# Arguments: -# base The file path to make the destination relative to. -# dst The destination file path -# -# Results: -# The path of the destination file, relative to the base file. - -proc ::fileutil::relativeUrl {base dst} { - # Like 'relative', but for links from _inside_ a file to a - # different file. - - if {![string equal [file pathtype $base] [file pathtype $dst]]} { - return -code error "Unable to compute relation for paths of different pathtypes: [file pathtype $base] vs. [file pathtype $dst], ($base vs. $dst)" - } - - set base [lexnormalize [file join [pwd] $base]] - set dst [lexnormalize [file join [pwd] $dst]] - - set basedir [file dirname $base] - set dstdir [file dirname $dst] - - set dstdir [relative $basedir $dstdir] - - # dstdir == '.' on input => dstdir output has trailing './'. Strip - # this superfluous segment off. - - if {[string equal $dstdir "."]} { - return [file tail $dst] - } elseif {[string equal [file tail $dstdir] "."]} { - return [file join [file dirname $dstdir] [file tail $dst]] - } else { - return [file join $dstdir [file tail $dst]] - } -} - -# ::fileutil::fullnormalize -- -# -# Normalizes a path completely. I.e. a symlink in the last -# element is resolved as well, not only symlinks in the higher -# elements. -# -# Arguments: -# path The path to normalize -# -# Results: -# The input path with all symlinks resolved. - -proc ::fileutil::fullnormalize {path} { - # When encountering symlinks in a file copy operation Tcl copies - # the link, not the contents of the file it references. There are - # situations there this is not acceptable. For these this command - # resolves all symbolic links in the path, including in the last - # element of the path. A "file copy" using the return value of - # this command copies an actual file, it will not encounter - # symlinks. - - # BUG / WORKAROUND. Using the / instead of the join seems to work - # around a bug in the path handling on windows which can break the - # core 'file normalize' for symbolic links. This was exposed by - # the find testsuite which could not reproduced outside. I believe - # that there is some deep path bug in the core triggered under - # special circumstances. Use of / likely forces a refresh through - # the string rep and so avoids the problem with the path intrep. - - return [file dirname [Normalize $path/__dummy__]] - #return [file dirname [Normalize [file join $path __dummy__]]] -} diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/http-2.10b1.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/http-2.10b1.tm deleted file mode 100644 index 6c3c068c..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/http-2.10b1.tm +++ /dev/null @@ -1,5457 +0,0 @@ -# http.tcl -- -# -# Client-side HTTP for GET, POST, and HEAD commands. These routines can -# be used in untrusted code that uses the Safesock security policy. -# These procedures use a callback interface to avoid using vwait, which -# is not defined in the safe base. -# -# See the file "license.terms" for information on usage and redistribution of -# this file, and for a DISCLAIMER OF ALL WARRANTIES. - -package require Tcl 8.6- -# Keep this in sync with pkgIndex.tcl and with the install directories in -# Makefiles -package provide http 2.10b1 - -namespace eval http { - # Allow resourcing to not clobber existing data - - variable http - if {![info exists http]} { - array set http { - -accept */* - -cookiejar {} - -pipeline 1 - -postfresh 0 - -proxyhost {} - -proxyport {} - -proxyfilter http::ProxyRequired - -proxynot {} - -proxyauth {} - -repost 0 - -threadlevel 0 - -urlencoding utf-8 - -zip 1 - } - # We need a useragent string of this style or various servers will - # refuse to send us compressed content even when we ask for it. This - # follows the de-facto layout of user-agent strings in current browsers. - # Safe interpreters do not have ::tcl_platform(os) or - # ::tcl_platform(osVersion). - if {[interp issafe]} { - set http(-useragent) "Mozilla/5.0\ - (Windows; U;\ - Windows NT 10.0)\ - http/[package provide http] Tcl/[package provide Tcl]" - } else { - set http(-useragent) "Mozilla/5.0\ - ([string totitle $::tcl_platform(platform)]; U;\ - $::tcl_platform(os) $::tcl_platform(osVersion))\ - http/[package provide http] Tcl/[package provide Tcl]" - } - } - - proc init {} { - # Set up the map for quoting chars. RFC3986 Section 2.3 say percent - # encode all except: "... percent-encoded octets in the ranges of - # ALPHA (%41-%5A and %61-%7A), DIGIT (%30-%39), hyphen (%2D), period - # (%2E), underscore (%5F), or tilde (%7E) should not be created by URI - # producers ..." - for {set i 0} {$i <= 256} {incr i} { - set c [format %c $i] - if {![string match {[-._~a-zA-Z0-9]} $c]} { - set map($c) %[format %.2X $i] - } - } - # These are handled specially - set map(\n) %0D%0A - variable formMap [array get map] - - # Create a map for HTTP/1.1 open sockets - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - if {[info exists socketMapping]} { - # Close open sockets on re-init. Do not permit retries. - foreach {url sock} [array get socketMapping] { - unset -nocomplain socketClosing($url) - unset -nocomplain socketPlayCmd($url) - CloseSocket $sock - } - } - - # CloseSocket should have unset the socket* arrays, one element at - # a time. Now unset anything that was overlooked. - # Traces on "unset socketRdState(*)" will call CancelReadPipeline and - # cancel any queued responses. - # Traces on "unset socketWrState(*)" will call CancelWritePipeline and - # cancel any queued requests. - array unset socketMapping - array unset socketRdState - array unset socketWrState - array unset socketRdQueue - array unset socketWrQueue - array unset socketPhQueue - array unset socketClosing - array unset socketPlayCmd - array unset socketCoEvent - array unset socketProxyId - array set socketMapping {} - array set socketRdState {} - array set socketWrState {} - array set socketRdQueue {} - array set socketWrQueue {} - array set socketPhQueue {} - array set socketClosing {} - array set socketPlayCmd {} - array set socketCoEvent {} - array set socketProxyId {} - return - } - init - - variable urlTypes - if {![info exists urlTypes]} { - set urlTypes(http) [list 80 ::http::socket] - } - - variable encodings [string tolower [encoding names]] - # This can be changed, but iso8859-1 is the RFC standard. - variable defaultCharset - if {![info exists defaultCharset]} { - set defaultCharset "iso8859-1" - } - - # Force RFC 3986 strictness in geturl url verification? - variable strict - if {![info exists strict]} { - set strict 1 - } - - # Let user control default keepalive for compatibility - variable defaultKeepalive - if {![info exists defaultKeepalive]} { - set defaultKeepalive 0 - } - - # Regular expression used to parse cookies - variable CookieRE {(?x) # EXPANDED SYNTAX - \s* # Ignore leading spaces - ([^][\u0000- ()<>@,;:\\""/?={}\u007f-\uffff]+) # Match the name - = # LITERAL: Equal sign - ([!\u0023-+\u002D-:<-\u005B\u005D-~]*) # Match the value - (?: - \s* ; \s* # LITERAL: semicolon - ([^\u0000]+) # Match the options - )? - } - - variable TmpSockCounter 0 - variable ThreadCounter 0 - - variable reasonDict [dict create {*}{ - 100 Continue - 101 {Switching Protocols} - 102 Processing - 103 {Early Hints} - 200 OK - 201 Created - 202 Accepted - 203 {Non-Authoritative Information} - 204 {No Content} - 205 {Reset Content} - 206 {Partial Content} - 207 Multi-Status - 208 {Already Reported} - 226 {IM Used} - 300 {Multiple Choices} - 301 {Moved Permanently} - 302 Found - 303 {See Other} - 304 {Not Modified} - 305 {Use Proxy} - 306 (Unused) - 307 {Temporary Redirect} - 308 {Permanent Redirect} - 400 {Bad Request} - 401 Unauthorized - 402 {Payment Required} - 403 Forbidden - 404 {Not Found} - 405 {Method Not Allowed} - 406 {Not Acceptable} - 407 {Proxy Authentication Required} - 408 {Request Timeout} - 409 Conflict - 410 Gone - 411 {Length Required} - 412 {Precondition Failed} - 413 {Content Too Large} - 414 {URI Too Long} - 415 {Unsupported Media Type} - 416 {Range Not Satisfiable} - 417 {Expectation Failed} - 418 (Unused) - 421 {Misdirected Request} - 422 {Unprocessable Content} - 423 Locked - 424 {Failed Dependency} - 425 {Too Early} - 426 {Upgrade Required} - 428 {Precondition Required} - 429 {Too Many Requests} - 431 {Request Header Fields Too Large} - 451 {Unavailable For Legal Reasons} - 500 {Internal Server Error} - 501 {Not Implemented} - 502 {Bad Gateway} - 503 {Service Unavailable} - 504 {Gateway Timeout} - 505 {HTTP Version Not Supported} - 506 {Variant Also Negotiates} - 507 {Insufficient Storage} - 508 {Loop Detected} - 510 {Not Extended (OBSOLETED)} - 511 {Network Authentication Required} - }] - - variable failedProxyValues { - binary - body - charset - coding - connection - connectionRespFlag - currentsize - host - http - httpResponse - meta - method - querylength - queryoffset - reasonPhrase - requestHeaders - requestLine - responseCode - state - status - tid - totalsize - transfer - type - } - - namespace export geturl config reset wait formatQuery postError quoteString - namespace export register unregister registerError - namespace export requestLine requestHeaders requestHeaderValue - namespace export responseLine responseHeaders responseHeaderValue - namespace export responseCode responseBody responseInfo reasonPhrase - # - Legacy aliases, were never exported: - # data, code, mapReply, meta, ncode - # - Callable from outside (e.g. from TLS) by fully-qualified name, but - # not exported: - # socket - # - Useful, but never exported (and likely to have naming collisions): - # size, status, cleanup, error, init - # Comments suggest that "init" can be used for re-initialisation, - # although the command is undocumented. - # - Never exported, renamed from lower-case names: - # GetTextLine, MakeTransformationChunked. -} - -# http::Log -- -# -# Debugging output -- define this to observe HTTP/1.1 socket usage. -# Should echo any args received. -# -# Arguments: -# msg Message to output -# -if {[info command http::Log] eq {}} {proc http::Log {args} {}} - -# http::register -- -# -# See documentation for details. -# -# Arguments: -# proto URL protocol prefix, e.g. https -# port Default port for protocol -# command Command to use to create socket -# Results: -# list of port and command that was registered. - -proc http::register {proto port command} { - variable urlTypes - set urlTypes([string tolower $proto]) [list $port $command] -} - -# http::unregister -- -# -# Unregisters URL protocol handler -# -# Arguments: -# proto URL protocol prefix, e.g. https -# Results: -# list of port and command that was unregistered. - -proc http::unregister {proto} { - variable urlTypes - set lower [string tolower $proto] - if {![info exists urlTypes($lower)]} { - return -code error "unsupported url type \"$proto\"" - } - set old $urlTypes($lower) - unset urlTypes($lower) - return $old -} - -# http::config -- -# -# See documentation for details. -# -# Arguments: -# args Options parsed by the procedure. -# Results: -# TODO - -proc http::config {args} { - variable http - set options [lsort [array names http -*]] - set usage [join $options ", "] - if {[llength $args] == 0} { - set result {} - foreach name $options { - lappend result $name $http($name) - } - return $result - } - set options [string map {- ""} $options] - set pat ^-(?:[join $options |])$ - if {[llength $args] == 1} { - set flag [lindex $args 0] - if {![regexp -- $pat $flag]} { - return -code error "Unknown option $flag, must be: $usage" - } - return $http($flag) - } elseif {[llength $args] % 2} { - return -code error "If more than one argument is supplied, the\ - number of arguments must be even" - } else { - foreach {flag value} $args { - if {![regexp -- $pat $flag]} { - return -code error "Unknown option $flag, must be: $usage" - } - if {($flag eq {-threadlevel}) && ($value ni {0 1 2})} { - return -code error {Option -threadlevel must be 0, 1 or 2} - } - set http($flag) $value - } - return - } -} - -# ------------------------------------------------------------------------------ -# Proc http::reasonPhrase -# ------------------------------------------------------------------------------ -# Command to return the IANA-recommended "reason phrase" for a HTTP Status Code. -# Information obtained from: -# https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml -# -# Arguments: -# code - A valid HTTP Status Code (integer from 100 to 599) -# -# Return Value: the reason phrase -# ------------------------------------------------------------------------------ - -proc http::reasonPhrase {code} { - variable reasonDict - if {![regexp -- {^[1-5][0-9][0-9]$} $code]} { - set msg {argument must be a three-digit integer from 100 to 599} - return -code error $msg - } - if {[dict exists $reasonDict $code]} { - set reason [dict get $reasonDict $code] - } else { - set reason Unassigned - } - return $reason -} - -# http::Finish -- -# -# Clean up the socket and eval close time callbacks -# -# Arguments: -# token Connection token. -# errormsg (optional) If set, forces status to error. -# skipCB (optional) If set, don't call the -command callback. This -# is useful when geturl wants to throw an exception instead -# of calling the callback. That way, the same error isn't -# reported to two places. -# -# Side Effects: -# May close the socket. - -proc http::Finish {token {errormsg ""} {skipCB 0}} { - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - variable $token - upvar 0 $token state - global errorInfo errorCode - set closeQueue 0 - if {$errormsg ne ""} { - set state(error) [list $errormsg $errorInfo $errorCode] - set state(status) "error" - } - if {[info commands ${token}--EventCoroutine] ne {}} { - rename ${token}--EventCoroutine {} - } - if {[info commands ${token}--SocketCoroutine] ne {}} { - rename ${token}--SocketCoroutine {} - } - if {[info exists state(socketcoro)]} { - Log $token Cancel socket after-idle event (Finish) - after cancel $state(socketcoro) - unset state(socketcoro) - } - - # Is this an upgrade request/response? - set upgradeResponse \ - [expr { [info exists state(upgradeRequest)] - && $state(upgradeRequest) - && [info exists state(http)] - && ([ncode $token] eq {101}) - && [info exists state(connection)] - && ("upgrade" in $state(connection)) - && [info exists state(upgrade)] - && ("" ne $state(upgrade)) - }] - - if { ($state(status) eq "timeout") - || ($state(status) eq "error") - || ($state(status) eq "eof") - } { - set closeQueue 1 - set connId $state(socketinfo) - if {[info exists state(sock)]} { - set sock $state(sock) - CloseSocket $state(sock) $token - } else { - # When opening the socket and calling http::reset - # immediately, the socket may not yet exist. - # Test http-4.11 may come here. - } - if {$state(tid) ne {}} { - # When opening the socket in a thread, and calling http::reset - # immediately, the thread may still exist. - # Test http-4.11 may come here. - thread::release $state(tid) - set state(tid) {} - } else { - } - } elseif {$upgradeResponse} { - # Special handling for an upgrade request/response. - # - geturl ensures that this is not a "persistent" socket used for - # multiple HTTP requests, so a call to KeepSocket is not needed. - # - Leave socket open, so a call to CloseSocket is not needed either. - # - Remove fileevent bindings. The caller will set its own bindings. - # - THE CALLER MUST PROCESS THE UPGRADED SOCKET IN THE CALLBACK COMMAND - # PASSED TO http::geturl AS -command callback. - catch {fileevent $state(sock) readable {}} - catch {fileevent $state(sock) writable {}} - } elseif { - ([info exists state(-keepalive)] && !$state(-keepalive)) - || ([info exists state(connection)] && ("close" in $state(connection))) - } { - set closeQueue 1 - set connId $state(socketinfo) - if {[info exists state(sock)]} { - set sock $state(sock) - CloseSocket $state(sock) $token - } else { - # When opening the socket and calling http::reset - # immediately, the socket may not yet exist. - # Test http-4.11 may come here. - } - } elseif { - ([info exists state(-keepalive)] && $state(-keepalive)) - && ([info exists state(connection)] && ("close" ni $state(connection))) - } { - KeepSocket $token - } - if {[info exists state(after)]} { - after cancel $state(after) - unset state(after) - } - if {[info exists state(-command)] && (!$skipCB) - && (![info exists state(done-command-cb)])} { - set state(done-command-cb) yes - if { [catch {namespace eval :: $state(-command) $token} err] - && ($errormsg eq "") - } { - set state(error) [list $err $errorInfo $errorCode] - set state(status) error - } - } - - if { $closeQueue - && [info exists socketMapping($connId)] - && ($socketMapping($connId) eq $sock) - } { - http::CloseQueuedQueries $connId $token - # This calls Unset. Other cases do not need the call. - } - return -} - -# http::KeepSocket - -# -# Keep a socket in the persistent sockets table and connect it to its next -# queued task if possible. Otherwise leave it idle and ready for its next -# use. -# -# If $socketClosing(*), then ("close" in $state(connection)) and therefore -# this command will not be called by Finish. -# -# Arguments: -# token Connection token. - -proc http::KeepSocket {token} { - variable http - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - variable $token - upvar 0 $token state - set tk [namespace tail $token] - - # Keep this socket open for another request ("Keep-Alive"). - # React if the server half-closes the socket. - # Discussion is in http::geturl. - catch {fileevent $state(sock) readable [list http::CheckEof $state(sock)]} - - # The line below should not be changed in production code. - # It is edited by the test suite. - set TEST_EOF 0 - if {$TEST_EOF} { - # ONLY for testing reaction to server eof. - # No server timeouts will be caught. - catch {fileevent $state(sock) readable {}} - } - - if { [info exists state(socketinfo)] - && [info exists socketMapping($state(socketinfo))] - } { - set connId $state(socketinfo) - # The value "Rready" is set only here. - set socketRdState($connId) Rready - - if { $state(-pipeline) - && [info exists socketRdQueue($connId)] - && [llength $socketRdQueue($connId)] - } { - # The usual case for pipelined responses - if another response is - # queued, arrange to read it. - set token3 [lindex $socketRdQueue($connId) 0] - set socketRdQueue($connId) [lrange $socketRdQueue($connId) 1 end] - - #Log pipelined, GRANT read access to $token3 in KeepSocket - set socketRdState($connId) $token3 - ReceiveResponse $token3 - - # Other pipelined cases. - # - The test above ensures that, for the pipelined cases in the two - # tests below, the read queue is empty. - # - In those two tests, check whether the next write will be - # nonpipeline. - } elseif { - $state(-pipeline) - && [info exists socketWrState($connId)] - && ($socketWrState($connId) eq "peNding") - - && [info exists socketWrQueue($connId)] - && [llength $socketWrQueue($connId)] - && (![set token3 [lindex $socketWrQueue($connId) 0] - set ${token3}(-pipeline) - ] - ) - } { - # This case: - # - Now it the time to run the "pending" request. - # - The next token in the write queue is nonpipeline, and - # socketWrState has been marked "pending" (in - # http::NextPipelinedWrite or http::geturl) so a new pipelined - # request cannot jump the queue. - # - # Tests: - # - In this case the read queue (tested above) is empty and this - # "pending" write token is in front of the rest of the write - # queue. - # - The write state is not Wready and therefore appears to be busy, - # but because it is "pending" we know that it is reserved for the - # first item in the write queue, a non-pipelined request that is - # waiting for the read queue to empty. That has now happened: so - # give that request read and write access. - set conn [set ${token3}(connArgs)] - #Log nonpipeline, GRANT r/w access to $token3 in KeepSocket - set socketRdState($connId) $token3 - set socketWrState($connId) $token3 - set socketWrQueue($connId) [lrange $socketWrQueue($connId) 1 end] - # Connect does its own fconfigure. - fileevent $state(sock) writable [list http::Connect $token3 {*}$conn] - #Log ---- $state(sock) << conn to $token3 for HTTP request (c) - - } elseif { - $state(-pipeline) - && [info exists socketWrState($connId)] - && ($socketWrState($connId) eq "peNding") - - } { - # Should not come here. The second block in the previous "elseif" - # test should be tautologous (but was needed in an earlier - # implementation) and will be removed after testing. - # If we get here, the value "pending" was assigned in error. - # This error would block the queue for ever. - Log ^X$tk <<<<< Error in queueing of requests >>>>> - token $token - - } elseif { - $state(-pipeline) - && [info exists socketWrState($connId)] - && ($socketWrState($connId) eq "Wready") - - && [info exists socketWrQueue($connId)] - && [llength $socketWrQueue($connId)] - && (![set token3 [lindex $socketWrQueue($connId) 0] - set ${token3}(-pipeline) - ] - ) - } { - # This case: - # - The next token in the write queue is nonpipeline, and - # socketWrState is Wready. Get the next event from socketWrQueue. - # Tests: - # - In this case the read state (tested above) is Rready and the - # write state (tested here) is Wready - there is no "pending" - # request. - # Code: - # - The code is the same as the code below for the nonpipelined - # case with a queued request. - set conn [set ${token3}(connArgs)] - #Log nonpipeline, GRANT r/w access to $token3 in KeepSocket - set socketRdState($connId) $token3 - set socketWrState($connId) $token3 - set socketWrQueue($connId) [lrange $socketWrQueue($connId) 1 end] - # Connect does its own fconfigure. - fileevent $state(sock) writable [list http::Connect $token3 {*}$conn] - #Log ---- $state(sock) << conn to $token3 for HTTP request (c) - - } elseif { - (!$state(-pipeline)) - && [info exists socketWrQueue($connId)] - && [llength $socketWrQueue($connId)] - && ("close" ni $state(connection)) - } { - # If not pipelined, (socketRdState eq Rready) tells us that we are - # ready for the next write - there is no need to check - # socketWrState. Write the next request, if one is waiting. - # If the next request is pipelined, it receives premature read - # access to the socket. This is not a problem. - set token3 [lindex $socketWrQueue($connId) 0] - set conn [set ${token3}(connArgs)] - #Log nonpipeline, GRANT r/w access to $token3 in KeepSocket - set socketRdState($connId) $token3 - set socketWrState($connId) $token3 - set socketWrQueue($connId) [lrange $socketWrQueue($connId) 1 end] - # Connect does its own fconfigure. - fileevent $state(sock) writable [list http::Connect $token3 {*}$conn] - #Log ---- $state(sock) << conn to $token3 for HTTP request (d) - - } elseif {(!$state(-pipeline))} { - set socketWrState($connId) Wready - # Rready and Wready and idle: nothing to do. - } - - } else { - CloseSocket $state(sock) $token - # There is no socketMapping($state(socketinfo)), so it does not matter - # that CloseQueuedQueries is not called. - } - return -} - -# http::CheckEof - -# -# Read from a socket and close it if eof. -# The command is bound to "fileevent readable" on an idle socket, and -# "eof" is the only event that should trigger the binding, occurring when -# the server times out and half-closes the socket. -# -# A read is necessary so that [eof] gives a meaningful result. -# Any bytes sent are junk (or a bug). - -proc http::CheckEof {sock} { - set junk [read $sock] - set n [string length $junk] - if {$n} { - Log "WARNING: $n bytes received but no HTTP request sent" - } - - if {[catch {eof $sock} res] || $res} { - # The server has half-closed the socket. - # If a new write has started, its transaction will fail and - # will then be error-handled. - CloseSocket $sock - } - return -} - -# http::CloseSocket - -# -# Close a socket and remove it from the persistent sockets table. If -# possible an http token is included here but when we are called from a -# fileevent on remote closure we need to find the correct entry - hence -# the "else" block of the first "if" command. - -proc http::CloseSocket {s {token {}}} { - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - set tk [namespace tail $token] - - catch {fileevent $s readable {}} - set connId {} - if {$token ne ""} { - variable $token - upvar 0 $token state - if {[info exists state(socketinfo)]} { - set connId $state(socketinfo) - } - } else { - set map [array get socketMapping] - set ndx [lsearch -exact $map $s] - if {$ndx >= 0} { - incr ndx -1 - set connId [lindex $map $ndx] - } - } - if { ($connId ne {}) - && [info exists socketMapping($connId)] - && ($socketMapping($connId) eq $s) - } { - Log "Closing connection $connId (sock $socketMapping($connId))" - if {[catch {close $socketMapping($connId)} err]} { - Log "Error closing connection: $err" - } else { - } - if {$token eq {}} { - # Cases with a non-empty token are handled by Finish, so the tokens - # are finished in connection order. - http::CloseQueuedQueries $connId - } else { - } - } else { - Log "Closing socket $s (no connection info)" - if {[catch {close $s} err]} { - Log "Error closing socket: $err" - } else { - } - } - return -} - -# http::CloseQueuedQueries -# -# connId - identifier "domain:port" for the connection -# token - (optional) used only for logging -# -# Called from http::CloseSocket and http::Finish, after a connection is closed, -# to clear the read and write queues if this has not already been done. - -proc http::CloseQueuedQueries {connId {token {}}} { - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - ##Log CloseQueuedQueries $connId $token - if {![info exists socketMapping($connId)]} { - # Command has already been called. - # Don't come here again - especially recursively. - return - } - - # Used only for logging. - if {$token eq {}} { - set tk {} - } else { - set tk [namespace tail $token] - } - - if { [info exists socketPlayCmd($connId)] - && ($socketPlayCmd($connId) ne {ReplayIfClose Wready {} {}}) - } { - # Before unsetting, there is some unfinished business. - # - If the server sent "Connection: close", we have stored the command - # for retrying any queued requests in socketPlayCmd, so copy that - # value for execution below. socketClosing(*) was also set. - # - Also clear the queues to prevent calls to Finish that would set the - # state for the requests that will be retried to "finished with error - # status". - # - At this stage socketPhQueue is empty. - set unfinished $socketPlayCmd($connId) - set socketRdQueue($connId) {} - set socketWrQueue($connId) {} - } else { - set unfinished {} - } - - Unset $connId - - if {$unfinished ne {}} { - Log ^R$tk Any unfinished transactions (excluding $token) failed \ - - token $token - unfinished $unfinished - {*}$unfinished - # Calls ReplayIfClose. - } - return -} - -# http::Unset -# -# The trace on "unset socketRdState(*)" will call CancelReadPipeline -# and cancel any queued responses. -# The trace on "unset socketWrState(*)" will call CancelWritePipeline -# and cancel any queued requests. - -proc http::Unset {connId} { - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - unset socketMapping($connId) - unset socketRdState($connId) - unset socketWrState($connId) - unset -nocomplain socketRdQueue($connId) - unset -nocomplain socketWrQueue($connId) - unset -nocomplain socketClosing($connId) - unset -nocomplain socketPlayCmd($connId) - unset -nocomplain socketProxyId($connId) - return -} - -# http::reset -- -# -# See documentation for details. -# -# Arguments: -# token Connection token. -# why Status info. -# -# Side Effects: -# See Finish - -proc http::reset {token {why reset}} { - variable $token - upvar 0 $token state - set state(status) $why - catch {fileevent $state(sock) readable {}} - catch {fileevent $state(sock) writable {}} - Finish $token - if {[info exists state(error)]} { - set errorlist $state(error) - unset state - eval ::error $errorlist - # i.e. error msg errorInfo errorCode - } - return -} - -# http::geturl -- -# -# Establishes a connection to a remote url via http. -# -# Arguments: -# url The http URL to goget. -# args Option value pairs. Valid options include: -# -blocksize, -validate, -headers, -timeout -# Results: -# Returns a token for this connection. This token is the name of an -# array that the caller should unset to garbage collect the state. - -proc http::geturl {url args} { - variable urlTypes - - # - If ::tls::socketCmd has its default value "::socket", change it to the - # new value ::http::socketForTls. - # - If the old value is different, then it has been modified either by the - # script or by the Tcl installation, and replaced by a new command. The - # script or installation that modified ::tls::socketCmd is also - # responsible for integrating ::http::socketForTls into its own "new" - # command, if it wishes to do so. - # - Commands that open a socket: - # - ::socket - basic - # - ::http::socket - can use a thread to avoid blockage by slow DNS - # lookup. See http::config option -threadlevel. - # - ::http::socketForTls - as ::http::socket, but can also open a socket - # for HTTPS/TLS through a proxy. - - if {[info exists ::tls::socketCmd] && ($::tls::socketCmd eq {::socket})} { - set ::tls::socketCmd ::http::socketForTls - } - - set token [CreateToken $url {*}$args] - variable $token - upvar 0 $token state - - AsyncTransaction $token - - # -------------------------------------------------------------------------- - # Synchronous Call to http::geturl - # -------------------------------------------------------------------------- - # - If the call to http::geturl is asynchronous, it is now complete (apart - # from delivering the return value). - # - If the call to http::geturl is synchronous, the command must now wait - # for the HTTP transaction to be completed. The call to http::wait uses - # vwait, which may be inappropriate if the caller makes other HTTP - # requests in the background. - # -------------------------------------------------------------------------- - - if {![info exists state(-command)]} { - # geturl does EVERYTHING asynchronously, so if the user - # calls it synchronously, we just do a wait here. - http::wait $token - - if {![info exists state]} { - # If we timed out then Finish has been called and the users - # command callback may have cleaned up the token. If so we end up - # here with nothing left to do. - return $token - } elseif {$state(status) eq "error"} { - # Something went wrong while trying to establish the connection. - # Clean up after events and such, but DON'T call the command - # callback (if available) because we're going to throw an - # exception from here instead. - set err [lindex $state(error) 0] - cleanup $token - return -code error $err - } - } - - return $token -} - -# ------------------------------------------------------------------------------ -# Proc http::CreateToken -# ------------------------------------------------------------------------------ -# Command to convert arguments into an initialised request token. -# The return value is the variable name of the token. -# -# Other effects: -# - Sets ::http::http(usingThread) if not already done -# - Sets ::http::http(uid) if not already done -# - Increments ::http::http(uid) -# - May increment ::http::TmpSockCounter -# - Alters ::http::socketPlayCmd, ::http::socketWrQueue if a -keepalive 1 -# request is appended to the queue of a persistent socket that is already -# scheduled to close. -# This also sets state(alreadyQueued) to 1. -# - Alters ::http::socketPhQueue if a -keepalive 1 request is appended to the -# queue of a persistent socket that has not yet been created (and is therefore -# represented by a placeholder). -# This also sets state(ReusingPlaceholder) to 1. -# ------------------------------------------------------------------------------ - -proc http::CreateToken {url args} { - variable http - variable urlTypes - variable defaultCharset - variable defaultKeepalive - variable strict - variable TmpSockCounter - - # Initialize the state variable, an array. We'll return the name of this - # array as the token for the transaction. - - if {![info exists http(usingThread)]} { - set http(usingThread) 0 - } - if {![info exists http(uid)]} { - set http(uid) 0 - } - set token [namespace current]::[incr http(uid)] - ##Log Starting http::geturl - token $token - variable $token - upvar 0 $token state - set tk [namespace tail $token] - reset $token - Log ^A$tk URL $url - token $token - - # Process command options. - - array set state { - -binary false - -blocksize 8192 - -queryblocksize 8192 - -validate 0 - -headers {} - -timeout 0 - -type application/x-www-form-urlencoded - -queryprogress {} - -protocol 1.1 - -guesstype 0 - binary 0 - state created - meta {} - method {} - coding {} - currentsize 0 - totalsize 0 - querylength 0 - queryoffset 0 - type application/octet-stream - body {} - status "" - http "" - httpResponse {} - responseCode {} - reasonPhrase {} - connection keep-alive - tid {} - requestHeaders {} - requestLine {} - transfer {} - proxyUsed none - } - set state(-keepalive) $defaultKeepalive - set state(-strict) $strict - # These flags have their types verified [Bug 811170] - array set type { - -binary boolean - -blocksize integer - -guesstype boolean - -queryblocksize integer - -strict boolean - -timeout integer - -validate boolean - -headers list - } - set state(charset) $defaultCharset - set options { - -binary -blocksize -channel -command -guesstype -handler -headers -keepalive - -method -myaddr -progress -protocol -query -queryblocksize - -querychannel -queryprogress -strict -timeout -type -validate - } - set usage [join [lsort $options] ", "] - set options [string map {- ""} $options] - set pat ^-(?:[join $options |])$ - foreach {flag value} $args { - if {[regexp -- $pat $flag]} { - # Validate numbers - if { [info exists type($flag)] - && (![string is $type($flag) -strict $value]) - } { - unset $token - return -code error \ - "Bad value for $flag ($value), must be $type($flag)" - } - if {($flag eq "-headers") && ([llength $value] % 2 != 0)} { - unset $token - return -code error "Bad value for $flag ($value), number\ - of list elements must be even" - } - set state($flag) $value - } else { - unset $token - return -code error "Unknown option $flag, can be: $usage" - } - } - - # Make sure -query and -querychannel aren't both specified - - set isQueryChannel [info exists state(-querychannel)] - set isQuery [info exists state(-query)] - if {$isQuery && $isQueryChannel} { - unset $token - return -code error "Can't combine -query and -querychannel options!" - } - - # Validate URL, determine the server host and port, and check proxy case - # Recognize user:pass@host URLs also, although we do not do anything with - # that info yet. - - # URLs have basically four parts. - # First, before the colon, is the protocol scheme (e.g. http) - # Second, for HTTP-like protocols, is the authority - # The authority is preceded by // and lasts up to (but not including) - # the following / or ? and it identifies up to four parts, of which - # only one, the host, is required (if an authority is present at all). - # All other parts of the authority (user name, password, port number) - # are optional. - # Third is the resource name, which is split into two parts at a ? - # The first part (from the single "/" up to "?") is the path, and the - # second part (from that "?" up to "#") is the query. *HOWEVER*, we do - # not need to separate them; we send the whole lot to the server. - # Both, path and query are allowed to be missing, including their - # delimiting character. - # Fourth is the fragment identifier, which is everything after the first - # "#" in the URL. The fragment identifier MUST NOT be sent to the server - # and indeed, we don't bother to validate it (it could be an error to - # pass it in here, but it's cheap to strip). - # - # An example of a URL that has all the parts: - # - # http://jschmoe:xyzzy@www.bogus.net:8000/foo/bar.tml?q=foo#changes - # - # The "http" is the protocol, the user is "jschmoe", the password is - # "xyzzy", the host is "www.bogus.net", the port is "8000", the path is - # "/foo/bar.tml", the query is "q=foo", and the fragment is "changes". - # - # Note that the RE actually combines the user and password parts, as - # recommended in RFC 3986. Indeed, that RFC states that putting passwords - # in URLs is a Really Bad Idea, something with which I would agree utterly. - # RFC 9110 Sec 4.2.4 goes further than this, and deprecates the format - # "user:password@". It is retained here for backward compatibility, - # but its use is not recommended. - # - # From a validation perspective, we need to ensure that the parts of the - # URL that are going to the server are correctly encoded. This is only - # done if $state(-strict) is true (inherited from $::http::strict). - - set URLmatcher {(?x) # this is _expanded_ syntax - ^ - (?: (\w+) : ) ? # - (?: // - (?: - ( - [^@/\#?]+ # - ) @ - )? - ( # - [^/:\#?]+ | # host name or IPv4 address - \[ [^/\#?]+ \] # IPv6 address in square brackets - ) - (?: : (\d+) )? # - )? - ( [/\?] [^\#]*)? # (including query) - (?: \# (.*) )? # - $ - } - - # Phase one: parse - if {![regexp -- $URLmatcher $url -> proto user host port srvurl]} { - unset $token - return -code error "Unsupported URL: $url" - } - # Phase two: validate - set host [string trim $host {[]}]; # strip square brackets from IPv6 address - if {$host eq ""} { - # Caller has to provide a host name; we do not have a "default host" - # that would enable us to handle relative URLs. - unset $token - return -code error "Missing host part: $url" - # Note that we don't check the hostname for validity here; if it's - # invalid, we'll simply fail to resolve it later on. - } - if {$port ne "" && $port > 65535} { - unset $token - return -code error "Invalid port number: $port" - } - # The user identification and resource identification parts of the URL can - # have encoded characters in them; take care! - if {$user ne ""} { - # Check for validity according to RFC 3986, Appendix A - set validityRE {(?xi) - ^ - (?: [-\w.~!$&'()*+,;=:] | %[0-9a-f][0-9a-f] )+ - $ - } - if {$state(-strict) && ![regexp -- $validityRE $user]} { - unset $token - # Provide a better error message in this error case - if {[regexp {(?i)%(?![0-9a-f][0-9a-f]).?.?} $user bad]} { - return -code error \ - "Illegal encoding character usage \"$bad\" in URL user" - } - return -code error "Illegal characters in URL user" - } - } - if {$srvurl ne ""} { - # RFC 3986 allows empty paths (not even a /), but servers - # return 400 if the path in the HTTP request doesn't start - # with / , so add it here if needed. - if {[string index $srvurl 0] ne "/"} { - set srvurl /$srvurl - } - # Check for validity according to RFC 3986, Appendix A - set validityRE {(?xi) - ^ - # Path part (already must start with / character) - (?: [-\w.~!$&'()*+,;=:@/] | %[0-9a-f][0-9a-f] )* - # Query part (optional, permits ? characters) - (?: \? (?: [-\w.~!$&'()*+,;=:@/?] | %[0-9a-f][0-9a-f] )* )? - $ - } - if {$state(-strict) && ![regexp -- $validityRE $srvurl]} { - unset $token - # Provide a better error message in this error case - if {[regexp {(?i)%(?![0-9a-f][0-9a-f])..} $srvurl bad]} { - return -code error \ - "Illegal encoding character usage \"$bad\" in URL path" - } - return -code error "Illegal characters in URL path" - } - if {![regexp {^[^?#]+} $srvurl state(path)]} { - set state(path) / - } - } else { - set srvurl / - set state(path) / - } - if {$proto eq ""} { - set proto http - } - set lower [string tolower $proto] - if {![info exists urlTypes($lower)]} { - unset $token - return -code error "Unsupported URL type \"$proto\"" - } - set defport [lindex $urlTypes($lower) 0] - set defcmd [lindex $urlTypes($lower) 1] - - if {$port eq ""} { - set port $defport - } - if {![catch {$http(-proxyfilter) $host} proxy]} { - set phost [lindex $proxy 0] - set pport [lindex $proxy 1] - } else { - set phost {} - set pport {} - } - - # OK, now reassemble into a full URL - set url ${proto}:// - if {$user ne ""} { - append url $user - append url @ - } - append url $host - if {$port != $defport} { - append url : $port - } - append url $srvurl - # Don't append the fragment! RFC 7230 Sec 5.1 - set state(url) $url - - # Proxy connections aren't shared among different hosts. - set state(socketinfo) $host:$port - - # Save the accept types at this point to prevent a race condition. [Bug - # c11a51c482] - set state(accept-types) $http(-accept) - - # Check whether this is an Upgrade request. - set connectionValues [SplitCommaSeparatedFieldValue \ - [GetFieldValue $state(-headers) Connection]] - set connectionValues [string tolower $connectionValues] - set upgradeValues [SplitCommaSeparatedFieldValue \ - [GetFieldValue $state(-headers) Upgrade]] - set state(upgradeRequest) [expr { "upgrade" in $connectionValues - && [llength $upgradeValues] >= 1}] - set state(connectionValues) $connectionValues - - if {$isQuery || $isQueryChannel} { - # It's a POST. - # A client wishing to send a non-idempotent request SHOULD wait to send - # that request until it has received the response status for the - # previous request. - if {$http(-postfresh)} { - # Override -keepalive for a POST. Use a new connection, and thus - # avoid the small risk of a race against server timeout. - set state(-keepalive) 0 - } else { - # Allow -keepalive but do not -pipeline - wait for the previous - # transaction to finish. - # There is a small risk of a race against server timeout. - set state(-pipeline) 0 - } - } elseif {$state(upgradeRequest)} { - # It's an upgrade request. Method must be GET (untested). - # Force -keepalive to 0 so the connection is not made over a persistent - # socket, i.e. one used for multiple HTTP requests. - set state(-keepalive) 0 - } else { - # It's a non-upgrade GET or HEAD. - set state(-pipeline) $http(-pipeline) - } - - # We cannot handle chunked encodings with -handler, so force HTTP/1.0 - # until we can manage this. - if {[info exists state(-handler)]} { - set state(-protocol) 1.0 - } - - # RFC 7320 A.1 - HTTP/1.0 Keep-Alive is problematic. We do not support it. - if {$state(-protocol) eq "1.0"} { - set state(connection) close - set state(-keepalive) 0 - } - - # Handle proxy requests here for http:// but not for https:// - # The proxying for https is done in the ::http::socketForTls command. - # A proxy request for http:// needs the full URL in the HTTP request line, - # including the server name. - # The *tls* test below attempts to describe protocols in addition to - # "https on port 443" that use HTTP over TLS. - if {($phost ne "") && (![string match -nocase *tls* $defcmd])} { - set srvurl $url - set targetAddr [list $phost $pport] - set state(proxyUsed) HttpProxy - # The value of state(proxyUsed) none|HttpProxy depends only on the - # all-transactions http::config settings and on the target URL. - # Even if this is a persistent socket there is no need to change the - # value of state(proxyUsed) for other transactions that use the socket: - # they have the same value already. - } else { - set targetAddr [list $host $port] - } - - set sockopts [list -async] - - # Pass -myaddr directly to the socket command - if {[info exists state(-myaddr)]} { - lappend sockopts -myaddr $state(-myaddr) - } - - set state(connArgs) [list $proto $phost $srvurl] - set state(openCmd) [list {*}$defcmd {*}$sockopts -type $token {*}$targetAddr] - - # See if we are supposed to use a previously opened channel. - # - In principle, ANY call to http::geturl could use a previously opened - # channel if it is available - the "Connection: keep-alive" header is a - # request to leave the channel open AFTER completion of this call. - # - In fact, we try to use an existing channel only if -keepalive 1 -- this - # means that at most one channel is left open for each value of - # $state(socketinfo). This property simplifies the mapping of open - # channels. - set reusing 0 - set state(alreadyQueued) 0 - set state(ReusingPlaceholder) 0 - if {$state(-keepalive)} { - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - if {[info exists socketMapping($state(socketinfo))]} { - # - If the connection is idle, it has a "fileevent readable" binding - # to http::CheckEof, in case the server times out and half-closes - # the socket (http::CheckEof closes the other half). - # - We leave this binding in place until just before the last - # puts+flush in http::Connected (GET/HEAD) or http::Write (POST), - # after which the HTTP response might be generated. - - if { [info exists socketClosing($state(socketinfo))] - && $socketClosing($state(socketinfo)) - } { - # socketClosing(*) is set because the server has sent a - # "Connection: close" header. - # Do not use the persistent socket again. - # Since we have only one persistent socket per server, and the - # old socket is not yet dead, add the request to the write queue - # of the dying socket, which will be replayed by ReplayIfClose. - # Also add it to socketWrQueue(*) which is used only if an error - # causes a call to Finish. - set reusing 1 - set sock $socketMapping($state(socketinfo)) - set state(proxyUsed) $socketProxyId($state(socketinfo)) - Log "reusing closing socket $sock for $state(socketinfo) - token $token" - - set state(alreadyQueued) 1 - lassign $socketPlayCmd($state(socketinfo)) com0 com1 com2 com3 - lappend com3 $token - set socketPlayCmd($state(socketinfo)) [list $com0 $com1 $com2 $com3] - lappend socketWrQueue($state(socketinfo)) $token - ##Log socketPlayCmd($state(socketinfo)) is $socketPlayCmd($state(socketinfo)) - ##Log socketWrQueue($state(socketinfo)) is $socketWrQueue($state(socketinfo)) - } elseif { - [catch {fconfigure $socketMapping($state(socketinfo))}] - && (![SockIsPlaceHolder $socketMapping($state(socketinfo))]) - } { - ###Log "Socket $socketMapping($state(socketinfo)) for $state(socketinfo)" - # FIXME Is it still possible for this code to be executed? If - # so, this could be another place to call TestForReplay, - # rather than discarding the queued transactions. - Log "WARNING: socket for $state(socketinfo) was closed\ - - token $token" - Log "WARNING - if testing, pay special attention to this\ - case (GH) which is seldom executed - token $token" - - # This will call CancelReadPipeline, CancelWritePipeline, and - # cancel any queued requests, responses. - Unset $state(socketinfo) - } else { - # Use the persistent socket. - # - The socket may not be ready to write: an earlier request might - # still be still writing (in the pipelined case) or - # writing/reading (in the nonpipeline case). This possibility - # is handled by socketWrQueue later in this command. - # - The socket may not yet exist, and be defined with a placeholder. - set reusing 1 - set sock $socketMapping($state(socketinfo)) - set state(proxyUsed) $socketProxyId($state(socketinfo)) - if {[SockIsPlaceHolder $sock]} { - set state(ReusingPlaceholder) 1 - lappend socketPhQueue($sock) $token - } else { - } - Log "reusing open socket $sock for $state(socketinfo) - token $token" - } - # Do not automatically close the connection socket. - set state(connection) keep-alive - } - } - - set state(reusing) $reusing - unset reusing - - if {![info exists sock]} { - # N.B. At this point ([info exists sock] == $state(reusing)). - # This will no longer be true after we set a value of sock here. - # Give the socket a placeholder name. - set sock HTTP_PLACEHOLDER_[incr TmpSockCounter] - } - set state(sock) $sock - - if {$state(reusing)} { - # Define these for use (only) by http::ReplayIfDead if the persistent - # connection has died. - set state(tmpConnArgs) $state(connArgs) - set state(tmpState) [array get state] - set state(tmpOpenCmd) $state(openCmd) - } - return $token -} - - -# ------------------------------------------------------------------------------ -# Proc ::http::SockIsPlaceHolder -# ------------------------------------------------------------------------------ -# Command to return 0 if the argument is a genuine socket handle, or 1 if is a -# placeholder value generated by geturl or ReplayCore before the real socket is -# created. -# -# Arguments: -# sock - either a valid socket handle or a placeholder value -# -# Return Value: 0 or 1 -# ------------------------------------------------------------------------------ - -proc http::SockIsPlaceHolder {sock} { - expr {[string range $sock 0 16] eq {HTTP_PLACEHOLDER_}} -} - - -# ------------------------------------------------------------------------------ -# state(reusing) -# ------------------------------------------------------------------------------ -# - state(reusing) is set by geturl, ReplayCore -# - state(reusing) is used by geturl, AsyncTransaction, OpenSocket, -# ConfigureNewSocket, and ScheduleRequest when creating and configuring the -# connection. -# - state(reusing) is used by Connect, Connected, Event x 2 when deciding -# whether to call TestForReplay. -# - Other places where state(reusing) is used: -# - Connected - if reusing and not pipelined, start the state(-timeout) -# timeout (when writing). -# - DoneRequest - if reusing and pipelined, send the next pipelined write -# - Event - if reusing and pipelined, start the state(-timeout) -# timeout (when reading). -# - Event - if (not reusing) and pipelined, send the next pipelined -# write. -# ------------------------------------------------------------------------------ - - -# ------------------------------------------------------------------------------ -# Proc http::AsyncTransaction -# ------------------------------------------------------------------------------ -# This command is called by geturl and ReplayCore to prepare the HTTP -# transaction prescribed by a suitably prepared token. -# -# Arguments: -# token - connection token (name of an array) -# -# Return Value: none -# ------------------------------------------------------------------------------ - -proc http::AsyncTransaction {token} { - variable $token - upvar 0 $token state - set tk [namespace tail $token] - - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - set sock $state(sock) - - # See comments above re the start of this timeout in other cases. - if {(!$state(reusing)) && ($state(-timeout) > 0)} { - set state(after) [after $state(-timeout) \ - [list http::reset $token timeout]] - } - - if { $state(-keepalive) - && (![info exists socketMapping($state(socketinfo))]) - } { - # This code is executed only for the first -keepalive request on a - # socket. It makes the socket persistent. - ##Log " PreparePersistentConnection" $token -- $sock -- DO - set DoLater [PreparePersistentConnection $token] - } else { - ##Log " PreparePersistentConnection" $token -- $sock -- SKIP - set DoLater {-traceread 0 -tracewrite 0} - } - - if {$state(ReusingPlaceholder)} { - # - This request was added to the socketPhQueue of a persistent - # connection. - # - But the connection has not yet been created and is a placeholder; - # - And the placeholder was created by an earlier request. - # - When that earlier request calls OpenSocket, its placeholder is - # replaced with a true socket, and it then executes the equivalent of - # OpenSocket for any subsequent requests that have - # $state(ReusingPlaceholder). - Log >J$tk after idle coro NO - ReusingPlaceholder - } elseif {$state(alreadyQueued)} { - # - This request was added to the socketWrQueue and socketPlayCmd - # of a persistent connection that will close at the end of its current - # read operation. - Log >J$tk after idle coro NO - alreadyQueued - } else { - Log >J$tk after idle coro YES - set CoroName ${token}--SocketCoroutine - set cancel [after idle [list coroutine $CoroName ::http::OpenSocket \ - $token $DoLater]] - dict set socketCoEvent($state(socketinfo)) $token $cancel - set state(socketcoro) $cancel - } - - return -} - - -# ------------------------------------------------------------------------------ -# Proc http::PreparePersistentConnection -# ------------------------------------------------------------------------------ -# This command is called by AsyncTransaction to initialise a "persistent -# connection" based upon a socket placeholder. It is called the first time the -# socket is associated with a "-keepalive" request. -# -# Arguments: -# token - connection token (name of an array) -# -# Return Value: - DoLater, a dictionary of boolean values listing unfinished -# tasks; to be passed to ConfigureNewSocket via OpenSocket. -# ------------------------------------------------------------------------------ - -proc http::PreparePersistentConnection {token} { - variable $token - upvar 0 $token state - - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - set DoLater {-traceread 0 -tracewrite 0} - set socketMapping($state(socketinfo)) $state(sock) - set socketProxyId($state(socketinfo)) $state(proxyUsed) - # - The value of state(proxyUsed) was set in http::CreateToken to either - # "none" or "HttpProxy". - # - $token is the first transaction to use this placeholder, so there are - # no other tokens whose (proxyUsed) must be modified. - - if {![info exists socketRdState($state(socketinfo))]} { - set socketRdState($state(socketinfo)) {} - # set varName ::http::socketRdState($state(socketinfo)) - # trace add variable $varName unset ::http::CancelReadPipeline - dict set DoLater -traceread 1 - } - if {![info exists socketWrState($state(socketinfo))]} { - set socketWrState($state(socketinfo)) {} - # set varName ::http::socketWrState($state(socketinfo)) - # trace add variable $varName unset ::http::CancelWritePipeline - dict set DoLater -tracewrite 1 - } - - if {$state(-pipeline)} { - #Log new, init for pipelined, GRANT write access to $token in geturl - # Also grant premature read access to the socket. This is OK. - set socketRdState($state(socketinfo)) $token - set socketWrState($state(socketinfo)) $token - } else { - # socketWrState is not used by this non-pipelined transaction. - # We cannot leave it as "Wready" because the next call to - # http::geturl with a pipelined transaction would conclude that the - # socket is available for writing. - #Log new, init for nonpipeline, GRANT r/w access to $token in geturl - set socketRdState($state(socketinfo)) $token - set socketWrState($state(socketinfo)) $token - } - - # Value of socketPhQueue() may have already been set by ReplayCore. - if {![info exists socketPhQueue($state(sock))]} { - set socketPhQueue($state(sock)) {} - } - set socketRdQueue($state(socketinfo)) {} - set socketWrQueue($state(socketinfo)) {} - set socketClosing($state(socketinfo)) 0 - set socketPlayCmd($state(socketinfo)) {ReplayIfClose Wready {} {}} - set socketCoEvent($state(socketinfo)) {} - set socketProxyId($state(socketinfo)) {} - - return $DoLater -} - -# ------------------------------------------------------------------------------ -# Proc ::http::OpenSocket -# ------------------------------------------------------------------------------ -# This command is called as a coroutine idletask to start the asynchronous HTTP -# transaction in most cases. For the exceptions, see the calling code in -# command AsyncTransaction. -# -# Arguments: -# token - connection token (name of an array) -# DoLater - dictionary of boolean values listing unfinished tasks -# -# Return Value: none -# ------------------------------------------------------------------------------ - -proc http::OpenSocket {token DoLater} { - variable $token - upvar 0 $token state - set tk [namespace tail $token] - - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - Log >K$tk Start OpenSocket coroutine - - if {![info exists state(-keepalive)]} { - # The request has already been cancelled by the calling script. - return - } - - set sockOld $state(sock) - - dict unset socketCoEvent($state(socketinfo)) $token - unset -nocomplain state(socketcoro) - - if {[catch { - if {$state(reusing)} { - # If ($state(reusing)) is true, then we do not need to create a new - # socket, even if $sockOld is only a placeholder for a socket. - set sock $sockOld - } else { - # set sock in the [catch] below. - set pre [clock milliseconds] - ##Log pre socket opened, - token $token - ##Log $state(openCmd) - token $token - set sock [namespace eval :: $state(openCmd)] - set state(sock) $sock - # Normal return from $state(openCmd) always returns a valid socket. - # A TLS proxy connection with 407 or other failure from the - # proxy server raises an error. - - # Initialisation of a new socket. - ##Log post socket opened, - token $token - ##Log socket opened, now fconfigure - token $token - set delay [expr {[clock milliseconds] - $pre}] - if {$delay > 3000} { - Log socket delay $delay - token $token - } - fconfigure $sock -translation {auto crlf} \ - -buffersize $state(-blocksize) - if {[package vsatisfies [package provide Tcl] 9.0-]} { - fconfigure $sock -profile tcl8 - } - ##Log socket opened, DONE fconfigure - token $token - } - - Log "Using $sock for $state(socketinfo) - token $token" \ - [expr {$state(-keepalive)?"keepalive":""}] - - # Code above has set state(sock) $sock - ConfigureNewSocket $token $sockOld $DoLater - ##Log OpenSocket success $sock - token $token - } result errdict]} { - ##Log OpenSocket failed $result - token $token - # There may be other requests in the socketPhQueue. - # Prepare socketPlayCmd so that Finish will replay them. - if { ($state(-keepalive)) && (!$state(reusing)) - && [info exists socketPhQueue($sockOld)] - && ($socketPhQueue($sockOld) ne {}) - } { - if {$socketMapping($state(socketinfo)) ne $sockOld} { - Log "WARNING: this code should not be reached.\ - {$socketMapping($state(socketinfo)) ne $sockOld}" - } - set socketPlayCmd($state(socketinfo)) [list ReplayIfClose Wready {} $socketPhQueue($sockOld)] - set socketPhQueue($sockOld) {} - } - if {[string range $result 0 20] eq {proxy connect failed:}} { - # - The HTTPS proxy did not create a socket. The pre-existing value - # (a "placeholder socket") is unchanged. - # - The proxy returned a valid HTTP response to the failed CONNECT - # request, and http::SecureProxyConnect copied this to $token, - # and also set ${token}(connection) set to "close". - # - Remove the error message $result so that Finish delivers this - # HTTP response to the caller. - set result {} - } - Finish $token $result - # Because socket creation failed, the placeholder "socket" must be - # "closed" and (if persistent) removed from the persistent sockets - # table. In the {proxy connect failed:} case Finish does this because - # the value of ${token}(connection) is "close". In the other cases here, - # it does so because $result is non-empty. - } - ##Log Leaving http::OpenSocket coroutine [info coroutine] - token $token - return -} - - -# ------------------------------------------------------------------------------ -# Proc ::http::ConfigureNewSocket -# ------------------------------------------------------------------------------ -# Command to initialise a newly-created socket. Called only from OpenSocket. -# -# This command is called by OpenSocket whenever a genuine socket (sockNew) has -# been opened for for use by HTTP. It does two things: -# (1) If $token uses a placeholder socket, this command replaces the placeholder -# socket with the real socket, not only in $token but in all other requests -# that use the same placeholder. -# (2) It calls ScheduleRequest to schedule each request that uses the socket. -# -# -# Value of sockOld/sockNew can be "sock" (genuine socket) or "ph" (placeholder). -# sockNew is ${token}(sock) -# sockOld sockNew CASES -# sock sock (if $reusing, and sockOld is sock) -# ph sock (if (not $reusing), and sockOld is ph) -# ph ph (if $reusing, and sockOld is ph) - not called in this case -# sock ph (cannot occur unless a bug) - not called in this case -# (if (not $reusing), and sockOld is sock) - illogical -# -# Arguments: -# token - connection token (name of an array) -# sockOld - handle or placeholder used for a socket before the call to -# OpenSocket -# DoLater - dictionary of boolean values listing unfinished tasks -# -# Return Value: none -# ------------------------------------------------------------------------------ - -proc http::ConfigureNewSocket {token sockOld DoLater} { - variable $token - upvar 0 $token state - set tk [namespace tail $token] - - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - set reusing $state(reusing) - set sock $state(sock) - set proxyUsed $state(proxyUsed) - ##Log " ConfigureNewSocket" $token $sockOld ... -- $reusing $sock $proxyUsed - - if {(!$reusing) && ($sock ne $sockOld)} { - # Replace the placeholder value sockOld with sock. - - if { [info exists socketMapping($state(socketinfo))] - && ($socketMapping($state(socketinfo)) eq $sockOld) - } { - set socketMapping($state(socketinfo)) $sock - set socketProxyId($state(socketinfo)) $proxyUsed - # tokens that use the placeholder $sockOld are updated below. - ##Log set socketMapping($state(socketinfo)) $sock - } - - # Now finish any tasks left over from PreparePersistentConnection on - # the connection. - # - # The "unset" traces are fired by init (clears entire arrays), and - # by http::Unset. - # Unset is called by CloseQueuedQueries and (possibly never) by geturl. - # - # CancelReadPipeline, CancelWritePipeline call http::Finish for each - # token. - # - # FIXME If Finish is placeholder-aware, these traces can be set earlier, - # in PreparePersistentConnection. - - if {[dict get $DoLater -traceread]} { - set varName ::http::socketRdState($state(socketinfo)) - trace add variable $varName unset ::http::CancelReadPipeline - } - if {[dict get $DoLater -tracewrite]} { - set varName ::http::socketWrState($state(socketinfo)) - trace add variable $varName unset ::http::CancelWritePipeline - } - } - - # Do this in all cases. - ScheduleRequest $token - - # Now look at all other tokens that use the placeholder $sockOld. - if { (!$reusing) - && ($sock ne $sockOld) - && [info exists socketPhQueue($sockOld)] - } { - ##Log " ConfigureNewSocket" $token scheduled, now do $socketPhQueue($sockOld) - foreach tok $socketPhQueue($sockOld) { - # 1. Amend the token's (sock). - ##Log set ${tok}(sock) $sock - set ${tok}(sock) $sock - set ${tok}(proxyUsed) $proxyUsed - - # 2. Schedule the token's HTTP request. - # Every token in socketPhQueue(*) has reusing 1 alreadyQueued 0. - set ${tok}(reusing) 1 - set ${tok}(alreadyQueued) 0 - ScheduleRequest $tok - } - set socketPhQueue($sockOld) {} - } - ##Log " ConfigureNewSocket" $token DONE - - return -} - - -# ------------------------------------------------------------------------------ -# The values of array variables socketMapping etc. -# ------------------------------------------------------------------------------ -# connId "$host:$port" -# socketMapping($connId) the handle or placeholder for the socket that is used -# for "-keepalive 1" requests to $connId. -# socketRdState($connId) the token that is currently reading from the socket. -# Other values: Rready (ready for next token to read). -# socketWrState($connId) the token that is currently writing to the socket. -# Other values: Wready (ready for next token to write), -# peNding (would be ready for next write, except that -# the integrity of a non-pipelined transaction requires -# waiting until the read(s) in progress are finished). -# socketRdQueue($connId) List of tokens that are queued for reading later. -# socketWrQueue($connId) List of tokens that are queued for writing later. -# socketPhQueue($sock) List of tokens that are queued to use a placeholder -# socket, when the real socket has not yet been created. -# socketClosing($connId) (boolean) true iff a server response header indicates -# that the server will close the connection at the end of -# the current response. -# socketPlayCmd($connId) The command to execute to replay pending and -# part-completed transactions if the socket closes early. -# socketCoEvent($connId) Identifier for the "after idle" event that will launch -# an OpenSocket coroutine to open or re-use a socket. -# socketProxyId($connId) The type of proxy that this socket uses: values are -# those of state(proxyUsed) i.e. none, HttpProxy, -# SecureProxy, and SecureProxyFailed. -# The value is not used for anything by http, its purpose -# is to set the value of state() for caller information. -# ------------------------------------------------------------------------------ - - -# ------------------------------------------------------------------------------ -# Using socketWrState(*), socketWrQueue(*), socketRdState(*), socketRdQueue(*) -# ------------------------------------------------------------------------------ -# The element socketWrState($connId) has a value which is either the name of -# the token that is permitted to write to the socket, or "Wready" if no -# token is permitted to write. -# -# The code that sets the value to Wready immediately calls -# http::NextPipelinedWrite, which examines socketWrQueue($connId) and -# processes the next request in the queue, if there is one. The value -# Wready is not found when the interpreter is in the event loop unless the -# socket is idle. -# -# The element socketRdState($connId) has a value which is either the name of -# the token that is permitted to read from the socket, or "Rready" if no -# token is permitted to read. -# -# The code that sets the value to Rready then examines -# socketRdQueue($connId) and processes the next request in the queue, if -# there is one. The value Rready is not found when the interpreter is in -# the event loop unless the socket is idle. -# ------------------------------------------------------------------------------ - - -# ------------------------------------------------------------------------------ -# Proc http::ScheduleRequest -# ------------------------------------------------------------------------------ -# Command to either begin the HTTP request, or add it to the appropriate queue. -# Called from two places in ConfigureNewSocket. -# -# Arguments: -# token - connection token (name of an array) -# -# Return Value: none -# ------------------------------------------------------------------------------ - -proc http::ScheduleRequest {token} { - variable $token - upvar 0 $token state - set tk [namespace tail $token] - - Log >L$tk ScheduleRequest - - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - set Unfinished 0 - - set reusing $state(reusing) - set sockNew $state(sock) - - # The "if" tests below: must test against the current values of - # socketWrState, socketRdState, and so the tests must be done here, - # not earlier in PreparePersistentConnection. - - if {$state(alreadyQueued)} { - # The request has been appended to the queue of a persistent socket - # (that is scheduled to close and have its queue replayed). - # - # A write may or may not be in progress. There is no need to set - # socketWrState to prevent another call stealing write access - all - # subsequent calls on this socket will come here because the socket - # will close after the current read, and its - # socketClosing($connId) is 1. - ##Log "HTTP request for token $token is queued" - - } elseif { $reusing - && $state(-pipeline) - && ($socketWrState($state(socketinfo)) ne "Wready") - } { - ##Log "HTTP request for token $token is queued for pipelined use" - lappend socketWrQueue($state(socketinfo)) $token - - } elseif { $reusing - && (!$state(-pipeline)) - && ($socketWrState($state(socketinfo)) ne "Wready") - } { - # A write is queued or in progress. Lappend to the write queue. - ##Log "HTTP request for token $token is queued for nonpipeline use" - lappend socketWrQueue($state(socketinfo)) $token - - } elseif { $reusing - && (!$state(-pipeline)) - && ($socketWrState($state(socketinfo)) eq "Wready") - && ($socketRdState($state(socketinfo)) ne "Rready") - } { - # A read is queued or in progress, but not a write. Cannot start the - # nonpipeline transaction, but must set socketWrState to prevent a - # pipelined request jumping the queue. - ##Log "HTTP request for token $token is queued for nonpipeline use" - #Log re-use nonpipeline, GRANT delayed write access to $token in geturl - set socketWrState($state(socketinfo)) peNding - lappend socketWrQueue($state(socketinfo)) $token - - } else { - if {$reusing && $state(-pipeline)} { - #Log new, init for pipelined, GRANT write access to $token in geturl - # DO NOT grant premature read access to the socket. - # set socketRdState($state(socketinfo)) $token - set socketWrState($state(socketinfo)) $token - } elseif {$reusing} { - # socketWrState is not used by this non-pipelined transaction. - # We cannot leave it as "Wready" because the next call to - # http::geturl with a pipelined transaction would conclude that the - # socket is available for writing. - #Log new, init for nonpipeline, GRANT r/w access to $token in geturl - set socketRdState($state(socketinfo)) $token - set socketWrState($state(socketinfo)) $token - } else { - } - - # Process the request now. - # - Command is not called unless $state(sock) is a real socket handle - # and not a placeholder. - # - All (!$reusing) cases come here. - # - Some $reusing cases come here too if the connection is - # marked as ready. Those $reusing cases are: - # $reusing && ($socketWrState($state(socketinfo)) eq "Wready") && - # EITHER !$pipeline && ($socketRdState($state(socketinfo)) eq "Rready") - # OR $pipeline - # - #Log ---- $state(socketinfo) << conn to $token for HTTP request (a) - ##Log " ScheduleRequest" $token -- fileevent $state(sock) writable for $token - # Connect does its own fconfigure. - - lassign $state(connArgs) proto phost srvurl - - if {[catch { - fileevent $state(sock) writable \ - [list http::Connect $token $proto $phost $srvurl] - } res opts]} { - # The socket no longer exists. - ##Log bug -- socket gone -- $res -- $opts - } - - } - - return -} - - -# ------------------------------------------------------------------------------ -# Proc http::SendHeader -# ------------------------------------------------------------------------------ -# Command to send a request header, and keep a copy in state(requestHeaders) -# for debugging purposes. -# -# Arguments: -# token - connection token (name of an array) -# key - header name -# value - header value -# -# Return Value: none -# ------------------------------------------------------------------------------ - -proc http::SendHeader {token key value} { - variable $token - upvar 0 $token state - set tk [namespace tail $token] - set sock $state(sock) - lappend state(requestHeaders) [string tolower $key] $value - puts $sock "$key: $value" - return -} - -# http::Connected -- -# -# Callback used when the connection to the HTTP server is actually -# established. -# -# Arguments: -# token State token. -# proto What protocol (http, https, etc.) was used to connect. -# phost Are we using keep-alive? Non-empty if yes. -# srvurl Service-local URL that we're requesting -# Results: -# None. - -proc http::Connected {token proto phost srvurl} { - variable http - variable urlTypes - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - variable $token - upvar 0 $token state - set tk [namespace tail $token] - - if {$state(reusing) && (!$state(-pipeline)) && ($state(-timeout) > 0)} { - set state(after) [after $state(-timeout) \ - [list http::reset $token timeout]] - } - - # Set back the variables needed here. - set sock $state(sock) - set isQueryChannel [info exists state(-querychannel)] - set isQuery [info exists state(-query)] - regexp {^(.+):([^:]+)$} $state(socketinfo) {} host port - - set lower [string tolower $proto] - set defport [lindex $urlTypes($lower) 0] - - # Send data in cr-lf format, but accept any line terminators. - # Initialisation to {auto *} now done in geturl, KeepSocket and DoneRequest. - # We are concerned here with the request (write) not the response (read). - lassign [fconfigure $sock -translation] trRead trWrite - fconfigure $sock -translation [list $trRead crlf] \ - -buffersize $state(-blocksize) - if {[package vsatisfies [package provide Tcl] 9.0-]} { - fconfigure $sock -profile tcl8 - } - - # The following is disallowed in safe interpreters, but the socket is - # already in non-blocking mode in that case. - - catch {fconfigure $sock -blocking off} - set how GET - if {$isQuery} { - set state(querylength) [string length $state(-query)] - if {$state(querylength) > 0} { - set how POST - set contDone 0 - } else { - # There's no query data. - unset state(-query) - set isQuery 0 - } - } elseif {$state(-validate)} { - set how HEAD - } elseif {$isQueryChannel} { - set how POST - # The query channel must be blocking for the async Write to - # work properly. - fconfigure $state(-querychannel) -blocking 1 -translation binary - set contDone 0 - } - if {[info exists state(-method)] && ($state(-method) ne "")} { - set how $state(-method) - } - set accept_types_seen 0 - - Log ^B$tk begin sending request - token $token - - if {[catch { - if {[info exists state(bypass)]} { - set state(method) [lindex [split $state(bypass) { }] 0] - set state(requestHeaders) {} - set state(requestLine) $state(bypass) - } else { - set state(method) $how - set state(requestHeaders) {} - set state(requestLine) "$how $srvurl HTTP/$state(-protocol)" - } - puts $sock $state(requestLine) - set hostValue [GetFieldValue $state(-headers) Host] - if {$hostValue ne {}} { - # Allow Host spoofing. [Bug 928154] - regexp {^[^:]+} $hostValue state(host) - SendHeader $token Host $hostValue - } elseif {$port == $defport} { - # Don't add port in this case, to handle broken servers. [Bug - # #504508] - set state(host) $host - SendHeader $token Host $host - } else { - set state(host) $host - SendHeader $token Host "$host:$port" - } - SendHeader $token User-Agent $http(-useragent) - if {($state(-protocol) > 1.0) && $state(-keepalive)} { - # Send this header, because a 1.1 server is not compelled to treat - # this as the default. - set ConnVal keep-alive - } elseif {($state(-protocol) > 1.0)} { - # RFC2616 sec 8.1.2.1 - set ConnVal close - } else { - # ($state(-protocol) <= 1.0) - # RFC7230 A.1 - # Some server implementations of HTTP/1.0 have a faulty - # implementation of RFC 2068 Keep-Alive. - # Don't leave this to chance. - # For HTTP/1.0 we have already "set state(connection) close" - # and "state(-keepalive) 0". - set ConnVal close - } - # Proxy authorisation (cf. mod by Anders Ramdahl to autoproxy by - # Pat Thoyts). - if {($http(-proxyauth) ne {}) && ($state(proxyUsed) eq {HttpProxy})} { - SendHeader $token Proxy-Authorization $http(-proxyauth) - } - # RFC7230 A.1 - "clients are encouraged not to send the - # Proxy-Connection header field in any requests" - set accept_encoding_seen 0 - set content_type_seen 0 - set connection_seen 0 - foreach {key value} $state(-headers) { - set value [string map [list \n "" \r ""] $value] - set key [string map {" " -} [string trim $key]] - if {[string equal -nocase $key "host"]} { - continue - } - if {[string equal -nocase $key "accept-encoding"]} { - set accept_encoding_seen 1 - } - if {[string equal -nocase $key "accept"]} { - set accept_types_seen 1 - } - if {[string equal -nocase $key "content-type"]} { - set content_type_seen 1 - } - if {[string equal -nocase $key "content-length"]} { - set contDone 1 - set state(querylength) $value - } - if { [string equal -nocase $key "connection"] - && [info exists state(bypass)] - } { - # Value supplied in -headers overrides $ConnVal. - set connection_seen 1 - } elseif {[string equal -nocase $key "connection"]} { - # Remove "close" or "keep-alive" and use our own value. - # In an upgrade request, the upgrade is not guaranteed. - # Value "close" or "keep-alive" tells the server what to do - # if it refuses the upgrade. We send a single "Connection" - # header because some websocket servers, e.g. civetweb, reject - # multiple headers. Bug [d01de3281f] of tcllib/websocket. - set connection_seen 1 - set listVal $state(connectionValues) - if {[set pos [lsearch $listVal close]] != -1} { - set listVal [lreplace $listVal $pos $pos] - } - if {[set pos [lsearch $listVal keep-alive]] != -1} { - set listVal [lreplace $listVal $pos $pos] - } - lappend listVal $ConnVal - set value [join $listVal {, }] - } - if {[string length $key]} { - SendHeader $token $key $value - } - } - # Allow overriding the Accept header on a per-connection basis. Useful - # for working with REST services. [Bug c11a51c482] - if {!$accept_types_seen} { - SendHeader $token Accept $state(accept-types) - } - if { (!$accept_encoding_seen) - && (![info exists state(-handler)]) - && $http(-zip) - } { - SendHeader $token Accept-Encoding gzip,deflate - } elseif {!$accept_encoding_seen} { - SendHeader $token Accept-Encoding identity - } else { - } - if {!$connection_seen} { - SendHeader $token Connection $ConnVal - } - if {$isQueryChannel && ($state(querylength) == 0)} { - # Try to determine size of data in channel. If we cannot seek, the - # surrounding catch will trap us - - set start [tell $state(-querychannel)] - seek $state(-querychannel) 0 end - set state(querylength) \ - [expr {[tell $state(-querychannel)] - $start}] - seek $state(-querychannel) $start - } - - # Note that we don't do Cookie2; that's much nastier and not normally - # observed in practice either. It also doesn't fix the multitude of - # bugs in the basic cookie spec. - if {$http(-cookiejar) ne ""} { - set cookies "" - set separator "" - foreach {key value} [{*}$http(-cookiejar) \ - getCookies $proto $host $state(path)] { - append cookies $separator $key = $value - set separator "; " - } - if {$cookies ne ""} { - SendHeader $token Cookie $cookies - } - } - - # Flush the request header and set up the fileevent that will either - # push the POST data or read the response. - # - # fileevent note: - # - # It is possible to have both the read and write fileevents active at - # this point. The only scenario it seems to affect is a server that - # closes the connection without reading the POST data. (e.g., early - # versions TclHttpd in various error cases). Depending on the - # platform, the client may or may not be able to get the response from - # the server because of the error it will get trying to write the post - # data. Having both fileevents active changes the timing and the - # behavior, but no two platforms (among Solaris, Linux, and NT) behave - # the same, and none behave all that well in any case. Servers should - # always read their POST data if they expect the client to read their - # response. - - if {$isQuery || $isQueryChannel} { - # POST method. - if {!$content_type_seen} { - SendHeader $token Content-Type $state(-type) - } - if {!$contDone} { - SendHeader $token Content-Length $state(querylength) - } - puts $sock "" - flush $sock - # Flush flushes the error in the https case with a bad handshake: - # else the socket never becomes writable again, and hangs until - # timeout (if any). - - lassign [fconfigure $sock -translation] trRead trWrite - fconfigure $sock -translation [list $trRead binary] - fileevent $sock writable [list http::Write $token] - # The http::Write command decides when to make the socket readable, - # using the same test as the GET/HEAD case below. - } else { - # GET or HEAD method. - if { (![catch {fileevent $sock readable} binding]) - && ($binding eq [list http::CheckEof $sock]) - } { - # Remove the "fileevent readable" binding of an idle persistent - # socket to http::CheckEof. We can no longer treat bytes - # received as junk. The server might still time out and - # half-close the socket if it has not yet received the first - # "puts". - fileevent $sock readable {} - } - puts $sock "" - flush $sock - Log ^C$tk end sending request - token $token - # End of writing (GET/HEAD methods). The request has been sent. - - DoneRequest $token - } - - } err]} { - # The socket probably was never connected, OR the connection dropped - # later, OR https handshake error, which may be discovered as late as - # the "flush" command above... - Log "WARNING - if testing, pay special attention to this\ - case (GI) which is seldom executed - token $token" - if {[info exists state(reusing)] && $state(reusing)} { - # The socket was closed at the server end, and closed at - # this end by http::CheckEof. - if {[TestForReplay $token write $err a]} { - return - } else { - Finish $token {failed to re-use socket} - } - - # else: - # This is NOT a persistent socket that has been closed since its - # last use. - # If any other requests are in flight or pipelined/queued, they will - # be discarded. - } elseif {$state(status) eq ""} { - # https handshake errors come here, for - # Tcl 8.7 without http::SecureProxyConnect, and for Tcl 8.6. - set msg [registerError $sock] - registerError $sock {} - if {$msg eq {}} { - set msg {failed to use socket} - } - Finish $token $msg - } elseif {$state(status) ne "error"} { - Finish $token $err - } - } - return -} - -# http::registerError -# -# Called (for example when processing TclTLS activity) to register -# an error for a connection on a specific socket. This helps -# http::Connected to deliver meaningful error messages, e.g. when a TLS -# certificate fails verification. -# -# Usage: http::registerError socket ?newValue? -# -# "set" semantics, except that a "get" (a call without a new value) for a -# non-existent socket returns {}, not an error. - -proc http::registerError {sock args} { - variable registeredErrors - - if { ([llength $args] == 0) - && (![info exists registeredErrors($sock)]) - } { - return - } elseif { ([llength $args] == 1) - && ([lindex $args 0] eq {}) - } { - unset -nocomplain registeredErrors($sock) - return - } - set registeredErrors($sock) {*}$args -} - -# http::DoneRequest -- -# -# Command called when a request has been sent. It will arrange the -# next request and/or response as appropriate. -# -# If this command is called when $socketClosing(*), the request $token -# that calls it must be pipelined and destined to fail. - -proc http::DoneRequest {token} { - variable http - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - variable $token - upvar 0 $token state - set tk [namespace tail $token] - set sock $state(sock) - - # If pipelined, connect the next HTTP request to the socket. - if {$state(reusing) && $state(-pipeline)} { - # Enable next token (if any) to write. - # The value "Wready" is set only here, and - # in http::Event after reading the response-headers of a - # non-reusing transaction. - # Previous value is $token. It cannot be pending. - set socketWrState($state(socketinfo)) Wready - - # Now ready to write the next pipelined request (if any). - http::NextPipelinedWrite $token - } else { - # If pipelined, this is the first transaction on this socket. We wait - # for the response headers to discover whether the connection is - # persistent. (If this is not done and the connection is not - # persistent, we SHOULD retry and then MUST NOT pipeline before knowing - # that we have a persistent connection - # (rfc2616 8.1.2.2)). - } - - # Connect to receive the response, unless the socket is pipelined - # and another response is being sent. - # This code block is separate from the code below because there are - # cases where socketRdState already has the value $token. - if { $state(-keepalive) - && $state(-pipeline) - && [info exists socketRdState($state(socketinfo))] - && ($socketRdState($state(socketinfo)) eq "Rready") - } { - #Log pipelined, GRANT read access to $token in Connected - set socketRdState($state(socketinfo)) $token - } - - if { $state(-keepalive) - && $state(-pipeline) - && [info exists socketRdState($state(socketinfo))] - && ($socketRdState($state(socketinfo)) ne $token) - } { - # Do not read from the socket until it is ready. - ##Log "HTTP response for token $token is queued for pipelined use" - # If $socketClosing(*), then the caller will be a pipelined write and - # execution will come here. - # This token has already been recorded as "in flight" for writing. - # When the socket is closed, the read queue will be cleared in - # CloseQueuedQueries and so the "lappend" here has no effect. - lappend socketRdQueue($state(socketinfo)) $token - } else { - # In the pipelined case, connection for reading depends on the - # value of socketRdState. - # In the nonpipeline case, connection for reading always occurs. - ReceiveResponse $token - } - return -} - -# http::ReceiveResponse -# -# Connects token to its socket for reading. - -proc http::ReceiveResponse {token} { - variable $token - upvar 0 $token state - set tk [namespace tail $token] - set sock $state(sock) - - #Log ---- $state(socketinfo) >> conn to $token for HTTP response - lassign [fconfigure $sock -translation] trRead trWrite - fconfigure $sock -translation [list auto $trWrite] \ - -buffersize $state(-blocksize) - if {[package vsatisfies [package provide Tcl] 9.0-]} { - fconfigure $sock -profile tcl8 - } - Log ^D$tk begin receiving response - token $token - - coroutine ${token}--EventCoroutine http::Event $sock $token - if {[info exists state(-handler)] || [info exists state(-progress)]} { - fileevent $sock readable [list http::EventGateway $sock $token] - } else { - fileevent $sock readable ${token}--EventCoroutine - } - return -} - - -# http::EventGateway -# -# Bug [c2dc1da315]. -# - Recursive launch of the coroutine can occur if a -handler or -progress -# callback is used, and the callback command enters the event loop. -# - To prevent this, the fileevent "binding" is disabled while the -# coroutine is in flight. -# - If a recursive call occurs despite these precautions, it is not -# trapped and discarded here, because it is better to report it as a -# bug. -# - Although this solution is believed to be sufficiently general, it is -# used only if -handler or -progress is specified. In other cases, -# the coroutine is called directly. - -proc http::EventGateway {sock token} { - variable $token - upvar 0 $token state - fileevent $sock readable {} - catch {${token}--EventCoroutine} res opts - if {[info commands ${token}--EventCoroutine] ne {}} { - # The coroutine can be deleted by completion (a non-yield return), by - # http::Finish (when there is a premature end to the transaction), by - # http::reset or http::cleanup, or if the caller set option -channel - # but not option -handler: in the last case reading from the socket is - # now managed by commands ::http::Copy*, http::ReceiveChunked, and - # http::MakeTransformationChunked. - # - # Catch in case the coroutine has closed the socket. - catch {fileevent $sock readable [list http::EventGateway $sock $token]} - } - - # If there was an error, re-throw it. - return -options $opts $res -} - - -# http::NextPipelinedWrite -# -# - Connecting a socket to a token for writing is done by this command and by -# command KeepSocket. -# - If another request has a pipelined write scheduled for $token's socket, -# and if the socket is ready to accept it, connect the write and update -# the queue accordingly. -# - This command is called from http::DoneRequest and http::Event, -# IF $state(-pipeline) AND (the current transfer has reached the point at -# which the socket is ready for the next request to be written). -# - This command is called when a token has write access and is pipelined and -# keep-alive, and sets socketWrState to Wready. -# - The command need not consider the case where socketWrState is set to a token -# that does not yet have write access. Such a token is waiting for Rready, -# and the assignment of the connection to the token will be done elsewhere (in -# http::KeepSocket). -# - This command cannot be called after socketWrState has been set to a -# "pending" token value (that is then overwritten by the caller), because that -# value is set by this command when it is called by an earlier token when it -# relinquishes its write access, and the pending token is always the next in -# line to write. - -proc http::NextPipelinedWrite {token} { - variable http - variable socketRdState - variable socketWrState - variable socketWrQueue - variable socketClosing - variable $token - upvar 0 $token state - set connId $state(socketinfo) - - if { [info exists socketClosing($connId)] - && $socketClosing($connId) - } { - # socketClosing(*) is set because the server has sent a - # "Connection: close" header. - # Behave as if the queues are empty - so do nothing. - } elseif { $state(-pipeline) - && [info exists socketWrState($connId)] - && ($socketWrState($connId) eq "Wready") - - && [info exists socketWrQueue($connId)] - && [llength $socketWrQueue($connId)] - && ([set token2 [lindex $socketWrQueue($connId) 0] - set ${token2}(-pipeline) - ] - ) - } { - # - The usual case for a pipelined connection, ready for a new request. - #Log pipelined, GRANT write access to $token2 in NextPipelinedWrite - set conn [set ${token2}(connArgs)] - set socketWrState($connId) $token2 - set socketWrQueue($connId) [lrange $socketWrQueue($connId) 1 end] - # Connect does its own fconfigure. - fileevent $state(sock) writable [list http::Connect $token2 {*}$conn] - #Log ---- $connId << conn to $token2 for HTTP request (b) - - # In the tests below, the next request will be nonpipeline. - } elseif { $state(-pipeline) - && [info exists socketWrState($connId)] - && ($socketWrState($connId) eq "Wready") - - && [info exists socketWrQueue($connId)] - && [llength $socketWrQueue($connId)] - && (![ set token3 [lindex $socketWrQueue($connId) 0] - set ${token3}(-pipeline) - ] - ) - - && [info exists socketRdState($connId)] - && ($socketRdState($connId) eq "Rready") - } { - # The case in which the next request will be non-pipelined, and the read - # and write queues is ready: which is the condition for a non-pipelined - # write. - set conn [set ${token3}(connArgs)] - #Log nonpipeline, GRANT r/w access to $token3 in NextPipelinedWrite - set socketRdState($connId) $token3 - set socketWrState($connId) $token3 - set socketWrQueue($connId) [lrange $socketWrQueue($connId) 1 end] - # Connect does its own fconfigure. - fileevent $state(sock) writable [list http::Connect $token3 {*}$conn] - #Log ---- $state(sock) << conn to $token3 for HTTP request (c) - - } elseif { $state(-pipeline) - && [info exists socketWrState($connId)] - && ($socketWrState($connId) eq "Wready") - - && [info exists socketWrQueue($connId)] - && [llength $socketWrQueue($connId)] - && (![set token2 [lindex $socketWrQueue($connId) 0] - set ${token2}(-pipeline) - ] - ) - } { - # - The case in which the next request will be non-pipelined, but the - # read queue is NOT ready. - # - A read is queued or in progress, but not a write. Cannot start the - # nonpipeline transaction, but must set socketWrState to prevent a new - # pipelined request (in http::geturl) jumping the queue. - # - Because socketWrState($connId) is not set to Wready, the assignment - # of the connection to $token2 will be done elsewhere - by command - # http::KeepSocket when $socketRdState($connId) is set to "Rready". - - #Log re-use nonpipeline, GRANT delayed write access to $token in NextP.. - set socketWrState($connId) peNding - } - return -} - -# http::CancelReadPipeline -# -# Cancel pipelined responses on a closing "Keep-Alive" socket. -# -# - Called by a variable trace on "unset socketRdState($connId)". -# - The variable relates to a Keep-Alive socket, which has been closed. -# - Cancels all pipelined responses. The requests have been sent, -# the responses have not yet been received. -# - This is a hard cancel that ends each transaction with error status, -# and closes the connection. Do not use it if you want to replay failed -# transactions. -# - N.B. Always delete ::http::socketRdState($connId) before deleting -# ::http::socketRdQueue($connId), or this command will do nothing. -# -# Arguments -# As for a trace command on a variable. - -proc http::CancelReadPipeline {name1 connId op} { - variable socketRdQueue - ##Log CancelReadPipeline $name1 $connId $op - if {[info exists socketRdQueue($connId)]} { - set msg {the connection was closed by CancelReadPipeline} - foreach token $socketRdQueue($connId) { - set tk [namespace tail $token] - Log ^X$tk end of response "($msg)" - token $token - set ${token}(status) eof - Finish $token ;#$msg - } - set socketRdQueue($connId) {} - } - return -} - -# http::CancelWritePipeline -# -# Cancel queued events on a closing "Keep-Alive" socket. -# -# - Called by a variable trace on "unset socketWrState($connId)". -# - The variable relates to a Keep-Alive socket, which has been closed. -# - In pipelined or nonpipeline case: cancels all queued requests. The -# requests have not yet been sent, the responses are not due. -# - This is a hard cancel that ends each transaction with error status, -# and closes the connection. Do not use it if you want to replay failed -# transactions. -# - N.B. Always delete ::http::socketWrState($connId) before deleting -# ::http::socketWrQueue($connId), or this command will do nothing. -# -# Arguments -# As for a trace command on a variable. - -proc http::CancelWritePipeline {name1 connId op} { - variable socketWrQueue - - ##Log CancelWritePipeline $name1 $connId $op - if {[info exists socketWrQueue($connId)]} { - set msg {the connection was closed by CancelWritePipeline} - foreach token $socketWrQueue($connId) { - set tk [namespace tail $token] - Log ^X$tk end of response "($msg)" - token $token - set ${token}(status) eof - Finish $token ;#$msg - } - set socketWrQueue($connId) {} - } - return -} - -# http::ReplayIfDead -- -# -# - A query on a re-used persistent socket failed at the earliest opportunity, -# because the socket had been closed by the server. Keep the token, tidy up, -# and try to connect on a fresh socket. -# - The connection is monitored for eof by the command http::CheckEof. Thus -# http::ReplayIfDead is needed only when a server event (half-closing an -# apparently idle connection), and a client event (sending a request) occur at -# almost the same time, and neither client nor server detects the other's -# action before performing its own (an "asynchronous close event"). -# - To simplify testing of http::ReplayIfDead, set TEST_EOF 1 in -# http::KeepSocket, and then http::ReplayIfDead will be called if http::geturl -# is called at any time after the server timeout. -# -# Arguments: -# token Connection token. -# -# Side Effects: -# Use the same token, but try to open a new socket. - -proc http::ReplayIfDead {token doing} { - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - variable $token - upvar 0 $token state - - Log running http::ReplayIfDead for $token $doing - - # 1. Merge the tokens for transactions in flight, the read (response) queue, - # and the write (request) queue. - - set InFlightR {} - set InFlightW {} - - # Obtain the tokens for transactions in flight. - if {$state(-pipeline)} { - # Two transactions may be in flight. The "read" transaction was first. - # It is unlikely that the server would close the socket if a response - # was pending; however, an earlier request (as well as the present - # request) may have been sent and ignored if the socket was half-closed - # by the server. - - if { [info exists socketRdState($state(socketinfo))] - && ($socketRdState($state(socketinfo)) ne "Rready") - } { - lappend InFlightR $socketRdState($state(socketinfo)) - } elseif {($doing eq "read")} { - lappend InFlightR $token - } - - if { [info exists socketWrState($state(socketinfo))] - && $socketWrState($state(socketinfo)) ni {Wready peNding} - } { - lappend InFlightW $socketWrState($state(socketinfo)) - } elseif {($doing eq "write")} { - lappend InFlightW $token - } - - # Report any inconsistency of $token with socket*state. - if { ($doing eq "read") - && [info exists socketRdState($state(socketinfo))] - && ($token ne $socketRdState($state(socketinfo))) - } { - Log WARNING - ReplayIfDead pipelined token $token $doing \ - ne socketRdState($state(socketinfo)) \ - $socketRdState($state(socketinfo)) - - } elseif { - ($doing eq "write") - && [info exists socketWrState($state(socketinfo))] - && ($token ne $socketWrState($state(socketinfo))) - } { - Log WARNING - ReplayIfDead pipelined token $token $doing \ - ne socketWrState($state(socketinfo)) \ - $socketWrState($state(socketinfo)) - } - } else { - # One transaction should be in flight. - # socketRdState, socketWrQueue are used. - # socketRdQueue should be empty. - - # Report any inconsistency of $token with socket*state. - if {$token ne $socketRdState($state(socketinfo))} { - Log WARNING - ReplayIfDead nonpipeline token $token $doing \ - ne socketRdState($state(socketinfo)) \ - $socketRdState($state(socketinfo)) - } - - # Report the inconsistency that socketRdQueue is non-empty. - if { [info exists socketRdQueue($state(socketinfo))] - && ($socketRdQueue($state(socketinfo)) ne {}) - } { - Log WARNING - ReplayIfDead nonpipeline token $token $doing \ - has read queue socketRdQueue($state(socketinfo)) \ - $socketRdQueue($state(socketinfo)) ne {} - } - - lappend InFlightW $socketRdState($state(socketinfo)) - set socketRdQueue($state(socketinfo)) {} - } - - set newQueue {} - lappend newQueue {*}$InFlightR - lappend newQueue {*}$socketRdQueue($state(socketinfo)) - lappend newQueue {*}$InFlightW - lappend newQueue {*}$socketWrQueue($state(socketinfo)) - - - # 2. Tidy up token. This is a cut-down form of Finish/CloseSocket. - # Do not change state(status). - # No need to after cancel state(after) - either this is done in - # ReplayCore/ReInit, or Finish is called. - - catch {close $state(sock)} - Unset $state(socketinfo) - - # 2a. Tidy the tokens in the queues - this is done in ReplayCore/ReInit. - # - Transactions, if any, that are awaiting responses cannot be completed. - # They are listed for re-sending in newQueue. - # - All tokens are preserved for re-use by ReplayCore, and their variables - # will be re-initialised by calls to ReInit. - # - The relevant element of socketMapping, socketRdState, socketWrState, - # socketRdQueue, socketWrQueue, socketClosing, socketPlayCmd will be set - # to new values in ReplayCore. - - ReplayCore $newQueue - return -} - -# http::ReplayIfClose -- -# -# A request on a socket that was previously "Connection: keep-alive" has -# received a "Connection: close" response header. The server supplies -# that response correctly, but any later requests already queued on this -# connection will be lost when the socket closes. -# -# This command takes arguments that represent the socketWrState, -# socketRdQueue and socketWrQueue for this connection. The socketRdState -# is not needed because the server responds in full to the request that -# received the "Connection: close" response header. -# -# Existing request tokens $token (::http::$n) are preserved. The caller -# will be unaware that the request was processed this way. - -proc http::ReplayIfClose {Wstate Rqueue Wqueue} { - Log running http::ReplayIfClose for $Wstate $Rqueue $Wqueue - - if {$Wstate in $Rqueue || $Wstate in $Wqueue} { - Log WARNING duplicate token in http::ReplayIfClose - token $Wstate - set Wstate Wready - } - - # 1. Create newQueue - set InFlightW {} - if {$Wstate ni {Wready peNding}} { - lappend InFlightW $Wstate - } - ##Log $Rqueue -- $InFlightW -- $Wqueue - set newQueue {} - lappend newQueue {*}$Rqueue - lappend newQueue {*}$InFlightW - lappend newQueue {*}$Wqueue - - # 2. Cleanup - none needed, done by the caller. - - ReplayCore $newQueue - return -} - -# http::ReInit -- -# -# Command to restore a token's state to a condition that -# makes it ready to replay a request. -# -# Command http::geturl stores extra state in state(tmp*) so -# we don't need to do the argument processing again. -# -# The caller must: -# - Set state(reusing) and state(sock) to their new values after calling -# this command. -# - Unset state(tmpState), state(tmpOpenCmd) if future calls to ReplayCore -# or ReInit are inappropriate for this token. Typically only one retry -# is allowed. -# The caller may also unset state(tmpConnArgs) if this value (and the -# token) will be used immediately. The value is needed by tokens that -# will be stored in a queue. -# -# Arguments: -# token Connection token. -# -# Return Value: (boolean) true iff the re-initialisation was successful. - -proc http::ReInit {token} { - variable $token - upvar 0 $token state - - if {!( - [info exists state(tmpState)] - && [info exists state(tmpOpenCmd)] - && [info exists state(tmpConnArgs)] - ) - } { - Log FAILED in http::ReInit via ReplayCore - NO tmp vars for $token - return 0 - } - - if {[info exists state(after)]} { - after cancel $state(after) - unset state(after) - } - if {[info exists state(socketcoro)]} { - Log $token Cancel socket after-idle event (ReInit) - after cancel $state(socketcoro) - unset state(socketcoro) - } - - # Don't alter state(status) - this would trigger http::wait if it is in use. - set tmpState $state(tmpState) - set tmpOpenCmd $state(tmpOpenCmd) - set tmpConnArgs $state(tmpConnArgs) - foreach name [array names state] { - if {$name ne "status"} { - unset state($name) - } - } - - # Don't alter state(status). - # Restore state(tmp*) - the caller may decide to unset them. - # Restore state(tmpConnArgs) which is needed for connection. - # state(tmpState), state(tmpOpenCmd) are needed only for retries. - - dict unset tmpState status - array set state $tmpState - set state(tmpState) $tmpState - set state(tmpOpenCmd) $tmpOpenCmd - set state(tmpConnArgs) $tmpConnArgs - - return 1 -} - -# http::ReplayCore -- -# -# Command to replay a list of requests, using existing connection tokens. -# -# Abstracted from http::geturl which stores extra state in state(tmp*) so -# we don't need to do the argument processing again. -# -# Arguments: -# newQueue List of connection tokens. -# -# Side Effects: -# Use existing tokens, but try to open a new socket. - -proc http::ReplayCore {newQueue} { - variable TmpSockCounter - - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - if {[llength $newQueue] == 0} { - # Nothing to do. - return - } - - ##Log running ReplayCore for {*}$newQueue - set newToken [lindex $newQueue 0] - set newQueue [lrange $newQueue 1 end] - - # 3. Use newToken, and restore its values of state(*). Do not restore - # elements tmp* - we try again only once. - - set token $newToken - variable $token - upvar 0 $token state - - if {![ReInit $token]} { - Log FAILED in http::ReplayCore - NO tmp vars - Log ReplayCore reject $token - Finish $token {cannot send this request again} - return - } - - set tmpState $state(tmpState) - set tmpOpenCmd $state(tmpOpenCmd) - set tmpConnArgs $state(tmpConnArgs) - unset state(tmpState) - unset state(tmpOpenCmd) - unset state(tmpConnArgs) - - set state(reusing) 0 - set state(ReusingPlaceholder) 0 - set state(alreadyQueued) 0 - Log ReplayCore replay $token - - # Give the socket a placeholder name before it is created. - set sock HTTP_PLACEHOLDER_[incr TmpSockCounter] - set state(sock) $sock - - # Move the $newQueue into the placeholder socket's socketPhQueue. - set socketPhQueue($sock) {} - foreach tok $newQueue { - if {[ReInit $tok]} { - set ${tok}(reusing) 1 - set ${tok}(sock) $sock - lappend socketPhQueue($sock) $tok - Log ReplayCore replay $tok - } else { - Log ReplayCore reject $tok - set ${tok}(reusing) 1 - set ${tok}(sock) NONE - Finish $tok {cannot send this request again} - } - } - - AsyncTransaction $token - - return -} - -# Data access functions: -# Data - the URL data -# Status - the transaction status: ok, reset, eof, timeout, error -# Code - the HTTP transaction code, e.g., 200 -# Size - the size of the URL data - -proc http::responseBody {token} { - variable $token - upvar 0 $token state - return $state(body) -} -proc http::status {token} { - if {![info exists $token]} { - return "error" - } - variable $token - upvar 0 $token state - return $state(status) -} -proc http::responseLine {token} { - variable $token - upvar 0 $token state - return $state(http) -} -proc http::requestLine {token} { - variable $token - upvar 0 $token state - return $state(requestLine) -} -proc http::responseCode {token} { - variable $token - upvar 0 $token state - if {[regexp {[0-9]{3}} $state(http) numeric_code]} { - return $numeric_code - } else { - return $state(http) - } -} -proc http::size {token} { - variable $token - upvar 0 $token state - return $state(currentsize) -} -proc http::requestHeaders {token args} { - set lenny [llength $args] - if {$lenny > 1} { - return -code error {usage: ::http::requestHeaders token ?headerName?} - } else { - return [Meta $token request {*}$args] - } -} -proc http::responseHeaders {token args} { - set lenny [llength $args] - if {$lenny > 1} { - return -code error {usage: ::http::responseHeaders token ?headerName?} - } else { - return [Meta $token response {*}$args] - } -} -proc http::requestHeaderValue {token header} { - Meta $token request $header VALUE -} -proc http::responseHeaderValue {token header} { - Meta $token response $header VALUE -} -proc http::Meta {token who args} { - variable $token - upvar 0 $token state - - if {$who eq {request}} { - set whom requestHeaders - } elseif {$who eq {response}} { - set whom meta - } else { - return -code error {usage: ::http::Meta token request|response ?headerName ?VALUE??} - } - - set header [string tolower [lindex $args 0]] - set how [string tolower [lindex $args 1]] - set lenny [llength $args] - if {$lenny == 0} { - return $state($whom) - } elseif {($lenny > 2) || (($lenny == 2) && ($how ne {value}))} { - return -code error {usage: ::http::Meta token request|response ?headerName ?VALUE??} - } else { - set result {} - set combined {} - foreach {key value} $state($whom) { - if {$key eq $header} { - lappend result $key $value - append combined $value {, } - } - } - if {$lenny == 1} { - return $result - } else { - return [string range $combined 0 end-2] - } - } -} - - -# ------------------------------------------------------------------------------ -# Proc http::responseInfo -# ------------------------------------------------------------------------------ -# Command to return a dictionary of the most useful metadata of a HTTP -# response. -# -# Arguments: -# token - connection token (name of an array) -# -# Return Value: a dict. See man page http(n) for a description of each item. -# ------------------------------------------------------------------------------ - -proc http::responseInfo {token} { - variable $token - upvar 0 $token state - set result {} - foreach {key origin name} { - stage STATE state - status STATE status - responseCode STATE responseCode - reasonPhrase STATE reasonPhrase - contentType STATE type - binary STATE binary - redirection RESP location - upgrade STATE upgrade - error ERROR - - postError STATE posterror - method STATE method - charset STATE charset - compression STATE coding - httpRequest STATE -protocol - httpResponse STATE httpResponse - url STATE url - connectionRequest REQ connection - connectionResponse RESP connection - connectionActual STATE connection - transferEncoding STATE transfer - totalPost STATE querylength - currentPost STATE queryoffset - totalSize STATE totalsize - currentSize STATE currentsize - proxyUsed STATE proxyUsed - } { - if {$origin eq {STATE}} { - if {[info exists state($name)]} { - dict set result $key $state($name) - } else { - # Should never come here - dict set result $key {} - } - } elseif {$origin eq {REQ}} { - dict set result $key [requestHeaderValue $token $name] - } elseif {$origin eq {RESP}} { - dict set result $key [responseHeaderValue $token $name] - } elseif {$origin eq {ERROR}} { - # Don't flood the dict with data. The command ::http::error is - # available. - if {[info exists state(error)]} { - set msg [lindex $state(error) 0] - } else { - set msg {} - } - dict set result $key $msg - } else { - # Should never come here - dict set result $key {} - } - } - return $result -} -proc http::error {token} { - variable $token - upvar 0 $token state - if {[info exists state(error)]} { - return $state(error) - } - return -} -proc http::postError {token} { - variable $token - upvar 0 $token state - if {[info exists state(postErrorFull)]} { - return $state(postErrorFull) - } - return -} - -# http::cleanup -# -# Garbage collect the state associated with a transaction -# -# Arguments -# token The token returned from http::geturl -# -# Side Effects -# Unsets the state array. - -proc http::cleanup {token} { - variable $token - upvar 0 $token state - if {[info commands ${token}--EventCoroutine] ne {}} { - rename ${token}--EventCoroutine {} - } - if {[info commands ${token}--SocketCoroutine] ne {}} { - rename ${token}--SocketCoroutine {} - } - if {[info exists state(after)]} { - after cancel $state(after) - unset state(after) - } - if {[info exists state(socketcoro)]} { - Log $token Cancel socket after-idle event (cleanup) - after cancel $state(socketcoro) - unset state(socketcoro) - } - if {[info exists state]} { - unset state - } - return -} - -# http::Connect -# -# This callback is made when an asynchronous connection completes. -# -# Arguments -# token The token returned from http::geturl -# -# Side Effects -# Sets the status of the connection, which unblocks -# the waiting geturl call - -proc http::Connect {token proto phost srvurl} { - variable $token - upvar 0 $token state - set tk [namespace tail $token] - - if {[catch {eof $state(sock)} tmp] || $tmp} { - set err "due to unexpected EOF" - } elseif {[set err [fconfigure $state(sock) -error]] ne ""} { - # set err is done in test - } else { - # All OK - set state(state) connecting - fileevent $state(sock) writable {} - ::http::Connected $token $proto $phost $srvurl - return - } - - # Error cases. - Log "WARNING - if testing, pay special attention to this\ - case (GJ) which is seldom executed - token $token" - if {[info exists state(reusing)] && $state(reusing)} { - # The socket was closed at the server end, and closed at - # this end by http::CheckEof. - if {[TestForReplay $token write $err b]} { - return - } - - # else: - # This is NOT a persistent socket that has been closed since its - # last use. - # If any other requests are in flight or pipelined/queued, they will - # be discarded. - } - Finish $token "connect failed: $err" - return -} - -# http::Write -# -# Write POST query data to the socket -# -# Arguments -# token The token for the connection -# -# Side Effects -# Write the socket and handle callbacks. - -proc http::Write {token} { - variable http - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - variable $token - upvar 0 $token state - set tk [namespace tail $token] - set sock $state(sock) - - # Output a block. Tcl will buffer this if the socket blocks - set done 0 - if {[catch { - # Catch I/O errors on dead sockets - - if {[info exists state(-query)]} { - # Chop up large query strings so queryprogress callback can give - # smooth feedback. - if { $state(queryoffset) + $state(-queryblocksize) - >= $state(querylength) - } { - # This will be the last puts for the request-body. - if { (![catch {fileevent $sock readable} binding]) - && ($binding eq [list http::CheckEof $sock]) - } { - # Remove the "fileevent readable" binding of an idle - # persistent socket to http::CheckEof. We can no longer - # treat bytes received as junk. The server might still time - # out and half-close the socket if it has not yet received - # the first "puts". - fileevent $sock readable {} - } - } - puts -nonewline $sock \ - [string range $state(-query) $state(queryoffset) \ - [expr {$state(queryoffset) + $state(-queryblocksize) - 1}]] - incr state(queryoffset) $state(-queryblocksize) - if {$state(queryoffset) >= $state(querylength)} { - set state(queryoffset) $state(querylength) - set done 1 - } - } else { - # Copy blocks from the query channel - - set outStr [read $state(-querychannel) $state(-queryblocksize)] - if {[eof $state(-querychannel)]} { - # This will be the last puts for the request-body. - if { (![catch {fileevent $sock readable} binding]) - && ($binding eq [list http::CheckEof $sock]) - } { - # Remove the "fileevent readable" binding of an idle - # persistent socket to http::CheckEof. We can no longer - # treat bytes received as junk. The server might still time - # out and half-close the socket if it has not yet received - # the first "puts". - fileevent $sock readable {} - } - } - puts -nonewline $sock $outStr - incr state(queryoffset) [string length $outStr] - if {[eof $state(-querychannel)]} { - set done 1 - } - } - } err opts]} { - # Do not call Finish here, but instead let the read half of the socket - # process whatever server reply there is to get. - set state(posterror) $err - set info [dict get $opts -errorinfo] - set code [dict get $opts -code] - set state(postErrorFull) [list $err $info $code] - set done 1 - } - - if {$done} { - catch {flush $sock} - fileevent $sock writable {} - Log ^C$tk end sending request - token $token - # End of writing (POST method). The request has been sent. - - DoneRequest $token - } - - # Callback to the client after we've completely handled everything. - - if {[string length $state(-queryprogress)]} { - namespace eval :: $state(-queryprogress) \ - [list $token $state(querylength) $state(queryoffset)] - } - return -} - -# http::Event -# -# Handle input on the socket. This command is the core of -# the coroutine commands ${token}--EventCoroutine that are -# bound to "fileevent $sock readable" and process input. -# -# Arguments -# sock The socket receiving input. -# token The token returned from http::geturl -# -# Side Effects -# Read the socket and handle callbacks. - -proc http::Event {sock token} { - variable http - variable socketMapping - variable socketRdState - variable socketWrState - variable socketRdQueue - variable socketWrQueue - variable socketPhQueue - variable socketClosing - variable socketPlayCmd - variable socketCoEvent - variable socketProxyId - - variable $token - upvar 0 $token state - set tk [namespace tail $token] - while 1 { - yield - ##Log Event call - token $token - - if {![info exists state]} { - Log "Event $sock with invalid token '$token' - remote close?" - if {!([catch {eof $sock} tmp] || $tmp)} { - if {[set d [read $sock]] ne ""} { - Log "WARNING: additional data left on closed socket\ - - token $token" - } else { - } - } else { - } - Log ^X$tk end of response (token error) - token $token - CloseSocket $sock - return - } else { - } - if {$state(state) eq "connecting"} { - ##Log - connecting - token $token - if { $state(reusing) - && $state(-pipeline) - && ($state(-timeout) > 0) - && (![info exists state(after)]) - } { - set state(after) [after $state(-timeout) \ - [list http::reset $token timeout]] - } else { - } - - if {[catch {gets $sock state(http)} nsl]} { - Log "WARNING - if testing, pay special attention to this\ - case (GK) which is seldom executed - token $token" - if {[info exists state(reusing)] && $state(reusing)} { - # The socket was closed at the server end, and closed at - # this end by http::CheckEof. - - if {[TestForReplay $token read $nsl c]} { - return - } else { - } - # else: - # This is NOT a persistent socket that has been closed since - # its last use. - # If any other requests are in flight or pipelined/queued, - # they will be discarded. - } else { - # https handshake errors come here, for - # Tcl 8.7 with http::SecureProxyConnect. - set msg [registerError $sock] - registerError $sock {} - if {$msg eq {}} { - set msg $nsl - } - Log ^X$tk end of response (error) - token $token - Finish $token $msg - return - } - } elseif {$nsl >= 0} { - ##Log - connecting 1 - token $token - set state(state) "header" - } elseif { ([catch {eof $sock} tmp] || $tmp) - && [info exists state(reusing)] - && $state(reusing) - } { - # The socket was closed at the server end, and we didn't notice. - # This is the first read - where the closure is usually first - # detected. - - if {[TestForReplay $token read {} d]} { - return - } else { - } - - # else: - # This is NOT a persistent socket that has been closed since its - # last use. - # If any other requests are in flight or pipelined/queued, they - # will be discarded. - } else { - } - } elseif {$state(state) eq "header"} { - if {[catch {gets $sock line} nhl]} { - ##Log header failed - token $token - Log ^X$tk end of response (error) - token $token - Finish $token $nhl - return - } elseif {$nhl == 0} { - ##Log header done - token $token - Log ^E$tk end of response headers - token $token - # We have now read all headers - # We ignore HTTP/1.1 100 Continue returns. RFC2616 sec 8.2.3 - if { ($state(http) == "") - || ([regexp {^\S+\s(\d+)} $state(http) {} x] && $x == 100) - } { - set state(state) "connecting" - continue - # This was a "return" in the pre-coroutine code. - } else { - } - - # We have $state(http) so let's split it into its components. - if {[regexp {^HTTP/(\S+) ([0-9]{3}) (.*)$} $state(http) \ - -> httpResponse responseCode reasonPhrase] - } { - set state(httpResponse) $httpResponse - set state(responseCode) $responseCode - set state(reasonPhrase) $reasonPhrase - } else { - set state(httpResponse) $state(http) - set state(responseCode) $state(http) - set state(reasonPhrase) $state(http) - } - - if { ([info exists state(connection)]) - && ([info exists socketMapping($state(socketinfo))]) - && ("keep-alive" in $state(connection)) - && ($state(-keepalive)) - && (!$state(reusing)) - && ($state(-pipeline)) - } { - # Response headers received for first request on a - # persistent socket. Now ready for pipelined writes (if - # any). - # Previous value is $token. It cannot be "pending". - set socketWrState($state(socketinfo)) Wready - http::NextPipelinedWrite $token - } else { - } - - # Once a "close" has been signaled, the client MUST NOT send any - # more requests on that connection. - # - # If either the client or the server sends the "close" token in - # the Connection header, that request becomes the last one for - # the connection. - - if { ([info exists state(connection)]) - && ([info exists socketMapping($state(socketinfo))]) - && ("close" in $state(connection)) - && ($state(-keepalive)) - } { - # The server warns that it will close the socket after this - # response. - ##Log WARNING - socket will close after response for $token - # Prepare data for a call to ReplayIfClose. - Log $token socket will close after this transaction - # 1. Cancel socket-assignment coro events that have not yet - # launched, and add the tokens to the write queue. - if {[info exists socketCoEvent($state(socketinfo))]} { - foreach {tok can} $socketCoEvent($state(socketinfo)) { - lappend socketWrQueue($state(socketinfo)) $tok - unset -nocomplain ${tok}(socketcoro) - after cancel $can - Log $tok Cancel socket after-idle event (Event) - Log Move $tok from socketCoEvent to socketWrQueue and cancel its after idle coro - } - set socketCoEvent($state(socketinfo)) {} - } else { - } - - if { ($socketRdQueue($state(socketinfo)) ne {}) - || ($socketWrQueue($state(socketinfo)) ne {}) - || ($socketWrState($state(socketinfo)) ni - [list Wready peNding $token]) - } { - set InFlightW $socketWrState($state(socketinfo)) - if {$InFlightW in [list Wready peNding $token]} { - set InFlightW Wready - } else { - set msg "token ${InFlightW} is InFlightW" - ##Log $msg - token $token - } - set socketPlayCmd($state(socketinfo)) \ - [list ReplayIfClose $InFlightW \ - $socketRdQueue($state(socketinfo)) \ - $socketWrQueue($state(socketinfo))] - - # - All tokens are preserved for re-use by ReplayCore. - # - Queues are preserved in case of Finish with error, - # but are not used for anything else because - # socketClosing(*) is set below. - # - Cancel the state(after) timeout events. - foreach tokenVal $socketRdQueue($state(socketinfo)) { - if {[info exists ${tokenVal}(after)]} { - after cancel [set ${tokenVal}(after)] - unset ${tokenVal}(after) - } else { - } - # Tokens in the read queue have no (socketcoro) to - # cancel. - } - } else { - set socketPlayCmd($state(socketinfo)) \ - {ReplayIfClose Wready {} {}} - } - - # Do not allow further connections on this socket (but - # geturl can add new requests to the replay). - set socketClosing($state(socketinfo)) 1 - } else { - } - - set state(state) body - - # According to - # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Connection - # any comma-separated "Connection:" list implies keep-alive, but I - # don't see this in the RFC so we'll play safe and - # scan any list for "close". - # Done here to support combining duplicate header field's values. - if { [info exists state(connection)] - && ("close" ni $state(connection)) - && ("keep-alive" ni $state(connection)) - } { - lappend state(connection) "keep-alive" - } else { - } - - # If doing a HEAD, then we won't get any body - if {$state(-validate)} { - Log ^F$tk end of response for HEAD request - token $token - set state(state) complete - Eot $token - return - } elseif { - ($state(method) eq {CONNECT}) - && [string is integer -strict $state(responseCode)] - && ($state(responseCode) >= 200) - && ($state(responseCode) < 300) - } { - # A successful CONNECT response has no body. - # (An unsuccessful CONNECT has headers and body.) - # The code below is abstracted from Eot/Finish, but - # keeps the socket open. - catch {fileevent $state(sock) readable {}} - catch {fileevent $state(sock) writable {}} - set state(state) complete - set state(status) ok - if {[info commands ${token}--EventCoroutine] ne {}} { - rename ${token}--EventCoroutine {} - } - if {[info commands ${token}--SocketCoroutine] ne {}} { - rename ${token}--SocketCoroutine {} - } - if {[info exists state(socketcoro)]} { - Log $token Cancel socket after-idle event (Finish) - after cancel $state(socketcoro) - unset state(socketcoro) - } - if {[info exists state(after)]} { - after cancel $state(after) - unset state(after) - } - if { [info exists state(-command)] - && (![info exists state(done-command-cb)]) - } { - set state(done-command-cb) yes - if {[catch {namespace eval :: $state(-command) $token} err]} { - set state(error) [list $err $errorInfo $errorCode] - set state(status) error - } - } - return - } else { - } - - # - For non-chunked transfer we may have no body - in this case - # we may get no further file event if the connection doesn't - # close and no more data is sent. We can tell and must finish - # up now - not later - the alternative would be to wait until - # the server times out. - # - In this case, the server has NOT told the client it will - # close the connection, AND it has NOT indicated the resource - # length EITHER by setting the Content-Length (totalsize) OR - # by using chunked Transfer-Encoding. - # - Do not worry here about the case (Connection: close) because - # the server should close the connection. - # - IF (NOT Connection: close) AND (NOT chunked encoding) AND - # (totalsize == 0). - - if { (!( [info exists state(connection)] - && ("close" in $state(connection)) - ) - ) - && ($state(transfer) eq {}) - && ($state(totalsize) == 0) - } { - set msg {body size is 0 and no events likely - complete} - Log "$msg - token $token" - set msg {(length unknown, set to 0)} - Log ^F$tk end of response body {*}$msg - token $token - set state(state) complete - Eot $token - return - } else { - } - - # We have to use binary translation to count bytes properly. - lassign [fconfigure $sock -translation] trRead trWrite - fconfigure $sock -translation [list binary $trWrite] - - if { - $state(-binary) || [IsBinaryContentType $state(type)] - } { - # Turn off conversions for non-text data. - set state(binary) 1 - } else { - } - if {[info exists state(-channel)]} { - if {$state(binary) || [llength [ContentEncoding $token]]} { - fconfigure $state(-channel) -translation binary - } else { - } - if {![info exists state(-handler)]} { - # Initiate a sequence of background fcopies. - fileevent $sock readable {} - rename ${token}--EventCoroutine {} - CopyStart $sock $token - return - } else { - } - } else { - } - } elseif {$nhl > 0} { - # Process header lines. - ##Log header - token $token - $line - if {[regexp -nocase {^([^:]+):(.+)$} $line x key value]} { - set key [string tolower $key] - switch -- $key { - content-type { - set state(type) [string trim [string tolower $value]] - # Grab the optional charset information. - if {[regexp -nocase \ - {charset\s*=\s*\"((?:[^""]|\\\")*)\"} \ - $state(type) -> cs]} { - set state(charset) [string map {{\"} \"} $cs] - } else { - regexp -nocase {charset\s*=\s*(\S+?);?} \ - $state(type) -> state(charset) - } - } - content-length { - set state(totalsize) [string trim $value] - } - content-encoding { - set state(coding) [string trim $value] - } - transfer-encoding { - set state(transfer) \ - [string trim [string tolower $value]] - } - proxy-connection - - connection { - # RFC 7230 Section 6.1 states that a comma-separated - # list is an acceptable value. - if {![info exists state(connectionRespFlag)]} { - # This is the first "Connection" response header. - # Scrub the earlier value set by iniitialisation. - set state(connectionRespFlag) {} - set state(connection) {} - } - foreach el [SplitCommaSeparatedFieldValue $value] { - lappend state(connection) [string tolower $el] - } - } - upgrade { - set state(upgrade) [string trim $value] - } - set-cookie { - if {$http(-cookiejar) ne ""} { - ParseCookie $token [string trim $value] - } else { - } - } - } - lappend state(meta) $key [string trim $value] - } else { - } - } else { - } - } else { - # Now reading body - ##Log body - token $token - if {[catch { - if {[info exists state(-handler)]} { - set n [namespace eval :: $state(-handler) [list $sock $token]] - ##Log handler $n - token $token - # N.B. the protocol has been set to 1.0 because the -handler - # logic is not expected to handle chunked encoding. - # FIXME Allow -handler with 1.1 on dechunked stacked chan. - if {$state(totalsize) == 0} { - # We know the transfer is complete only when the server - # closes the connection - i.e. eof is not an error. - set state(state) complete - } else { - } - if {![string is integer -strict $n]} { - if 1 { - # Do not tolerate bad -handler - fail with error - # status. - set msg {the -handler command for http::geturl must\ - return an integer (the number of bytes\ - read)} - Log ^X$tk end of response (handler error) -\ - token $token - Eot $token $msg - } else { - # Tolerate the bad -handler, and continue. The - # penalty: - # (a) Because the handler returns nonsense, we know - # the transfer is complete only when the server - # closes the connection - i.e. eof is not an - # error. - # (b) http::size will not be accurate. - # (c) The transaction is already downgraded to 1.0 - # to avoid chunked transfer encoding. It MUST - # also be forced to "Connection: close" or the - # HTTP/1.0 equivalent; or it MUST fail (as - # above) if the server sends - # "Connection: keep-alive" or the HTTP/1.0 - # equivalent. - set n 0 - set state(state) complete - } - } else { - } - } elseif {[info exists state(transfer_final)]} { - # This code forgives EOF in place of the final CRLF. - set line [GetTextLine $sock] - set n [string length $line] - set state(state) complete - if {$n > 0} { - # - HTTP trailers (late response headers) are permitted - # by Chunked Transfer-Encoding, and can be safely - # ignored. - # - Do not count these bytes in the total received for - # the response body. - Log "trailer of $n bytes after final chunk -\ - token $token" - append state(transfer_final) $line - set n 0 - } else { - Log ^F$tk end of response body (chunked) - token $token - Log "final chunk part - token $token" - Eot $token - } - } elseif { [info exists state(transfer)] - && ($state(transfer) eq "chunked") - } { - ##Log chunked - token $token - set size 0 - set hexLenChunk [GetTextLine $sock] - #set ntl [string length $hexLenChunk] - if {[string trim $hexLenChunk] ne ""} { - scan $hexLenChunk %x size - if {$size != 0} { - ##Log chunk-measure $size - token $token - set chunk [BlockingRead $sock $size] - set n [string length $chunk] - if {$n >= 0} { - append state(body) $chunk - incr state(log_size) [string length $chunk] - ##Log chunk $n cumul $state(log_size) -\ - token $token - } else { - } - if {$size != [string length $chunk]} { - Log "WARNING: mis-sized chunk:\ - was [string length $chunk], should be\ - $size - token $token" - set n 0 - set state(connection) close - Log ^X$tk end of response (chunk error) \ - - token $token - set msg {error in chunked encoding - fetch\ - terminated} - Eot $token $msg - } else { - } - # CRLF that follows chunk. - # If eof, this is handled at the end of this proc. - GetTextLine $sock - } else { - set n 0 - set state(transfer_final) {} - } - } else { - # Line expected to hold chunk length is empty, or eof. - ##Log bad-chunk-measure - token $token - set n 0 - set state(connection) close - Log ^X$tk end of response (chunk error) - token $token - Eot $token {error in chunked encoding -\ - fetch terminated} - } - } else { - ##Log unchunked - token $token - if {$state(totalsize) == 0} { - # We know the transfer is complete only when the server - # closes the connection. - set state(state) complete - set reqSize $state(-blocksize) - } else { - # Ask for the whole of the unserved response-body. - # This works around a problem with a tls::socket - for - # https in keep-alive mode, and a request for - # $state(-blocksize) bytes, the last part of the - # resource does not get read until the server times out. - set reqSize [expr { $state(totalsize) - - $state(currentsize)}] - - # The workaround fails if reqSize is - # capped at $state(-blocksize). - # set reqSize [expr {min($reqSize, $state(-blocksize))}] - } - set c $state(currentsize) - set t $state(totalsize) - ##Log non-chunk currentsize $c of totalsize $t -\ - token $token - set block [read $sock $reqSize] - set n [string length $block] - if {$n >= 0} { - append state(body) $block - ##Log non-chunk [string length $state(body)] -\ - token $token - } else { - } - } - # This calculation uses n from the -handler, chunked, or - # unchunked case as appropriate. - if {[info exists state]} { - if {$n >= 0} { - incr state(currentsize) $n - set c $state(currentsize) - set t $state(totalsize) - ##Log another $n currentsize $c totalsize $t -\ - token $token - } else { - } - # If Content-Length - check for end of data. - if { - ($state(totalsize) > 0) - && ($state(currentsize) >= $state(totalsize)) - } { - Log ^F$tk end of response body (unchunked) -\ - token $token - set state(state) complete - Eot $token - } else { - } - } else { - } - } err]} { - Log ^X$tk end of response (error ${err}) - token $token - Finish $token $err - return - } else { - if {[info exists state(-progress)]} { - namespace eval :: $state(-progress) \ - [list $token $state(totalsize) $state(currentsize)] - } else { - } - } - } - - # catch as an Eot above may have closed the socket already - # $state(state) may be connecting, header, body, or complete - if {(![catch {eof $sock} eof]) && $eof} { - # [eof sock] succeeded and the result was 1 - ##Log eof - token $token - if {[info exists $token]} { - set state(connection) close - if {$state(state) eq "complete"} { - # This includes all cases in which the transaction - # can be completed by eof. - # The value "complete" is set only in http::Event, and it is - # used only in the test above. - Log ^F$tk end of response body (unchunked, eof) -\ - token $token - Eot $token - } else { - # Premature eof. - Log ^X$tk end of response (unexpected eof) - token $token - Eot $token eof - } - } else { - # open connection closed on a token that has been cleaned up. - Log ^X$tk end of response (token error) - token $token - CloseSocket $sock - } - } else { - # EITHER [eof sock] failed - presumed done by Eot - # OR [eof sock] succeeded and the result was 0 - } - } - return -} - -# http::TestForReplay -# -# Command called if eof is discovered when a socket is first used for a -# new transaction. Typically this occurs if a persistent socket is used -# after a period of idleness and the server has half-closed the socket. -# -# token - the connection token returned by http::geturl -# doing - "read" or "write" -# err - error message, if any -# caller - code to identify the caller - used only in logging -# -# Return Value: boolean, true iff the command calls http::ReplayIfDead. - -proc http::TestForReplay {token doing err caller} { - variable http - variable $token - upvar 0 $token state - set tk [namespace tail $token] - if {$doing eq "read"} { - set code Q - set action response - set ing reading - } else { - set code P - set action request - set ing writing - } - - if {$err eq {}} { - set err "detect eof when $ing (server timed out?)" - } - - if {$state(method) eq "POST" && !$http(-repost)} { - # No Replay. - # The present transaction will end when Finish is called. - # That call to Finish will abort any other transactions - # currently in the write queue. - # For calls from http::Event this occurs when execution - # reaches the code block at the end of that proc. - set msg {no retry for POST with http::config -repost 0} - Log reusing socket failed "($caller)" - $msg - token $token - Log error - $err - token $token - Log ^X$tk end of $action (error) - token $token - return 0 - } else { - # Replay. - set msg {try a new socket} - Log reusing socket failed "($caller)" - $msg - token $token - Log error - $err - token $token - Log ^$code$tk Any unfinished (incl this one) failed - token $token - ReplayIfDead $token $doing - return 1 - } -} - -# http::IsBinaryContentType -- -# -# Determine if the content-type means that we should definitely transfer -# the data as binary. [Bug 838e99a76d] -# -# Arguments -# type The content-type of the data. -# -# Results: -# Boolean, true if we definitely should be binary. - -proc http::IsBinaryContentType {type} { - lassign [split [string tolower $type] "/;"] major minor - if {$major eq "text"} { - return false - } - # There's a bunch of XML-as-application-format things about. See RFC 3023 - # and so on. - if {$major eq "application"} { - set minor [string trimright $minor] - if {$minor in {"json" "xml" "xml-external-parsed-entity" "xml-dtd"}} { - return false - } - } - # Not just application/foobar+xml but also image/svg+xml, so let us not - # restrict things for now... - if {[string match "*+xml" $minor]} { - return false - } - return true -} - -proc http::ParseCookie {token value} { - variable http - variable CookieRE - variable $token - upvar 0 $token state - - if {![regexp $CookieRE $value -> cookiename cookieval opts]} { - # Bad cookie! No biscuit! - return - } - - # Convert the options into a list before feeding into the cookie store; - # ugly, but quite easy. - set realopts {hostonly 1 path / secure 0 httponly 0} - dict set realopts origin $state(host) - dict set realopts domain $state(host) - foreach option [split [regsub -all {;\s+} $opts \u0000] \u0000] { - regexp {^(.*?)(?:=(.*))?$} $option -> optname optval - switch -exact -- [string tolower $optname] { - expires { - if {[catch { - #Sun, 06 Nov 1994 08:49:37 GMT - dict set realopts expires \ - [clock scan $optval -format "%a, %d %b %Y %T %Z"] - }] && [catch { - # Google does this one - #Mon, 01-Jan-1990 00:00:00 GMT - dict set realopts expires \ - [clock scan $optval -format "%a, %d-%b-%Y %T %Z"] - }] && [catch { - # This is in the RFC, but it is also in the original - # Netscape cookie spec, now online at: - # - #Sunday, 06-Nov-94 08:49:37 GMT - dict set realopts expires \ - [clock scan $optval -format "%A, %d-%b-%y %T %Z"] - }]} {catch { - #Sun Nov 6 08:49:37 1994 - dict set realopts expires \ - [clock scan $optval -gmt 1 -format "%a %b %d %T %Y"] - }} - } - max-age { - # Normalize - if {[string is integer -strict $optval]} { - dict set realopts expires [expr {[clock seconds] + $optval}] - } - } - domain { - # From the domain-matches definition [RFC 2109, section 2]: - # Host A's name domain-matches host B's if [...] - # A is a FQDN string and has the form NB, where N is a - # non-empty name string, B has the form .B', and B' is a - # FQDN string. (So, x.y.com domain-matches .y.com but - # not y.com.) - if {$optval ne "" && ![string match *. $optval]} { - dict set realopts domain [string trimleft $optval "."] - dict set realopts hostonly [expr { - ! [string match .* $optval] - }] - } - } - path { - if {[string match /* $optval]} { - dict set realopts path $optval - } - } - secure - httponly { - dict set realopts [string tolower $optname] 1 - } - } - } - dict set realopts key $cookiename - dict set realopts value $cookieval - {*}$http(-cookiejar) storeCookie $realopts -} - -# http::GetTextLine -- -# -# Get one line with the stream in crlf mode. -# Used if Transfer-Encoding is chunked, to read the line that -# reports the size of the following chunk. -# Empty line is not distinguished from eof. The caller must -# be able to handle this. -# -# Arguments -# sock The socket receiving input. -# -# Results: -# The line of text, without trailing newline - -proc http::GetTextLine {sock} { - set tr [fconfigure $sock -translation] - lassign $tr trRead trWrite - fconfigure $sock -translation [list crlf $trWrite] - set r [BlockingGets $sock] - fconfigure $sock -translation $tr - return $r -} - -# http::BlockingRead -# -# Replacement for a blocking read. -# The caller must be a coroutine. -# Used when we expect to read a chunked-encoding -# chunk of known size. - -proc http::BlockingRead {sock size} { - if {$size < 1} { - return - } - set result {} - while 1 { - set need [expr {$size - [string length $result]}] - set block [read $sock $need] - set eof [expr {[catch {eof $sock} tmp] || $tmp}] - append result $block - if {[string length $result] >= $size || $eof} { - return $result - } else { - yield - } - } -} - -# http::BlockingGets -# -# Replacement for a blocking gets. -# The caller must be a coroutine. -# Empty line is not distinguished from eof. The caller must -# be able to handle this. - -proc http::BlockingGets {sock} { - while 1 { - set count [gets $sock line] - set eof [expr {[catch {eof $sock} tmp] || $tmp}] - if {$count >= 0 || $eof} { - return $line - } else { - yield - } - } -} - -# http::CopyStart -# -# Error handling wrapper around fcopy -# -# Arguments -# sock The socket to copy from -# token The token returned from http::geturl -# -# Side Effects -# This closes the connection upon error - -proc http::CopyStart {sock token {initial 1}} { - upvar 0 $token state - if {[info exists state(transfer)] && $state(transfer) eq "chunked"} { - foreach coding [ContentEncoding $token] { - if {$coding eq {deflateX}} { - # Use the standards-compliant choice. - set coding2 decompress - } else { - set coding2 $coding - } - lappend state(zlib) [zlib stream $coding2] - } - MakeTransformationChunked $sock [namespace code [list CopyChunk $token]] - } else { - if {$initial} { - foreach coding [ContentEncoding $token] { - if {$coding eq {deflateX}} { - # Use the standards-compliant choice. - set coding2 decompress - } else { - set coding2 $coding - } - zlib push $coding2 $sock - } - } - if {[catch { - # FIXME Keep-Alive on https tls::socket with unchunked transfer - # hangs until the server times out. A workaround is possible, as for - # the case without -channel, but it does not use the neat "fcopy" - # solution. - fcopy $sock $state(-channel) -size $state(-blocksize) -command \ - [list http::CopyDone $token] - } err]} { - Finish $token $err - } - } - return -} - -proc http::CopyChunk {token chunk} { - upvar 0 $token state - if {[set count [string length $chunk]]} { - incr state(currentsize) $count - if {[info exists state(zlib)]} { - foreach stream $state(zlib) { - set chunk [$stream add $chunk] - } - } - puts -nonewline $state(-channel) $chunk - if {[info exists state(-progress)]} { - namespace eval :: [linsert $state(-progress) end \ - $token $state(totalsize) $state(currentsize)] - } - } else { - Log "CopyChunk Finish - token $token" - if {[info exists state(zlib)]} { - set excess "" - foreach stream $state(zlib) { - catch { - $stream put -finalize $excess - set excess "" - set overflood "" - while {[set overflood [$stream get]] ne ""} { append excess $overflood } - } - } - puts -nonewline $state(-channel) $excess - foreach stream $state(zlib) { $stream close } - unset state(zlib) - } - Eot $token ;# FIX ME: pipelining. - } - return -} - -# http::CopyDone -# -# fcopy completion callback -# -# Arguments -# token The token returned from http::geturl -# count The amount transferred -# -# Side Effects -# Invokes callbacks - -proc http::CopyDone {token count {error {}}} { - variable $token - upvar 0 $token state - set sock $state(sock) - incr state(currentsize) $count - if {[info exists state(-progress)]} { - namespace eval :: $state(-progress) \ - [list $token $state(totalsize) $state(currentsize)] - } - # At this point the token may have been reset. - if {[string length $error]} { - Finish $token $error - } elseif {[catch {eof $sock} iseof] || $iseof} { - Eot $token - } else { - CopyStart $sock $token 0 - } - return -} - -# http::Eot -# -# Called when either: -# a. An eof condition is detected on the socket. -# b. The client decides that the response is complete. -# c. The client detects an inconsistency and aborts the transaction. -# -# Does: -# 1. Set state(status) -# 2. Reverse any Content-Encoding -# 3. Convert charset encoding and line ends if necessary -# 4. Call http::Finish -# -# Arguments -# token The token returned from http::geturl -# force (previously) optional, has no effect -# reason - "eof" means premature EOF (not EOF as the natural end of -# the response) -# - "" means completion of response, with or without EOF -# - anything else describes an error condition other than -# premature EOF. -# -# Side Effects -# Clean up the socket - -proc http::Eot {token {reason {}}} { - variable $token - upvar 0 $token state - if {$reason eq "eof"} { - # Premature eof. - set state(status) eof - set reason {} - } elseif {$reason ne ""} { - # Abort the transaction. - set state(status) $reason - } else { - # The response is complete. - set state(status) ok - } - - if {[string length $state(body)] > 0} { - if {[catch { - foreach coding [ContentEncoding $token] { - if {$coding eq {deflateX}} { - # First try the standards-compliant choice. - set coding2 decompress - if {[catch {zlib $coding2 $state(body)} result]} { - # If that fails, try the MS non-compliant choice. - set coding2 inflate - set state(body) [zlib $coding2 $state(body)] - } else { - # error {failed at standards-compliant deflate} - set state(body) $result - } - } else { - set state(body) [zlib $coding $state(body)] - } - } - } err]} { - Log "error doing decompression for token $token: $err" - Finish $token $err - return - } - - if {!$state(binary)} { - # If we are getting text, set the incoming channel's encoding - # correctly. iso8859-1 is the RFC default, but this could be any - # IANA charset. However, we only know how to convert what we have - # encodings for. - - set enc [CharsetToEncoding $state(charset)] - if {$enc ne "binary"} { - if {[package vsatisfies [package provide Tcl] 9.0-]} { - set state(body) [encoding convertfrom -profile tcl8 $enc $state(body)] - } else { - set state(body) [encoding convertfrom $enc $state(body)] - } - } - - # Translate text line endings. - set state(body) [string map {\r\n \n \r \n} $state(body)] - } - if {[info exists state(-guesstype)] && $state(-guesstype)} { - GuessType $token - } - } - Finish $token $reason - return -} - - -# ------------------------------------------------------------------------------ -# Proc http::GuessType -# ------------------------------------------------------------------------------ -# Command to attempt limited analysis of a resource with undetermined -# Content-Type, i.e. "application/octet-stream". This value can be set for two -# reasons: -# (a) by the server, in a Content-Type header -# (b) by http::geturl, as the default value if the server does not supply a -# Content-Type header. -# -# This command converts a resource if: -# (1) it has type application/octet-stream -# (2) it begins with an XML declaration "?" -# (3) one tag is named "encoding" and has a recognised value; or no "encoding" -# tag exists (defaulting to utf-8) -# -# RFC 9110 Sec. 8.3 states: -# "If a Content-Type header field is not present, the recipient MAY either -# assume a media type of "application/octet-stream" ([RFC2046], Section 4.5.1) -# or examine the data to determine its type." -# -# The RFC goes on to describe the pitfalls of "MIME sniffing", including -# possible security risks. -# -# Arguments: -# token - connection token -# -# Return Value: (boolean) true iff a change has been made -# ------------------------------------------------------------------------------ - -proc http::GuessType {token} { - variable $token - upvar 0 $token state - - if {$state(type) ne {application/octet-stream}} { - return 0 - } - - set body $state(body) - # e.g. { ...} - - if {![regexp -nocase -- {^<[?]xml[[:space:]][^>?]*[?]>} $body match]} { - return 0 - } - # e.g. {} - - set contents [regsub -- {[[:space:]]+} $match { }] - set contents [string range [string tolower $contents] 6 end-2] - # e.g. {version="1.0" encoding="utf-8"} - # without excess whitespace or upper-case letters - - if {![regexp -- {^([^=" ]+="[^"]+" )+$} "$contents "]} { - return 0 - } - # The application/xml default encoding: - set res utf-8 - - set tagList [regexp -all -inline -- {[^=" ]+="[^"]+"} $contents] - foreach tag $tagList { - regexp -- {([^=" ]+)="([^"]+)"} $tag -> name value - if {$name eq {encoding}} { - set res $value - } - } - set enc [CharsetToEncoding $res] - if {$enc eq "binary"} { - return 0 - } - if {[package vsatisfies [package provide Tcl] 9.0-]} { - set state(body) [encoding convertfrom -profile tcl8 $enc $state(body)] - } else { - set state(body) [encoding convertfrom $enc $state(body)] - } - set state(body) [string map {\r\n \n \r \n} $state(body)] - set state(type) application/xml - set state(binary) 0 - set state(charset) $res - return 1 -} - - -# http::wait -- -# -# See documentation for details. -# -# Arguments: -# token Connection token. -# -# Results: -# The status after the wait. - -proc http::wait {token} { - variable $token - upvar 0 $token state - - if {![info exists state(status)] || $state(status) eq ""} { - # We must wait on the original variable name, not the upvar alias - vwait ${token}(status) - } - - return [status $token] -} - -# http::formatQuery -- -# -# See documentation for details. Call http::formatQuery with an even -# number of arguments, where the first is a name, the second is a value, -# the third is another name, and so on. -# -# Arguments: -# args A list of name-value pairs. -# -# Results: -# TODO - -proc http::formatQuery {args} { - if {[llength $args] % 2} { - return \ - -code error \ - -errorcode [list HTTP BADARGCNT $args] \ - {Incorrect number of arguments, must be an even number.} - } - set result "" - set sep "" - foreach i $args { - append result $sep [quoteString $i] - if {$sep eq "="} { - set sep & - } else { - set sep = - } - } - return $result -} - -# http::quoteString -- -# -# Do x-www-urlencoded character mapping -# -# Arguments: -# string The string the needs to be encoded -# -# Results: -# The encoded string - -proc http::quoteString {string} { - variable http - variable formMap - - # The spec says: "non-alphanumeric characters are replaced by '%HH'". Use - # a pre-computed map and [string map] to do the conversion (much faster - # than [regsub]/[subst]). [Bug 1020491] - - if {[package vsatisfies [package provide Tcl] 9.0-]} { - set string [encoding convertto -profile tcl8 $http(-urlencoding) $string] - } else { - set string [encoding convertto $http(-urlencoding) $string] - } - return [string map $formMap $string] -} - -# http::ProxyRequired -- -# Default proxy filter. -# -# Arguments: -# host The destination host -# -# Results: -# The current proxy settings - -proc http::ProxyRequired {host} { - variable http - if {(![info exists http(-proxyhost)]) || ($http(-proxyhost) eq {})} { - return - } - if {![info exists http(-proxyport)] || ($http(-proxyport) eq {})} { - set port 8080 - } else { - set port $http(-proxyport) - } - - # Simple test (cf. autoproxy) for hosts that must be accessed directly, - # not through the proxy server. - foreach domain $http(-proxynot) { - if {[string match -nocase $domain $host]} { - return {} - } - } - return [list $http(-proxyhost) $port] -} - -# http::CharsetToEncoding -- -# -# Tries to map a given IANA charset to a tcl encoding. If no encoding -# can be found, returns binary. -# - -proc http::CharsetToEncoding {charset} { - variable encodings - - set charset [string tolower $charset] - if {[regexp {iso-?8859-([0-9]+)} $charset -> num]} { - set encoding "iso8859-$num" - } elseif {[regexp {iso-?2022-(jp|kr)} $charset -> ext]} { - set encoding "iso2022-$ext" - } elseif {[regexp {shift[-_]?jis} $charset]} { - set encoding "shiftjis" - } elseif {[regexp {(?:windows|cp)-?([0-9]+)} $charset -> num]} { - set encoding "cp$num" - } elseif {$charset eq "us-ascii"} { - set encoding "ascii" - } elseif {[regexp {(?:iso-?)?lat(?:in)?-?([0-9]+)} $charset -> num]} { - switch -- $num { - 5 {set encoding "iso8859-9"} - 1 - 2 - 3 { - set encoding "iso8859-$num" - } - default { - set encoding "binary" - } - } - } else { - # other charset, like euc-xx, utf-8,... may directly map to encoding - set encoding $charset - } - set idx [lsearch -exact $encodings $encoding] - if {$idx >= 0} { - return $encoding - } else { - return "binary" - } -} - - -# ------------------------------------------------------------------------------ -# Proc http::ContentEncoding -# ------------------------------------------------------------------------------ -# Return the list of content-encoding transformations we need to do in order. -# - # -------------------------------------------------------------------------- - # Options for Accept-Encoding, Content-Encoding: the switch command - # -------------------------------------------------------------------------- - # The symbol deflateX allows http to attempt both versions of "deflate", - # unless there is a -channel - for a -channel, only "decompress" is tried. - # Alternative/extra lines for switch: - # The standards-compliant version of "deflate" can be chosen with: - # deflate { lappend r decompress } - # The Microsoft non-compliant version of "deflate" can be chosen with: - # deflate { lappend r inflate } - # The previously used implementation of "compress", which appears to be - # incorrect and is rarely used by web servers, can be chosen with: - # compress - x-compress { lappend r decompress } - # -------------------------------------------------------------------------- -# -# Arguments: -# token - Connection token. -# -# Return Value: list -# ------------------------------------------------------------------------------ - -proc http::ContentEncoding {token} { - upvar 0 $token state - set r {} - if {[info exists state(coding)]} { - foreach coding [split $state(coding) ,] { - switch -exact -- $coding { - deflate { lappend r deflateX } - gzip - x-gzip { lappend r gunzip } - identity {} - br { - return -code error\ - "content-encoding \"br\" not implemented" - } - default { - Log "unknown content-encoding \"$coding\" ignored" - } - } - } - } - return $r -} - -proc http::ReceiveChunked {chan command} { - set data "" - set size -1 - yield - while {1} { - chan configure $chan -translation {crlf binary} - while {[gets $chan line] < 1} { yield } - chan configure $chan -translation {binary binary} - if {[scan $line %x size] != 1} { - return -code error "invalid size: \"$line\"" - } - set chunk "" - while {$size && ![chan eof $chan]} { - set part [chan read $chan $size] - incr size -[string length $part] - append chunk $part - } - if {[catch { - uplevel #0 [linsert $command end $chunk] - }]} { - http::Log "Error in callback: $::errorInfo" - } - if {[string length $chunk] == 0} { - # channel might have been closed in the callback - catch {chan event $chan readable {}} - return - } - } -} - -# http::SplitCommaSeparatedFieldValue -- -# Return the individual values of a comma-separated field value. -# -# Arguments: -# fieldValue Comma-separated header field value. -# -# Results: -# List of values. -proc http::SplitCommaSeparatedFieldValue {fieldValue} { - set r {} - foreach el [split $fieldValue ,] { - lappend r [string trim $el] - } - return $r -} - - -# http::GetFieldValue -- -# Return the value of a header field. -# -# Arguments: -# headers Headers key-value list -# fieldName Name of header field whose value to return. -# -# Results: -# The value of the fieldName header field -# -# Field names are matched case-insensitively (RFC 7230 Section 3.2). -# -# If the field is present multiple times, it is assumed that the field is -# defined as a comma-separated list and the values are combined (by separating -# them with commas, see RFC 7230 Section 3.2.2) and returned at once. -proc http::GetFieldValue {headers fieldName} { - set r {} - foreach {field value} $headers { - if {[string equal -nocase $fieldName $field]} { - if {$r eq {}} { - set r $value - } else { - append r ", $value" - } - } - } - return $r -} - -proc http::MakeTransformationChunked {chan command} { - coroutine [namespace current]::dechunk$chan ::http::ReceiveChunked $chan $command - chan event $chan readable [namespace current]::dechunk$chan - return -} - -interp alias {} http::data {} http::responseBody -interp alias {} http::code {} http::responseLine -interp alias {} http::mapReply {} http::quoteString -interp alias {} http::meta {} http::responseHeaders -interp alias {} http::metaValue {} http::responseHeaderValue -interp alias {} http::ncode {} http::responseCode - - -# ------------------------------------------------------------------------------ -# Proc http::socketForTls -# ------------------------------------------------------------------------------ -# Command to use in place of ::socket as the value of ::tls::socketCmd. -# This command does the same as http::socket, and also handles https connections -# through a proxy server. -# -# Notes. -# - The proxy server works differently for https and http. This implementation -# is for https. The proxy for http is implemented in http::CreateToken (in -# code that was previously part of http::geturl). -# - This code implicitly uses the tls options set for https in a call to -# http::register, and does not need to call commands tls::*. This simple -# implementation is possible because tls uses a callback to ::socket that can -# be redirected by changing the value of ::tls::socketCmd. -# -# Arguments: -# args - as for ::socket -# -# Return Value: a socket identifier -# ------------------------------------------------------------------------------ - -proc http::socketForTls {args} { - variable http - set host [lindex $args end-1] - set port [lindex $args end] - if { ($http(-proxyfilter) ne {}) - && (![catch {$http(-proxyfilter) $host} proxy]) - } { - set phost [lindex $proxy 0] - set pport [lindex $proxy 1] - } else { - set phost {} - set pport {} - } - if {$phost eq ""} { - set sock [::http::socket {*}$args] - } else { - set sock [::http::SecureProxyConnect {*}$args $phost $pport] - } - return $sock -} - - -# ------------------------------------------------------------------------------ -# Proc http::SecureProxyConnect -# ------------------------------------------------------------------------------ -# Command to open a socket through a proxy server to a remote server for use by -# tls. The caller must perform the tls handshake. -# -# Notes -# - Based on patch supplied by Melissa Chawla in ticket 1173760, and -# Proxy-Authorization header cf. autoproxy by Pat Thoyts. -# - Rewritten as a call to http::geturl, because response headers and body are -# needed if the CONNECT request fails. CONNECT is implemented for this case -# only, by state(bypass). -# - FUTURE WORK: give http::geturl a -connect option for a general CONNECT. -# - The request header Proxy-Connection is discouraged in RFC 7230 (June 2014), -# RFC 9112 (June 2022). -# -# Arguments: -# args - as for ::socket, ending in host, port; with proxy host, proxy -# port appended. -# -# Return Value: a socket identifier -# ------------------------------------------------------------------------------ - -proc http::SecureProxyConnect {args} { - variable http - variable ConnectVar - variable ConnectCounter - variable failedProxyValues - set varName ::http::ConnectVar([incr ConnectCounter]) - - # Extract (non-proxy) target from args. - set host [lindex $args end-3] - set port [lindex $args end-2] - set args [lreplace $args end-3 end-2] - - # Proxy server URL for connection. - # This determines where the socket is opened. - set phost [lindex $args end-1] - set pport [lindex $args end] - if {[string first : $phost] != -1} { - # IPv6 address, wrap it in [] so we can append :pport - set phost "\[${phost}\]" - } - set url http://${phost}:${pport} - # Elements of args other than host and port are not used when - # AsyncTransaction opens a socket. Those elements are -async and the - # -type $tokenName for the https transaction. Option -async is used by - # AsyncTransaction anyway, and -type $tokenName should not be propagated: - # the proxy request adds its own -type value. - - set targ [lsearch -exact $args -type] - if {$targ != -1} { - # Record in the token that this is a proxy call. - set token [lindex $args $targ+1] - upvar 0 ${token} state - set tim $state(-timeout) - set state(proxyUsed) SecureProxyFailed - # This value is overwritten with "SecureProxy" below if the CONNECT is - # successful. If it is unsuccessful, the socket will be closed - # below, and so in this unsuccessful case there are no other transactions - # whose (proxyUsed) must be updated. - } else { - set tim 0 - } - if {$tim == 0} { - # Do not use infinite timeout for the proxy. - set tim 30000 - } - - # Prepare and send a CONNECT request to the proxy, using - # code similar to http::geturl. - set requestHeaders [list Host $host] - lappend requestHeaders Connection keep-alive - if {$http(-proxyauth) != {}} { - lappend requestHeaders Proxy-Authorization $http(-proxyauth) - } - - set token2 [CreateToken $url -keepalive 0 -timeout $tim \ - -headers $requestHeaders -command [list http::AllDone $varName]] - variable $token2 - upvar 0 $token2 state2 - - # Kludges: - # Setting this variable overrides the HTTP request line and also allows - # -headers to override the Connection: header set by -keepalive. - # The arguments "-keepalive 0" ensure that when Finish is called for an - # unsuccessful request, the socket is always closed. - set state2(bypass) "CONNECT $host:$port HTTP/1.1" - - AsyncTransaction $token2 - - if {[info coroutine] ne {}} { - # All callers in the http package are coroutines launched by - # the event loop. - # The cwait command requires a coroutine because it yields - # to the caller; $varName is traced and the coroutine resumes - # when the variable is written. - cwait $varName - } else { - return -code error {code must run in a coroutine} - # For testing with a non-coroutine caller outside the http package. - # vwait $varName - } - unset $varName - - if { ($state2(state) ne "complete") - || ($state2(status) ne "ok") - || (![string is integer -strict $state2(responseCode)]) - } { - set msg {the HTTP request to the proxy server did not return a valid\ - and complete response} - if {[info exists state2(error)]} { - append msg ": " [lindex $state2(error) 0] - } - cleanup $token2 - return -code error $msg - } - - set code $state2(responseCode) - - if {($code >= 200) && ($code < 300)} { - # All OK. The caller in package tls will now call "tls::import $sock". - # The cleanup command does not close $sock. - # Other tidying was done in http::Event. - - # If this is a persistent socket, any other transactions that are - # already marked to use the socket will have their (proxyUsed) updated - # when http::OpenSocket calls http::ConfigureNewSocket. - set state(proxyUsed) SecureProxy - set sock $state2(sock) - cleanup $token2 - return $sock - } - - if {$targ != -1} { - # Non-OK HTTP status code; token is known because option -type - # (cf. targ) was passed through tcltls, and so the useful - # parts of the proxy's response can be copied to state(*). - # Do not copy state2(sock). - # Return the proxy response to the caller of geturl. - foreach name $failedProxyValues { - if {[info exists state2($name)]} { - set state($name) $state2($name) - } - } - set state(connection) close - set msg "proxy connect failed: $code" - # - This error message will be detected by http::OpenSocket and will - # cause it to present the proxy's HTTP response as that of the - # original $token transaction, identified only by state(proxyUsed) - # as the response of the proxy. - # - The cases where this would mislead the caller of http::geturl are - # given a different value of msg (below) so that http::OpenSocket will - # treat them as errors, but will preserve the $token array for - # inspection by the caller. - # - Status code 305 (Proxy Required) was deprecated for security reasons - # in RFC 2616 (June 1999) and in any case should never be served by a - # proxy. - # - Other 3xx responses from the proxy are inappropriate, and should not - # occur. - # - A 401 response from the proxy is inappropriate, and should not - # occur. It would be confusing if returned to the caller. - - if {($code >= 300) && ($code < 400)} { - set msg "the proxy server responded to the HTTP request with an\ - inappropriate $code redirect" - set loc [responseHeaderValue $token2 location] - if {$loc ne {}} { - append msg "to " $loc - } - } elseif {($code == 401)} { - set msg "the proxy server responded to the HTTP request with an\ - inappropriate 401 request for target-host credentials" - } else { - } - } else { - set msg "connection to proxy failed with status code $code" - } - - # - ${token2}(sock) has already been closed because -keepalive 0. - # - Error return does not pass the socket ID to the - # $token transaction, which retains its socket placeholder. - cleanup $token2 - return -code error $msg -} - -proc http::AllDone {varName args} { - set $varName done - return -} - - -# ------------------------------------------------------------------------------ -# Proc http::socket -# ------------------------------------------------------------------------------ -# This command is a drop-in replacement for ::socket. -# Arguments and return value as for ::socket. -# -# Notes. -# - http::socket is specified in place of ::socket by the definition of urlTypes -# in the namespace header of this file (http.tcl). -# - The command makes a simple call to ::socket unless the user has called -# http::config to change the value of -threadlevel from the default value 0. -# - For -threadlevel 1 or 2, if the Thread package is available, the command -# waits in the event loop while the socket is opened in another thread. This -# is a workaround for bug [824251] - it prevents http::geturl from blocking -# the event loop if the DNS lookup or server connection is slow. -# - FIXME Use a thread pool if connections are very frequent. -# - FIXME The peer thread can transfer the socket only to the main interpreter -# in the present thread. Therefore this code works only if this script runs -# in the main interpreter. In a child interpreter, the parent must alias a -# command to ::http::socket in the child, run http::socket in the parent, -# and then transfer the socket to the child. -# - The http::socket command is simple, and can easily be replaced with an -# alternative command that uses a different technique to open a socket while -# entering the event loop. -# - Unexpected behaviour by thread::send -async (Thread 2.8.6). -# An error in thread::send -async causes return of just the error message -# (not the expected 3 elements), and raises a bgerror in the main thread. -# Hence wrap the command with catch as a precaution. -# ------------------------------------------------------------------------------ - -proc http::socket {args} { - variable ThreadVar - variable ThreadCounter - variable http - - LoadThreadIfNeeded - - set targ [lsearch -exact $args -type] - if {$targ != -1} { - set token [lindex $args $targ+1] - set args [lreplace $args $targ $targ+1] - upvar 0 $token state - } - - if {!$http(usingThread)} { - # Use plain "::socket". This is the default. - return [eval ::socket $args] - } - - set defcmd ::socket - set sockargs $args - set script " - set code \[catch { - [list proc ::SockInThread {caller defcmd sockargs} [info body ::http::SockInThread]] - [list ::SockInThread [thread::id] $defcmd $sockargs] - } result opts\] - list \$code \$opts \$result - " - - set state(tid) [thread::create] - set varName ::http::ThreadVar([incr ThreadCounter]) - thread::send -async $state(tid) $script $varName - Log >T Thread Start Wait $args -- coro [info coroutine] $varName - if {[info coroutine] ne {}} { - # All callers in the http package are coroutines launched by - # the event loop. - # The cwait command requires a coroutine because it yields - # to the caller; $varName is traced and the coroutine resumes - # when the variable is written. - cwait $varName - } else { - return -code error {code must run in a coroutine} - # For testing with a non-coroutine caller outside the http package. - # vwait $varName - } - Log >U Thread End Wait $args -- coro [info coroutine] $varName [set $varName] - thread::release $state(tid) - set state(tid) {} - set result [set $varName] - unset $varName - if {(![string is list $result]) || ([llength $result] != 3)} { - return -code error "result from peer thread is not a list of\ - length 3: it is \n$result" - } - lassign $result threadCode threadDict threadResult - if {($threadCode != 0)} { - # This is an error in thread::send. Return the lot. - return -options $threadDict -code error $threadResult - } - - # Now the results of the catch in the peer thread. - lassign $threadResult catchCode errdict sock - - if {($catchCode == 0) && ($sock ni [chan names])} { - return -code error {Transfer of socket from peer thread failed.\ - Check that this script is not running in a child interpreter.} - } - return -options $errdict -code $catchCode $sock -} - -# The commands below are dependencies of http::socket and -# http::SecureProxyConnect and are not used elsewhere. - -# ------------------------------------------------------------------------------ -# Proc http::LoadThreadIfNeeded -# ------------------------------------------------------------------------------ -# Command to load the Thread package if it is needed. If it is needed and not -# loadable, the outcome depends on $http(-threadlevel): -# value 0 => Thread package not required, no problem -# value 1 => operate as if -threadlevel 0 -# value 2 => error return -# -# Arguments: none -# Return Value: none -# ------------------------------------------------------------------------------ - -proc http::LoadThreadIfNeeded {} { - variable http - if {$http(usingThread) || ($http(-threadlevel) == 0)} { - return - } - if {[catch {package require Thread}]} { - if {$http(-threadlevel) == 2} { - set msg {[http::config -threadlevel] has value 2,\ - but the Thread package is not available} - return -code error $msg - } - return - } - set http(usingThread) 1 - return -} - - -# ------------------------------------------------------------------------------ -# Proc http::SockInThread -# ------------------------------------------------------------------------------ -# Command http::socket is a ::socket replacement. It defines and runs this -# command, http::SockInThread, in a peer thread. -# -# Arguments: -# caller -# defcmd -# sockargs -# -# Return value: list of values that describe the outcome. The return is -# intended to be a normal (non-error) return in all cases. -# ------------------------------------------------------------------------------ - -proc http::SockInThread {caller defcmd sockargs} { - package require Thread - - set catchCode [catch {eval $defcmd $sockargs} sock errdict] - if {$catchCode == 0} { - set catchCode [catch {thread::transfer $caller $sock; set sock} sock errdict] - } - return [list $catchCode $errdict $sock] -} - - -# ------------------------------------------------------------------------------ -# Proc http::cwaiter::cwait -# ------------------------------------------------------------------------------ -# Command to substitute for vwait, without the ordering issues. -# A command that uses cwait must be a coroutine that is launched by an event, -# e.g. fileevent or after idle, and has no calling code to be resumed upon -# "yield". It cannot return a value. -# -# Arguments: -# varName - fully-qualified name of the variable that the calling script -# will write to resume the coroutine. Any scalar variable or -# array element is permitted. -# coroName - (optional) name of the coroutine to be called when varName is -# written - defaults to this coroutine -# timeout - (optional) timeout value in ms -# timeoutValue - (optional) value to assign to varName if there is a timeout -# -# Return Value: none -# ------------------------------------------------------------------------------ - -namespace eval http::cwaiter { - namespace export cwait - variable log {} - variable logOn 0 -} - -proc http::cwaiter::cwait { - varName {coroName {}} {timeout {}} {timeoutValue {}} -} { - set thisCoro [info coroutine] - if {$thisCoro eq {}} { - return -code error {cwait cannot be called outside a coroutine} - } - if {$coroName eq {}} { - set coroName $thisCoro - } - if {[string range $varName 0 1] ne {::}} { - return -code error {argument varName must be fully qualified} - } - if {$timeout eq {}} { - set toe {} - } elseif {[string is integer -strict $timeout] && ($timeout > 0)} { - set toe [after $timeout [list set $varName $timeoutValue]] - } else { - return -code error {if timeout is supplied it must be a positive integer} - } - - set cmd [list ::http::cwaiter::CwaitHelper $varName $coroName $toe] - trace add variable $varName write $cmd - CoLog "Yield $varName $coroName" - yield - CoLog "Resume $varName $coroName" - return -} - - -# ------------------------------------------------------------------------------ -# Proc http::cwaiter::CwaitHelper -# ------------------------------------------------------------------------------ -# Helper command called by the trace set by cwait. -# - Ignores the arguments added by trace. -# - A simple call to $coroName works, and in error cases gives a suitable stack -# trace, but because it is inside a trace the headline error message is -# something like {can't set "::Result(6)": error}, not the actual -# error. So let the trace command return. -# - Remove the trace immediately. We don't want multiple calls. -# ------------------------------------------------------------------------------ - -proc http::cwaiter::CwaitHelper {varName coroName toe args} { - CoLog "got $varName for $coroName" - set cmd [list ::http::cwaiter::CwaitHelper $varName $coroName $toe] - trace remove variable $varName write $cmd - after cancel $toe - - after 0 $coroName - return -} - - -# ------------------------------------------------------------------------------ -# Proc http::cwaiter::LogInit -# ------------------------------------------------------------------------------ -# Call this command to initiate debug logging and clear the log. -# ------------------------------------------------------------------------------ - -proc http::cwaiter::LogInit {} { - variable log - variable logOn - set log {} - set logOn 1 - return -} - -proc http::cwaiter::LogRead {} { - variable log - return $log -} - -proc http::cwaiter::CoLog {msg} { - variable log - variable logOn - if {$logOn} { - append log $msg \n - } - return -} - -namespace eval http { - namespace import ::http::cwaiter::* -} - -# Local variables: -# indent-tabs-mode: t -# End: diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/natsort-0.1.1.5.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/natsort-0.1.1.5.tm deleted file mode 100644 index 0dcf57e7..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/natsort-0.1.1.5.tm +++ /dev/null @@ -1,1886 +0,0 @@ -#! /usr/bin/env tclsh - - -package require flagfilter -namespace import ::flagfilter::check_flags - -namespace eval natsort { - proc scriptdir {} { - set possibly_linked_script [file dirname [file normalize [file join [info script] ...]]] - if {[file isdirectory $possibly_linked_script]} { - return $possibly_linked_script - } else { - return [file dirname $possibly_linked_script] - } - } - tcl::tm::add [scriptdir] -} - - -namespace eval natsort { - variable stacktrace_on 0 - - proc do_error {msg {then error}} { - #note we exit or error out even if debug selected - as every do_error call is meant to interrupt code processing at the site of call - #this is not just a 'logging' call even though it has log-like descriptors - lassign $then type code - if {$code eq ""} { - set code 1 - } - set type [string tolower $type] - set levels [list debug info notice warn error critical] - if {$type in [concat $levels exit]} { - puts stderr "|$type> $msg" - } else { - puts stderr "|>natsort_call_err> unable to interpret 2nd argument to do_error: '$then' should be one of '$levels' or 'exit '" - } - if {$::tcl_interactive} { - #may not always be desirable - but assumed to be more useful not to exit despite request, to aid in debugging - if {[string tolower $type] eq "exit"} { - puts stderr " (exit suppressed due to tcl_interactive - raising error instead)" - if {![string is digit -strict $code]} { - puts stderr "|>natsort_call_err> unable to interpret 2nd argument to do_error: '$then' should be: 'exit '" - } - } - return -code error $msg - } else { - if {$type ne "exit"} { - return -code error $msg - } else { - if {[string is digit -strict $code]} { - exit $code - } else { - puts stderr "|>natsort_call_err> unable to interpret 2nd argument to do_error: '$then' should be 'error' or 'exit '" - return -code error $msg - } - } - } - } - - - - - - - variable debug 0 - variable testlist - set testlist { - 00.test-firstposition.txt - 0001.blah.txt - 1.test-sorts-after-all-leadingzero-number-one-equivs.txt - 1010.thousand-and-ten.second.txt - 01010.thousand-and-ten.first.txt - 0001.aaa.txt - 001.zzz.txt - 08.octal.txt-last-octal - 008.another-octal-first-octal.txt - 08.again-second-octal.txt - 001.a.txt - 0010.reconfig.txt - 010.etc.txt - 005.etc.01.txt - 005.Etc.02.txt - 005.123.abc.txt - 200.somewhere.txt - 2zzzz.before-somewhere.txt - 00222-after-somewhere.txt - 005.00010.abc.txt - 005.a3423bc.00010.abc.txt - 005.001.abc.txt - 005.etc.1010.txt - 005.etc.010.txt - 005.etc.10.txt - " 005.etc.10.txt" - 005.etc.001.txt - 20.somewhere.txt - 4611686018427387904999999999-bignum.txt - 4611686018427387903-bigishnum.txt - 9223372036854775807-bigint.txt - etca-a - etc-a - etc2-a - a0001blah.txt - a010.txt - winlike-sort-difference-0.1.txt - winlike-sort-difference-0.1.1.txt - a1.txt - b1-a0001blah.txt - b1-a010.txt - b1-a1.txt - -a1.txt - --a1.txt - --a10.txt - 2.high-two.yml - 02.higher-two.yml - reconfig.txt - _common.stuff.txt - CASETEST.txt - casetest.txt - something.txt - some~thing.txt - someathing.txt - someThing.txt - thing.txt - thing_revised.txt - thing-revised.txt - "thing revised.txt" - "spacetest.txt" - " spacetest.txt" - " spacetest.txt" - "spacetest2.txt" - "spacetest 2.txt" - "spacetest02.txt" - name.txt - name2.txt - "name .txt" - "name2 .txt" - blah.txt - combined.txt - a001.txt - .test - .ssh - "Feb 10.txt" - "Feb 8.txt" - 1ab23v23v3r89ad8a8a8a9d.txt - "Folder (10)/file.tar.gz" - "Folder/file.tar.gz" - "Folder (1)/file (1).tar.gz" - "Folder (1)/file.tar.gz" - "Folder (01)/file.tar.gz" - "Folder1/file.tar.gz" - "Folder(1)/file.tar.gz" - - } - lappend testlist "Some file.txt" - lappend testlist " Some extra file1.txt" - lappend testlist " Some extra file01.txt" - lappend testlist " some extra file1.txt" - lappend testlist " Some extra file003.txt" - lappend testlist " Some file.txt" - lappend testlist "Some extra file02.txt" - lappend testlist "Program Files (x86)" - lappend testlist "01999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999991-bigger-pathologically-bignum.txt" - lappend testlist "199999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999-pathologically-bignum.txt" - lappend testlist "29999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999-smaller-pathologically-bignum.txt" - lappend testlist "199999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999-pathologically-bignum.txt with (more 1.txt" - lappend testlist "199999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999-pathologically-bignum.txt with (more 01.txt" - lappend testlist "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1.pathological-num-nonnum-swapping-leadzero-should-be-first.txt" - lappend testlist "a1a1a1a1a1a1a1a1a1a1a1a01a1a1a1a1a1a1a1a1a1a1a1a1.pathological-num-nonnum-swapping-leadzero-should-be-first.txt" - lappend testlist "b1b1b1b1.txt" - lappend testlist "b1b01z1z1.txt" - lappend testlist "c1c111c1.txt" - lappend testlist "c1c1c1c1.txt" - - namespace eval overtype { - proc right {args} { - # @d !todo - implement overflow, length checks etc - - if {[llength $args] < 2} { - error {usage: ?-overflow [1|0]? undertext overtext} - } - foreach {undertext overtext} [lrange $args end-1 end] break - - set opt(-overflow) 0 - array set opt [lrange $args 0 end-2] - - - set olen [string length $overtext] - set ulen [string length $undertext] - - if {$opt(-overflow)} { - return [string range $undertext 0 end-$olen]$overtext - } else { - if {$olen > $ulen} { - set diff [expr {$olen - $ulen}] - return [string range $undertext 0 end-$olen][string range $overtext 0 end-$diff] - } else { - return [string range $undertext 0 end-$olen]$overtext - } - } - } - proc left {args} { - # @c overtype starting at left (overstrike) - # @c can/should we use something like this?: 'format "%-*s" $len $overtext - - if {[llength $args] < 2} { - error {usage: ?-overflow [1|0]? ?-ellipsis [1|0]? ?-ellipsistext ...? undertext overtext} - } - foreach {undertext overtext} [lrange $args end-1 end] break - - set opt(-ellipsis) 0 - set opt(-ellipsistext) {...} - set opt(-overflow) 0 - array set opt [lrange $args 0 end-2] - - - set len [string length $undertext] - set overlen [string length $overtext] - set diff [expr {$overlen - $len}] - - #puts stdout "====================>overtype: datalen:$len overlen:$overlen diff:$diff" - #puts stdout "====================>overtype: data: $overtext" - if {$diff > 0} { - if {$opt(-overflow)} { - return $overtext - } else { - if {$opt(-ellipsis)} { - return [overtype::left [string range $overtext 0 [expr {$len -1}]] $opt(-ellipsistext)] - } else { - return [string range $overtext 0 [expr {$len -1}]] - } - } - } else { - return "$overtext[string range $undertext $overlen end]" - } - } - - } - - #considered using hex to make large numbers more compact for viewing in debug output - but it's not that much shorter and probably obscures more than it helps. - proc hex2dec {largeHex} { - set res 0 - set largeHex [string map [list _ ""] $largeHex] - foreach hexDigit [split $largeHex {}] { - set new 0x$hexDigit - set res [expr {16*$res + $new}] - } - return $res - } - proc dec2hex {decimalNumber} { - format %4.4llX $decimalNumber - } - proc trimzero {number} { - set trimmed [string trimleft $number 0] - if {[string length $trimmed] == 0} { - set trimmed 0 - } - return $trimmed - } - #todo - consider human numeric split - #e.g consider SI suffixes k|KMGTPEZY in that order - - #in this context, for natural sorting - numeric segments don't contain underscores or other punctuation such as . - + etc. - #review - what about unicode equivalents such as wide numerals \UFF10 to \UFF19? unicode normalization? - proc split_numeric_segments {name} { - set segments [list] - while {[string length $name]} { - if {[scan $name {%[0-9]%n} chunk len] == 2} { - lappend segments $chunk - set name [string range $name $len end] - } - if {[scan $name {%[^0-9]%n} chunk len] == 2} { - lappend segments $chunk - set name [string range $name $len end] - } - } - return $segments - } - - proc padleft {str count {ch " "}} { - set val [string repeat $ch $count] - append val $str - set diff [expr {max(0,$count - [string length $str])}] - set offset [expr {max(0,$count - $diff)}] - set val [string range $val $offset end] - } - - - # Sqlite may have limited collation sequences available in default builds. - # with custom builds - there may be others such as 'natsort' - see https://sqlite.org/forum/forumpost/e4dc6f3331 - # This is of limited use with the few builtin collations available in 2023 ie binary,nocase & rtrim - # but may provide a quicker,flexible sort option, especially if/when more collation sequences are added to sqlite - # There are also prebuilt packages such as sqlite3-icu which allows things like "SELECT icu_load_collation('en_AU', 'australian');" - proc sort_sqlite {stringlist args} { - package require sqlite3 - - - set args [check_flags -caller natsort_sqlite -defaults [list -db :memory: -collate nocase -winlike 0 -topchars "\uFFFF" -debug 0 -splitchars [list / . - _] -extras {all}] -values $args] - set db [string trim [dict get $args -db]] - set collate [string trim [dict get $args -collate]] - set debug [string trim [dict get $args -debug]] - set topchars [string trim [dict get $args -topchars]] - - set topdot [expr {"." in $topchars}] - set topunderscore [expr {"_" in $topchars}] - - - sqlite3 db_sort_basic $db - set orderedlist [list] - db_sort_basic eval [string map [list %collate% $collate] {create table sqlitesort(index0 text COLLATE %collate%, name text COLLATE %collate%)}] - foreach nm $stringlist { - set segments [split_numeric_segments $nm] - set index "" - set s 0 - foreach seg $segments { - if {($s == 0) && ![string length [string trim $seg]]} { - #don't index leading space - } elseif {($s == 0) && ($topunderscore) && [string match _* [string trim $seg]]} { - append index "[padleft "0" 5]-d -100 topunderscore " - append index [string trim $seg] - } elseif {($s == 0) && ($topdot) && [string match .* [string trim $seg]]} { - append index "[padleft "0" 5]-d -50 topdot " - append index [string trim $seg] - } else { - if {[string is digit [string trim $seg]]} { - set basenum [trimzero [string trim $seg]] - set lengthindex "[padleft [string length $basenum] 5]-d" - append index "$lengthindex " - #append index [padleft $basenum 40] - append index $basenum - } else { - append index [string trim $seg] - } - } - incr s - } - puts stdout ">>$index" - db_sort_basic eval {insert into sqlitesort values($index,$nm)} - } - db_sort_basic eval [string map [list %collate% $collate] {select name from sqlitesort order by index0 COLLATE %collate% ASC, name COLLATE %collate% ASC }] { - lappend orderedlist $name - } - db_sort_basic close - return $orderedlist - } - - proc get_leading_char_count {str char} { - #todo - something more elegant? regex? - set count 0 - foreach c [split $str "" ] { - if {$c eq $char} { - incr count - } else { - break - } - } - return $count - } - proc stacktrace {} { - set stack "Stack trace:\n" - for {set i 1} {$i < [info level]} {incr i} { - set lvl [info level -$i] - set pname [lindex $lvl 0] - append stack [string repeat " " $i]$pname - - if {![catch {info args $pname} pargs]} { - foreach value [lrange $lvl 1 end] arg $pargs { - - if {$value eq ""} { - if {$arg != 0} { - info default $pname $arg value - } - } - append stack " $arg='$value'" - } - } else { - append stack " !unknown vars for $pname" - } - - append stack \n - } - return $stack - } - - proc get_char_count {str char} { - expr {[string length $str]-[string length [string map [list $char {}] $str]]} - } - - proc build_key {chunk splitchars topdict tagconfig debug} { - variable stacktrace_on - if {$stacktrace_on} { - puts stderr "+++>[stacktrace]" - } - - set index_map [list - "" _ ""] - #e.g - need to maintain the order - #a b.txt - #a book.txt - #ab.txt - #abacus.txt - - - set original_splitchars [dict get $tagconfig original_splitchars] - - # tag_dashes test moved from loop - review - set tag_dashes 0 - if {![string length [dict get $tagconfig last_part_text_tag]]} { - #winlike - set tag_dashes 1 - } - if {("-" ni $original_splitchars)} { - set tag_dashes 1 - } - if {$debug >= 3} { - puts stdout "START build_key chunk : $chunk" - puts stdout "START build_key splitchars : $splitchars $topdict $tagconfig NO tag dashes" - } - - - ## index_map will have no effect if we've already split on the char anyway(?) - #foreach m [dict keys $index_map] { - # if {$m in $original_splitchars} { - # dict unset index_map $m - # } - #} - - #if {![string length $chunk]} return - - set result "" - if {![llength $splitchars]} { - #no more structural splits - but we need to examine numeric/non-numeric segments at the lowest level. - # we are at a leaf in the recursive split hierarchy - - set s "" ;#we never actually split on "" (unless that was put in splitchars.. but it probably shouldn't be) - set parts [list $chunk] ;#important to treat as list or leading/trailing whitespace lost - - - } else { - set s [lindex $splitchars 0] - if {"spudbucket$s" in "[split $chunk {}]"} { - error "dead-branch spudbucket" - set partindex [build_key $chunk [lrange $splitchars 1 end] $topdict $tagconfig $debug] - if {[dict get $tagconfig showsplits]} { - set pfx "(1${s}=)" ;# = sorts before _ - set partindex ${pfx}$partindex - } - - return $partindex - } else { - set parts_below_index "" - - if {$s ni [split $chunk ""]} { - #$s can be an empty string - set parts [list $chunk] - } else { - set parts [split $chunk $s] ;#whitespace preserved - even if splitting on s that is not in string. - } - #assert - we have a splitchar $s that is in the chunk - so at least one part - if {(![string length $s] || [llength $parts] == 0)} { - error "buld_key assertion false empty split char and/or no parts" - } - - set pnum 1 ;# 1 based for clarity of reading index in debug output - set subpart_count [llength $parts] - - set sub_splits [lrange $splitchars 1 end] ;#pass same splitchars to each subpart - foreach p $parts { - set partindex [build_key $p $sub_splits $topdict $tagconfig $debug] - set lastpart [expr {$pnum == $subpart_count}] - - - ####################### - set showsplits [dict get $tagconfig showsplits] - #split prefixing experiment - maybe not suitable for general use - as it affects sort order - #note that pfx must be consistent until last one, no matter how many partnumbers there are in total. - # we don't want to influence sort order before reaching end. - #e.g for: - #(1.=)... - #(1._)...(2._)...(3.=) - #(1._)...(2.=) - #Note that this is probably more suitable for highly structure dependant sorts where the results are maybe less.. natural. - if {$showsplits} { - if {$lastpart} { - set pfx "(${pnum}${s}_" - #set pfx "(${pnum}${s}=)" ;# = sorts before _ - } else { - set pfx "(${pnum}${s}_" - } - append parts_below_index $pfx - } - ####################### - - if {$lastpart} { - if {[string length $p] && [string is digit $p]} { - set last_part_tag "<22${s}>" - } else { - set last_part_tag "<33${s}>" - } - - set last_part_text_tag [dict get $tagconfig last_part_text_tag] - #for -winlike 1 there is no tag configured. Windows explorer likes to put things in the order: - # module-0.1.1.tm - # module-0.1.1.2.tm - # module-0.1.tm - # arguably -winlike 0 is more natural/human - # module-0.1.tm - # module-0.1.1.tm - # module-0.1.1.2.tm - - if {[string length $last_part_text_tag]} { - #replace only the first text-tag (<30>) from the subpart_index - if {[string match "<30?>*" $partindex]} { - #give textual string index a specific tag for last part in split only. e.g <130> for lower than integers - set partindex "<130>[string range $partindex 5 end]" - } - #append parts_below_index $last_part_tag - } - #set partindex $last_part_tag$partindex - - - } - append parts_below_index $partindex - - - - if {$showsplits} { - if {$lastpart} { - set suffix "${pnum}${s}=)" ;# = sorts before _ - } else { - set suffix "${pnum}${s}_)" - } - append parts_below_index $suffix - } - - - incr pnum - } - append parts_below_index "" ;# don't add anything at the tail that may perturb sort order - - if {$debug >= 3} { - set pad [string repeat " " 20] - puts stdout "END build_key chunk : $chunk " - puts stdout "END build_key splitchars : $splitchars $topdict $tagconfig NO tag dashes" - puts stdout "END build_key ret below_index: $parts_below_index" - } - return $parts_below_index - - - } - } - - - - #puts stdout ">>>chunk:'$chunk'<<< split-on:$s parts: '$parts' splitchars: $splitchars -topdict:$topdict" - - - - - - #if {$chunk eq ""} { - # puts "___________________________________________!!!____" - #} - #puts stdout "-->chunk:$chunk $s parts:$parts" - - #puts stdout "---chunk:'$chunk' part:'$part' parts:'$parts' s:'$s'" - - - - - set segments [split_numeric_segments $chunk] ;#! - set stringindex "" - set segnum 0 - foreach seg $segments { - #puts stdout "=================---->seg:$seg segments:$segments" - #-strict ? - if {[string length $seg] && [string is digit $seg]} { - set basenum [trimzero [string trim $seg]] - set lengthindex "[padleft [string length $basenum] 4]d" - #append stringindex "<20>$lengthindex $basenum $seg" - } else { - set c1 [string range $seg 0 0] - #puts stdout "==============> c1'$c1' topdict: $topdict stringindex:$stringindex" - - if {$c1 in [dict keys $topdict]} { - set tag [dict get $topdict $c1] - #append stringindex "${tag}$c1" - #set seg [string range $seg 1 end] - } - #textindex - set leader "<30>" - set idx $seg - set idx [string trim $idx] - set idx [string tolower $idx] - set idx [string map $index_map $idx] - - - - - - #set the X-c count to match the length of the index - not the raw data - set lengthindex "[padleft [string length $idx] 4]c" - - #append stringindex "${leader}$idx $lengthindex $texttail" - } - } - - if {[llength $parts] != 1} { - error "build_key assertion fail llength parts != 1 parts:$parts" - } - - set segtail_clearance_buffer " " ;#space to clear other split indicators if using showsplits - set segtail $segtail_clearance_buffer - append segtail "\[" - set grouping "" - set pnum 0 - foreach p $parts { - set sublen_list [list] - set subsegments [split_numeric_segments $p] - set i 0 - - set partsorter "" - foreach sub $subsegments { - ##don't trim spaces here - it would be inconsistent. Some subs are pure whitespace - others have internal whitespace. e.g "a4 400b a b2" becomes "a 4 { } 400 {b a b} 2" - #mapping away all whitespace would be consistent, but not necessarily desirable. If it's in the index_map it'll happen anyway - so we don't do it manually here except for evaluating conditions. - set test_trim [string trim $sub] - set str $sub - set str [string tolower $str] - set str [string map $index_map $str] - if {[string length $test_trim] && [string is digit $test_trim]} { - append partsorter [trimzero $str] - } else { - append partsorter "$str" - } - append partsorter - } - - - foreach sub $subsegments { - - if {[string length $sub] && [string is digit $sub]} { - set basenum [trimzero [string trim $sub]] - set subequivs $basenum - set lengthindex "[padleft [string length $subequivs] 4]d " - set idx "$lengthindex [padleft $basenum 10]" ;#todo - cycle through data and determine longest - set tail [overtype::left [string repeat " " 10] $sub] - #set tail "" - } else { - set idx "" - - - set lookahead [lindex $subsegments $i+1] - if {![string length $lookahead]} { - set zeronum "[padleft 0 4]d0" - } else { - set zeronum "" - } - set subequivs $sub - #set subequivs [string trim $subequivs] - set subequivs [string tolower $subequivs] - set subequivs [string map $index_map $subequivs] - - append idx $subequivs - append idx $zeronum - - set idx $subequivs - - - # - - set ch "-" - if {$tag_dashes} { - #puts stdout "____TAG DASHES" - #winlike - set numleading [get_leading_char_count $seg $ch] - if {$numleading > 0} { - set texttail "<31-leading[padleft $numleading 4]$ch>" - } else { - set texttail "<30>" - } - set numothers [expr {[get_char_count $seg $ch] - $numleading}] - if {$debug >= 2} { - puts stdout "____dashcount: [get_char_count $seg $ch] numothers: $numothers" - } - if {$numothers > 0} { - append texttail "<31-others[padleft $numothers 4]$ch>" - } else { - append textail "<30>" - } - } else { - set texttail "<30>" - } - - - - - #set idx $partsorter - set tail "" - #set tail [string tolower $sub] ;#raw - #set tail $partsorter - #append tail ":[string tolower $p]" ;#we need the whole part - even though it makes the index much larger. !todo - tagconfig switch to toggle case sensitive sorting - } - - append grouping "$idx $tail|$s" - incr i - } - - - - - - if {$p eq ""} { - # no subsegments.. - set zeronum "[padleft 0 4]d0" - #append grouping "\u000$zerotail" - append grouping ".$zeronum" - } - - #append grouping | - #append grouping $s - #foreach len $sublen_list { - # append segtail "<[padleft $len 3]>" - #} - incr pnum - } - set grouping [string trimright $grouping $s] - append grouping "[padleft [llength $parts] 4]" - append segtail $grouping - - - #append segtail " <[padleft [llength $parts] 4]>" - - append segtail "\]" - - - #if {[string length $seg] && [string is digit $seg]} { - # append segtail "<20>" - #} else { - # append segtail "<30>" - #} - append stringindex $segtail - - incr segnum - - - - - lappend indices $stringindex - - if {[llength $indices] > 1} { - puts stderr "INDICES [llength $indices]: $stringindex" - error "build_key assertion error deadconcept indices" - } - - #topchar handling on splitter characters - #set c1 [string range $chunk 0 0] - if {$s in [dict keys $topdict]} { - set tag [dict get $topdict $s] - set joiner [string map [list ">" "$s>"] ${tag}] - #we have split on this character $s so if the first part is empty string then $s was a leading character - # we need to bring a tag out front for this, or it will be dominated by the leading sections-remaing tag - # (since the empty string produces no tag of it's own - ?) - if {[string length [lindex $parts 0]] == 0} { - set prefix ${joiner} - } else { - set prefix "" - } - } else { - #use standard character-data positioning tag if no override from topdict - set joiner "<30J>$s" - set prefix "" - } - - - set contentindex $prefix[join $indices $joiner] - if {[string length $s]} { - set split_indicator "" - } else { - set split_indicator "" - - } - if {![string length $s]} { - set s ~ - } - - #return "[overtype::left [string repeat { } 80] $contentindex][overtype::left [string repeat { } 10] [list $s $chunk]]" - #return $contentindex$split_indicator - #return [overtype::left [string repeat - 40] $contentindex] - - if {$debug >= 3} { - puts stdout "END build_key chunk : $chunk" - puts stdout "END build_key splitchars : $splitchars $topdict $tagconfig NO tag dashes" - puts stdout "END build_key ret contentidx : $contentindex" - } - return $contentindex - } - - #---------------------------------------- - #line-processors - data always last argument - opts can be empty string - #all processor should accept empty opts and ignore opts if they don't use them - proc _lineinput_as_tcl1 {opts line} { - set out "" - foreach i $line { - append out "$i " - } - set out [string range $out 0 end-1] - return $out - } - #should be equivalent to above - proc _lineinput_as_tcl {opts line} { - return [concat {*}$line] - } - #will put extra tcl quoting if it was already tcl-shaped e.g text "a" -> {"a"} - proc _lineoutput_as_tcl {opts line} { - return [regexp -inline -all {\S+} $line] - } - - proc _lineinput_as_raw {opts line} { - return $line - } - proc _lineoutput_as_raw {opts line} { - return $line - } - - #words is opposite of tcl - proc _lineinput_as_words {opts line} { - #wordlike_parts - return [regexp -inline -all {\S+} $line] - } - proc _lineoutput_as_words {opts line} { - return [concat {*}$line] - } - - #opts same as tcllib csv::split - except without the 'line' element - #?-alternate? ?sepChar? ?delChar? - proc _lineinput_as_csv {opts line} { - package require csv - if {[lindex $opts 0] eq "-alternate"} { - return [csv::split -alternate $line {*}[lrange $opts 1 end]] - } else { - return [csv::split $line {*}$opts] - } - } - #opts same as tcllib csv::join - #?sepChar? ?delChar? ?delMode? - proc _lineoutput_as_csv {opts line} { - package require csv - return [csv::join $line {*}$opts] - } - #---------------------------------------- - proc sort {stringlist args} { - #puts stdout "natsort::sort args: $args" - variable debug - if {![llength $stringlist]} return - - #allow pass through of the check_flags flag -debugargs so it can be set by the caller - set debugargs 0 - if {[set posn [lsearch $args -debugargs]] >=0} { - if {$posn == [llength $args]-1} { - #-debugargs at tail of list - set debugargs 1 - } else { - set debugargs [lindex $args $posn+1] - } - } - - #-return flagged|defaults doesn't work Review. - #flagfilter global processor/allocator not working 2023-08 - set args [check_flags \ - -caller natsort::sort \ - -return supplied|defaults \ - -debugargs $debugargs \ - -defaults [list -collate nocase \ - -winlike 0 \ - -splits "\uFFFF" \ - -topchars {. _} \ - -showsplits 1 \ - -sortmethod ascii \ - -collate "\uFFFF" \ - -inputformat raw \ - -inputformatapply {index data} \ - -inputformatoptions "" \ - -outputformat raw \ - -outputformatoptions "" \ - -cols "\uFFFF" \ - -debug 0 -db "" -stacktrace 0 -splits "\uFFFF" -showsplits 0] \ - -required {all} \ - -extras {none} \ - -commandprocessors {} \ - -values $args] - - #csv unimplemented - - set winlike [dict get $args -winlike] - set topchars [dict get $args -topchars] - set cols [dict get $args -cols] - set debug [dict get $args -debug] - set stacktrace [dict get $args -stacktrace] - set showsplits [dict get $args -showsplits] - set splits [dict get $args -splits] - set sortmethod [dict get $args -sortmethod] - set opt_collate [dict get $args -collate] - set opt_inputformat [dict get $args -inputformat] - set opt_inputformatapply [dict get $args -inputformatapply] - set opt_inputformatoptions [dict get $args -inputformatoptions] - set opt_outputformat [dict get $args -outputformat] - set opt_outputformatoptions [dict get $args -outputformatoptions] - dict unset args -showsplits - dict unset args -splits - if {$debug} { - puts stdout "natsort::sort processed_args: $args" - if {$debug == 1} { - puts stdout "natsort::sort - try also -debug 2, -debug 3" - } - } - - #set sortmethod "-dictionary" ;# sorts a2b before a001b - possibly other strangenesses that are hard to reason about - - if {$sortmethod in [list dictionary ascii]} { - set sortmethod "-$sortmethod" - # -ascii is default for tcl lsort. - } else { - set sortmethod "-ascii" - } - - set allowed_collations [list nocase] - if {$opt_collate ne "\uFFFF"} { - if {$opt_collate ni $allowed_collations} { - error "natsort::sort unknown value for -collate option. Only acceptable value(s): $allowed_collations" - } - set nocaseopt "-$opt_collate" - } else { - set nocaseopt "" - } - set allowed_inputformats [list tcl raw csv words] - if {$opt_inputformat ni $allowed_inputformats} { - error "natsort::sort unknown value for -inputformat option. Only acceptable value(s): $allowed_inputformats" - } - set allowed_outputformats [list tcl raw csv words] - if {$opt_inputformat ni $allowed_outputformats} { - error "natsort::sort unknown value for -outputformat option. Only acceptable value(s): $allowed_outputformats" - } - - # - set winsplits [list / . _] - set commonsplits [list / . _ -] - #set commonsplits [list] - - set tagconfig [dict create] - dict set tagconfig last_part_text_tag "<19>" - if {$winlike} { - set splitchars $winsplits - #windows explorer sorts leading spaces at the top - which doesn't seem very helpful for keeping things together - but the explorer doesn't seem able to create leading spaces anyway. - set wintop [list "(" ")" { } {.} {_}] ;#windows specific order - foreach t $topchars { - if {$t ni $wintop} { - lappend wintop $t - } - } - set topchars $wintop - dict set tagconfig last_part_text_tag "" - } else { - set splitchars $commonsplits - } - if {$splits ne "\uFFFF"} { - set splitchars $splits - } - dict set tagconfig original_splitchars $splitchars - dict set tagconfig showsplits $showsplits - - #create topdict - set i 0 - set topdict [dict create] - foreach c $topchars { - incr i ;#start at 01 so that 00 reserved for final-split tag (allows x-0.1.txt to sort above x-0.1.1.txt by default. Use tagconfig to change, or choose -winlike 1 for explorer-like sorting) - dict set topdict $c "<0$i>" - } - set keylist [list] - - - if {$opt_inputformat eq "tcl"} { - set lineinput_transform [list _lineinput_as_tcl $opt_inputformatoptions] - } elseif {$opt_inputformat eq "csv"} { - set lineinput_transform [list _lineinput_as_csv $opt_inputformatoptions] - } elseif {$opt_inputformat eq "raw"} { - set lineinput_transform [list _lineinput_as_raw $opt_inputformatoptions] - } elseif {$opt_inputformat eq "words"} { - set lineinput_transform [list _lineinput_as_words $opt_inputformatoptions] - } - if {$opt_outputformat eq "tcl"} { - set lineoutput_transform [list _lineoutput_as_tcl $opt_outputformatoptions] - } elseif {$opt_outputformat eq "csv"} { - set lineoutput_transform [list _lineoutput_as_csv $opt_outputformatoptions] - } elseif {$opt_outputformat eq "raw"} { - set lineoutput_transform [list _lineoutput_as_raw $opt_outputformatoptions] - } elseif {$opt_outputformat eq "words"} { - set lineoutput_transform [list _lineoutput_as_words $opt_outputformatoptions] - } - - - if {("data" in $opt_inputformatapply) || ("index" in $opt_inputformatapply)} { - if {$opt_inputformat eq "raw"} { - set tf_stringlist $stringlist - } else { - set tf_stringlist [list] - foreach v $stringlist { - lappend tf_stringlist [{*}$lineinput_transform $v] - } - } - if {"data" in $opt_inputformatapply} { - set tf_data_stringlist $tf_stringlist - } else { - set tf_data_stringlist $stringlist - } - if {"index" in $opt_inputformatapply} { - set tf_index_stringlist $tf_stringlist - } else { - set tf_index_stringlist $stringlist - } - } else { - set tf_data_stringlist $stringlist - set tf_index_stringlist $stringlist - } - - - - if {$stacktrace} { - puts stdout [natsort::stacktrace] - set natsort::stacktrace_on 1 - } - if {$cols eq "\uFFFF"} { - set colkeys [lmap v $stringlist {}] - } else { - set colkeys [list] - foreach v $tf_index_stringlist { - set lineparts $v - set k [list] - foreach c $cols { - lappend k [lindex $lineparts $c] - } - lappend colkeys [join $k "_"] ;#use a common-split char - Review - } - } - #puts stdout "colkeys: $colkeys" - - if {$opt_inputformat eq "raw"} { - #no inputformat was applied - can just use stringlist - foreach value $stringlist ck $colkeys { - set contentindex [build_key $value $splitchars $topdict $tagconfig $debug] - set colindex [build_key $ck $splitchars $topdict $tagconfig $debug] - lappend keylist ${colindex}-${contentindex}-$value ;#note: entire raw value used for final sort disambiguation (can be whitespace that was ignored in indexing) - } - } else { - foreach keyinput $tf_index_stringlist datavalue $tf_data_stringlist ck $colkeys { - #data may or may not have been transformed - #column index may or may not have been built with transformed data - - set contentindex [build_key $keyinput $splitchars $topdict $tagconfig $debug] - set colindex [build_key $ck $splitchars $topdict $tagconfig $debug] - lappend keylist ${colindex}-${contentindex}-$datavalue ;#note: entire value used for final sort disambiguation (can be whitespace that was ignored in indexing) - } - } - #puts stderr "keylist: $keylist" - - ################################################################################################### - # Use the generated keylist to do the actual sorting - # select either the transformed or raw data as the corresponding output - ################################################################################################### - if {[string length $nocaseopt]} { - set sortcommand [list lsort $sortmethod $nocaseopt -indices $keylist] - } else { - set sortcommand [list lsort $sortmethod -indices $keylist] - } - if {$opt_outputformat eq "raw"} { - #raw output means no further transformations - it doesn't mean there wasn't a transform applied on the input side - #use the tf_data_stringlist in the output - which will be the same as the input stringlist if no input transform applied for data. - #(Also - it may or may not have been *sorted* on transformed data depending on whether 'index' was in $opt_inputformatapply) - foreach idx [{*}$sortcommand] { - lappend result [lindex $tf_data_stringlist $idx] - } - } else { - #we need to apply an output format - #The data may or may not have been transformed at input - foreach idx [{*}$sortcommand] { - lappend result [{*}$lineoutput_transform [lindex $tf_data_stringlist $idx]] - } - } - ################################################################################################### - - - - - - if {$debug >= 2} { - set screen_width 250 - set max_val 0 - set max_idx 0 - ##### calculate colum widths - foreach i [{*}$sortcommand] { - set len_val [string length [lindex $stringlist $i]] - if {$len_val > $max_val} { - set max_val $len_val - } - set len_idx [string length [lindex $keylist $i]] - if {$len_idx > $max_idx} { - set max_idx $len_idx - } - } - #### - set l_width [expr {$max_val + 1}] - set leftcol [string repeat " " $l_width] - set r_width [expr {$screen_width - $l_width - 1}] - set rightcol [string repeat " " $r_width] - set str [overtype::left $leftcol RAW] - puts stdout " $str Index with possibly transformed data at tail" - foreach i [{*}$sortcommand] { - #puts stdout "|d> [overtype::left $leftcol [lindex $stringlist $i] ] [lindex $keylist $i]" - set index [lindex $keylist $i] - set len_idx [string length $index] - set rowcount [expr {$len_idx / $r_width}] - if {($len_idx % $r_width) > 0} { - incr rowcount - } - set rows [list] - for {set r 0} {$r < $rowcount} {incr r} { - lappend rows [string range $index 0 $r_width-$r] - set index [string range $index $r_width end] - } - - set r 0 - foreach idxpart $rows { - if {$r == 0} { - #use the untransformed stringlist - set str [overtype::left $leftcol [lindex $stringlist $i]] - } else { - set str [overtype::left $leftcol ...]] - } - puts stdout " $str $idxpart" - incr r - } - #puts stdout "|> '[lindex $stringlist $i]'" - #puts stdout "|> [lindex $keylist $i]" - } - - puts stdout "|debug> topdict: $topdict" - puts stdout "|debug> splitchars: $splitchars" - } - return $result - } - - - - #Note that although leading whitespace isn't a commonly used feature of filesystem names - it's possible at least on FreeBSD,windows and linux so we should try to handle it sensibly. - proc sort_experiment {stringlist args} { - package require sqlite3 - - variable debug - set args [check_flags -caller natsort::sort \ - -defaults [dict create -db :memory: -collate nocase -nullvalue "->NULL<" -winlike 0 -topchars [list] -debug 0] \ - -extras {all} \ - -values $args] - set db [string trim [dict get $args -db]] - set collate [string trim [dict get $args -collate]] - set winlike [string trim [dict get $args -winlike]] - set debug [string trim [dict get $args -debug]] - set nullvalue [string trim [dict get $args -nullvalue]] - - - set topchars [string trim [dict get $args -topchars]] - - set topdot [expr {"." in $topchars}] - set topunderscore [expr {"_" in $topchars}] - - - sqlite3 db_natsort2 $db - #-- - #our table must handle the name with the greatest number of numeric/non-numeric splits. - #This means a single list member with pathological naming e.g a1a1a1a1a1a1a1a1a1a1a1.txt could greatly extend the number of columns and indices and affect performance. - #review: could be optimised to aggregate the tail into a single index, as the the extra columns won't assist in ordering, but make the table and query bigger. - # we should probably determine the longest common sequence of splits in the input list and add only one more index for the segment after that. - set maxsegments 0 - #-- - set prefix "idx" - - #note - there will be more columns in the sorting table than segments. - # (a segment equals one of the numeric & non-numeric string portions returned from 'split_numeric_sgements') - #--------------------------- - # consider - # a123b.v1.2.txt - # a123b.v1.3beta1.txt - # these have the following segments: - # a 123 b.v 1 . 2 .txt - # a 123 b.v 1 . 3 beta 1 .txt - #--------------------------- - # The first string has 7 segments (numbered 0 to 6) - # the second string has 9 segments - # - # for example when the data has any elements in a segment position that are numeric (e.g 0001 123) - # - then an index column with numeric equivalents will be created (e.g 0001 becomes 1), and any non-numeric values in that column will get mapped to a negative value (for special cases) or a high value such as NULL (with NULLS LAST sql support) - # - # when a segment - - #cycle through all strings - we cannot build tabledef as we go because the column type depends on whether all segments for a particular column are text vs int-equivalent. - array set segmentinfo {} - foreach nm $stringlist { - set segments [split_numeric_segments $nm] - if {![string length [string trim [lindex $segments 0]]]} { - if {[string is digit [string trim [lindex $segments 1]]]} { - #name is whitespace followed by a digit - special case - ignore the whitespace for numbers only. (whitespace still goes through to name column though) - set segments [lrange $segments 1 end] - } - } - - - set c 0 ;#start of index columns - if {[llength $segments] > $maxsegments} { - set maxsegments [llength $segments] - } - foreach seg $segments { - set seg [string trim $seg] - set column_exists [info exists segmentinfo($c,type)] - if {[string is digit $seg]} { - if {$column_exists} { - #override it (may currently be text or int) - set segmentinfo($c,type) "int" - } else { - #new column - set segmentinfo($c,name) ${prefix}$c - set segmentinfo($c,type) "int" - } - } else { - #text never overrides int - if {!$column_exists} { - set segmentinfo($c,name) ${prefix}$c - set segmentinfo($c,type) "text" - } - } - incr c - } - } - if {$debug} { - puts stdout "Largest number of num/non-num segments in data: $maxsegments" - #parray segmentinfo - } - - # - set tabledef "" - set ordered_column_names [list] - set ordered_segmentinfo_tags [lsort -dictionary [array names segmentinfo *]] - foreach k $ordered_segmentinfo_tags { - lassign [split $k ,] c tag - if {$tag eq "type"} { - set type [set segmentinfo($k)] - if {$type eq "int"} { - append tabledef "$segmentinfo($c,name) int," - } else { - append tabledef "$segmentinfo($c,name) text COLLATE $collate," - } - append tabledef "raw$c text COLLATE $collate," - lappend ordered_column_names $segmentinfo($c,name) - lappend ordered_column_names raw$c ;#additional index column not in segmentinfo - } - if {$tag eq "name"} { - #lappend ordered_column_names $segmentinfo($k) - } - } - append tabledef "name text" - - #puts stdout "tabledef:$tabledef" - - - db_natsort2 eval [string map [list %tabledef% $tabledef] {create table natsort(%tabledef%)}] - - - foreach nm $stringlist { - array unset intdata - array set intdata {} - array set rawdata {} - #init array and build sql values string - set sql_insert "insert into natsort values(" - for {set i 0} {$i < $maxsegments} {incr i} { - set intdata($i) "" - set rawdata($i) "" - append sql_insert "\$intdata($i),\$rawdata($i)," - } - append sql_insert "\$nm" ;#don't manipulate name value in any way - e.g must leave all whitespace as the sort must return exactly the same elements as in the original list. - append sql_insert ")" - - set segments [split_numeric_segments $nm] - if {![string length [string trim [lindex $segments 0]]]} { - if {[string is digit [string trim [lindex $segments 1]]]} { - #name is whitespace followed by a digit - special case - ignore the whitespace for numbers only. (whitespace still goes through to name column though) - set segments [lrange $segments 1 end] - } - } - set values "" - set c 0 - foreach seg $segments { - if {[set segmentinfo($c,type)] eq "int"} { - if {[string is digit [string trim $seg]]} { - set intdata($c) [trimzero [string trim $seg]] - } else { - catch {unset intdata($c)} ;#set NULL - sorts last - if {($c == 0) && ($topunderscore) && [string match _* [string trim $seg]]} { - set intdata($c) -100 - } - if {($c == 0) && ($topdot) && [string match .* [string trim $seg]]} { - set intdata($c) -50 - } - } - set rawdata($c) [string trim $seg] - } else { - #pure text column - #set intdata($c) [string trim $seg] ;#ignore leading/trailing whitespace - we sort first on trimmed version, then refine with the sort on rawdata index - #catch {unset indata($c)} - set indata($c) [string trim $seg] - set rawdata($c) $seg - } - #set rawdata($c) [string trim $seg]# - #set rawdata($c) $seg - incr c - } - db_natsort2 eval $sql_insert - } - - set orderedlist [list] - - if {$debug} { - db_natsort2 eval {select * from pragma_table_info('natsort')} rowdata { - parray rowdata - } - } - set orderby "order by " - - foreach cname $ordered_column_names { - if {[string match "idx*" $cname]} { - append orderby "$cname ASC NULLS LAST," - } else { - append orderby "$cname ASC," - } - } - append orderby " name ASC" - #append orderby " NULLS LAST" ;#?? - - #e.g "order by idx0 ASC, raw0 ASC, idx1 ASC .... name ASC" - if {$debug} { - puts stdout "orderby clause: $orderby" - } - db_natsort2 eval [string map [list %orderby% $orderby] {select * from natsort %orderby%}] rowdata { - set line "- " - #parray rowdata - set columnnames $rowdata(*) - #puts stdout "columnnames: $columnnames" - #[lsort -dictionary [array names rowdata] - append line "$rowdata(name) \n" - foreach nm $columnnames { - if {$nm ne "name"} { - append line "$nm: $rowdata($nm) " - } - } - #puts stdout $line - #puts stdout "$rowdata(name)" - lappend orderedlist $rowdata(name) - } - - db_natsort2 close - return $orderedlist - } -} - - -#application section e.g this file might be linked from /usr/local/bin/natsort -namespace eval natsort { - namespace import ::flagfilter::check_flags - - proc called_directly_namematch {} { - global argv0 - #see https://wiki.tcl-lang.org/page/main+script - #trailing ... let's us resolve symlinks in last component of the path (could be something else like ___ but ... seems unlikely to collide with anything in the filesystem) - if {[info exists argv0] - && - [file dirname [file normalize [file join [info script] ...]]] - eq - [file dirname [file normalize [file join $argv0 ...]]] - } { - return 1 - } else { - #puts stdout "norm info script: [file dirname [file normalize [file join [info script] ...]]]" - #puts stdout "norm argv0 : [file dirname [file normalize [file join $argv0 ...]]]" - return 0 - } - } - #Review issues around comparing names vs using inodes (esp with respect to samba shares) - proc called_directly_inodematch {} { - global argv0 - if {[info exists argv0] - && [file exists [info script]] && [file exists $argv0]} { - file stat $argv0 argv0Info - file stat [info script] scriptInfo - expr {$argv0Info(dev) == $scriptInfo(dev) - && $argv0Info(ino) == $scriptInfo(ino)} - } else { - return 0 - } - } - - set is_namematch [called_directly_namematch] - set is_inodematch [called_directly_inodematch] - #### - #review - reliability of mechanisms to determine direct calls - # we don't want application being called when being used as a library, but we need it to run if called directly or from symlinks etc - #-- choose a policy and leave the others commented. - #set is_called_directly $is_namematch - #set is_called_directly $is_inodematch - set is_called_directly [expr {$is_namematch || $is_inodematch}] - #set is_called_directly [expr {$is_namematch && $is_inodematch}] - ### - - - #puts stdout "called_directly_name: [called_directly_namematch] called_directly_inode: [called_directly_inodematch]" - - - # - - - proc test_pass_fail_message {pass {additional ""}} { - variable test_fail_msg - variable test_pass_msg - if {$pass} { - puts stderr $test_pass_msg - } else { - puts stderr $test_fail_msg - } - puts stderr $additional - } - - variable test_fail_msg "XXXXXXXXXXXX FAIL XXXXXXXXXXXXX" - variable test_pass_msg "------------ PASS -------------" - proc test_sort_1 {args} { - package require struct::list - puts stderr "---$args" - set args [check_flags -caller natsort:test_sort_1 -defaults [list -collate nocase -topchars "\uFFFF" -winlike 0 -debug 0 -stacktrace 0 -showsplits 0 ] -values $args] - - puts stderr "test_sort_1 got args: $args" - - set unsorted_input { - 2.2.2 - 2.2.2.2 - 1a.1.1 - 1a.2.1.1 - 1.12.1 - 1.2.1.1 - 1.02.1.1 - 1.002b.1.1 - 1.1.1.2 - 1.1.1.1 - } - set input { -1.1.1 -1.1.1.2 -1.002b.1.1 -1.02.1.1 -1.2.1.1 -1.12.1 -1a.1.1 -1a.2.1.1 -2.2.2 -2.2.2.2 - } - - set sorted [natsort::sort $input {*}$args] - set is_match [struct::list equal $input $sorted] - - set msg "windows-explorer order" - - test_pass_fail_message $is_match $msg - puts stdout [string repeat - 40] - puts stdout INPUT - puts stdout [string repeat - 40] - foreach item $input { - puts stdout $item - } - puts stdout [string repeat - 40] - puts stdout OUTPUT - puts stdout [string repeat - 40] - foreach item $sorted { - puts stdout $item - } - test_pass_fail_message $is_match $msg - return [expr {!$is_match}] - } - proc test_sort_showsplits {args} { - package require struct::list - - set args [check_flags -caller natsort:test_sort_1 \ - -defaults [list -collate nocase -topchars "\uFFFF" -winlike 0 -debug 0 -stacktrace 0 -showsplits 1 ] \ - -extras {all} \ - -values $args] - - set input1 { - a-b.txt - a.b.c.txt - b.c-txt - } - - - set input2 { - a.b.c.txt - a-b.txt - b.c-text - } - - foreach {msg testlist } [list "custom-order" $input1 "windows-explorer (should work with -winlike 1)" $input2] { - set sorted [natsort::sort $testlist {*}$args] - set is_match [struct::list equal $testlist $sorted] - - test_pass_fail_message $is_match $msg - puts stderr "INPUT" - puts stderr "[string repeat - 40]" - foreach item $testlist { - puts stdout $item - } - puts stderr "[string repeat - 40]" - puts stderr "OUTPUT" - puts stderr "[string repeat - 40]" - foreach item $sorted { - puts stdout $item - } - - test_pass_fail_message $is_match $msg - } - - #return [expr {!$is_match}] - - } - - #tcl dispatch order - non flag items up front - #trailing flags are paired even if supplied as solo flags e.g -l becomes -l 1 - proc commandline_ls {args} { - set operands [list] - set posn 0 - foreach a $args { - if {![string match -* $a]} { - lappend operands $a - } else { - set flag1_posn $posn - break - } - incr posn - } - set args [lrange $args $flag1_posn end] - - - set debug 0 - set posn [lsearch $args -debug] - if {$posn > 0} { - if {[lindex $args $posn+1]} { - set debug [lindex $args $posn+1] - } - } - if {$debug} { - puts stderr "|debug>commandline_ls got $args" - } - - #if first operand not supplied - replace it with current working dir - if {[lindex $operands 0] eq "\uFFFF"} { - lset operands 0 [pwd] - } - - set targets [list] - foreach op $operands { - if {$op ne "\uFFFF"} { - set opchars [split [file tail $op] ""] - if {"?" in $opchars || "*" in $opchars} { - lappend targets $op - } else { - #actual file or dir - set targetitem $op - set targetitem [file normalize $op] - if {![file exists $targetitem]} { - if {$debug} { - puts stderr "|debug>commandline_ls Unable to access path '$targetitem'" - } - } - lappend targets $targetitem - if {$debug} { - puts stderr "|debug>commandline_ls listing for $targetitem" - } - } - } - } - set args [check_flags -caller commandline_ls \ - -return flagged|defaults \ - -debugargs 0 \ - -defaults [list -collate nocase -algorithm sort -topchars "\uFFFF" -winlike 0 -debug 0 -stacktrace 0 -showsplits 0 -algorithm sort] \ - -required {all} \ - -extras {all} \ - -soloflags {-v -l} \ - -commandprocessors {} \ - -values $args ] - if {$debug} { - puts stderr "|debug>args: $args" - } - - - set algorithm [dict get $args -algorithm] - dict unset args -algorithm - - set allfolders [list] - set allfiles [list] - foreach item $targets { - if {[file exists $item]} { - if {[file type $item] eq "directory"} { - set dotfolders [glob -nocomplain -directory $item -type {d} -tail .*] - set folders [glob -nocomplain -directory $item -type {d} -tail *] - set allfolders [concat $allfolders $dotfolders $folders] - - set dotfiles [glob -nocomplain -directory $item -type {f} -tail .*] - set files [glob -nocomplain -directory $item -type {f} -tail *] - set allfiles [concat $allfiles $dotfiles $files] - } else { - #file (or link?) - set files [glob -nocomplain -directory [file dirname $item] -tail [file tail $item]] - set allfiles [concat $allfiles $files] - } - } else { - set folders [glob -nocomplain -directory $item -type {d} -tail [file tail $item]] - set allfolders [concat $allfolders $folders] - set files [glob -nocomplain -directory [file dirname $item] -tail [file tail $item]] - set allfiles [concat $allfiles $files] - } - } - - - set sorted_folders [natsort::sort $allfolders {*}$args] - set sorted_files [natsort::sort $allfiles {*}$args] - - foreach fold $sorted_folders { - puts stdout $fold - } - foreach file $sorted_files { - puts stdout $file - } - - return "-- ok printed to stdout [llength $sorted_folders] folders and [llength $sorted_files] files --" - } - - package require argp - argp::registerArgs commandline_test { - { -showsplits boolean 0} - { -stacktrace boolean 0} - { -debug boolean 0} - { -winlike boolean 0} - { -db string ":memory:"} - { -collate string "nocase"} - { -algorithm string "sort"} - { -topchars string "\uFFFF"} - { -testlist string {10 1 30 3}} - } - argp::setArgsNeeded commandline_test {-stacktrace} - proc commandline_test {test args} { - variable testlist - puts stdout "commandline_test got $args" - argp::parseArgs opts - puts stdout "commandline_test got [array get opts]" - set args [check_flags -caller natsort_commandline \ - -return flagged|defaults \ - -defaults [list -db :memory: -collate nocase -testlist $testlist -algorithm sort -topchars "\uFFFF" -winlike 0 -debug 0 -stacktrace 0 -showsplits 0] \ - -values $args] - - if {[string tolower $test] in [list "1" "true"]} { - set test "sort" - } else { - if {![llength [info commands $test]]} { - error "test $test not found" - } - } - dict unset args -test - set stacktrace [dict get $args -stacktrace] - # dict unset args -stacktrace - - set argtestlist [dict get $args -testlist] - dict unset args -testlist - - - set debug [dict get $args -debug] - - set collate [dict get $args -collate] - set db [dict get $args -db] - set winlike [dict get $args -winlike] - set topchars [dict get $args -topchars] - - - puts stderr "|test>-----start natsort::$test--- input list size : [llength $argtestlist]" - #set resultlist [$test $argtestlist -db $db -collate $collate -topchars $topchars -winlike $winlike] - set resultlist [$test $argtestlist {*}$args] - foreach nm $resultlist { - puts stdout $nm - } - puts stdout "|test>-----end natsort::$test--- sorted list size: [llength $resultlist]" - return "test end" - } - proc commandline_runtests {runtests args} { - set argvals [check_flags -caller commandline_runtests \ - -defaults [list -collate nocase -algorithm sort -topchars "\uFFFF" -winlike 0 -debug 0 -stacktrace 0 -showsplits "\uFFFF" -runtests 1] \ - -values $args] - - puts stderr "runtests args: $argvals" - - #set runtests [dict get $argvals -runtests] - dict unset argvals -runtests - dict unset argvals -algorithm - - puts stderr "runtests args: $argvals" - #exit 0 - - set test_prefix "::natsort::test_sort_" - - if {$runtests eq "1"} { - set runtests "*" - } - - - set testcommands [info commands ${test_prefix}${runtests}] - if {![llength $testcommands]} { - puts stderr "No test commands matched -runtests argument '$runtests'" - puts stderr "Use 1 to run all tests" - set alltests [info commands ${test_prefix}*] - puts stderr "Valid tests are:" - - set prefixlen [string length $test_prefix] - foreach t $alltests { - set shortname [string range $t $prefixlen end] - puts stderr "$t = -runtests $shortname" - } - - } else { - foreach cmd $testcommands { - puts stderr [string repeat - 40] - puts stderr "calling $cmd with args: '$argvals'" - puts stderr [string repeat - 40] - $cmd {*}$argvals - } - } - exit 0 - } - proc help {args} { - puts stdout "natsort::help got '$args'" - return "Help not implemented" - } - proc natsort_pipe {args} { - #PIPELINE to take input list on stdin and write sorted list to stdout - #strip - from arglist - #set args [check_flags -caller natsort_pipeline \ - # -return all \ - # -defaults [list -db :memory: -collate nocase -algorithm sort -topchars "\uFFFF" -winlike 0 -debug 0 -stacktrace 0 -showsplits 0] \ - # -values $args] - - - set debug [dict get $args -debug] - if {$debug} { - puts stderr "|debug> natsort_pipe got args:'$args'" - } - set algorithm [dict get $args -algorithm] - dict unset args -algorithm - - set proclist [info commands ::natsort::sort*] - set algos [list] - foreach p $proclist { - lappend algos [namespace tail $p] - } - if {$algorithm ni [list {*}$proclist {*}$algos]} { - do_error "valid sort mechanisms: $algos" 2 - } - - - set input_list [list] - while {![eof stdin]} { - if {[gets stdin line] > 0} { - lappend input_list $line - } else { - if {[eof stdin]} { - - } else { - after 10 - } - } - } - - if {$debug} { - puts stderr "|debug> received [llength $input_list] list elements" - } - - set resultlist [$algorithm $input_list {*}$args] - if {$debug} { - puts stderr "|debug> returning [llength $resultlist] list elements" - } - foreach r $resultlist { - puts stdout $r - } - #exit 0 - - } - if {($is_called_directly)} { - set cmdprocessors { - {helpfinal {match "^help$" dispatch natsort::help}} - {helpfinal {sub -topic default "NONE"}} - } - #set args [check_flags \ - # -caller test1 \ - # -debugargs 2 \ - # -return arglist \ - # -defaults [list -collate nocase -algorithm sort -topchars "\uFFFF" -winlike 0 -debug 0 -stacktrace 0 -showsplits 0] \ - # -required {none} \ - # -extras {all} \ - # -commandprocessors $cmdprocessors \ - # -values $::argv ] - interp alias {} do_filter {} ::flagfilter::check_flags - - #mashopts are generally single-letter opts that can be run together e.g -l -d as -ld - set cmdprocessors { - {helpcmd {match "^help$" dispatch natsort::help singleopts {-v}}} - {helpcmd {sub -operand default \uFFFF singleopts {-l}}} - {lscmd {match "^ls$" dispatch natsort::commandline_ls dispatchtype tcl dispatchglobal 1 mashopts {-l -a} singleopts {-l -a} pairopts {} longopts {--color=always}}} - {lscmd {sub dir default "\uFFFF"}} - {lscmd {sub dir2 default "\uFFFF"}} - {lscmd {sub dir3 default "\uFFFF"}} - {lscmd {sub dir4 default "\uFFFF"}} - {lscmd {sub dir5 default "\uFFFF"}} - {lscmd {sub dir6 default "\uFFFF"}} - {runtests {match "^-tests$" dispatch natsort::commandline_runtests singleopts {-l}}} - {runtests {sub testname default "1" singleopts {-l}}} - {pipecmd {match "^-$" dispatch natsort::natsort_pipe dispatchtype tcl}} - } - set arglist [do_filter \ - -debugargs 0 \ - -debugargsonerror 2 \ - -caller cline_dispatch1 \ - -return all \ - -soloflags {-v -x} \ - -defaults [list -collate nocase -algorithm sort -topchars "\uFFFF" -winlike 0 -debug 0 -stacktrace 0 -showsplits 0 ] \ - -required {all} \ - -extras {all} \ - -commandprocessors $cmdprocessors \ - -values $::argv ] - - - #mashopts are generally single-letter opts that can be run together e.g -l -d as -ld - set cmdprocessors { - {testcmd {match "^test$" dispatch natsort::commandline_test singleopts {-l}}} - {testcmd {sub testname default "1" singleopts {-l}}} - } - set arglist [check_flags \ - -debugargs 0 \ - -caller cline_dispatch2 \ - -return all \ - -soloflags {-v -l} \ - -defaults [list -collate nocase -algorithm sort -testlist "1 2 3 10" -topchars "\uFFFF" -winlike 0 -debug 0 -stacktrace 0 -showsplits 0 ] \ - -required {all} \ - -extras {all} \ - -commandprocessors $cmdprocessors \ - -values $::argv ] - - - - - #set cmdprocessors [list] - #set args [check_flags -caller test1 -defaults [list -collate nocase -algorithm sort -topchars "\uFFFF" -winlike 0 -debug 0 -stacktrace 0 -showsplits 0] -required {all} -extras {none} -commandprocessors $cmdprocessors -values $::argv ] - - #set args [check_flags -caller test1 -defaults [list -collate nocase -algorithm sort -topchars "\uFFFF" -winlike 0 -debug 0 -stacktrace 0 -showsplits 0] -required {all} -extras {none} -commandprocessors {-cmd {-cmd -cmdarg1 -default "."} {-cmd -cmdarg2 -default j}} -values $::argv ] - #set args [check_flags -caller test1 -defaults [list -collate nocase -algorithm sort -topchars "\uFFFF" -winlike 0 -debug 0 -stacktrace 0 -showsplits 0] -required {all} -extras {none} -commandprocessors {{-cmd -default help} {-cmd -cmdarg1 -default "."} {-cmd -cmdarg2 -default j}} -values $::argv ] - #set args [check_flags -caller test1 -defaults [list -collate nocase -algorithm sort -topchars "\uFFFF" -winlike 0 -debug 0 -stacktrace 0 -showsplits 0] -required {all} -extras {none} -commandprocessors {ls {ls lsdir -default "\uFFFF"}} -values $::argv ] - - exit 0 - - if {$::argc} { - - } - } -} - - -package provide natsort [namespace eval natsort { - variable version - set version 0.1.1.5 -}] - - diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/oolib-0.1.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/oolib-0.1.tm deleted file mode 100644 index 3756fceb..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/oolib-0.1.tm +++ /dev/null @@ -1,195 +0,0 @@ -#JMN - api should be kept in sync with package patternlib where possible -# -package provide oolib [namespace eval oolib { - variable version - set version 0.1 -}] - -namespace eval oolib { - oo::class create collection { - variable o_data ;#dict - variable o_alias - constructor {} { - set o_data [dict create] - } - method info {} { - return [dict info $o_data] - } - method count {} { - return [dict size $o_data] - } - method isEmpty {} { - expr {[dict size $o_data] == 0} - } - method names {{globOrIdx {}}} { - if {[llength $globOrIdx]} { - if {[string is integer -strict $globOrIdx]} { - if {$idx < 0} { - set idx "end-[expr {abs($idx + 1)}]" - } - if {[catch {lindex [dict keys $o_data] $idx} result]} { - error "[self object] no such index : '$idx'" - } else { - return $result - } - } else { - #glob - return [lsearch -glob -all -inline [dict keys $o_data] $globOrIdx] - } - } else { - return [dict keys $o_data] - } - } - #like names but without globbing - method keys {} { - dict keys $o_data - } - method key {{posn 0}} { - if {$posn < 0} { - set posn "end-[expr {abs($posn + 1)}]" - } - if {[catch {lindex [dict keys $o_data] $posn} result]} { - error "[self object] no such index : '$posn'" - } else { - return $result - } - } - method hasKey {key} { - dict exists $o_data $key - } - method get {} { - return $o_data - } - method items {} { - return [dict values $o_data] - } - method item {key} { - if {[string is integer -strict $key]} { - if {$key > 0} { - set valposn [expr {(2*$key) +1}] - return [lindex $o_data $valposn] - } else { - set key "end-[expr {abs($key + 1)}]" - return [lindex [dict keys $o_data] $key] - } - } - if {[dict exists $o_data $key]} { - return [dict get $o_data $key] - } - } - #inverse lookup - method itemKeys {value} { - set value_indices [lsearch -all [dict values $o_data] $value] - set keylist [list] - foreach i $value_indices { - set idx [expr {(($i + 1) *2) -2}] - lappend keylist [lindex $o_data $idx] - } - return $keylist - } - method search {value args} { - set matches [lsearch {*}$args [dict values $o_data] $value] - if {"-inline" in $args} { - return $matches - } else { - set keylist [list] - foreach i $matches { - set idx [expr {(($i + 1) *2) -2}] - lappend keylist [lindex $o_data $idx] - } - return $keylist - } - } - #review - see patternlib. Is the intention for aliases to be configurable independent of whether the target exists? - method alias {newAlias existingKeyOrAlias} { - if {[string is integer -strict $newAlias]} { - error "[self object] collection key alias cannot be integer" - } - if {[string length $existingKeyOrAlias]} { - set o_alias($newAlias) $existingKeyOrAlias - } else { - unset o_alias($newAlias) - } - } - method aliases {{key ""}} { - if {[string length $key]} { - set result [list] - foreach {n v} [array get o_alias] { - if {$v eq $key} { - lappend result $n $v - } - } - return $result - } else { - return [array get o_alias] - } - } - #if the supplied index is an alias, return the underlying key; else return the index supplied. - method realKey {idx} { - if {[catch {set o_alias($idx)} key]} { - return $idx - } else { - return $key - } - } - method add {value key} { - if {[string is integer -strict $key]} { - error "[self object] collection key must not be an integer. Use another structure if integer keys required" - } - if {[dict exists $o_data $key]} { - error "[self object] col_processors object error: key '$key' already exists in collection" - } - dict set o_data $key $value - return [expr {[dict size $o_data] - 1}] ;#return index of item - } - method remove {idx {endRange ""}} { - if {[string length $endRange]} { - error "[self object] collection error: ranged removal not yet implemented.. remove one item at a time" - } - if {[string is integer -strict $idx]} { - if {$idx < 0} { - set idx "end-[expr {abs($idx+1)}]" - } - set key [lindex [dict keys $o_data] $idx] - set posn $idx - } else { - set key $idx - set posn [lsearch -exact [dict keys $o_data] $key] - if {$posn < 0} { - error "[self object] no such index: '$idx' in this collection" - } - } - dict unset o_data $key - return - } - method clear {} { - set o_data [dict create] - return - } - method reverse {} { - set dictnew [dict create] - foreach k [lreverse [dict keys $o_data]] { - dict set dictnew $k [dict get $o_data $k] - } - set o_data $dictnew - return - } - #review - cmd as list vs cmd as script? - method map {cmd} { - set seed [list] - dict for {k v} $o_data { - lappend seed [uplevel #0 [list {*}$cmd $v]] - } - return $seed - } - method objectmap {cmd} { - set seed [list] - dict for {k v} $o_data { - lappend seed [uplevel #0 [list $v {*}$cmd]] - } - return $seed - } - } - -} - diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/cap-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/cap-0.1.0.tm deleted file mode 100644 index 1c70a227..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/cap-0.1.0.tm +++ /dev/null @@ -1,663 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::cap 0.1.0 -# Meta platform tcl -# Meta description pkg capability register -# Meta license BSD -# @@ Meta End - - -#*** !doctools -#[manpage_begin punkshell_module_punk::cap 0 0.1.0] -#[copyright "2023 JMNoble - BSD licensed"] -#[titledesc {capability provider and handler plugin system}] -#[moddesc {punk capabilities plugin system}] -#[require punk::cap] -#[description] -#[keywords module capability plugin] -#[section Overview] -#[para]punk::cap provides management of named capabilities and the provider packages and handler packages that implement a pluggable capability. -#[para]see also [uri https://core.tcl-lang.org/tcllib/doc/trunk/embedded/md/tcllib/files/modules/pluginmgr/pluginmgr.md {tcllib pluginmgr}] for an alternative which uses safe interpreters -#[subsection Concepts] -#[para]A [term capability] may be something like providing a folder of files, or just a data dictionary, and/or an API -# -#[para][term {capability handler}] - a package/namespace which may provide validation and standardised ways of looking up provider data -# registered (or not) using register_capabilityname -# -#[para][term {capability provider}] - a package which registers as providing one or more capablities. -#[para]registered using register_package -#the capabilitylist is a list of 2-element lists where the first element is the capabilityname and the second element is a (possibly empty) dict of data relevant to that capability -#A capabilityname may appear multiple times. ie a package may register that it provides the capability with multiple datasets. - - -#*** !doctools -#[section API] - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz -package require oolib - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::cap { - variable pkgcapsdeclared [dict create] - variable pkgcapsaccepted [dict create] - variable caps [dict create] - namespace eval class { - if {[info commands [namespace current]::interface_caphandler.registry] eq ""} { - #*** !doctools - #[subsection {Namespace punk::cap::class}] - #[para] class definitions - #[list_begin itemized] [comment {- punk::cap::class groupings -}] - # [item] - # [para] [emph {handler_classes}] - # [list_begin enumerated] - - oo::class create [namespace current]::interface_caphandler.registry { - #*** !doctools - #[enum] CLASS [class interface_caphandler.registry] - #[list_begin definitions] - # [para] [emph METHODS] - method pkg_register {pkg capname capdict fullcapabilitylist} { - #*** !doctools - #[call class::interface_caphandler.registry [method pkg_register] [arg pkg] [arg capname] [arg capdict] [arg fullcapabilitylist]] - #handler may override and return 0 (indicating don't register)e.g if pkg capdict data wasn't valid - #overridden handler must be able to handle multiple calls for same pkg - but it may return 1 or 0 as it wishes. - return 1 ;#default to permit - } - method pkg_unregister {pkg} { - #*** !doctools - #[call class::interface_caphandler.registry [method pkg_unregister] [arg pkg]] - return ;#unregistration return is ignored - review - } - #*** !doctools - #[list_end] - } - - oo::class create [namespace current]::interface_caphandler.sysapi { - #*** !doctools - #[enum] CLASS [class interface_caphandler.sysapi] - #[list_begin definitions] - # [para] [emph METHODS] - - - #*** !doctools - #[list_end] - } - - #*** !doctools - # [list_end] [comment {- end enumeration handler classes -}] - - #*** !doctools - # [item] - # [para] [emph {provider_classes}] - # [list_begin enumerated] - - #Provider classes - oo::class create [namespace current]::interface_capprovider.registration { - #*** !doctools - # [enum] CLASS [class interface_cappprovider.registration] - # [para]Your provider package will need to instantiate this object under a sub-namespace called [namespace capsystem] within your package namespace. - # [para]If your package namespace is mypackages::providerpkg then the object command would be at mypackages::providerpkg::capsystem::capprovider.registration - # [para]Example code for your provider package to evaluate within its namespace: - # [example { - #namespace eval capsystem { - # if {[info commands capprovider.registration] eq ""} { - # punk::cap::class::interface_capprovider.registration create capprovider.registration - # oo::objdefine capprovider.registration { - # method get_declarations {} { - # set decls [list] - # lappend decls [list punk.templates {relpath ../templates}] - # lappend decls [list another_capability_name {somekey blah key2 etc}] - # return $decls - # } - # } - # } - #} - #}] - #[para] The above example declares that your package can be registered as a provider for the capabilities named 'punk.templates' and 'another_capability_name' - # [list_begin definitions] - # [para] [emph METHODS] - method get_declarations {} { - #*** - #[call class::interface_capprovider.registration [method get_declarations]] - #[para] This method must be overridden by your provider using oo::objdefine cappprovider.registration as in the example above. - # There must be at least one 2-element list in the result for the provider to be registerable. - #[para]The first element of the list is the capabilityname - which can be custom to your provider/handler packages - or a well-known name that other authors may use/implement. - #[para]The second element is a dictionary of keys specific to the capability being implemented. It may be empty if the any potential capability handlers for the named capability don't require registration data. - error "interface_capprovider.registration not implemented by provider" - } - #*** !doctools - # [list_end] - } - - oo::class create [namespace current]::interface_capprovider.provider { - #*** !doctools - # [enum] CLASS [class interface_capprovider.provider] - # [para] Your provider package will need to instantiate this directly under it's own namespace with the command name of [emph {provider}] - # [example { - # namespace eval mypackages::providerpkg { - # punk::cap::class::interface_capprovider.provider create provider mypackages::providerpkg - # } - # }] - # [list_begin definitions] - # [para] [emph METHODS] - variable provider_pkg - variable registrationobj - constructor {providerpkg} { - #*** !doctools - #[call class::interface_capprovider.provider [method constructor] [arg providerpkg]] - variable provider_pkg - if {$providerpkg in [list "" "::"]} { - error "interface_capprovider.provider constructor error. Invalid provider '$providerpkg'" - } - if {![namespace exists ::$providerpkg]} { - error "interface_capprovider.provider constructor error. Invalid provider '$providerpkg' - matching namespace not found" - } - - set registrationobj ::${providerpkg}::capsystem::capprovider.registration - if {[info commands $registrationobj] eq ""} { - error "capprovider.provider constructor error. Missing capprovider.registration interface at '$obj' (command not found) interface_capprovider.regstration instantiation must precede interface_capprovider.provider" - } - - set provider_pkg [string trim $providerpkg ""] - - } - method register {{capabilityname_glob *}} { - #*** !doctools - #[comment {- -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---}] - #[call class::interface_capprovider.provider [method register] [opt capabilityname_glob]] - # - #[para]This is the mechanism by which a user of your provider package will register your package as a provider of the capability named. - # - #[para]A user of your provider may elect to register all your declared capabilities: - #[example { - # package require mypackages::providerpkg - # mypackages::providerpkg::provider register * - #}] - #[para] Or a specific capability may be registered: - #[example { - # package require mypackages::providerpkg - # mypackages::providerpkg::provider register another_capability_name - #}] - # - variable provider_pkg - set all_decls [$registrationobj get_declarations] - set register_decls [lsearch -all -inline -index 0 $all_decls $capabilityname_glob] - punk::cap::register_package $provider_pkg $register_decls - } - method capabilities {} { - #*** !doctools - #[comment {- -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---}] - #[call class::interface_capprovider.provider [method capabilities]] - #[para] return a list of capabilities supported by this provider package - variable provider_pkg - variable registrationobj - - set capabilities [list] - set decls [$registrationobj get_declarations] - foreach decl $decls { - lassign $decl capname capdict - if {$capname ni $capabilities} { - lappend capabilities $capname - } - } - return $capabilities - } - #*** !doctools - # [list_end] [comment {- end class definitions -}] - } - #*** !doctools - # [list_end] [comment {- end enumeration provider_classes }] - #[list_end] [comment {- end itemized list punk::cap::class groupings -}] - } - } ;# end namespace class - - #*** !doctools - #[subsection {Namespace punk::cap}] - #[para] Main punk::cap API for client programs interested in using capability handler packages and associated (registered) provider packages - #[list_begin definitions] - - #Not all capability names have to be registered. - #A package registering as a provider using register_package can include capabilitynames in it's capabilitylist which have no associated handler. - #such unregistered capabilitynames may be used just to flag something, or have datamembers significant to callers cooperatively interested in that capname. - #we allow registering a capability with an empty handler (capnamespace) - but this means another handler could be registered later. - proc register_capabilityname {capname capnamespace} { - variable caps - variable pkgcapsdeclared - variable pkgcapsaccepted - if {$capnamespace ne ""} { - #normalize with leading :: in case caller passed in package name rather than fully qualified namespace - if {![string match ::* $capnamespace]} { - set capnamespace ::$capnamespace - } - } - #allow register of existing capname iff there is no current handler - #as handlers can be used to validate during provider registration - ideally handlers should be registered before any pkgs call register_package - #we allow loading a handler later though - but will need to validate existing data from pkgs that have already registered as providers - if {[set hdlr [capability_get_handler $capname]] ne ""} { - error "register_capabilityname cannot register capability:$capname with handler:$capnamespace. There is already a registered handler:$hdlr" - } - #assert: capnamespace may or may not be empty string, capname may or may not already exist in caps dict, caps $capname providers may have existing entries. - dict set caps $capname handler $capnamespace - if {![dict exists $caps $capname providers]} { - dict set caps $capname providers [list] - } - if {[llength [set providers [dict get $caps $capname providers]]]} { - #some provider(s) were in place before the handler was registered - if {[set capreg [punk::cap::capsystem::get_caphandler_registry $capname]] ne ""} { - foreach pkg $providers { - set fullcapabilitylist [dict get $pkgcapsdeclared $pkg] - foreach capspec $fullcapabilitylist { - lassign $capspec cn capdict - if {$cn ne $capname} { - continue - } - set do_register [$capreg pkg_register $pkg $capdict $fullcapabilitylist] - set list_accepted [dict get $pkgcapsaccepted $pkg] - if {$do_register} { - if {$capspec ni $list_accepted} { - dict lappend pkgcapsaccepted $pkg $capspec - } - } else { - set posn [lsearch $list_accepted $capspec] - if {$posn >=0} { - set list_accepted [lreplace $list_accepted $posn $posn] - dict set pkgcapsaccepted $pkg $list_accepted - } - } - } - #check if any accepted for this cap and remove from caps as necessary - set count 0 - foreach accepted_capspec [dict get $pkgcapsaccepted $pkg] { - if {[lindex $accepted_capspec 0] eq $capname} { - incr count - } - } - if {$count == 0} { - set pkgposn [lsearch $providers $pkg] - if {$pkgposn >= 0} { - set updated_providers [lreplace $providers $posn $posn] - dict set caps $capname providers $updated_providers - } - } - } - - } - - } - } - proc capability_exists {capname} { - #*** !doctools - # [call [fun capability_exists] [arg capname]] - # Return a boolean indicating if the named capability exists (0|1) - variable caps - return [dict exists $caps $capname] - } - proc capability_has_handler {capname} { - #*** !doctools - # [call [fun capability_has_handler] [arg capname]] - #Return a boolean indicating if the named capability has a handler package installed (0|1) - variable caps - return [expr {[dict exists $caps $capname handler] && [dict get $caps $capname handler] ne ""}] - } - proc capability_get_handler {capname} { - #*** !doctools - # [call [fun capability_get_handler] [arg capname]] - #Return the base namespace of the active handler package for the named capability. - #[para] The base namespace for a handler will always be the package name, but prefixed with :: - variable caps - if {[dict exists $caps $capname]} { - return [dict get $caps $capname handler] - } - return "" - } - proc call_handler {capname args} { - if {[set handler [capability_get_handler $capname]] eq ""} { - error "punk::cap::call_handler $capname $args - no handler registered for capability $capname" - } - set obj ${handler}::api_$capname - $obj [lindex $args 0] {*}[lrange $args 1 end] - } - proc get_providers {capname} { - variable caps - if {[dict exists $caps $capname]} { - return [dict get $caps $capname providers] - } - return [list] - } - - #register package with arbitrary capnames from capabilitylist - #The registered pkg is a module that provides some service to that capname. Possibly just data members, that the capability will use. - proc register_package {pkg capabilitylist args} { - variable pkgcapsdeclared - variable pkgcapsaccepted - variable caps - set defaults [dict create\ - -nowarnings false - ] - dict for {k v} $args { - if {$k ni $defaults} { - error "Unrecognized option $k. Known options [dict keys $defaults]" - } - } - set opts [dict merge $defaults $args] - set warnings [expr {! [dict get $opts -nowarnings]}] - - if {[string match ::* $pkg]} { - set pkg [string range $pkg 2 end] - } - if {[dict exists $pkgcapsaccepted $pkg]} { - set pkg_already_accepted [dict get $pkgcapsaccepted $pkg] - } else { - set pkg_already_accepted [list] - } - package require $pkg - set providerapi ::${pkg}::provider - if {[info commands $providerapi] eq ""} { - error "register_package error. pkg '$pkg' doesn't seem to be a punk::cap capability provider (no object found at $providerapi)" - } - set defined_caps [$providerapi capabilities] - #for each capability - # - ensure 1st element is a single word - # - ensure that if 2nd element (capdict) is present - it is dict shaped - foreach capspec $capabilitylist { - lassign $capspec capname capdict - - if {$warnings} { - if {$capname ni $defined_caps} { - puts stderr "WARNING: pkg '$pkg' doesn't declare support for capability '$capname'." - } - } - if {[llength $capname] !=1} { - error "register_package error. pkg: '$pkg' An entry in the capability list doesn't appear to have a single-word name. Problematic entry:'$capspec'" - } - if {[expr {[llength $capdict] %2 != 0}]} { - error "register_package error. pkg:'$pkg' The second element for capname:'$capname' doesn't appear to be a valid dict. Problematic entry: '$capspec'" - } - if {$capspec in $pkg_already_accepted} { - #review - multiple handlers? if so - will need to record which handler(s) accepted the capspec - if {$warnings} { - puts stderr "WARNING: register_package pkg $pkg already has capspec marked as accepted: $capspec" - } - continue - } - if {[dict exists $caps $capname]} { - set cap_pkgs [dict get $caps $capname providers] - } else { - dict set caps $capname [dict create handler "" providers [list]] - set cap_pkgs [list] - } - #todo - if there's a caphandler - call it's init/validation callback for the pkg - set do_register 1 ;#default assumption unless vetoed by handler - if {[set capreg [punk::cap::capsystem::get_caphandler_registry $capname]] ne ""} { - #Note that the interface_caphandler.registry instance must be able to handle multiple calls for same pkg - set do_register [$capreg pkg_register $pkg $capname $capdict $capabilitylist] - } - if {$do_register} { - if {$pkg ni $cap_pkgs} { - lappend cap_pkgs $pkg - dict set caps $capname providers $cap_pkgs - } - dict lappend pkgcapsaccepted $pkg $capspec ;#if pkg is being registered prior to handler-registration - the handler may undo this entry - } - } - #another call to register_pkg with same pkg may have been made (most likely with different capname) so we must append - but check not already present - #dict lappend pkgcapsdeclared $pkg $capabilitylist - if {[dict exists $pkgcapsdeclared $pkg]} { - set capspecs [dict get $pkgcapsdeclared $pkg] - foreach spec $capspecs { - if {$spec ni $capspecs} { - lappend capspecs $spec - } - } - dict set pkgcapsdeclared $pkg $capspecs - } else { - dict set pkgcapsdeclared $pkg $capabilitylist - } - } - - #todo! - proc unregister_package {pkg {capname *}} { - variable pkgcapsdeclared - variable caps - if {[string match ::* $pkg]} { - set pkg [string range $pkg 2 end] - } - if {[dict exists $pkgcapsdeclared $pkg]} { - #remove corresponding entries in caps - set capabilitylist [dict get $pkgcapsdeclared $pkg] - foreach c $capabilitylist { - set do_unregister 1 - lassign $c capname _capdict - set cap_info [dict get $caps $capname] - set pkglist [dict get $cap_info providers] - set posn [lsearch $pkglist $pkg] - if {$posn >= 0} { - if {[set capreg [punk::cap::capsystem::get_caphandler_registry $capname]] ne ""} { - #review - # it seems not useful to allow the callback to block this unregister action - #the pkg may have multiple datasets for each capname so callback will only be called for first dataset we encounter - #vetoing unregister would make this more complex for no particular advantage - #if per dataset deregistration required this should probably be a separate thing - $capreg pkg_unregister $pkg $capname - } - set pkglist [lreplace $pkglist $posn $posn] - dict set caps $capname providers $pkglist - } - } - #delete the main registration record - dict unset pkgcapsdeclared $pkg - } - } - - proc pkgcap {pkg} { - variable pkgcapsdeclared - variable pkgcapsaccepted - if {[string match ::* $pkg]} { - set pkg [string range $pkg 2 end] - } - if {[dict exists $pkgcapsdeclared $pkg]} { - set accepted "" - if {[dict exists $pkgcapsaccepted $pkg]} { - set accepted [dict get $pkgcapsaccepted $pkg] - } - return [dict create declared [dict get $pkgcapsdeclared $pkg] accepted $accepted] - } else { - return - } - } - proc pkgcaps {} { - variable pkgcapsdeclared - variable pkgcapsaccepted - set result [dict create] - foreach {pkg capsdeclared} $pkgcapsdeclared { - set accepted "" - if {[dict exists $pkgcapsaccepted $pkg]} { - set accepted [dict get $pkgcapsaccepted $pkg] - } - dict set result $pkg declared $capsdeclared - dict set result $pkg accepted $accepted - } - return $result - } - - proc capability {capname} { - variable caps - if {[dict exists $caps $capname]} { - return [dict get $caps $capname] - } - return "" - } - proc capabilities {{glob *}} { - variable caps - set capnames [lsort [dict keys $caps $glob]] - set cap_list [list] - foreach capname $capnames { - lappend cap_list [list $capname [dict get $caps $capname]] - } - return $cap_list - } - - proc capabilitynames {{glob *}} { - variable caps - return [lsort [dict keys $caps $glob]] - } - #return only those capnames which have at least one provider - proc capabilitynames_provided {{glob *}} { - variable caps - set keys [lsort [dict keys $caps $glob]] - set cap_list [list] - foreach k $keys { - if {[llength [dict get $caps $k providers]] > 0} { - lappend cap_list $k - } - } - return $cap_list - } - #*** !doctools - #[list_end] [comment {- end definitions for namespace punk::cap -}] - - namespace eval advanced { - #*** !doctools - #[subsection {Namespace punk::cap::advanced}] - #[para] punk::cap::advanced API. Functions here are generally not the preferred way to interact with punk::cap. - #[para] In some cases they may allow interaction in less safe ways or may allow use of features that are unavailable in the base namespace. - #[para] Some functions are here because they are only marginally or rarely useful, and they are here to keep the base API simple. - #[list_begin definitions] - - proc promote_provider {pkg} { - #*** !doctools - # [call advanced::[fun promote_provider] [arg pkg]] - #[para]Move the named provider package to the preferred end of the list (tail). - #[para]The active handler may or may not utilise this for preferencing. See documentation for the specific handler package to confirm. - #[para] - #[para] promote/demote doesn't always make a lot of sense .. should preferably be configurable per capapbility for multicap provider pkgs - #[para]The idea is to provide a crude way to preference/depreference packages independently of order the packages were loaded - #e.g a caller or cap-handler can ascribe some meaning to the order of the 'providers' key returned from punk::cap::capabilities - #[para]The order of providers will be the order the packages were loaded & registered - #[para]the naming: "promote vs demote" operates on a latest-package-in-list has higher preference assumption (matching last pkg loaded) - #[para]Each capability handler could and should implement specific preferencing methods within its own API if finer control needed. - #In some cases the preference/loading order may be inapplicable/irrelevant to a particular capability anyway. - #[para]As this is just a basic mechanism, which can't support independent per-cap preferencing for multi-cap packages - - # it only allows putting the pkgs to the head or tail of the lists. - #[para]Whether particular caps or users of caps do anything with this ordering is dependent on the cap-handler and/or calling code. - variable pkgcapsdeclared - variable caps - if {[string match ::* $pkg]} { - set pkg [string range $pkg 2 end] - } - if {![dict exists $pkgcapsdeclared $pkg]} { - error "punk::cap::promote_package error pkg'$pkg' not registered. Use register_package \$pkg first" - } - if {[dict size $pkgcapsdeclared] > 1} { - set pkginfo [dict get $pkgcapsdeclared $pkg] - #remove and re-add at end of dict - dict unset pkgcapsdeclared $pkg - dict set pkgcapsdeclared $pkg $pkginfo - dict for {cap cap_info} $caps { - set cap_pkgs [dict get $cap_info providers] - if {$pkg in $cap_pkgs} { - set posn [lsearch $cap_pkgs $pkg] - if {$posn >=0} { - #rewrite package list with pkg at tail of list for this capability - set cap_pkgs [lreplace $cap_pkgs $posn $posn] - lappend cap_pkgs $pkg - dict set caps $cap providers $cap_pkgs - } - } - } - } - } - proc demote_provider {pkg} { - #*** !doctools - # [call advanced::[fun demote_provider] [arg pkg]] - #[para]Move the named provider package to the preferred end of the list (tail). - #[para]The active handler may or may not utilise this for preferencing. See documentation for the specific handler package to confirm. - variable pkgcapsdeclared - variable caps - if {[string match ::* $pkg]} { - set pkg [string range $pkg 2 end] - } - if {![dict exists $pkgcapsdeclared $pkg]} { - error "punk::cap::promote_package error pkg'$pkg' not registered. Use register_package \$pkg first" - } - if {[dict size $pkgcapsdeclared] > 1} { - set pkginfo [dict get $pkgcapsdeclared $pkg] - #remove and re-add at start of dict - dict unset pkgcapsdeclared $pkg - dict set pkgcapsdeclared $pkg $pkginfo - set pkgcapsdeclared [dict merge [dict create $pkg $pkginfo] $pkgcapsdeclared] - dict for {cap cap_info} $caps { - set cap_pkgs [dict get $cap_info providers] - if {$pkg in $cap_pkgs} { - set posn [lsearch $cap_pkgs $pkg] - if {$posn >=0} { - #rewrite package list with pkg at head of list for this capability - set cap_pkgs [lreplace $cap_pkgs $posn $posn] - set cap_pkgs [list $pkg {*}$cap_pkgs] - dict set caps $cap providers $cap_pkgs - } - } - } - } - } - - #*** !doctools - #[list_end] - } - - -#*** !doctools -#[section Internal] - - namespace eval capsystem { - #*** !doctools - #[subsection {Namespace punk::cap::capsystem}] - #[para] Internal functions used to communicate between punk::cap and capability handlers - #[list_begin definitions] - proc get_caphandler_registry {capname} { - set ns [::punk::cap::capability_get_handler $capname]::capsystem - if {[namespace exists ${ns}]} { - if {[info command ${ns}::caphandler.registry] ne ""} { - if {[info object isa object ${ns}::caphandler.registry]} { - return ${ns}::caphandler.registry - } - } - } - return "" - } - #*** !doctools - #[list_end] - } -} - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::cap [namespace eval punk::cap { - variable version - variable pkg punk::cap - set version 0.1.0 - variable README.md [string map [list %pkg% $pkg %ver% $version] { - # punk capabilities system - ## pkg: %pkg% version: %ver% - - punk::cap base namespace - }] - return $version -}] -return - -#*** !doctools -#[manpage_end] diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/cap/handlers/caphandler-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/cap/handlers/caphandler-0.1.0.tm deleted file mode 100644 index 8fdce944..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/cap/handlers/caphandler-0.1.0.tm +++ /dev/null @@ -1,52 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::cap::handlers::caphandler 0.1.0 -# Meta platform tcl -# Meta license -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::cap::handlers::caphandler { - - - - -} - - - - - - - - - - - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::cap::handlers::caphandler [namespace eval punk::cap::handlers::caphandler { - variable pkg punk::cap::handlers::caphandler - variable version - set version 0.1.0 -}] -return \ No newline at end of file diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/cap/handlers/scriptlibs-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/cap/handlers/scriptlibs-0.1.0.tm deleted file mode 100644 index 8298ec18..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/cap/handlers/scriptlibs-0.1.0.tm +++ /dev/null @@ -1,52 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::cap::handlers::scriptlibs 0.1.0 -# Meta platform tcl -# Meta license -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::cap::handlers::scriptlibs { - - - - -} - - - - - - - - - - - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::cap::handlers::scriptlibs [namespace eval punk::cap::handlers::scriptlibs { - variable pkg punk::cap::handlers::scriptlibs - variable version - set version 0.1.0 -}] -return \ No newline at end of file diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/cap/handlers/templates-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/cap/handlers/templates-0.1.0.tm deleted file mode 100644 index 75a925dd..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/cap/handlers/templates-0.1.0.tm +++ /dev/null @@ -1,145 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::cap::handlers::templates 0.1.0 -# Meta platform tcl -# Meta license -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -#register using: -# punk::cap::register_capabilityname templates ::punk::cap::handlers::templates - -#By convention and for consistency, we don't register here during package loading - but require the calling app to do it. -# (even if it tends to be done immediately after package require anyway) -# registering capability handlers can involve validating existing provider data and is best done explicitly as required. -# It is also possible for a capability handler to be registered to handle more than one capabilityname - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::cap::handlers::templates { - namespace eval capsystem { - #interfaces for punk::cap to call into - if {[info commands caphandler.registry] eq ""} { - punk::cap::class::interface_caphandler.registry create caphandler.registry - oo::objdefine caphandler.registry { - method pkg_register {pkg capname capdict caplist} { - #caplist may not be complete set - which somewhat reduces its utility here regarding any decisions based on the context of this capname/capdict (review - remove this arg?) - - # -- --- --- --- --- --- --- ---- --- - # validation of capdict - # -- --- --- --- --- --- --- ---- --- - if {![dict exists $capdict relpath]} { - puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - package '$pkg' is attempting to register with punk::cap as a provider of '$capname' capability, but is missing 'relpath' key" - return 0 - } - set provide_statement [package ifneeded $pkg [package require $pkg]] - set tmfile [lindex $provide_statement end] - if {![file exists $tmfile]} { - puts stderr "punk::cap::handlers::templates::capsystem::pkg_register WARNING - unable to determine base folder for package '$pkg' which is attempting to register with punk::cap as a provider of '$capname' capability" - return 0 - } - set tpath [file normalize [file join $tmfile [dict get $capdict relpath]]] ;#relpath is relative to the tm *file* - not it's containing folder - if {![file isdirectory $tpath]} { - puts stderr "punk::cap::handlers::templates::capsystem pkg_register WARNING - unable to validate relpath location [dict get $capdict relpath] ($tpath) for package '$pkg' which is attempting to register with punk::cap as a provider of '$capname' capability" - return 0 - } - - - # -- --- --- --- --- --- --- ---- --- - # update package internal data - # -- --- --- --- --- --- --- ---- --- - if {$capname ni $::punk::cap::handlers::templates::handled_caps} { - lappend ::punk::cap::handlers::templates::handled_caps $capname - } - set cname [string map [list . _] $capname] - upvar ::punk::cap::handlers::templates::pkg_folders_$cname pfolders - dict lappend pfolders $pkg $tpath - - - # -- --- --- --- --- --- --- ---- --- - # instantiation of api at punk::cap::handlers::templates::api_$capname - # -- --- --- --- --- --- --- ---- --- - if {[info commands ::punk::cap::handlers::templates::$capname] eq ""} { - punk::cap::handlers::templates::class::api create ::punk::cap::handlers::templates::api_$capname $capname - } - - return 1 - } - method pkg_unregister {pkg} { - upvar ::punk::cap::handlers::templates::handled_caps hcaps - foreach capname $hcaps { - set cname [string map [list . _] $capname] - upvar ::punk::cap::handlers::templates::pkg_folders_$cname pfolders - dict unset pfolders $pkg - #destroy api objects? - } - } - } - } - } - - variable handled_caps [list] - #variable pkg_folders [dict create] - - # -- --- --- --- --- --- --- - #handler api for clients of this capability - called via punk::cap::call_handler ?args? - # -- --- --- --- --- --- --- - namespace export * - namespace eval class { - oo::class create api { - #return a dict keyed on folder with source pkg as value - constructor {capname} { - variable capabilityname - variable cname - set cname [string map [list . _] $capname] - set capabilityname $capname - } - method folders {} { - variable capabilityname - variable cname - upvar punk::cap::handlers::templates::pkg_folders_$cname pkg_folders - package require punk::cap - set capinfo [punk::cap::capability $capabilityname] - # e.g {punk.templates {handler punk::mix::templates providers ::somepkg}} - - #use the order of pkgs as registered with punk::cap - may have been modified with punk::cap::promote_package/demote_package - set pkgs [dict get $capinfo providers] - set folderdict [dict create] - foreach pkg $pkgs { - foreach pfolder [dict get $pkg_folders $pkg] { - dict set folderdict $pfolder [list source $pkg sourcetype package] - } - } - return $folderdict - } - } - } - - - -} - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::cap::handlers::templates [namespace eval punk::cap::handlers::templates { - variable pkg punk::cap::handlers::templates - variable version - set version 0.1.0 -}] -return \ No newline at end of file diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/docgen-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/docgen-0.1.0.tm deleted file mode 100644 index f4d26342..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/docgen-0.1.0.tm +++ /dev/null @@ -1,71 +0,0 @@ - -# -*- tcl -* -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::docgen 0.1.0 -# Meta platform tcl -# Meta license BSD -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz - -package require punk::repo - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::docgen { - proc get_doctools_comments {fname} { - #does no validation of doctools commands - #existence of string match #\**!doctools is taken as evidence enough that the file has inline doctools - review - if {![file exists $fname]} { - error "get_doctools_comments file '$fname' not found" - } - set fd [open $fname r] - set data [read $fd] - close $fd - if {![string match "*#\**!doctools*" $data]} { - return - } - set data [string map [list \r\n \n] $data] - set in_doctools 0 - set doctools "" - foreach ln [split $data \n] { - set ln [string trim $ln] - if {$in_doctools && [string index $ln 0] != "#"} { - set in_doctools 0 - } elseif {[string range $ln 0 1] == "#*"} { - #todo - process doctools ordering hints in tail of line - set in_doctools 1 - } elseif {$in_doctools} { - append doctools [string range $ln 1 end] \n - } - } - return $doctools - } - #todo - proc autogen_doctools_comments {fname} {} - # - will probably need to use something like parsetcl - as we won't be able to reliably source in an interp without side-effects and use info body etc. - # - mechanism will be to autodocument namespaces, procs, methods where no #*** doctools indication present - but use existing doctools comments for that particular item if it is present. - - - -} - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::docgen [namespace eval punk::docgen { - variable pkg punk::docgen - variable version - set version 0.1.0 -}] -return \ No newline at end of file diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/du-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/du-0.1.0.tm deleted file mode 100644 index fa02b3a9..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/du-0.1.0.tm +++ /dev/null @@ -1,1284 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::du 0.1.0 -# Meta platform tcl -# Meta license BSD -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz -package require punk::mix::base - - -namespace eval punk::du { - variable has_twapi 0 -} -if {"windows" eq $::tcl_platform(platform)} { - package require zzzload - zzzload::pkg_require twapi - - if {[catch {package require twapi}]} { - puts stderr "Warning: punk::du - unable to load twapi. Disk operations may be much slower on windows without the twapi package" - } else { - set punk::du::has_twapi 1 - } - #package require punk::winpath -} - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::du { - - - proc dirlisting {folderpath args} { - set defaults [dict create\ - -glob *\ - ] - set opts [dict merge $defaults $args] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_glob [dict get $opts -glob] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - if {[lib::pathcharacterlen $folderpath] == 0} { - set folderpath [pwd] - } elseif {[file pathtype $folderpath] ne "absolute"} { - #file normalize relativelly slow - avoid in inner loops - #set folderpath [file normalize $folderpath] - - } - #run whichever of du_dirlisting_twapi, du_dirlisting_generic, du_dirlisting_unix has been activated - set dirinfo [active::du_dirlisting $folderpath {*}$opts] - } - - - - #Note that unix du seems to do depth-first - which makese sense when piping.. as output can be emitted as we go rather than requiring sort at end. - #breadth-first with sort can be quite fast .. but memory usage can easily get out of control - proc du { args } { - variable has_twapi - package require struct::set - - - if 0 { - switch -exact [llength $args] { - 0 { - set dir . - set switch -k - } - 1 { - set dir $args - set switch -k - } - 2 { - set switch [lindex $args 0] - set dir [lindex $args 1] - } - default { - set msg "only one switch and one dir " - append msg "currently supported" - return -code error $msg - } - } - - set switch [string tolower $switch] - - set -b 1 - set -k 1024 - set -m [expr 1024*1024] - } - - - set opts $args - # flags in args are solos (or longopts --something=somethingelse) or sometimes pairopts - # we don't currently support mashopts (ie -xy vs separate -x -y) - - - #------------------------------------------------------- - # process any pairopts first and remove the pair - # (may also process some solo-opts) - - set opt_depth -1 - if {[set posn [lsearch $opts -d]] >= 0} { - set opt_depth [lindex $opts $posn+1] - set opts [lreplace $opts $posn $posn+1] - } - foreach o $opts { - if {[string match --max-depth=* $o]} { - set opt_depth [lindex [split $o =] 1] - if {![string is integer -strict $opt_depth]} { - error "--max-depth=n n must be an integer" - } - } - } - #------------------------------------------------------- - #only solos and longopts remain in the opts now - - - set lastarg [lindex $opts end] - if {[string length $lastarg] && (![string match -* $lastarg])} { - set dir $lastarg - set opts [lrange $opts 0 end-1] - } else { - set dir . - set opts $opts - } - foreach a $opts { - if {![string match -* $a]} { - error "unrecognized option '$a'" - } - } - - set -b 1 - set -k 1024 - set -m [expr 1024*1024] - set switch -k ;#default (same as unix) - set lc_opts [string tolower $opts] - - - - if {"-b" in $lc_opts} { - set switch -b - } elseif {"-k" in $lc_opts} { - set switch -k - } elseif {"-m" in $lc_opts} { - set switch -m - } - set opt_progress 0 - if {"--prog" in $lc_opts || "--progress" in $lc_opts} { - set opt_progress 1 - } - set opt_extra 0 - if {"--extra" in $lc_opts} { - set opt_extra 1 - } - set opt_vfs 0 - #This configures whether to enter a vfsmount point - #It will have no effect if cwd already with a vfs mount point - as then opt_vfs will be set to 1 automatically anyway. - if {"--vfs" in $lc_opts} { - set opt_vfs 1 - } - - - - set result [list] - - set dir_depths_remaining [list] - - set is_windows [expr {$::tcl_platform(platform) eq "windows"}] - set zero [expr {0}] - - # ## ### ### ### ### - # containerid and itemid - set folders [list] ;#we lookup string by index - lappend folders [file dirname $dir] - lappend folders $dir ;#itemindex 1 - # ## ### ### ### ### - if {![file isdirectory $dir]} { - lappend dir_depths_remaining [list $zero $zero [expr {1}] [lib::du_lit [file tail $dir]] [file size $dir]] - #set ary($dir,bytes) [file size $dir] - set leveldircount 0 - } else { - lappend dir_depths_remaining [list $zero $zero [expr {1}] [lib::du_lit [file tail $dir]] $zero] - set leveldircount 1 - } - set level [expr {0}] - set nextlevel [expr {1}] - #dir_depths list structure - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - #0 1 2 3 4 5 - #i_depth i_containerid i_itemid i_item i_size i_index - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - set i_depth [expr {0}] - set i_containerid [expr {1}] - set i_itemid [expr {2}] - set i_item [expr {3}] - set i_size [expr {4}] - set i_index [expr {5}] - - - set listlength [llength $dir_depths_remaining] - set diridx 0 - #this is a breadth-first algorithm - while {$leveldircount > 0} { - set leveldirs 0 - set levelfiles 0 - for {set i $diridx} {$i < $listlength} {incr i} { - #lassign [lindex $dir_depths_remaining $i] _d containeridx folderidx itm bytecount - set folderidx [lindex $dir_depths_remaining $i $i_itemid] - set folderpath [lindex $folders $folderidx] - #puts stderr ->$folderpath - #if {$i >= 20} { - #return - #} - - #twapi supports gathering file sizes during directory contents traversal - #for dirlisting methods that return an empty list in filesizes whilst files has entries - we will need to populate it below - #e.g tcl glob based dirlisting doesn't support gathering file sizes at the same time - - set in_vfs 0 - if {[package provide vfs] ne ""} { - foreach vfsmount [vfs::filesystem info] { - if {[file pathtype $folderpath] ne "absolute"} { - set testpath [file normalize $folderpath] - } else { - set testpath $folderpath - } - - if {[punk::mix::base::lib::path_a_atorbelow_b $testpath $vfsmount]} { - set in_vfs 1 - #if already descended to or below a vfs mount point - set opt_vfs true - set opt_vfs 1 - break - } - } - } - - if {$in_vfs} { - set du_info [lib::du_dirlisting_tclvfs $folderpath] - } else { - #run the activated function (proc imported to active namespace and renamed) - set du_info [active::du_dirlisting $folderpath] - } - - - set dirs [dict get $du_info dirs] - set files [dict get $du_info files] - set filesizes [dict get $du_info filesizes] - set vfsmounts [dict get $du_info vfsmounts] - #puts "---> vfsmounts $vfsmounts " - if {$opt_vfs} { - foreach vm $vfsmounts { - #puts stderr "vm: $vm" - #check if vfs is mounted over a file or a dir - if {$vm in $files} { - puts stderr "vfs mounted over file $vm" - set mposn [lsearch $files $vm] - set files [lreplace $files $mposn $mposn] - if {[llength $filesizes]} { - set filesizes [lreplace $filesizes $mposn $mposn] - } - } - if {$vm ni $dirs} { - puts stderr "treating $vm as dir" - lappend dirs $vm - } - } - } - - - incr leveldirs [llength $dirs] - incr levelfiles [llength $files] - - #lappend dir_depths_remaining {*}[lmap d $dirs {::list $nextdepth [lib::du_lit $cont/$itm] $d $zero}] - #folderidx is parent index for new dirs - lappend dir_depths_remaining {*}[lib::du_new_eachdir $dirs $nextlevel $folderidx] - - #we don't need to sort files (unless we add an option such as -a to du (?)) - set bytecount [expr {0}] - - if {[llength $files] && ![llength $filesizes]} { - #listing mechanism didn't supply corresponding sizes - foreach filename $files { - #incr bytecount [file size [file join $folderpath $filename] - incr bytecount [file size $filename] - } - } else { - set filesizes [lsearch -all -inline -not $filesizes[unset filesizes] na] ;#only legal non-number is na - set bytecount [tcl::mathop::+ {*}$filesizes] - } - - - #we can safely assume initial count was zero - lset dir_depths_remaining $i $i_size $bytecount - #incr diridx - } - #puts stdout "level: $level dirs: $leveldirs" - if {$opt_extra} { - puts stdout "level: $level dircount: $leveldirs filecount: $levelfiles" - } - incr level ;#zero based - set nextlevel [expr {$level + 1}] - set leveldircount [expr {[llength $dir_depths_remaining] - $listlength }]; #current - previous - while loop terminates when zero - #puts "diridx: $diridx i: $i rem: [llength $dir_depths_remaining] listlenth:$listlength levldircount: $leveldircount" - set diridx $i - set listlength [llength $dir_depths_remaining] - } - #puts stdout ">>> loop done" - #flush stdout - #puts stdout $dir_depths_remaining - set dirs_as_encountered $dir_depths_remaining ;#index is in sync with 'folders' list - set dir_depths_longfirst $dirs_as_encountered - - #store the index before sorting - for {set i 0} {$i < [llength $dir_depths_remaining]} {incr i} { - lset dir_depths_longfirst $i $i_index $i - } - set dir_depths_longfirst [lsort -integer -index 0 -decreasing $dir_depths_longfirst[set dir_depths_longfirst {}]] - - #store main index in the reducing list - set dir_depths_remaining $dir_depths_longfirst - for {set i 0} {$i < [llength $dir_depths_remaining]} {incr i} { - #stored index at position 3 - lset dir_depths_remaining $i $i_index $i - } - - #index 3 - #dir_depths_remaining -> dir_depths_longfirst -> dirs_as_encountered - - #puts stdout "initial dir_depths_remaining: $dir_depths_remaining" - - - #summing performance is not terrible but significant on large tree - the real time is for large trees in the main loop above - #update - on really large trees the reverse is true especiallyl now that twapi fixed the original speed issues.. todo - rework/simplify below - review natsort - # - #TODO - reconsider sorting by depth.. lreverse dirs_as_encountered should work.. - if {[llength $dir_depths_longfirst] > 1} { - set i 0 - foreach dd $dir_depths_longfirst { - lassign $dd d parentidx folderidx item bytecount - #set nm $cont/$item - set nm [lindex $folders $folderidx] - set dnext [expr {$d +1}] - set nextdepthposns [lsearch -all -integer -index 0 $dir_depths_remaining $dnext] - set nextdepthposns [lsort -integer -decreasing $nextdepthposns[set nextdepthposns {}]];#remove later elements first - foreach posn $nextdepthposns { - set id [lindex $dir_depths_remaining $posn $i_itemid] - set ndirname [lindex $folders $id] - #set ndirname $cont/$item - #set item [lindex $dir_depths_remaining $posn $i_item] - #set ndirname [lindex $ndir 1] - if {[string match $nm/* $ndirname]} { - #puts stdout "dir $nm adding subdir size $ndirname" - #puts stdout "incr $nm from $ary($nm,bytes) plus $ary($ndirname,bytes)" - incr bytecount [lindex $dir_depths_remaining $posn $i_size] - set dir_depths_remaining [lreplace $dir_depths_remaining[set dir_depths_remaining {}] $posn $posn] - } - } - lset dir_depths_longfirst $i $i_size $bytecount - set p [lsearch -index $i_index -integer $dir_depths_remaining $i] - lset dir_depths_remaining $p $i_size $bytecount - #set ary($nm,bytes) $bytecount - incr i - } - } - #set dir_depths_longfirst [lsort -index 1 -decreasing $dir_depths_longfirst] - # - - set retval [list] - #copy across the bytecounts - for {set i 0} {$i < [llength $dir_depths_longfirst]} {incr i} { - set posn [lindex $dir_depths_longfirst $i $i_index] - set bytes [lindex $dir_depths_longfirst $i $i_size] - lset dirs_as_encountered $posn $i_size $bytes - } - foreach dirinfo [lreverse $dirs_as_encountered] { - set id [lindex $dirinfo $i_itemid] - set depth [lindex $dirinfo $i_depth] - if {($opt_depth >= 0) && $depth > $opt_depth} { - continue - } - set path [lindex $folders $id] - #set path $cont/$item - set item [lindex $dirinfo $i_item] - set bytes [lindex $dirinfo $i_size] - set size [expr {$bytes / [set $switch]}] - lappend retval [list $size $path] - } - # copyright 2002 by The LIGO Laboratory - return $retval - } - namespace eval active { - variable functions [list du_dirlisting ""] - variable functions_known [dict create] - - #known functions from lib namespace - dict set functions_known du_dirlisting [list du_dirlisting_twapi du_dirlisting_generic du_dirlisting_unix du_dirlisting_undecided] - - proc show_functions {} { - variable functions - variable functions_known - set msg "" - dict for {callname implementations} $functions_known { - append msg "callname: $callname" \n - foreach imp $implementations { - if {[dict get $functions $callname] eq $imp} { - append msg " $imp (active)" \n - } else { - append msg " $imp" \n - } - } - } - return $msg - } - proc set_active_function {callname implementation} { - variable functions - variable functions_known - if {$callname ni [dict keys $functions_known]} { - error "unknown function callname $callname" - } - if {$implementation ni [dict get $functions_known $callname]} { - error "unknown implementation $implementation for callname $callname" - } - dict set functions $callname $implementation - - catch {rename ::punk::du::active::$callname ""} - namespace eval ::punk::du::active [string map [list %imp% $implementation %call% $callname] { - namespace import ::punk::du::lib::%imp% - rename %imp% %call% - }] - - return $implementation - } - proc get_active_function {callname} { - variable functions - variable functions_known - if {$callname ni [dict keys $functions_known]} { - error "unknown function callname $callname known functions: [dict keys $functions_known]" - } - return [dict get $functions $callname] - } - - - #where we import & the appropriate du_listing.. function for the platform - } - namespace eval lib { - variable du_literal - variable winfile_attributes [list 16 directory 32 archive 1024 reparse_point 18 [list directory hidden] 34 [list archive hidden] ] - #caching this is faster than calling twapi api each time.. unknown if twapi is calculating from bitmask - or calling windows api - #we could work out all flags and calculate from bitmask.. but it's not necessarily going to be faster than some simple caching mechanism like this - - proc decode_win_attributes {bitmask} { - variable winfile_attributes - if {[dict exists $winfile_attributes $bitmask]} { - return [dict get $winfile_attributes $bitmask] - } else { - #list/dict shimmering? - return [lindex [lappend winfile_attributes $bitmask [twapi::decode_file_attributes $bitmask]] end] - } - } - proc attributes_twapi {path {detail basic}} { - try { - set iterator [twapi::find_file_open $path -detail $detail] ;# -detail full only adds data to the altname field - if {[twapi::find_file_next $iterator iteminfo]} { - set attrinfo [decode_win_attributes [dict get $iteminfo attrs]] - set result [dict create -archive 0 -hidden 0 -longname $path -readonly 0 -shortname {} -system 0] - if {"hidden" in $attrinfo} { - dict set result -hidden 1 - } - if {"system" in $attrinfo} { - dict set result -system 1 - } - if {"readonly" in $attrinfo} { - dict set result -readonly 1 - } - dict set result -shortname [dict get $iteminfo altname] - dict set result -rawflags $attrinfo - set extras [list] - #foreach prop {ctime atime mtime size} { - # lappend extras $prop [dict get $iteminfo $prop] - #} - #dict set result -extras $extras - dict set result -raw $iteminfo - return $result - } else { - error "could not read attributes for $path" - } - } finally { - catch {twapi::find_file_close $iterator} - } - } - - #todo - review 'errors' key. We have errors relating to containing folder and args vs per child-item errors - additional key needed? - namespace export du_dirlisting_twapi du_dirlisting_generic du_dirlisting_unix du_dirlisting_undecided - # get listing without using unix-tools (may not be installed on the windows system) - # this dirlisting is customised for du - so only retrieves dirs,files,filesizes (minimum work needed to perform du function) - # This also preserves path rep for elements in the dirs/folders keys etc - which can make a big difference in performance - proc du_dirlisting_twapi {folderpath args} { - set defaults [dict create\ - -glob *\ - -with_sizes 1\ - -with_times 1\ - ] - set opts [dict merge $defaults $args] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_glob [dict get $opts -glob] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_with_sizes [dict get $opts -with_sizes] - set ftypes [list f d l] - if {"$opt_with_sizes" in {0 1}} { - #don't use string is boolean - (f false vs f file!) - #only accept 0|1 - if {$opt_with_sizes} { - set sized_types $ftypes - } else { - set sized_types [list] - } - } else { - set sized_types $opt_with_sizes - } - if {[llength $sized_types]} { - foreach st $sized_types { - if {$st ni $ftypes} { - error "du_dirlisting_twapi unrecognized element in -with_sizes '$st'" - } - } - } - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_with_times [dict get $opts -with_times] - if {"$opt_with_times" in {0 1}} { - if {$opt_with_times} { - set timed_types $ftypes - } else { - set timed_types [list] - } - } else { - set timed_types $opt_with_times - } - if {[llength $timed_types]} { - foreach item $timed_types { - if {$item ni $ftypes} { - error "du_dirlisting_twapi unrecognised element in -with-times '$item'" - } - } - } - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - - set errors [dict create] - set altname "" ;#possible we have to use a different name e.g short windows name or dos-device path //?/ - # return it so it can be stored and tried as an alternative for problem paths - #puts stderr ">>> glob: $opt_glob" - #REVIEW! windows api pattern matchttps://www.red-gate.com/simple-talk/blogs/the-unexpected-behaviour-of-directoryinfo-getfiles-with-three-letter-extensions/hing is .. weird. partly due to 8.3 filenames - #https://www.red-gate.com/simple-talk/blogs/the-unexpected-behaviour-of-directoryinfo-getfiles-with-three-letter-extensions/ - #we will certainly need to check the resulting listing with our supplied glob.. but maybe we will have to change the glob passed to find_file_open too. - # using * all the time may be inefficient - so we might be able to avoid that in some cases. - try { - #glob of * will return dotfiles too on windows - set iterator [twapi::find_file_open [file join $folderpath $opt_glob] -detail basic] ;# -detail full only adds data to the altname field - } on error args { - try { - if {[string match "*denied*" $args]} { - #output similar format as unixy du - puts stderr "[file dirname $folderpath] '[file tail $folderpath]':$args" - dict lappend errors $folderpath $::errorCode - return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors] - } - if {[string match "*TWAPI_WIN32 59*" $::errorCode]} { - puts stderr "[file dirname $folderpath] '[file tail $folderpath]':$args (possibly blocked by permissions or share config e.g follow symlinks = no on samba)" - puts stderr " (errorcode: $::errorCode)\n" - dict lappend errors $folderpath $::errorCode - return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors] - } - - - #errorcode TWAPI_WIN32 2 {The system cannot find the file specified.} - #This can be a perfectly normal failure to match the glob.. which means we shouldn't really warn or error - #The find-all glob * won't get here because it returns . & .. - #so we should return immediately only if the glob has globchars ? or * but isn't equal to just "*" ? (review) - #Note that windows glob ? seems to return more than just single char results - it includes .. - which differs to tcl glob - #also ???? seems to returns items 4 or less - not just items exactly 4 long (review - where is this documented?) - if {$opt_glob ne "*" && [regexp {[?*]} $opt_glob]} { - if {[string match "*TWAPI_WIN32 2 *" $::errorCode]} { - #looks like an ordinary no results for chosen glob - return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors] - } - } - - - if {[set plen [pathcharacterlen $folderpath]] >= 250} { - set errmsg "error reading folder: $folderpath (len:$plen)\n" - append errmsg "error: $args" \n - append errmsg "errorcode: $::errorCode" \n - # re-fetch this folder with altnames - #file normalize - aside from being slow - will have problems with long paths - so this won't work. - #this function should only accept absolute paths - # - # - #Note: using -detail full only helps if the last segment of path has an altname.. - #To properly shorten we need to have kept track of altname all the way from the root! - #We can .. for now call Tcl's file attributes to get shortname of the whole path - it is *expensive* e.g 5ms for a long path on local ssd - #### SLOW - set fixedpath [dict get [file attributes $folderpath] -shortname] - #### SLOW - - - append errmsg "retrying with with windows altname '$fixedpath'" - puts stderr $errmsg - } else { - set errmsg "error reading folder: $folderpath (len:$plen)\n" - append errmsg "error: $args" \n - append errmsg "errorcode: $::errorCode" \n - set tmp_errors [list $::errorCode] - #possibly an illegal windows filename - easily happens on a machine with WSL or with drive mapped to unix share - #we can use //?/path dos device path - but not with tcl functions - #unfortunately we can't call find_file_open directly on the problem name - we have to call the parent folder and iterate through again.. - #this gets problematic as we go deeper unless we rewrite the .. but we can get at least one level further here - - set fixedtail "" - - set parent [file dirname $folderpath] - set badtail [file tail $folderpath] - set iterator [twapi::find_file_open [file join $parent *] -detail full] ;#retrieve with altnames - while {[twapi::find_file_next $iterator iteminfo]} { - set nm [dict get $iteminfo name] - if {$nm eq $badtail} { - set fixedtail [dict get $iteminfo altname] - break - } - } - - if {![string length $fixedtail]} { - dict lappend errors $folderpath {*}$tmp_errors - puts stderr "[file dirname $folderpath] '[file tail $folderpath]':$args (Unable to retrieve altname to progress further with path - returning no contents for this folder)" - return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors] - } - - #twapi as at 2023-08 doesn't seem to support //?/ dos device paths.. - #Tcl can test only get as far as testing existence of illegal name by prefixing with //?/ - but can't glob inside it - #we can call file attributes on it - but we get no shortname (but we could get shortname for parent that way) - #so the illegalname_fix doesn't really work here - #set fixedpath [punk::winpath::illegalname_fix $parent $fixedtail] - - #this has shortpath for the tail - but it's not the canonical-shortpath because we didn't call it on the $parent part REIEW. - set fixedpath [file join $parent $fixedtail] - append errmsg "retrying with with windows dos device path $fixedpath\n" - puts stderr $errmsg - - } - - if {[catch { - set iterator [twapi::find_file_open $fixedpath/* -detail basic] - } errMsg]} { - puts stderr "[file dirname $folderpath] '[file tail $folderpath]':$args (failed to read even with fixedpath:'$fixedpath')" - puts stderr " (errorcode: $::errorCode)\n" - puts stderr "$errMsg" - dict lappend errors $folderpath $::errorCode - return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors] - } - - - } on error args { - set errmsg "error reading folder: $folderpath\n" - append errmsg "error: $args" \n - append errmsg "errorInfo: $::errorInfo" \n - puts stderr "$errmsg" - puts stderr "FAILED to collect info for folder '$folderpath'" - #append errmsg "aborting.." - #error $errmsg - return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors] - - } - } - set dirs [list] - set files [list] - set filesizes [list] - set allsizes [dict create] - set alltimes [dict create] - - set links [list] - set flaggedhidden [list] - set flaggedsystem [list] - set flaggedreadonly [list] - - while {[twapi::find_file_next $iterator iteminfo]} { - set nm [dict get $iteminfo name] - #recheck glob - #review! - if {![string match $opt_glob $nm]} { - continue - } - set tail_altname [dict get $iteminfo altname] ;#altname of tail - not whole path - set attrinfo [decode_win_attributes [dict get $iteminfo attrs]] - #puts stderr "$iteminfo" - #puts stderr "$nm -> [dict get $iteminfo attrs] -> $attrinfo" - set ftype "" - #attributes applicable to any classification - set fullname [file_join_one $folderpath $nm] - - if {"hidden" in $attrinfo} { - lappend flaggedhidden $fullname - } - if {"system" in $attrinfo} { - lappend flaggedsystem $fullname - } - if {"readonly" in $attrinfo} { - lappend flaggedreadonly $fullname - } - - #main classification - if {"reparse_point" in $attrinfo} { - #this concept doesn't correspond 1-to-1 with unix links - #https://learn.microsoft.com/en-us/windows/win32/fileio/reparse-points - #review - and see which if any actually belong in the links key of our return - - - #One thing it could be, is a 'mounted folder' https://learn.microsoft.com/en-us/windows/win32/fileio/determining-whether-a-directory-is-a-volume-mount-point - # - #we will treat as zero sized for du purposes.. review - option -L for symlinks like BSD du? - #Note 'file readlink' can fail on windows - reporting 'invalid argument' - according to tcl docs, 'On systems that don't support symbolic links this option is undefined' - #The link may be viewable ok in windows explorer, and cmd.exe /c dir and unix tools such as ls - #if we need it without resorting to unix-tools that may not be installed: exec {*}[auto_execok dir] /A:L {c:\some\path} - #e.g (stripped of headers/footers and other lines) - #2022-10-02 04:07 AM priv [\\?\c:\repo\elixir\gameportal\apps\test\priv] - #Note we will have to parse beyond header fluff as /B strips the symlink info along with headers. - #du includes the size of the symlink - #but we can't get it with tcl's file size - #twapi doesn't seem to have anything to help read it either (?) - #the above was verified with a symlink that points to a non-existant folder.. mileage may vary for an actually valid link - # - #Note also - a shortcut created in explorer with drag and drop to an existant folder is a different animal to a symlink (file with .lnk extension) even though it looks the same in explorer window. - # - #links are techically files too, whether they point to a file/dir or nothing. - - lappend links $fullname - set ftype "l" - } elseif {"directory" in $attrinfo} { - if {$nm in {. ..}} { - continue - } - lappend dirs $fullname - set ftype "d" - } else { - - #review - is anything that isn't a reparse_point or a directory, some sort of 'file' in this context? What about the 'device' attribute? Can that occur in a directory listing of some sort? - lappend files $fullname - if {"f" in $sized_types} { - lappend filesizes [dict get $iteminfo size] - } - set ftype "f" - } - if {$ftype in $sized_types} { - dict set allsizes $fullname [dict create bytes [dict get $iteminfo size]] - } - if {$ftype in $timed_types} { - #convert time from windows (100ns units since jan 1, 1601) to Tcl time (seconds since Jan 1, 1970) - #We lose some precision by not passing the boolean to the large_system_time_to_secs_since_1970 function which returns fractional seconds - #but we need to maintain compatibility with other platforms and other tcl functions so if we want to return more precise times we will need another flag and/or result dict - dict set alltimes $fullname [dict create\ - c [twapi::large_system_time_to_secs_since_1970 [dict get $iteminfo ctime]]\ - a [twapi::large_system_time_to_secs_since_1970 [dict get $iteminfo atime]]\ - m [twapi::large_system_time_to_secs_since_1970 [dict get $iteminfo mtime]]\ - ] - } - } - twapi::find_file_close $iterator - set vfsmounts [get_vfsmounts_in_folder $folderpath] - - set effective_opts $opts - dict set effective_opts -with_times $timed_types - dict set effective_opts -with_sizes $sized_types - - #also determine whether vfs. file system x is *much* faster than file attributes - #whether or not there is a corresponding file/dir add any applicable mountpoints for the containing folder - return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes $filesizes sizes $allsizes times $alltimes flaggedhidden $flaggedhidden flaggedsystem $flaggedsystem flaggedreadonly $flaggedreadonly altname $altname opts $effective_opts errors $errors] - } - proc get_vfsmounts_in_folder {folderpath} { - set vfsmounts [list] - if {![llength [package provide vfs]]} { - return [list] - } - set fpath [punk::objclone $folderpath] - set is_rel 0 - if {[file pathtype $fpath] ne "absolute"} { - set fpath [file normalize $fpath] - set is_rel 1 - } - set known_vfs_mounts [vfs::filesystem info] - foreach mount $known_vfs_mounts { - if {[punk::mix::base::lib::path_a_above_b $fpath $mount]} { - if {([llength [file split $mount]] - [llength [file split $fpath]]) == 1} { - #the mount is in this folder - if {$is_rel} { - lappend vfsmounts [file join $folderpath [file tail $mount]] - } else { - lappend vfsmounts $mount - } - } - } - } - return $vfsmounts - } - #work around the horrible tilde-expansion thing (not needed for tcl 9+) - proc file_join_one {base newtail} { - if {[string index $newtail 0] ne {~}} { - return [file join $base $newtail] - } - return [file join $base ./$newtail] - } - - - #this is the cross-platform pure-tcl version - which calls glob multiple times to make sure it gets everythign it needs and can ignore everything it needs to. - #These repeated calls to glob will be a killer for performance - especially on a network share or when walking a large directory structure - proc du_dirlisting_generic {folderpath args} { - set defaults [dict create\ - -glob *\ - -with_sizes 0\ - -with_times 0\ - ] - set errors [dict create] - set known_opts [dict keys $defaults] - foreach k [dict keys $args] { - if {$k ni $known_opts} { - error "du_dirlisting_generic unknown-option $k" - } - } - set opts [dict merge $defaults $args] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_glob [dict get $opts -glob] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_with_sizes [dict get $opts -with_sizes] - set ftypes [list f d l] - if {"$opt_with_sizes" in {0 1}} { - #don't use string is boolean (false vs f problem where f indicates file) - if {$opt_with_sizes} { - set sized_types $ftypes - } else { - set sized_types [list] - } - } else { - set sized_types $opt_with_sizes - } - if {[llength $sized_types]} { - foreach st $sized_types { - if {$st ni $ftypes} { - error "du_dirlisting_generic unrecognized element in -with_sizes '$st'" - } - } - } - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_with_times [dict get $opts -with_times] - if {"$opt_with_times" in {0 1}} { - if {$opt_with_times} { - set timed_types $ftypes - } else { - set timed_types [list] - } - } else { - set timed_types $opt_with_times - } - if {[llength $timed_types]} { - foreach item $timed_types { - if {$item ni $ftypes} { - error "du_dirlisting_generic unrecognised element in -with-times '$item'" - } - } - } - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - # The repeated globs are a source of slowness for this function. - #TODO - we could minimize the number of globs if we know we need to do a file stat and/or file attributes on each entry anyway - #For the case where we don't need times,sizes or other metadata - it is faster to do multiple globs - #This all makes this function complicated to gather the required data efficiently. - - #note platform differences between what is considered hidden make this tricky. - # on windows 'glob .*' will not return some hidden dot items but will return . .. and glob -types hidden .* will not return some dotted items - # glob -types hidden * on windows will not necessarily return all dot files/folders - # unix-like platforms seem to consider all dot files as hidden so processing is more straightforward - # we need to process * and .* in the same glob calls and remove duplicates - # if we do * and .* in separate iterations of this loop we lose the ability to filter duplicates easily - - #note - with this design, we can't glob for all except dotfiles - this is for cross-platform consistency and efficiency (Review). - #dotfiles aren't considered hidden on all platforms - #some sort of antiglob is a possible enhancement and more consistent with the view that leading dot should be treated as any other filename character in this context. - if {$opt_glob eq "*"} { - #Note - glob operations seem to be faster than looped tests like 'file isdirectory' & 'file readlink' - #set parent [lindex $folders $folderidx] - set hdirs [glob -nocomplain -dir $folderpath -types {hidden d} * .*] - #set hdirs {} - set dirs [glob -nocomplain -dir $folderpath -types d * .*] - - set hlinks [glob -nocomplain -dir $folderpath -types {hidden l} * .*] - #set hlinks {} - set links [glob -nocomplain -dir $folderpath -types l * .*] ;#links may have dupes - we don't care. struct::set difference will remove - #set links [lsort -unique [concat $hlinks $links[unset links]]] - - set hfiles [glob -nocomplain -dir $folderpath -types {hidden f} * .*] - #set hfiles {} - set files [glob -nocomplain -dir $folderpath -types f * .*] - #set files {} - } else { - set hdirs [glob -nocomplain -dir $folderpath -types {hidden d} $opt_glob] - set dirs [glob -nocomplain -dir $folderpath -types d $opt_glob] - - set hlinks [glob -nocomplain -dir $folderpath -types {hidden l} $opt_glob] - set links [glob -nocomplain -dir $folderpath -types l $opt_glob] ;#links may have dupes - we don't care. struct::set difference will remove - - set hfiles [glob -nocomplain -dir $folderpath -types {hidden f} $opt_glob] - set files [glob -nocomplain -dir $folderpath -types f $opt_glob] - } - - #note struct::set difference produces unordered result - #struct::set difference removes duplicates - #remove links and . .. from directories, remove links from files - set files [struct::set difference [concat $hfiles $files[unset files]] $links] - set dirs [struct::set difference [concat $hdirs $dirs[unset dirs]] [concat $links [list [file join $folderpath .] [file join $folderpath ..] ]]] - set links [lsort -unique [concat $links $hlinks]] - - - #---- - set mdata_lists [du_get_metadata_lists $sized_types $timed_types $files $dirs $links] - - - - if {"windows" eq $::tcl_platform(platform)} { - set flaggedhidden [concat $hdirs $hfiles $hlinks] - } else { - #unix dotted files/folders are not 'flagged' as such - it's a convention - so the client can use the same convention to decide if something is hidden - #this allows us to return less data - but more importantly - reserve flaggedhidden for systems where such a flag exists and avoid conflating the different concepts of what is hidden - set flaggedhidden {} - } - - set vfsmounts [get_vfsmounts_in_folder $folderpath] - - set effective_opts $opts - dict set effective_opts -with_times $timed_types - dict set effective_opts -with_sizes $sized_types - - return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes [dict get $mdata_lists fsizes] sizes [dict get $mdata_lists allsizes] times [dict get $mdata_lists alltimes] flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $effective_opts errors $errors] - } - - proc du_dirlisting_tclvfs {folderpath args} { - set defaults [dict create\ - -glob *\ - -with_sizes 0\ - -with_times 0\ - ] - set opts [dict merge $defaults $args] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_glob [dict get $opts -glob] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_with_sizes [dict get $opts -with_sizes] - set ftypes [list f d l] - if {"$opt_with_sizes" in {0 1}} { - #don't use string is boolean (false vs f problem where f indicates file) - if {$opt_with_sizes} { - set sized_types $ftypes - } else { - set sized_types [list] - } - } else { - set sized_types $opt_with_sizes - } - if {[llength $sized_types]} { - foreach st $sized_types { - if {$st ni $ftypes} { - error "du_dirlisting_generic unrecognized element in -with_sizes '$st'" - } - } - } - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_with_times [dict get $opts -with_times] - if {"$opt_with_times" in {0 1}} { - if {$opt_with_times} { - set timed_types $ftypes - } else { - set timed_types [list] - } - } else { - set timed_types $opt_with_times - } - if {[llength $timed_types]} { - foreach item $timed_types { - if {$item ni $ftypes} { - error "du_dirlisting_generic unrecognised element in -with-times '$item'" - } - } - } - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - - set errors [dict create] - if {$opt_glob eq "*"} { - set dirs [glob -nocomplain -dir $folderpath -types d * .*] ;# also returns links to dirs - #review - how are links handled in vfs? presumably if the vfs is a full implementation they should work at least within the vfs? - set links [glob -nocomplain -dir $folderpath -types l * .*] ;# links may have dupes - we don't care. struct::set difference will remove - set files [glob -nocomplain -dir $folderpath -types f * .*] ;# also returns links to files - } else { - set dirs [glob -nocomplain -dir $folderpath -types d $opt_glob] - set links [glob -nocomplain -dir $folderpath -types l $opt_glob] - set files [glob -nocomplain -dir $folderpath -types f $opt_glob] - } - #remove any links from our dirs and files collections - set dirs [struct::set difference $dirs[unset dirs] [concat $links [list [file join $folderpath .] [file join $folderpath ..] ]]] - set files [struct::set difference $files[unset files] $links] - #nested vfs mount.. REVIEW - does anything need special handling? - set vfsmounts [get_vfsmounts_in_folder $folderpath] - - set mdata_lists [du_get_metadata_lists $sized_types $timed_types $files $dirs $links] - - - set effective_opts $opts - dict set effective_opts -with_times $timed_types - dict set effective_opts -with_sizes $sized_types - - return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes [dict get $mdata_lists fsizes] sizes [dict get $mdata_lists allsizes] times $alltimes flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $effective_opts errors $errors] - } - - #we can halve the number of round trips on unix-like systems, where 'hidden' always corresponds to dotted files - proc du_dirlisting_unix {folderpath args} { - set defaults [dict create\ - -glob *\ - -with_sizes 0\ - -with_times 0\ - ] - set errors [dict create] - dict lappend errors $folderpath "metadata support incomplete - prefer du_dirlisting_generic" - set opts [dict merge $defaults $args] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_glob [dict get $opts -glob] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_with_sizes [dict get $opts -with_sizes] - set ftypes [list f d l] - if {"$opt_with_sizes" in {0 1}} { - #don't use string is boolean (false vs f problem where f indicates file) - if {$opt_with_sizes} { - set sized_types $ftypes - } else { - set sized_types [list] - } - } else { - set sized_types $opt_with_sizes - } - if {[llength $sized_types]} { - foreach st $sized_types { - if {$st ni $ftypes} { - error "du_dirlisting_generic unrecognized element in -with_sizes '$st'" - } - } - } - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_with_times [dict get $opts -with_times] - if {"$opt_with_times" in {0 1}} { - if {$opt_with_times} { - set timed_types $ftypes - } else { - set timed_types [list] - } - } else { - set timed_types $opt_with_times - } - if {[llength $timed_types]} { - foreach item $timed_types { - if {$item ni $ftypes} { - error "du_dirlisting_generic unrecognised element in -with-times '$item'" - } - } - } - - #this is not consistent with standard way * works for unix folders - but it is consistent with behaviour of this facility on windows - if {$opt_glob eq "*"} { - set dirs [glob -nocomplain -dir $folderpath -types d * .*] ;# also returns links to dirs - set links [glob -nocomplain -dir $folderpath -types l * .*] ;# links may have dupes - we don't care. struct::set difference will remove - set files [glob -nocomplain -dir $folderpath -types f * .*] ;# also returns links to files - } else { - set dirs [glob -nocomplain -dir $folderpath -types d $opt_glob] - set links [glob -nocomplain -dir $folderpath -types l $opt_glob] - set files [glob -nocomplain -dir $folderpath -types f $opt_glob] - } - #remove any links from our dirs and files collections - set dirs [struct::set difference $dirs[unset dirs] [concat $links [list [file join $folderpath .] [file join $folderpath ..] ]]] - set files [struct::set difference $files[unset files] $links] - set vfsmounts [get_vfsmounts_in_folder $folderpath] - - set mdata_lists [du_get_metadata_lists $sized_types $timed_types $files $dirs $links] - - set effective_opts $opts - dict set effective_opts -with_times $timed_types - dict set effective_opts -with_sizes $sized_types - - - return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes [dict get $mdata_lists fsizes] sizes [dict get $mdata_lists allsizes] times [dict get $mdata_lists alltimes] flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors] - } - - #return fsizes,allsizes,alltimes metadata in same order as files,dirs,links lists - if specified in sized_types - proc du_get_metadata_lists {sized_types timed_types files dirs links} { - set meta_dict [dict create] - set meta_types [concat $sized_types $timed_types] - #known tcl stat keys 2023 - review - set empty_stat_dict [dict create atime {} ctime {} dev {} gid {} ino {} mode {} mtime {} nlink {} size {} type {} uid {}] - #make sure we call file stat only once per item - set statkeys [list] - if {[llength $meta_types]} { - foreach ft {f d l} lvar {files dirs links} { - if {"$ft" in $meta_types} { - foreach path [set $lvar] { - #caller may have read perm on the containing folder - but not on child item - so file stat could raise an error - if {![catch {file stat $path arrstat} errM]} { - dict set meta_dict $path [dict create shorttype $ft {*}[array get arrstat]] - } else { - dict lappend errors $path "file stat error: $errM" - dict set meta_dict $path [dict create shorttype $ft {*}$empty_stat_dict] - } - } - } - } - } - set fsizes [list] - set allsizes [dict create] - set alltimes [dict create] - #review birthtime field of stat? cross-platform differences ctime etc? - dict for {path pathinfo} $meta_dict { - set ft [dict get $pathinfo shorttype] - if {$ft in $sized_types} { - dict set allsizes $path [dict create bytes [dict get $pathinfo size]] - if {$ft eq "f"} { - #subst with na if empty? - lappend fsizes [dict get $pathinfo size] - } - } - if {$ft in $timed_types} { - dict set alltimes $path [dict create c [dict get $pathinfo ctime] a [dict get $pathinfo atime] m [dict get $pathinfo mtime]] - } - } - #todo - fix . The list lengths will presumably match but have empty values if failed to stat - if {"f" in $sized_types} { - if {[llength $fsizes] ne [llength $files]} { - dict lappend errors $folderpath "failed to retrieve all file sizes" - } - } - return [dict create fsizes $fsizes allsizes $allsizes alltimes $alltimes] - } - - - proc du_lit value { - variable du_literal - if {![info exists du_literal($value)]} { - set du_literal($value) $value - } - return $du_literal($value) - } - - #v1 - proc du_new_eachdirtail {dirtails depth parentfolderidx} { - set newlist {} - upvar folders folders - set parentpath [lindex $folders $parentfolderidx] - set newindex [llength $folders] - foreach dt $dirtails { - lappend folders [file join $parentpath [du_lit $dt]]; #store as a 'path' rather than a string (see tcl::unsupported::representation) - lappend newlist [::list $depth $parentfolderidx $newindex [du_lit $dt] [expr {0}]] - incr newindex - } - return $newlist - } - proc du_new_eachdir {dirpaths depth parentfolderidx} { - set newlist {} - upvar folders folders - set newindex [llength $folders] - foreach dp $dirpaths { - lappend folders $dp - #puts stdout "--->$dp" - lappend newlist [::list $depth $parentfolderidx $newindex [du_lit [file tail $dp]] [expr {0}]] - incr newindex - } - return $newlist - } - - #same implementation as punk::strlen - #get length of path which has internal rep of path - maintaining path/list rep without shimmering to string representation. - proc pathcharacterlen {pathrep} { - append str2 $pathrep {} - string length $str2 - } - #just an experiment - proc pathcharacterlen1 {pathrep} { - #This works - but is unnecessarily complex - set l 0 - set parts [file split $pathrep] - if {[llength $parts] < 2} { - return [string length [lindex $parts 0]] - } - foreach seg $parts { - incr l [string length $seg] - } - return [expr {$l + [llength $parts] -2}] - } - #slower - doesn't work for short paths like c:/ - proc pathcharacterlen2 {pathrep} { - return [tcl::mathop::+ {*}[lmap v [set plist [file split $pathrep]] {[string length $v]}] [llength $plist] -2] - } - - #Strip using lengths without examining path components - #without normalization is much faster - proc path_strip_alreadynormalized_prefixdepth {path prefix} { - set tail [lrange [file split $path] [llength [file split $prefix]] end] - if {[llength $tail]} { - return [file join {*}$tail] - } else { - return "" - } - } - - proc du_dirlisting_undecided {folderpath args} { - if {"windows" eq $::tcl_platform(platform)} { - #jmn disable twapi - #tailcall du_dirlisting_generic $folderpath {*}$args - - set loadstate [zzzload::pkg_require twapi] - if {$loadstate ni [list loading failed]} { - #either already loaded by zzload or ordinary package require - package require twapi ;#should be fast once twapi dll loaded in zzzload thread - set ::punk::du::has_twapi 1 - punk::du::active::set_active_function du_dirlisting du_dirlisting_twapi - tailcall du_dirlisting_twapi $folderpath {*}$args - } else { - if {$loadstate eq "failed"} { - puts stderr "punk::du defaulting to du_dirlisting_generic because twapi load failed" - punk::du::active::set_active_function du_dirlisting du_dirlisting_generic - } - tailcall du_dirlisting_generic $folderpath {*}$args - } - } else { - punk::du::active::set_active_function du_dirlisting du_dirlisting_unix - tailcall du_dirlisting_unix $folderpath {*}$args - } - } - - - } - package require natsort - #interp alias {} du {} .=args>* punk::du |> .=>1 natsort::sort -cols 1 |> list_as_lines * punk::du::du |> .=>1 natsort::sort -cols 1 -outputformat words |> list_as_lines * punk::du::du |> .=>1 natsort::sort -cols 1 -outputformat csv -outputformatoptions {\r\t\t\t} |> list_as_lines 0} { - set argvals [lrange $args 0 $numargs-1] - set args [lrange $args $numargs end] - } - if {[llength $argvals] < $numargs} { - error "wrong # args: $from_ns $subcommand requires args: $pos_argnames" - } - tailcall [namespace current] $subcommand {*}$argvals {*}$args -extension $from_ns - } else { - tailcall [namespace current] $subcommand {*}$args -extension $from_ns - } - } - proc _split_args {arglist} { - #don't assume arglist is fully paired. - set posn [lsearch $arglist -extension] - set opts [list] - if {$posn >= 0} { - if {$posn+2 <= [llength $arglist]} { - set opts [list -extension [lindex $arglist $posn+1]] - set argsremaining [lreplace $arglist $posn $posn+1] - } else { - #no value supplied to -extension - error "punk::mix::base::_split_args - no value found for option '-extension'. Supply a value or omit the option." - } - } else { - set argsremaining $arglist - } - - return [list opts $opts args $argsremaining] - } -} - - -#base API (potentially overridden functions - may also be called from overriding namespace) -#commands should either handle or silently ignore -extension -namespace eval punk::mix::base { - namespace ensemble create - namespace export help dostuff get_commands set_alias - namespace ensemble configure [namespace current] -unknown punk::mix::base::_unknown - proc get_commands {args} { - #--------- - #extension@@opts/@?@-extension,args@@args= [_split_args $args] ;#dependency on punk pipeline/patternmatching system - lassign [_split_args $args] _opts opts _args args - if {[dict exists $opts -extension]} { - set extension [dict get $opts -extension] - } else { - set extension "" - } - #--------- - if {![string length $extension]} { - set extension [namespace qualifiers [lindex [info level -1] 0]] - } - - set maincommands [list] - #extension may still be blank e.g if punk::mix::base::get_commands called directly - if {[string length $extension]} { - set nsmain $extension - #puts stdout "get_commands nsmain: $nsmain" - set parentpatterns [namespace eval $nsmain [list namespace export]] - set nscommands [list] - foreach p $parentpatterns { - lappend nscommands {*}[info commands ${nsmain}::$p] - } - foreach c $nscommands { - set cmd [namespace tail $c] - lappend maincommands $cmd - } - set maincommands [lsort $maincommands] - } - - - - - set nsbase [namespace current] - set basepatterns [namespace export] - #puts stdout "basepatterns:$basepatterns" - set nscommands [list] - foreach p $basepatterns { - lappend nscommands {*}[info commands ${nsbase}::$p] - } - - set basecommands [list] - foreach c $nscommands { - set cmd [namespace tail $c] - if {$cmd ni $maincommands} { - lappend basecommands $cmd - } - } - set basecommands [lsort $basecommands] - - - return [list main $maincommands base $basecommands] - } - proc help {args} { - #' **%ensemblecommand% help** *args* - #' - #' Help for ensemble commands in the command line interface - #' - #' - #' Arguments: - #' - #' * args - first word of args is the helptopic requested - usually a command name - #' - calling help with no arguments will list available commands - #' - #' Returns: help text (text) - #' - #' Examples: - #' - #' ``` - #' %ensemblecommand% help - #' ``` - #' - #' - - - #extension.= @@opts/@?@-extension,args@@args=>. [_split_args $args] {| - # >} inspect -label a {| - # >} .=e>end,data>end pipeswitch { - # pipecase ,0/1/#= $switchargs {| - # e/0 - # >} .=>. {set e} - # pipecase /1,1/1/#= $switchargs - #} |@@ok/result> " opts $opts] - } - if {$ftype ni [list file directory]} { - #review - links? - error "cksum_path error file type '$ftype' not supported" - } - - - set opt_cksum_algorithm [dict get $opts -cksum_algorithm] - if {$opt_cksum_algorithm ni [cksum_algorithms]} { - return [list error unsupported_cksum_algorithm cksum "" opts $opts] - } - set opt_cksum_acls [dict get $opts -cksum_acls] - if {$opt_cksum_acls} { - puts stderr "cksum_path is not yet able to cksum ACLs" - return - } - - set opt_cksum_meta [dict get $opts -cksum_meta] - set opt_use_tar [dict get $opts -cksum_usetar] - if {$ftype eq "file"} { - if {$opt_use_tar eq "auto"} { - if {$opt_cksum_meta eq "1"} { - set opt_use_tar 1 - } else { - #prefer no tar if meta not required - faster/simpler - #meta == auto or 0 - set opt_cksum_meta 0 - set opt_use_tar 0 - } - } elseif {$opt_use_tar eq "0"} { - if {$opt_cksum_meta eq "1"} { - puts stderr "cksum_path doesn't yet support a non-tar cksum with metadata for a file" - return [list error unsupported_meta_without_tar cksum "" opts $opts] - } else { - #meta == auto or 0 - set opt_cksum_meta 0 - } - } else { - #tar == 1 - if {$opt_cksum_meta eq "0"} { - puts stderr "cksum_path doesn't yet support a tar cksum without metadata for a file" - return [list error unsupported_tar_without_meta cksum "" opts $opts] - } else { - #meta == auto or 1 - set opt_cksum_meta 1 - } - } - } elseif {$ftype eq "directory"} { - if {$opt_use_tar eq "auto"} { - if {$opt_cksum_meta in [list "auto" "1"]} { - set opt_use_tar 1 - set opt_cksum_meta 1 - } else { - puts stderr "cksum_path doesn't yet support a content-only cksum of a folder structure. Currently only files supported without metadata. For folders use cksum_path -cksum_meta 1 or auto" - return [list error unsupported_directory_cksum_without_meta cksum "" opts $opts] - } - } elseif {$opt_use_tar eq "0"} { - puts stderr "cksum_path doesn't yet support a cksum of a folder structure without tar. Currently only files supported without metadata. For folders use cksum_path -cksum_meta 1 or auto with -cksum_usetar 1 or auto" - return [list error unsupported_directory_cksum_without_tar cksum "" opts $opts] - } else { - #tar 1 - if {$opt_cksum_meta eq "0"} { - puts stderr "cksum_path doesn't yet support a tar checksum of a folder structure without metadat. Currently only files supported without metadata. For folders use cksum_path -cksum_meta 1 or auto with -cksum_usetar 1 or auto" - return [list error unsupported_without_meta cksum "" opts $opts] - } else { - #meta == auto or 1 - set opt_cksum_meta 1 - } - } - } - - dict set opts_actual -cksum_meta $opt_cksum_meta - dict set opts_actual -cksum_usetar $opt_use_tar - - - if {$opt_use_tar} { - package require tar ;#from tcllib - } - - if {$path eq $base} { - #attempting to cksum at root/volume level of a filesystem.. extra work - #This needs fixing for general use.. not necessarily just for project repos - puts stderr "cksum_path doesn't yet support cksum of entire volume. (todo)" - return [list error unsupported_path opts $opts] - } - - if {$opt_cksum_algorithm eq "sha1"} { - package require sha1 - set cksum_command [list sha1::sha1 -hex -file] - } elseif {$opt_cksum_algorithm in [list "sha2" "sha256"]} { - package require sha256 - set cksum_command [list sha2::sha256 -hex -file] - } elseif {$opt_cksum_algorithm eq "md5"} { - package require md5 - set cksum_command [list md5::md5 -hex -file] - } elseif {$opt_cksum_algorithm eq "cksum"} { - package require cksum ;#tcllib - set cksum_command [list crc::cksum -format 0x%X -file] - } elseif {$opt_cksum_algorithm eq "adler32"} { - set cksum_command [list cksum_adler32_file] - } elseif {$opt_cksum_algorithm in [list "sha3" "sha3-256"]} { - #todo - replace with something that doesn't call another process - #set cksum_command [list apply {{file} {lindex [exec fossil sha3sum -256 $file] 0}}] - set cksum_command [list $sha3_implementation 256] - } elseif {$opt_cksum_algorithm in [list "sha3-224" "sha3-384" "sha3-512"]} { - set bits [lindex [split $opt_cksum_algorithm -] 1] - #set cksum_command [list apply {{bits file} {lindex [exec fossil sha3sum -$bits $file] 0}} $bits] - set cksum_command [list $sha3_implementation $bits] - } - - set cksum "" - if {$opt_use_tar != 0} { - set target [file tail $path] - set tmplocation [punk::mix::util::tmpdir] - set archivename $tmplocation/[punk::mix::util::tmpfile].tar - - cd $base ;#cd is process-wide.. keep cd in effect for as small a scope as possible. (review for thread issues) - - #temp emission to stdout.. todo - repl telemetry channel - puts stdout "cksum_path: creating temporary tar archive for $path" - puts stdout " at: $archivename .." - tar::create $archivename $target - if {$ftype eq "file"} { - set sizeinfo "(size [file size $target])" - } else { - set sizeinfo "(file type $ftype - size unknown)" - } - puts stdout "cksum_path: calculating cksum for $target $sizeinfo..." - set cksum [{*}$cksum_command $archivename] - #puts stdout "cksum_path: cleaning up.. " - file delete -force $archivename - cd $startdir - - } else { - #todo - if {$ftype eq "file"} { - if {$opt_cksum_meta} { - return [list error unsupported_opts_combo cksum "" opts $opts] - } else { - set cksum [{*}$cksum_command $path] - } - } else { - error "cksum_path unsupported $opts for path type [file type $path]" - } - } - set result [dict create] - dict set result cksum $cksum - dict set result opts $opts_actual - return $result - } - - #dict_path_cksum keyed on path - with value as a dict that must contain cksum key - but can contain other keys - #e.g -cksum_usetar which is one of the keys understood by the punk::mix::base::lib::cksum_path function - or unrelated keys which will also be passed through - #cksum only calculated for keys in dict where cksum is empty - ie return same dict but with empty cksums filled out. - #base can be empty string in which case paths must be absolute - proc fill_relativecksums_from_base_and_relativepathdict {base {dict_path_cksum {}}} { - if {$base eq ""} { - set error_paths [list] - dict for {path pathinfo} $dict_path_cksum { - if {[file pathtype $path] ne "absolute"} { - lappend error_paths $path - } - } - if {[llength $error_paths]} { - puts stderr "get_relativecksums_from_base_and_relativepathdict has empty base - and non-absolute paths in the supplied checksum dict - aborting" - puts stderr "error_paths: $error_paths" - error "fill_relativecksums_from_base_and_relativepathdict error: non-absolute paths when base empty. $error_paths" - } - } else { - if {[file pathtype $base] ne "absolute"} { - error "fill_relativecksums_from_base_and_relativepathdict error: base supplied but was not absolute path. $base" - } - #conversely now we have a base - so we require all paths are relative. - #We will ignore/disallow volume-relative - as these shouldn't be used here either - set error_paths [list] - dict for {path pathinfo} $dict_path_cksum { - if {[file pathtype $path] ne "relative"} { - lappend error_paths $path - } - } - if {[llength $error_paths]} { - puts stderr "fill_relativecksums_from_base_and_relativepathdict has a supplied absolute base path, but some of the paths in the supplied dict are not relative - aborting" - error "fill_relativecksums_from_base_and_relativepathdict error: non-relative paths when base supplied. $error_paths" - } - } - - - dict for {path pathinfo} $dict_path_cksum { - if {![dict exists $pathinfo cksum]} { - dict set pathinfo cksum "" - } else { - if {[dict get $pathinfo cksum] ne "" && ![cksum_is_tag [dict get $pathinfo cksum]]} { - continue ;#already filled with non-tag value - } - } - if {$base ne ""} { - set fullpath [file join $base $path] - } else { - set fullpath $path - } - - set ckopts [cksum_filter_opts {*}$pathinfo] - - if {![file exists $fullpath]} { - dict set dict_path_cksum $path cksum "" - } else { - set ckinfo [cksum_path $fullpath {*}$ckopts] - dict set dict_path_cksum $path cksum [dict get $ckinfo cksum] - dict set dict_path_cksum $path cksum_all_opts [dict get $ckinfo opts] - if {[dict exists $ckinfo error]} { - dict set dict_path_cksum $path cksum_error [dict get $ckinfo error] - } - } - } - - return $dict_path_cksum - } - #whether cksum is e.g - proc cksum_is_tag {cksum} { - expr {[string index $cksum 0] eq "<" && [string index $cksum end] eq ">"} - } - proc cksum_filter_opts {args} { - set ck_opt_names [dict keys [cksum_default_opts]] - set ck_opts [dict create] - dict for {k v} $args { - if {$k in $ck_opt_names} { - dict set ck_opts $k $v - } - } - return $ck_opts - } - - #convenience so caller doesn't have to pre-calculate the relative path from the base - #Note semantic difference from fill_relativecksums_from_base_and_relativepathdict (hence get_ vs fill_) - #Here we will raise an error if cksum exists and is not empty or a tag - whereas the multiple path version will ignore valid-looking prefilled cksum values - #base is the presumed location to store the checksum file. The caller should retain (normalize if relative) - proc get_relativecksum_from_base {base specifiedpath args} { - if {$base ne ""} { - #targetpath ideally should be within same project tree as base if base supplied - but not necessarily below it - #we don't necessarily want to restrict this to use in punk projects though - so we'll allow anything with a common prefix - if {[file pathtype $specifiedpath] eq "relative"} { - if {[file pathtype $base] eq "relative"} { - set normbase [file normalize $base] - set normtarg [file normalize [file join $normbase $specifiedpath]] - set targetpath $normtarg - set storedpath [punk::path::relative $normbase $normtarg] - } else { - set targetpath [file join $base $specifiedpath] - set storedpath $specifiedpath - } - } else { - #specifed absolute - if {[file pathtype $base] eq "relative"} { - #relative to cwd or to specifiedpath? For consistency it should arguably be cwd but a case could be made that when one path is relative it is in reference to the other - #there is a strong possibility that allowing this combination will cause confusion - better to disallow - error "get_relativecksum_from_base error: disallowed pathtype combination. Base must be empty or absolute when specified path is absolute" - } - #both absolute - compute relative path if they share a common prefix - set commonprefix [punk::mix::util::path_common_prefix $base $specifiedpath] - if {$commonprefix eq ""} { - #absolute base with no shared prefix doesn't make sense - we could ignore it - but better to error-out and require the caller specify an empty base - error "get_relativecksum_from_base error: base '$base' and specifiedpath '$specifiedpath' don't share a common root. Use empty-string for base if independent absolute path is required" - } - set targetpath $specifiedpath - set storedpath [punk::path::relative $base $specifiedpath] - - } - } else { - if {[file type $specifiedpath] eq "relative"} { - #if specifiedpath is relative - and we don't have a base, we now need to convert relative to cwd to an absolute path for storage - set targetpath [file normalize $specifiedpath] - set storedpath $targetpath - } else { - set targetpath $specifiedpath - set storedpath $targetpath - } - } - - # - #NOTE: specifiedpath can be a relative path (to cwd) when base is empty - #OR - a relative path when base itself is relative e.g base: somewhere targetpath somewhere/etc - #possibly also: base: somewhere targetpath: ../elsewhere/etc - # - #todo - write tests - - - if {([llength $args] % 2) != 0} { - error "get_relativecksum_from_base error. args supplied must be in the form of key-value pairs. received '$args' " - } - if {[dict exists $args cksum]} { - if {[dict get $args cksum] ne "" && ![cksum_is_tag [dict get $args cksum]]} { - error "get_relativecksum_from_base called with existing cksum value (and is not a tag or empty-value to be replaced) cksum: [dict get $args cksum] Set cksum to be empty, any tag such as or remove the key and try again." - } - } - - - set ckopts [cksum_filter_opts {*}$args] - set ckinfo [cksum_path $targetpath {*}$ckopts] - - set keyvals $args - dict set keyvals cksum [dict get $ckinfo cksum] - dict set keyvals cksum_all_opts [dict get $ckinfo opts] - if {[dict exists $ckinfo error]} { - dict set keyvals cksum_error [dict get $ckinfo error] - } - - #set relpath [punk::repo::path_strip_alreadynormalized_prefixdepth $fullpath $base] ;#empty base ok noop - #storedpath is relative if possible - return [dict create $storedpath $keyvals] - } - - #calculate the runtime checksum and vfs checksums - proc get_all_vfs_build_cksums {path} { - set buildfolder [get_build_workdir $path] - set cksum_base_folder [file dirname $buildfolder] ;#this is the /src folder - a reasonable base for our vfs cksums - set dict_cksums [dict create] - - set buildrelpath [punk::repo::path_strip_alreadynormalized_prefixdepth $buildfolder $cksum_base_folder] - set vfs_tail_list [glob -nocomplain -dir $cksum_base_folder -type d -tails *.vfs] - - foreach vfstail $vfs_tail_list { - set vname [file rootname $vfstail] - dict set dict_cksums $vfstail [list cksum ""] - dict set dict_cksums [file join $buildrelpath $vname.exe] [list cksum ""] - } - - set fullpath_buildruntime $buildfolder/buildruntime.exe - - set ckinfo_buildruntime [cksum_path $fullpath_buildruntime] - set ck [dict get $ckinfo_buildruntime cksum] - - - set relpath [file join $buildrelpath "buildruntime.exe"] - dict set dict_cksums $relpath [list cksum $ck] - - set dict_cksums [fill_relativecksums_from_base_and_relativepathdict $cksum_base_folder $dict_cksums] - - return $dict_cksums - } - - proc get_vfs_build_cksums_stored {vfsfolder} { - set vfscontainer [file dirname $vfsfolder] - set buildfolder $vfscontainer/_build - set vfs [file tail $vfsfolder] - set vname [file rootname $vfs] - set dict_vfs [list $vname.vfs "" $vname.exe "" buildruntime.exe ""] - set ckfile $buildfolder/$vname.cksums - if {[file exists $ckfile]} { - set data [punk::mix::util::fcat -translation binary $ckfile] - foreach ln [split $data \n] { - if {[string trim $ln] eq ""} {continue} - lassign $ln path cksum - dict set dict_vfs $path $cksum - } - } - return $dict_vfs - } - proc get_all_build_cksums_stored {path} { - set buildfolder [get_build_workdir $path] - - set vfscontainer [file dirname $buildfolder] - set vfslist [glob -nocomplain -dir $vfscontainer -type d -tail *.vfs] - set dict_cksums [dict create] - foreach vfs $vfslist { - set vname [file rootname $vfs] - set dict_vfs [get_vfs_build_cksums_stored $vfscontainer/$vfs] - - dict set dict_cksums $vname $dict_vfs - } - return $dict_cksums - } - - proc store_vfs_build_cksums {vfsfolder} { - if {![file isdirectory $vfsfolder]} { - error "Unable to find supplied vfsfolder: $vfsfolder" - } - set vfscontainer [file dirname $vfsfolder] - set buildfolder $vfscontainer/_build - set dict_vfs [get_vfs_build_cksums $vfsfolder] - set data "" - dict for {path cksum} $dict_vfs { - append data "$path $cksum" \n - } - set fd [open $buildfolder/$vname.cksums w] - chan configure $fd -translation binary - puts $fd $data - close $fd - return $dict_vfs - } - - - - } -} diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/cli-0.3.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/cli-0.3.tm deleted file mode 100644 index a845285c..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/cli-0.3.tm +++ /dev/null @@ -1,925 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::mix::cli 0.3 -# Meta platform tcl -# Meta license -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz -package require punk::repo -package require punkcheck ;#checksum and/or timestamp records - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ - - -namespace eval punk::mix::cli { - namespace eval temp_import { - } - namespace ensemble create - - package require punk::overlay - catch { - punk::overlay::import_commandset module . ::punk::mix::commandset::module - } - punk::overlay::import_commandset debug . ::punk::mix::commandset::debug - punk::overlay::import_commandset repo . ::punk::mix::commandset::repo - punk::overlay::import_commandset lib . ::punk::mix::commandset::loadedlib - - catch { - package require punk::mix::commandset::project - punk::overlay::import_commandset project . ::punk::mix::commandset::project - punk::overlay::import_commandset projects . ::punk::mix::commandset::project::collection - } - if {[catch { - package require punk::mix::commandset::layout - punk::overlay::import_commandset project.layout . ::punk::mix::commandset::layout - punk::overlay::import_commandset project.layouts . ::punk::mix::commandset::layout::collection - } errM]} { - puts stderr "error loading punk::mix::commandset::layout" - puts stderr $errM - } - if {[catch { - package require punk::mix::commandset::buildsuite - punk::overlay::import_commandset buildsuite . ::punk::mix::commandset::buildsuite - punk::overlay::import_commandset buildsuites . ::punk::mix::commandset::buildsuite::collection - } errM]} { - puts stderr "error loading punk::mix::commandset::buildsuite" - puts stderr $errM - } - punk::overlay::import_commandset scriptwrap . ::punk::mix::commandset::scriptwrap - if {[catch { - package require punk::mix::commandset::doc - punk::overlay::import_commandset doc . ::punk::mix::commandset::doc - punk::overlay::import_commandset "" "" ::punk::mix::commandset::doc::collection - } errM]} { - puts stderr "error loading punk::mix::commandset::doc" - puts stderr $errM - } - - - proc help {args} { - #set basehelp [punk::mix::base::help -extension [namespace current] {*}$args] - set basehelp [punk::mix::base help {*}$args] - #puts stdout "punk::mix help" - return $basehelp - } - - proc stat {{workingdir ""} args} { - dict set args -v 0 - punk::mix::cli::lib::get_status $workingdir {*}$args - } - proc status {{workingdir ""} args} { - dict set args -v 1 - punk::mix::cli::lib::get_status $workingdir {*}$args - } - - - - - - - -} - - -namespace eval punk::mix::cli { - - - #interp alias {} ::punk::mix::cli::project.new {} ::punk::mix::cli::new - - - - - - - - proc make {args} { - set startdir [pwd] - set project_base "" ;#empty for unknown - if {[punk::repo::is_git $startdir]} { - set project_base [punk::repo::find_git] - set sourcefolder $project_base/src - } elseif {[punk::repo::is_fossil $startdir]} { - set project_base [punk::repo::find_fossil] - set sourcefolder $project_base/src - } else { - if {[punk::repo::is_candidate $startdir]} { - set project_base [punk::repo::find_candidate] - set sourcefolder $project_base/src - puts stderr "WARNING - project not under git or fossil control" - puts stderr "Using base folder $project_base" - } else { - set sourcefolder $startdir - } - } - - #review - why can't we be anywhere in the project? - if {([file tail $sourcefolder] ne "src") || (![file exists $sourcefolder/make.tcl])} { - puts stderr "pmix make must be run from src folder containing make.tcl - unable to proceed (cwd: [pwd])" - if {[string length $project_base]} { - if {[file exists $project_base/src] && [string tolower [pwd]] ne [string tolower $project_base/src]} { - puts stderr "Try cd to $project_base/src" - } - } else { - if {[file exists $startdir/Makefile]} { - puts stdout "A Makefile exists at $startdir/Makefile." - if {"windows" eq $::tcl_platform(platform)} { - puts stdout "Try running: msys2 -ucrt64 -here -c \"make build\" or bash -c \"make build\"" - } else { - puts stdout "Try runing: make build" - } - } - } - return false - } - - if {![string length $project_base]} { - puts stderr "WARNING no git or fossil repository detected." - puts stderr "Using base folder $startdir" - set project_base $startdir - } - - set lc_this_exe [string tolower [info nameofexecutable]] - set lc_proj_bin [string tolower $project_base/bin] - set lc_build_bin [string tolower $project_base/src/_build] - - if {"project" in $args} { - set is_own_exe 0 - if {[string match "${lc_proj_bin}*" $lc_this_exe] || [string match "${lc_build_bin}" $lc_this_exe]} { - set is_own_exe 1 - puts stderr "WARNING - running make using executable that may be created by the project being built" - set answer [util::askuser "Do you want to proceed using this executable? (build will probably stop when it is unable to update the executable) Y|N"] - if {[string tolower $answer] ne "y"} { - puts stderr "mix new aborting due to user response '$answer' (required Y|y to proceed) use -confirm 0 to avoid prompts." - return - } - } - } - cd $sourcefolder - #use run so that stdout visible as it goes - if {![catch {run --timeout=55000 -debug [info nameofexecutable] $sourcefolder/make.tcl {*}$args} exitinfo]} { - #todo - notify if exit because of timeout! - puts stderr "exitinfo: $exitinfo" - set exitcode [dict get $exitinfo exitcode] - } else { - puts stderr "Error unable to determine exitcode. err: $exitinfo" - cd $startdir - return false - } - - cd $startdir - if {$exitcode != 0} { - puts stderr "FAILED with exitcode $exitcode" - return false - } else { - puts stdout "OK make finished " - return true - } - } - - proc Kettle {args} { - tailcall lib::kettle_call lib {*}$args - } - proc KettleShell {args} { - tailcall lib::kettle_call shell {*}$args - } - - - - namespace eval lib { - namespace path ::punk::mix::util - - - proc module_types {} { - #first in list is default for unspecified -type when creating new module - return [list plain tarjar zipkit] - } - - proc validate_modulename {modulename args} { - set defaults [list\ - -name_description modulename\ - ] - if {[llength $args] %2 != 0} {error "validate_modulename args must be name-value pairs: received '$args'"} - set known_opts [dict keys $defaults] - foreach k [dict keys $args] { - if {$k ni $known_opts} { - error "validate_modulename error: unknown option $k. known options: $known_opts" - } - } - set opts [dict merge $defaults $args] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_name_description [dict get $opts -name_description] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - - validate_name_not_empty_or_spaced $modulename -name_description $opt_name_description - set testname [string map [list :: ""] $modulename] - if {[string first : $testname] >=0} { - error "$opt_name_description '$modulename' can only contain paired colons" - } - set badchars [list - "$" "?" "*"] - foreach bc $badchars { - if {[string first $bc $modulename] >= 0} { - error "$opt_name_description '$modulename' can not contain character '$bc'" - } - } - return $modulename - } - - proc validate_projectname {projectname args} { - set defaults [list\ - -name_description projectname\ - ] - if {[llength $args] %2 != 0} {error "validate_modulename args must be name-value pairs: received '$args'"} - set known_opts [dict keys $defaults] - foreach k [dict keys $args] { - if {$k ni $known_opts} { - error "validate_modulename error: unknown option $k. known options: $known_opts" - } - } - set opts [dict merge $defaults $args] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_name_description [dict get $opts -name_description] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - validate_name_not_empty_or_spaced $projectname -name_description $opt_name_description - set reserved_words [list etc lib bin modules src doc vendorlib vendormodules embedded runtime _aside _build] - if {$projectname in $reserved_words } { - error "$opt_name_description '$projectname' cannot be one of reserved_words: $reserved_words" - } - if {[string first "::" $projectname] >= 0} { - error "$opt_name_description '$projectname' cannot contain namespace separator '::'" - } - return $projectname - } - proc validate_name_not_empty_or_spaced {name args} { - set defaults [list\ - -name_description projectname\ - ] - if {[llength $args] %2 != 0} {error "validate_modulename args must be name-value pairs: received '$args'"} - set known_opts [dict keys $defaults] - foreach k [dict keys $args] { - if {$k ni $known_opts} { - error "validate_modulename error: unknown option $k. known options: $known_opts" - } - } - set opts [dict merge $defaults $args] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_name_description [dict get $opts -name_description] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- - if {![string length $name]} { - error "$opt_name_description cannot be empty" - } - if {[string length [string map [list " " "" \n "" \r "" \t ""] $name]] != [string length $name]} { - error "$opt_name_description cannot contain whitespace" - } - return $name - } - - #split modulename (as present in a filename or namespaced name) into name/version ignoring leading namespace path - #ignore trailing .tm .TM if present - #if version doesn't pass validation - treat it as part of the modulename and return empty version string without error - #Up to caller to validate. - proc split_modulename_version {modulename} { - set lastpart [namespace tail $modulename] - set lastpart [file tail $lastpart] ;# should be ok to use file tail now that we've ensured no namespace components - if {[string equal -nocase [file extension $modulename] ".tm"]} { - set fileparts [split [file rootname $lastpart] -] - } else { - set fileparts [split $lastpart -] - } - if {[punk::mix::util::is_valid_tm_version [lindex $fileparts end]]} { - set versionsegment [lindex $fileparts end] - set namesegment [join [lrange $fileparts 0 end-1] -];#re-stitch - } else { - # - set namesegment [join $fileparts -] - set versionsegment "" - } - return [list $namesegment $versionsegment] - } - - proc get_status {{workingdir ""} args} { - set result "" - if {$workingdir ne ""} { - if {[file pathtype $workingdir] ne "absolute"} { - set workingdir [file normalize $workingdir] - } - set active_dir $workingdir - } else { - set active_dir [pwd] - } - set defaults [dict create\ - -v 1\ - ] - set opts [dict merge $defaults $args] - # -- --- --- --- --- --- --- --- --- - set opt_v [dict get $opts -v] - # -- --- --- --- --- --- --- --- --- - - - set repopaths [punk::repo::find_repos [pwd]] - set repos [dict get $repopaths repos] - if {![llength $repos]} { - append result [dict get $repopaths warnings] - } else { - append result [dict get $repopaths warnings] - lassign [lindex $repos 0] repopath repotypes - if {"fossil" in $repotypes} { - #review - multiple process launches to fossil a bit slow on windows.. - #could we query global db in one go instead? - # - set fossil_prog [auto_execok fossil] - append result "FOSSIL project based at $repopath with revision: [punk::repo::fossil_revision $repopath]" \n - set fosinfo [exec {*}$fossil_prog info] - append result [join [punk::repo::grep {repository:*} $fosinfo] \n] \n - - set fosrem [exec {*}$fossil_prog remote ls] - if {[string length $fosrem]} { - append result "Remotes:\n" - append result " " $fosrem \n - } - - - append result [join [punk::repo::grep {tags:*} $fosinfo] \n] \n - - set dbinfo [exec {*}$fossil_prog dbstat] - append result [join [punk::repo::grep {project-name:*} $dbinfo] \n] \n - append result [join [punk::repo::grep {tickets:*} $dbinfo] \n] \n - append result [join [punk::repo::grep {project-age:*} $dbinfo] \n] \n - append result [join [punk::repo::grep {latest-change:*} $dbinfo] \n] \n - append result [join [punk::repo::grep {files:*} $dbinfo] \n] \n - append result [join [punk::repo::grep {check-ins:*} $dbinfo] \n] \n - if {"project" in $repotypes} { - #punk project - if {![catch {package require textblock; package require patternpunk}]} { - set result [textblock::join [textblock::join [>punk . logo] " "] $result] - append result \n - } - } - - set timeline [exec fossil timeline -n 5 -t ci] - set timeline [string map [list \r\n \n] $timeline] - append result $timeline - if {$opt_v} { - set repostate [punk::repo::workingdir_state $repopath -repopaths $repopaths -repotypes fossil] - append result \n [punk::repo::workingdir_state_summary $repostate] - } - - } - #repotypes *could* be both git and fossil - so report both if so - if {"git" in $repotypes} { - append result "GIT project based at $repopath with revision: [punk::repo::git_revision $repopath]" \n - if {[string length [set git_prog [auto_execok git]]]} { - set git_remotes [exec {*}$git_prog remote -v] - append result $git_remotes - if {$opt_v} { - set repostate [punk::repo::workingdir_state $repopath -repopaths $repopaths -repotypes git] - append result \n [punk::repo::workingdir_state_summary $repostate] - } - } - } - - } - - return $result - } - - - proc build_modules_from_source_to_base {srcdir basedir args} { - set antidir [list "#*" "_aside" ".git" ".fossil*"] ;#exact or glob patterns for folders we don't want to search in. - set defaults [list\ - -installer punk::mix::cli::build_modules_from_source_to_base\ - -call-depth-internal 0\ - -max_depth 1000\ - -subdirlist {}\ - -punkcheck_eventobj "\uFFFF"\ - -glob *.tm\ - ] - set opts [dict merge $defaults $args] - - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - set installername [dict get $opts -installer] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - set CALLDEPTH [dict get $opts -call-depth-internal] - set max_depth [dict get $opts -max_depth] - set subdirlist [dict get $opts -subdirlist] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - set fileglob [dict get $opts -glob] - if {![string match "*.tm" $fileglob]} { - error "build_modules_from_source_to_base -glob '$fileglob' doesn't seem to target tcl modules." - } - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_punkcheck_eventobj [dict get $opts -punkcheck_eventobj] - - set magicversion [punk::mix::util::magic_tm_version] ;#deliberately large so given load-preference when testing - set module_list [list] - - if {[file tail [file dirname $srcdir]] ne "src"} { - puts stderr "ERROR build_modules_from_source_to_base can only be called with a srcdir that is a subfolder of your 'src' directory" - puts stderr "The .tm modules are namespaced based on their directory depth - so we need to start at the root" - puts stderr "To build a subtree of your modules - use an appropriate src/modules folder and pass in the -subdirlist." - puts stderr "e.g if your modules are based at /x/src/modules2 and you wish to build only the .tm files at /x/src/modules2/skunkworks/lib" - puts stderr "Use: >build_modules_from_source_to_base /x/src/modules2 /x/modules2 -subdirlist {skunkworks lib}" - exit 2 - } - set srcdirname [file tail $srcdir] - - set build [file dirname $srcdir]/_build/$srcdirname ;#relative to *original* srcdir - not current_source_dir - if {[llength $subdirlist] == 0} { - set target_module_dir $basedir - set current_source_dir $srcdir - } else { - set target_module_dir $basedir/[file join {*}$subdirlist] - set current_source_dir $srcdir/[file join {*}$subdirlist] - } - if {![file exists $target_module_dir]} { - error "build_modules_from_source_to_base from current source dir: '$current_source_dir'. Basedir:'$current_module_dir' doesn't exist or is empty" - } - if {![file exists $current_source_dir]} { - error "build_modules_from_source_to_base from current source dir:'$current_source_dir' doesn't exist or is empty" - } - - #---------------------------------------- - set punkcheck_file [file join $basedir/.punkcheck] - if {$CALLDEPTH == 0} { - - set config [dict create\ - -glob $fileglob\ - -max_depth 0\ - ] - #lassign [punkcheck::start_installer_event $punkcheck_file $installername $srcdir $basedir $config] _eventid punkcheck_eventid _recordset record_list - # -- --- - set installer [punkcheck::installtrack new $installername $punkcheck_file] - $installer set_source_target $srcdir $basedir - set event [$installer start_event $config] - # -- --- - - } else { - set event $opt_punkcheck_eventobj - } - #---------------------------------------- - - - - set src_modules [glob -nocomplain -dir $current_source_dir -type f -tail $fileglob] - - set did_skip 0 ;#flag for stdout/stderr formatting only - foreach m $src_modules { - set is_interesting 0 - if {[string match "foobar" $current_source_dir]} { - set is_interesting 1 - } - if {$is_interesting} { - puts "build_modules_from_source_to_base >>> module $current_source_dir/$m" - } - set fileparts [split [file rootname $m] -] - set tmfile_versionsegment [lindex $fileparts end] - if {$tmfile_versionsegment eq $magicversion} { - #rebuild the .tm from the #tarjar - set basename [join [lrange $fileparts 0 end-1] -] - set versionfile $current_source_dir/$basename-buildversion.txt - set versionfiledata "" - if {![file exists $versionfile]} { - puts stderr "\nWARNING: Missing buildversion text file: $versionfile" - puts stderr "Using version 0.1 - create $versionfile containing the desired version number as the top line to avoid this warning\n" - set module_build_version "0.1" - } else { - set fd [open $versionfile r] - set versionfiledata [read $fd]; close $fd - set ln0 [lindex [split $versionfiledata \n] 0] - set ln0 [string trim $ln0]; set ln0 [string trim $ln0 \r] - if {![util::is_valid_tm_version $ln0]} { - puts stderr "ERROR: build version '$ln0' specified in $versionfile is not suitable. Please ensure a proper version number is at first line of file" - exit 3 - } - set module_build_version $ln0 - } - - - if {[file exists $current_source_dir/#tarjar-$basename-$magicversion]} { - #TODO - file mkdir $buildfolder - - if {[file exists $current_source_dir/#tarjar-$basename-$magicversion/DESCRIPTION.txt]} { - - } else { - - } - #REVIEW - should be in same structure/depth as $target_module_dir in _build? - set tmfile $basedir/_build/$basename-$module_build_version.tm - file mkdir $basedir/_build - file delete -force $basedir/_build/#tarjar-$basename-$module_build_version - file delete -force $tmfile - - - file copy -force $current_source_dir/#tarjar-$basename-$magicversion $basedir/_build/#tarjar-$basename-$module_build_version - # - #bsdtar doesn't seem to work.. or I haven't worked out the right options? - #exec tar -cvf $basedir/_build/$basename-$module_build_version.tm $basedir/_build/#tarjar-$basename-$module_build_version - package require tar - tar::create $tmfile $basedir/_build/#tarjar-$basename-$module_build_version - if {![file exists $tmfile]} { - puts stdout "ERROR: Failed to build tarjar file $tmfile" - exit 4 - } - #copy the file? - #set target $target_module_dir/$basename-$module_build_version.tm - #file copy -force $tmfile $target - - lappend module_list $tmfile - } else { - #assume that either the .tm is not a tarjar - or the tarjar dir is capped (trailing #) and the .tm has been manually tarred. - if {[file exists $current_source_dir/#tarjar-$basename-${magicversion}#]} { - puts stderr "\nWarning: found 'capped' folder #tarjar-$basename-${magicversion}# - No attempt being made to update version in description.txt" - } - - #------------------------------ - # - #set target_relpath [punkcheck::lib::path_relative $basedir $target_module_dir/$basename-$module_build_version.tm] - #set file_record [punkcheck::installfile_begin $basedir $target_relpath $installername -eventid $punkcheck_eventid] - $event targetset_init INSTALL $target_module_dir/$basename-$module_build_version.tm - $event targetset_addsource $versionfile - $event targetset_addsource $current_source_dir/$m - - #set changed_list [list] - ## -- --- --- --- --- --- - #set source_relpath [punkcheck::lib::path_relative $basedir $versionfile] - #set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record] - ## -- --- --- --- --- --- - #set source_relpath [punkcheck::lib::path_relative $basedir $current_source_dir/$m] - #set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record] - ## -- --- --- --- --- --- - #set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]] - #set changed_list [dict get $changed_unchanged changed] - - - if {\ - [llength [dict get [$event targetset_source_changes] changed]]\ - || [llength [$event get_targets_exist]] < [llength [$event get_targets]]\ - } { - - #set file_record [punkcheck::installfile_started_install $basedir $file_record] - $event targetset_started - # -- --- --- --- --- --- - set target $target_module_dir/$basename-$module_build_version.tm - if {$did_skip} {set did_skip 0; puts -nonewline stdout \n} - puts stdout "copying module $current_source_dir/$m to $target as version: $module_build_version ([file tail $target])" - set fd [open $current_source_dir/$m r]; fconfigure $fd -translation binary; set data [read $fd]; close $fd - set data [string map [list $magicversion $module_build_version] $data] - set fdout [open $target w] - fconfigure $fdout -translation binary - puts -nonewline $fdout $data - close $fdout - #file copy -force $srcdir/$m $target - lappend module_list $target - # -- --- --- --- --- --- - #set file_record [punkcheck::installfile_finished_install $basedir $file_record] - $event targetset_end OK - } else { - if {$is_interesting} { - puts stdout "skipping module $current_source_dir/$m - no change in sources detected" - } - puts -nonewline stderr "." - set did_skip 1 - #set file_record [punkcheck::installfile_skipped_install $basedir $file_record] - $event targetset_end SKIPPED - } - - #------------------------------ - - } - - continue - } - - - if {![util::is_valid_tm_version $tmfile_versionsegment]} { - #last segment doesn't look even slightly versiony - fail. - puts stderr "ERROR: Unable to confirm file $current_source_dir/$m is a reasonably versioned .tm module - ABORTING." - exit 1 - } - - ##------------------------------ - ## - #set target_relpath [punkcheck::lib::path_relative $basedir $target_module_dir/$m] - #set file_record [punkcheck::installfile_begin $basedir $target_relpath $installername -eventid $punkcheck_eventid] - #set changed_list [list] - ## -- --- --- --- --- --- - #set source_relpath [punkcheck::lib::path_relative $basedir $current_source_dir/$m] - #set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record] - ## -- --- --- --- --- --- - #set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]] - #set changed_list [dict get $changed_unchanged changed] - - #---------- - $event targetset_init INSTALL $target_module_dir/$m - $event targetset_addsource $current_source_dir/$m - if {\ - [llength [dict get [$event targetset_source_changes] changed]]\ - || [llength [$event get_targets_exist]] < [llength [$event get_targets]]\ - } { - - #set file_record [punkcheck::installfile_started_install $basedir $file_record] - $event targetset_started - # -- --- --- --- --- --- - if {$did_skip} {set did_skip 0; puts -nonewline stdout \n} - lappend module_list $current_source_dir/$m - file copy -force $current_source_dir/$m $target_module_dir - puts stderr "Copied already versioned module $current_source_dir/$m to $target_module_dir" - # -- --- --- --- --- --- - #set file_record [punkcheck::installfile_finished_install $basedir $file_record] - $event targetset_end OK -note "already versioned module" - } else { - puts -nonewline stderr "." - set did_skip 1 - if {$is_interesting} { - puts stderr "$current_source_dir/$m [$event targetset_source_changes]" - } - #set file_record [punkcheck::installfile_skipped_install $basedir $file_record] - $event targetset_end SKIPPED - } - - } - if {$CALLDEPTH >= $max_depth} { - set subdirs [list] - } else { - set subdirs [glob -nocomplain -dir $current_source_dir -type d -tail *] - } - #puts stderr "subdirs: $subdirs" - foreach d $subdirs { - set skipdir 0 - foreach dg $antidir { - if {[string match $dg $d]} { - set skipdir 1 - continue - } - } - if {$skipdir} { - continue - } - if {![file exists $target_module_dir/$d]} { - file mkdir $target_module_dir/$d - } - lappend module_list {*}[build_modules_from_source_to_base $srcdir $basedir\ - -call-depth-internal [expr {$CALLDEPTH +1}]\ - -subdirlist [list {*}$subdirlist $d]\ - -punkcheck_eventobj $event\ - -glob $fileglob\ - ] - } - if {$did_skip} { - puts -nonewline stdout \n - } - if {$CALLDEPTH == 0} { - $event destroy - $installer destroy - } - return $module_list - } - - variable kettle_reset_bodies [dict create] - variable kettle_reset_args [dict create] - #We are abusing kettle to run in-process. - # when we change to another project we need recipes to be reloaded. - # Kettle rewrites some of it's own procs - stopping reloading of recipes when we change folders - #kettle_init stores the original proc bodies & args - proc kettle_init {} { - variable kettle_reset_bodies ;#dict - variable kettle_reset_args - set reset_procs [list\ - ::kettle::benchmarks\ - ::kettle::doc\ - ::kettle::figures\ - ::kettle::meta::scan\ - ::kettle::testsuite\ - ] - foreach p $reset_procs { - set b [info body $p] - if {[string match "*Overwrite self*" $b]} { - dict set kettle_reset_bodies $p $b - set argnames [info args $p] - set arglist [list] - foreach a $argnames { - if {[info default $p $a dval]} { - lappend arglist [list $a $dval] - } else { - lappend arglist $a - } - } - dict set kettle_reset_args $p $arglist - } - } - - } - #call kettle_reinit to ensure recipes point to current project - proc kettle_reinit {} { - variable kettle_reset_bodies - variable kettle_reset_args - foreach p [dict keys $kettle_reset_bodies] { - set b [dict get $kettle_reset_bodies $p] - set argl [dict get $kettle_reset_args $p] - uplevel 1 [list ::proc $p $argl $b] - } - #todo - determine standard recipes by examining standard.tcl instead of hard coding? - set standard_recipes [list\ - null\ - forever\ - list-recipes\ - help-recipes\ - help-dump\ - help-recipes\ - help\ - list\ - list-options\ - help-options\ - show-configuration\ - show-state\ - show\ - meta-status\ - gui\ - ] - #set ::kettle::recipe::recipe [dict create] - foreach r [dict keys $::kettle::recipe::recipe] { - if {$r ni $standard_recipes} { - dict unset ::kettle::recipe::recipe $r - } - } - } - proc kettle_call {calltype args} { - variable kettle_reset_bodies - if {$calltype ni [list lib shell]} { - error "pmix kettle_call 1st argument must be one of: 'lib' for direct use of kettle module or 'shell' to call as separate process" - } - if {$calltype eq "shell"} { - set kettleappfile [file dirname [info nameofexecutable]]/kettle - set kettlebatfile [file dirname [info nameofexecutable]]/kettle.bat - - if {(![file exists $kettleappfile]) && (![file exists $kettlebatfile])} { - error "pmix kettle_call unable to find installed kettle application file '$kettleappfile' (or '$kettlebatfile' if on windows)" - } - if {[file exists $kettleappfile]} { - set kettlescript $kettleappfile - } - if {$::tcl_platform(platform) eq "windows"} { - if {[file exists $kettlebatfile]} { - set kettlescript $kettlebatfile - } - } - } - set startdir [pwd] - if {![file exists $startdir/build.tcl]} { - error "pmix kettle must be run from a folder containing build.tcl (cwd: [pwd])" - } - if {[package provide kettle] eq ""} { - puts stdout "Loading kettle package - may be delay on first load ..." - package require kettle - kettle_init ;#store original procs for those kettle procs that rewrite themselves - } else { - if {[dict size $kettle_reset_bodies] == 0} { - #presumably package require kettle was called without calling our kettle_init hack. - kettle_init - } else { - #undo proc rewrites - kettle_reinit - } - } - set first [lindex $args 0] - if {[string match @* $first]} { - error "pmix kettle doesn't support special operations - try calling tclsh kettle directly" - } - if {$first eq "-f"} { - set args [lassign $args __ path] - } else { - set path $startdir/build.tcl - } - set opts [list] - - if {[lindex $args 0] eq "-trace"} { - set args [lrange $args 1 end] - lappend opts --verbose on - } - set goals [list] - - if {$calltype eq "lib"} { - file mkdir ~/.kettle - set dotfile ~/.kettle/config - if {[file exists $dotfile] && - [file isfile $dotfile] && - [file readable $dotfile]} { - ::kettle io trace {Loading dotfile $dotfile ...} - set args [list {*}[::kettle path cat $dotfile] {*}$args] - } - } - - #hardcoded kettle option names (::kettle option names) - retrieved using kettle::option names - #This is done so we don't have to load kettle lib for shell call (both loading as module and running shell are annoyingly SLOW) - #REVIEW - needs to be updated to keep in sync with kettle. - set knownopts [list\ - --exec-prefix --bin-dir --lib-dir --prefix --man-dir --html-dir --markdown-dir --include-dir \ - --ignore-glob --dry --verbose --machine --color --state --config --with-shell --log \ - --log-append --log-mode --with-dia --constraints --file --limitconstraints --tmatch --notfile --single --valgrind --tskip --repeats \ - --iters --collate --match --rmatch --with-doc-destination --with-git --target --test-include \ - ] - - while {[llength $args]} { - set o [lindex $args 0] - switch -glob -- $o { - --* { - #instead of using: kettle option known - if {$o ni $knownopts} { - error "Unable to process unknown option $o." {} [list KETTLE (pmix)] - } - lappend opts $o [lindex $args 1] - #::kettle::option set $o [lindex $args 1] - set args [lrange $args 2 end] - } - default { - lappend goals $o - set args [lrange $args 1 end] - } - } - } - - if {![llength $goals]} { - lappend goals help - } - if {"--prefix" ni [dict keys $opts]} { - dict set opts --prefix [file dirname $startdir] - } - if {$calltype eq "lib"} { - ::kettle status clear - ::kettle::option::set @kettle $startdir - foreach {o v} $opts { - ::kettle option set $o $v - } - ::kettle option set @srcscript $path - ::kettle option set @srcdir [file dirname $path] - ::kettle option set @goals $goals - #load standard recipes as listed in build.tcl - ::source $path - puts stderr "recipes: [::kettle recipe names]" - ::kettle recipe run {*}[::kettle option get @goals] - - set state [::kettle option get --state] - if {$state ne {}} { - puts stderr "saving kettle state: $state" - ::kettle status save $state - } - - } else { - #shell - puts stdout "Running external kettle process with args: $opts $goals" - run -n tclsh $kettlescript -f $path {*}$opts {*}$goals - } - - } - proc kettle_punk_recipes {} { - set txtdst ... - } - - } -} - - -namespace eval punk::mix::cli { - proc _cli {args} { - #don't use tailcall - base uses info level to determine caller - ::punk::mix::base::_cli {*}$args - } - variable default_command help - package require punk::mix::base - package require punk::overlay - punk::overlay::custom_from_base [namespace current] ::punk::mix::base -} - - - - - - - - - - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::mix::cli [namespace eval punk::mix::cli { - variable version - set version 0.3 -}] -return diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/buildsuite-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/buildsuite-0.1.0.tm deleted file mode 100644 index 883e02d2..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/buildsuite-0.1.0.tm +++ /dev/null @@ -1,152 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::mix::commandset::buildsuite 0.1.0 -# Meta platform tcl -# Meta license -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::mix::commandset::buildsuite { - namespace export * - proc projects {suite} { - set pathinfo [punk::repo::find_repos [pwd]] - set projectdir [dict get $pathinfo closest] - set suites_dir [file join $projectdir src buildsuites] - if {![file isdirectory [file join $suites_dir $suite]]} { - puts stderr "suite: $suite not found in buildsuites folder: $suites_dir" - return - } - set suite_dir [file join $suites_dir $suite] - set projects [glob -dir $suite_dir -type d -tails *] - - #use internal du which although breadth-first is generally faster - puts stdout "Examining source folders in $suite_dir." ;#A hint that something is happening in case sources are large - set du_info [punk::du::du -d 1 -b $suite_dir] - set du_sizes [dict create] - set suite_total_size "-" - foreach du_record $du_info { - if {[llength $du_record] != 2} { - #sanity precaution - punk::du::du should always output list of 2 element lists - at least with flags we're using - continue - } - set sz [lindex $du_record 0] - set path_parts [file split [lindex $du_record 1]] ;#should handle spaced-paths ok. - set s [lindex $path_parts end-1] - set p [lindex $path_parts end] - - #This handles case where a project folder is same name as suite e.g src/buildsuites/tcl/tcl - #so we can't just use tail as dict key. We could assume last record is always total - but - if {![string match -nocase $s $suite]} { - if {$s eq "buildsuites" && [string match -nocase $p $suite]} { - set suite_total_size $sz ;#this includes config files in suite base - so we don't really want to use this to report the total source size - } else { - #something else - shouldn't happen - puts stderr "Unexpected output from du in suite_dir: $suite_dir" - puts stderr "$du_record" - #try to continue anyway - } - } else { - dict set du_sizes $p $sz - } - } - - #build another dict for sizes where we ensure exactly one entry for each project exists and exclude total (don't blindly trust du output e.g in case weird filename/permission issue) - set psizes [list] - foreach p $projects { - if {[dict exists $du_sizes $p]} { - dict set psizes $p [dict get $du_sizes $p] - } else { - dict set psizes $p - - } - } - set total_source_size "-" - if {[catch { - set total_source_size [tcl::mathop::+ {*}[dict values $psizes]] - } errM]} { - puts stderr "Failed to calculate total source size. Errmsg: $errM" - } - package require overtype - - set title1 "Projects" - set widest1 [tcl::mathfunc::max {*}[lmap v [concat [list $title1] $projects] {punk::strlen $v}]] - set col1 [string repeat " " $widest1] - - set size_values [dict values $psizes] - # Title is probably widest - but go through the process anyway! - set title2 "Source Bytes" - set widest2 [tcl::mathfunc::max {*}[lmap v [concat [list $title2] $size_values] {punk::strlen $v}]] - set col2 [string repeat " " $widest2] - - - set output "" - append output "[overtype::left $col1 $title1] [overtype::right $col2 $title2]" \n - foreach p [lsort $projects] { - #todo - provide some basic info for each - last build time? last time-to-build? - append output "[overtype::left $col1 $p] [overtype::right $col2 [dict get $psizes $p]]" \n - } - append output "Total Source size: $total_source_size bytes" \n - return $output - } - - - namespace eval collection { - namespace export * - proc _default {{glob {}}} { - if {![string length $glob]} { - set glob * - } - #todo - review - we want the furthest not the closest if we are potentially inside a buildsuite project - set pathinfo [punk::repo::find_repos [pwd]] - set projectdir [dict get $pathinfo closest] - set suites_dir [file join $projectdir src buildsuites] - if {![file exists $suites_dir]} { - puts stderr "No buildsuites folder found at $suites_dir" - return - } - set suites [lsort [glob -dir $suites_dir -type d -tails *]] - if {$glob ne "*"} { - set suites [lsearch -all -inline $suites $glob] - } - return $suites - } - } - - -} - - - - - - - - - - - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::mix::commandset::buildsuite [namespace eval punk::mix::commandset::buildsuite { - variable version - set version 0.1.0 -}] -return diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/debug-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/debug-0.1.0.tm deleted file mode 100644 index 8ed735c1..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/debug-0.1.0.tm +++ /dev/null @@ -1,92 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::mix::commandset::debug 0.1.0 -# Meta platform tcl -# Meta license -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::mix::commandset::debug { - namespace export get paths - namespace path ::punk::mix::cli - - #Except for 'get' - all debug commands should emit to stdout - proc paths {} { - set out "" - puts stdout "find_repos output:" - set pathinfo [punk::repo::find_repos [pwd]] - pdict $pathinfo - - set projectdir [dict get $pathinfo closest] - set modulefolders [lib::find_source_module_paths $projectdir] - puts stdout "modulefolders: $modulefolders" - - set template_base_dict [punk::mix::base::lib::get_template_basefolders] - puts stdout "get_template_basefolders output:" - pdict $template_base_dict - return - } - - #call other debug command - but capture stdout as return value - proc get {args} { - set nm [lindex $args 0] - if {$nm eq ""} { - set nscmds [info commands [namespace current]::*] - set cmds [lmap v $nscmds {namespace tail $v}] - error "debug.get missing debug command argument. Try one of: $cmds" - return - } - set nextargs [lrange $args 1 end] - set out "" - if {[info commands [namespace current]::$nm] ne ""} { - append out [runout -n -tcl [namespace current]::$nm {*}$nextargs] \n - } else { - set nscmds [info commands [namespace current]::*] - set cmds [lmap v $nscmds {namespace tail $v}] - error "debug.get invalid debug command '$nm' Try one of: $cmds" - } - return $out - } - namespace eval lib { - - } - - -} - - - - - - - - - - - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::mix::commandset::debug [namespace eval punk::mix::commandset::debug { - variable version - set version 0.1.0 -}] -return diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/doc-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/doc-0.1.0.tm deleted file mode 100644 index 36a654b7..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/doc-0.1.0.tm +++ /dev/null @@ -1,286 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::mix::commandset::doc 0.1.0 -# Meta platform tcl -# Meta license -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz - -package require punk::path ;# for treefilenames, relative -package require punk::repo -package require punk::docgen ;#inline doctools - generate doctools .man files at src/docgen prior to using kettle to producing .html .md etc -package require punk::mix::cli ;#punk::mix::cli::lib used for kettle_call -#package require punk::mix::util ;#for path_relative - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::mix::commandset::doc { - namespace export * - - proc _default {} { - puts "documentation subsystem" - puts "commands: doc.build" - puts " build documentation from src/doc to src/embedded using the kettle build tool" - } - - proc build {} { - puts "build docs" - set projectdir [punk::repo::find_project] - if {$projectdir eq ""} { - puts stderr "No current project dir - unable to build docs" - return - } - #user may delete the comment containing "--- punk::docgen::overwrites" and then manually edit, and we won't overwrite - #we still generate output in src/docgen so user can diff and manually update if thats what they prefer - set oldfiles [punk::path::treefilenames $projectdir/src/doc _module_*.man] - foreach maybedoomed $oldfiles { - set fd [open $maybedoomed r] - set data [read $fd] - close $fd - if {[string match "*--- punk::docgen overwrites *" $data]} { - file delete -force $maybedoomed - } - } - set generated [lib::do_docgen modules] - if {[dict get $generated count] > 0} { - #review - set doclist [dict get $generated docs] - set source_base [dict get $generated base] - set target_base $projectdir/src/doc - foreach dinfo $doclist { - lassign $dinfo module fpath - set relpath [punk::path::relative $source_base $fpath] - set relfolder [file dirname $relpath] - if {$relfolder eq "."} { - set relfolder "" - } - file mkdir [file join $target_base $relfolder] - set target [file join $target_base $relfolder _module_[file tail $fpath]] - puts stderr "target --> $target" - if {![file exists $target]} { - file copy $fpath $target - } - } - } - - if {[file exists $projectdir/src/doc]} { - set original_wd [pwd] - cd $projectdir/src - #---------- - set installer [punkcheck::installtrack new project.new $projectdir/src/.punkcheck] - $installer set_source_target $projectdir/src/doc $projectdir/src/embedded - set event [$installer start_event {-install_step kettledoc}] - #use same virtual id "kettle_build_doc" as project.new - review best way to keep identifiers like this in sync. - $event targetset_init VIRTUAL kettle_build_doc ;#VIRTUAL - since there is no specific target file - and we don't know all the files that will be generated - $event targetset_addsource $projectdir/src/doc ;#whole doc tree is considered the source - #---------- - if {\ - [llength [dict get [$event targetset_source_changes] changed]]\ - } { - $event targetset_started - # -- --- --- --- --- --- - puts stdout "BUILDING DOCS at $projectdir/src/embedded from src/doc" - if {[catch { - - punk::mix::cli::lib::kettle_call lib doc - #Kettle doc - - } errM]} { - $event targetset_end FAILED -note "kettle_build_doc failed: $errM" - } else { - $event targetset_end OK - } - # -- --- --- --- --- --- - } else { - puts stderr "No change detected in src/doc" - $event targetset_end SKIPPED - } - $event end - $event destroy - $installer destroy - cd $original_wd - } else { - puts stderr "No doc folder found at $projectdir/src/doc" - } - } - proc status {} { - set projectdir [punk::repo::find_project] - if {$projectdir eq ""} { - puts stderr "No current project dir - unable to check doc status" - return - } - if {![file exists $projectdir/src/doc]} { - set result "No documentation source found. Expected .man files in doctools format at $projectdir/src/doc" - return $result - } - set original_wd [pwd] - cd $projectdir/src - puts stdout "Testing status of doctools source location $projectdir/src/doc ..." - flush stdout - #---------- - set installer [punkcheck::installtrack new project.new $projectdir/src/.punkcheck] - $installer set_source_target $projectdir/src/doc $projectdir/src/embedded - set event [$installer start_event {-install_step kettledoc}] - #use same virtual id "kettle_build_doc" as project.new - review best way to keep identifiers like this in sync. - $event targetset_init QUERY kettle_build_doc ;#usually VIRTUAL - since there is no specific target file - and we don't know all the files that will be generated - but here we use QUERY to ensure no writes to .punkcheck - set last_completion [$event targetset_last_complete] - - if {[llength $last_completion]} { - #adding a source causes it to be checksummed - $event targetset_addsource $projectdir/src/doc ;#whole doc tree is considered the source - #---------- - set changeinfo [$event targetset_source_changes] - if {\ - [llength [dict get $changeinfo changed]]\ - } { - puts stdout "changed" - puts stdout $changeinfo - } else { - puts stdout "No changes detected in $projectdir/src/doc tree" - } - } else { - #no previous completion-record for this target - must assume changed - no need to trigger checksumming - puts stdout "No existing record of doc build in .punkcheck. Assume it needs to be rebuilt." - } - - - $event destroy - $installer destroy - - cd $original_wd - } - proc validate {} { - #todo - run and validate punk::docgen output - set projectdir [punk::repo::find_project] - if {$projectdir eq ""} { - puts stderr "No current project dir - unable to check doc status" - return - } - if {![file exists $projectdir/src/doc]} { - set result "No documentation source found. Expected .man files in doctools format at $projectdir/src/doc" - return $result - } - set original_wd [pwd] - set docroot $projectdir/src/doc - cd $docroot - - dtplite validate $docroot - - #punk::mix::cli::lib::kettle_call lib validate-doc - - cd $original_wd - } - - namespace eval collection { - variable pkg - set pkg punk::mix::commandset::doc - - namespace export * - namespace path [namespace parent] - - } - - namespace eval lib { - variable pkg - set pkg punk::mix::commandset::doc - proc do_docgen {{project_subpath modules}} { - #Extract doctools comments from source code - set projectdir [punk::repo::find_project] - set output_base [file join $projectdir src docgen] - set codesource_path [file join $projectdir $project_subpath] - if {![file isdirectory $codesource_path]} { - puts stderr "WARNING punk::mix::commandset::doc unable to find codesource_path $codesource_path during do_docgen - skipping inline doctools generation" - return - } - if {[file isdirectory $output_base]} { - if {[catch { - file delete -force $output_base - }]} { - error "do_docgen failed to delete existing output base folder: $output_base" - } - } - file mkdir $output_base - - set matched_paths [punk::path::treefilenames $codesource_path *.tm -antiglob_paths {**/mix/templates/** **/mixtemplates/**}] - set count 0 - set newdocs [list] - set docgen_header_comments "" - append docgen_header_comments {[comment {--- punk::docgen generated from inline doctools comments ---}]} \n - append docgen_header_comments {[comment {--- punk::docgen DO NOT EDIT DOCS HERE UNLESS YOU REMOVE THESE COMMENT LINES ---}]} \n - append docgen_header_comments {[comment {--- punk::docgen overwrites this file ---}]} \n - foreach fullpath $matched_paths { - set doctools [punk::docgen::get_doctools_comments $fullpath] - if {$doctools ne ""} { - set fname [file tail $fullpath] - set mod_tail [file rootname $fname] - set relpath [punk::path::relative $codesource_path [file dirname $fullpath]] - if {$relpath eq "."} { - set relpath "" - } - set tailsegs [file split $relpath] - set module_fullname [join $tailsegs ::]::$mod_tail - set target_docname $fname.man - set this_outdir [file join $output_base $relpath] - - if {[string length $fname] > 99} { - #output needs to be tarballed to do checksum change tests in a reasonably straightforward and not-too-terribly slow way. - #hack - review. Determine exact limit - test if tcllib tar fixed or if it's a limit of the particular tar format - #work around tcllib tar filename length limit ( somewhere around 100?) This seems to be a limit on the length of a particular segment in the path.. not whole path length? - #this case only came up because docgen used to path munge to long filenames - but left because we know there is a limit and renaming fixes it - even if it's ugly - but still allows doc generation. - #review - if we're checking fname - should also test length of whole path and determine limits for tar - package require md5 - set target_docname [md5::md5 -hex $fullpath]_overlongfilename.man - puts stderr "WARNING - overlong file name - renaming $fullpath" - puts stderr " to [file dirname $fullpath]/$target_docname" - } - - file mkdir $this_outdir - puts stdout "saving [string length $doctools] bytes of doctools output from file $relpath/$fname" - set outfile [file join $this_outdir $target_docname] - set fd [open $outfile w] - fconfigure $fd -translation binary - puts -nonewline $fd $docgen_header_comments$doctools - close $fd - incr count - lappend newdocs [list $module_fullname $outfile] - } - } - return [list count $count docs $newdocs base $output_base] - } - - } -} - - - - - - - - - - - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::mix::commandset::doc [namespace eval punk::mix::commandset::doc { - variable pkg punk::mix::commandset::doc - variable version - set version 0.1.0 -}] -return \ No newline at end of file diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/layout-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/layout-0.1.0.tm deleted file mode 100644 index 0a9ff2d4..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/layout-0.1.0.tm +++ /dev/null @@ -1,188 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::mix::commandset::layout 0.1.0 -# Meta platform tcl -# Meta license -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz - -#sort of a circular dependency when commandset loaded by punk::mix::cli - that's ok, but this could theoretically be loaded by another cli and with another base -package require punk::mix -package require punk::mix::base - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::mix::commandset::layout { - namespace export * - - #per layout functions - proc files {layout} { - set allfiles [lib::layout_all_files $layout] - return [join $allfiles \n] - } - proc templatefiles {layout} { - set templatefiles [lib::layout_scan_for_template_files $layout] - return [join $templatefiles \n] - } - proc templatefiles.relative {layout} { - set template_base_dict [punk::mix::base::lib::get_template_basefolders] - - set bases_containing_layout [list] - dict for {tbase folderinfo} $template_base_dict { - if {[file exists $tbase/layouts/$layout]} { - lappend bases_containing_layout $tbase - } - } - if {![llength $bases_containing_layout]} { - puts stderr "Unable to locate folder for layout '$layout'" - puts stderr "searched [dict size $template_base_dict] template folders" - return - } - set tpldir [lindex $bases_containing_layout end] - - set layout_base $tpldir/layouts - set layout_dir [file join $layout_base $layout] - - set stripprefix [file normalize $layout_dir] - set templatefiles [lib::layout_scan_for_template_files $layout] - set tails [list] - foreach templatefullpath $templatefiles { - lappend tails [punk::repo::path_strip_alreadynormalized_prefixdepth $templatefullpath $stripprefix] - } - return [join $tails \n] - } - - #layout collection functions - to be imported with punk::overlay::import_commandset separately - namespace eval collection { - namespace export * - proc _default {{glob {}}} { - if {![string length $glob]} { - set glob * - } - set layouts [list] - #set tplfolderdict [punk::cap::call_handler punk.templates folders] - set tplfolderdict [punk::mix::base::lib::get_template_basefolders] - dict for {tdir folderinfo} $tplfolderdict { - set layout_base $tdir/layouts - #collect all layouts and use lsearch glob rather than the filesystem glob (avoid issues with dotted folder names) - set all_layouts [lsort [glob -nocomplain -dir $layout_base -type d -tail *]] - foreach match [lsearch -all -inline $all_layouts $glob] { - lappend layouts [list $match $folderinfo] - } - } - return [join [lsort -index 0 $layouts] \n] - } - - } - namespace eval lib { - proc layout_all_files {layout} { - set tplbasedict [punk::mix::base::lib::get_template_basefolders] - set layouts_found [list] - dict for {tplbase folderinfo} $tplbasedict { - if {[file isdirectory $tplbase/layouts/$layout]} { - lappend layouts_found $tplbase/layouts/$layout - } - } - if {![llength $layouts_found]} { - puts stderr "layout '$layout' not found." - puts stderr "searched [dict size $tplbasedict] template folders" - dict for {tplbase pkg} $tplbasedict { - puts stderr " - $tplbase $pkg" - } - return - } - set layoutfolder [lindex $layouts_found end] - - if {![file isdirectory $layoutfolder]} { - puts stderr "layout '$layout' not found in /layouts within one of template_folders. (get_template_folder returned: $tplbasedict)" - } - set file_list [list] - util::foreach-file $layoutfolder path { - lappend file_list $path - } - - return $file_list - } - - # - #todo - allow specifying which package the layout is from: e.g "punk::mix::templates project" ?? - proc layout_scan_for_template_files {layout {tags {}}} { - #equivalent for projects? punk::mix::commandset::module::lib::templates_dict -scriptpath "" - set tplbasedict [punk::mix::base::lib::get_template_basefolders] - set layouts_found [list] - dict for {tpldir pkg} $tplbasedict { - if {[file isdirectory $tpldir/layouts/$layout]} { - lappend layouts_found $tpldir/layouts/$layout - } - } - if {![llength $layouts_found]} { - puts stderr "layout '$layout' not found." - puts stderr "searched [dict size $tplbasedict] template folders" - dict for {tpldir pkg} $tplbasedict { - puts stderr " - $tpldir $pkg" - } - return - } - set layoutfolder [lindex $layouts_found end] - - #use last matching layout found. review silent if multiple? - if {![llength $tags]} { - #todo - get standard tags from somewhere - set tagnames [list project] - foreach tn $tagnames { - lappend tags [string cat % $tn %] - } - } - set file_list [list] - util::foreach-file $layoutfolder path { - set fd [open $path r] - fconfigure $fd -translation binary - set data [read $fd] - close $fd - foreach tag $tags { - if {[string match "*$tag*" $data]} { - lappend file_list $path - } - } - } - - return $file_list - } - } - - -} - - - - - - - - - - - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::mix::commandset::layout [namespace eval punk::mix::commandset::layout { - variable version - set version 0.1.0 -}] -return diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/loadedlib-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/loadedlib-0.1.0.tm deleted file mode 100644 index 3bbe8b47..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/loadedlib-0.1.0.tm +++ /dev/null @@ -1,529 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::mix::commandset::loadedlib 0.1.0 -# Meta platform tcl -# Meta license -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz -package require punk::ns - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::mix::commandset::loadedlib { - namespace export * - #search automatically wrapped in * * - can contain inner * ? globs - proc search {searchstring} { - catch {package require frobznodule666} ;#ensure pkg system has loaded/searched for everything - if {[catch {package require natsort}]} { - set has_natsort 0 - } else { - set has_natsort 1 - } - if {[regexp {[?*]} $searchstring]} { - #caller has specified specific glob pattern - use it - #todo - respect supplied case only if uppers present? require another flag? - set matches [lsearch -all -inline -nocase [package names] $searchstring] - } else { - #make it easy to search for anything - set matches [lsearch -all -inline -nocase [package names] "*$searchstring*"] - } - - set matchinfo [list] - foreach m $matches { - set versions [package versions $m] - if {$has_natsort} { - set versions [natsort::sort $versions] - } else { - set versions [lsort $versions] - } - lappend matchinfo [list $m $versions] - } - return [join [lsort $matchinfo] \n] - } - proc loaded.search {searchstring} { - set search_result [search $searchstring] - set all_libs [split $search_result \n] - set col1items [list] - set col2items [list] - set col3items [list] - foreach libinfo $all_libs { - if {[string trim $libinfo] eq ""} { - continue - } - set versions [lassign $libinfo libname] - if {[set ver [package provide $libname]] ne ""} { - lappend col1items $libname - lappend col2items $versions - lappend col3items $ver - } - } - - package require overtype - set title1 "Library" - set widest1 [tcl::mathfunc::max {*}[lmap v [concat [list $title1] $col1items] {string length $v}]] - set col1 [string repeat " " $widest1] - set title2 "Versions Avail." - set widest2 [tcl::mathfunc::max {*}[lmap v [concat [list $title2] $col2items] {string length $v}]] - set col2 [string repeat " " $widest2] - set title3 "Loaded Version" - set widest3 [tcl::mathfunc::max {*}[lmap v [concat [list $title3] $col3items] {string length $v}]] - set col3 [string repeat " " $widest3] - - - set tablewidth [expr {$widest1 + 1 + $widest2 + 1 + $widest3}] - - set table "" - append table [string repeat - $tablewidth] \n - append table "[overtype::left $col1 $title1] [overtype::left $col2 $title2] [overtype::left $col3 $title3]" \n - append table [string repeat - $tablewidth] \n - foreach c1 $col1items c2 $col2items c3 $col3items { - append table "[overtype::left $col1 $c1] [overtype::left $col2 $c2] [overtype::left $col3 $c3]" \n - } - - return $table - - - set loaded_libs [list] - foreach libinfo $all_libs { - if {[string trim $libinfo] eq ""} { - continue - } - set versions [lassign $libinfo libname] - if {[set ver [package provide $libname]] ne ""} { - lappend loaded_libs "$libname $versions (loaded $ver)" - } - } - return [join $loaded_libs \n] - } - - proc info {libname} { - if {[catch {package require natsort}]} { - set has_natsort 0 - } else { - set has_natsort 1 - } - catch {package require frobznodule666} ;#ensure pkg system has loaded/searched for everything - set pkgsknown [package names] - if {[set posn [lsearch $pkgsknown $libname]] >= 0} { - puts stdout "Found package [lindex $pkgsknown $posn]" - } else { - puts stderr "Package not found as available library/module - check tcl::tm::list and \$auto_path" - } - set versions [package versions [lindex $libname 0]] - if {$has_natsort} { - set versions [natsort::sort $versions] - } else { - set versions [lsort $versions] - } - if {![llength $versions]} { - puts stderr "No version numbers found for library/module $libname" - return false - } - puts stdout "Versions of $libname found: $versions" - set alphaposn [lsearch $versions "999999.*"] - if {$alphaposn >= 0} { - set alpha [lindex $versions $alphaposn] - #remove and tack onto beginning.. - set versions [lreplace $versions $alphaposn $alphaposn] - set versions [list $alpha {*}$versions] - } - foreach ver $versions { - set loadinfo [package ifneeded $libname $ver] - puts stdout "$libname $ver" - puts stdout "--- 'package ifneeded' script ---" - puts stdout $loadinfo - puts stdout "---" - } - return - } - - proc copyasmodule {library modulefoldername args} { - set defaults [list -askme 1] - set opts [dict merge $defaults $args] - set opt_askme [dict get $opts -askme] - - if {[catch {package require natsort}]} { - set has_natsort 0 - } else { - set has_natsort 1 - } - - catch {package require frobznodule666} ;#ensure pkg system has loaded/searched for everything - - if {[file pathtype $modulefoldername] eq "absolute"} { - if {![file exists $modulefoldername]} { - error "Path '$modulefoldername' not found. Enter a fully qualified path, or just the tail such as 'modules' if you are within the project to use /src/modules" - } - #use the target folder as the source of projectdir info - set pathinfo [punk::repo::find_repos $modulefoldername] - set projectdir [dict get $pathinfo closest] - set modulefolder_path $modulefoldername - } else { - #use the current working directory as the source of projectdir info - set pathinfo [punk::repo::find_repos [pwd]] - set projectdir [dict get $pathinfo closest] - if {$projectdir ne ""} { - set modulefolders [punk::mix::cli::lib::find_source_module_paths $projectdir] - foreach k [list modules vendormodules] { - set knownfolder [file join $projectdir src $k] - if {$knownfolder ni $modulefolders} { - lappend modulefolders $knownfolder - } - } - set mtails [list] - foreach path $modulefolders { - lappend mtails [file tail $path] - } - - #special case bootsupport/modules so it can be referred to as just bootsupport or bootsupport/modules - lappend modulefolders [file join $projectdir src bootsupport/modules] - - if {$modulefoldername ni $mtails && $modulefoldername ni "bootsupport bootsupport/modules"} { - set msg "Suplied modulefoldername '$modulefoldername' doesn't appear to be a known module folder within the project at $projectdir\n" - append msg "Known module folders: [lsort $mtails]\n" - append msg "Use a name from the above list, or a fully qualified path\n" - error $msg - } - - if {$modulefoldername eq "bootsupport"} { - set modulefoldername "bootsupport/modules" - } - set modulefolder_path [file join $projectdir src $modulefoldername] - } else { - set msg "No current project found at or above current directory\n" - append msg "Supplied modulefoldername '$modulefoldername' is a name or relative path - cannot use when outside a project." \n - append msg "Supply an absolute path for the target modulefolder, or try again from within a project directory" \n - error $msg - } - } - puts stdout "-----------------------------" - if {$projectdir ne ""} { - puts stdout "Using projectdir: $projectdir for lib.copyasmodule" - } else { - puts stdout "No current project." - } - puts stdout "-----------------------------" - if {![file exists $modulefolder_path]} { - error "Selected module folder path '$modulefolder_path' doesn't exist. Required subdirectories for namespaced modules will be created automatically - but base selected folder must exist first" - } - - - set libfound [lsearch -all -inline [package names] $library] - if {[llength $libfound] != 1 || ![string length $libfound]} { - error "Library must match exactly one entry in the list of package names visible to the current interpretor: found '$libfound'" - } - - set versions [package versions [lindex $libfound 0]] - if {$has_natsort} { - set versions [natsort::sort $versions] - } else { - set versions [lsort $versions] - } - if {![llength $versions]} { - error "No version numbers found for library/module $libfound - sorry, you will need to copy it across manually" - } - puts stdout "Versions of $libfound found: $versions" - set alphaposn [lsearch $versions "999999.*"] - if {$alphaposn >= 0} { - set alpha [lindex $versions $alphaposn] - #remove and tack onto beginning.. - set versions [lreplace $versions $alphaposn $alphaposn] - set versions [list $alpha {*}$versions] - } - - set ver [lindex $versions end] ;# todo - make selectable! don't assume tail is latest?.. package vcompare? - if {[llength $versions] > 1} { - puts stdout "Version selected: $ver" - } - - set loadinfo [package ifneeded $libfound $ver] - set loadinfo [string map [list \r\n \n] $loadinfo] - set loadinfo_lines [split $loadinfo \n] - if {[catch {llength $loadinfo}]} { - set loadinfo_is_listshaped 0 - } else { - set loadinfo_is_listshaped 1 - } - - #check for redirection to differently cased version of self - this is only detected if this is the only command in the package ifneeded result - #- must have matching version. REVIEW this requirement. Is there a legitimate reason to divert to a differently cased other-version? - set is_package_require_self_recased 0 - set is_package_require_diversion 0 - set lib_diversion_name "" - if {[llength $loadinfo_lines] == 1} { - #e.g Thread 3.0b1 diverts to thread 3.0b1 - set line1 [lindex $loadinfo_lines 0] - #check if multiparted with semicolon - #We need to distinguish "package require ; more stuff" from "package require ver> ;" possibly with trailing comment? - set parts [list] - if {[regexp {;} $line1]} { - foreach p [split $line1 {;}] { - set p [string trim $p] - if {[string length $p]} { - #only append parts with some content that doesn't look like a comment - if {![string match "#*" $p]} { - lappend parts $p - } - } - } - - } - if {[llength $parts] == 1} { - #seems like a lone package require statement. - #check if package require, package\trequire etc - if {[string match "package*require" [lrange $line1 0 1]]} { - set is_package_require_diversion 1 - if {[lindex $line1 2] eq "-exact"} { - #package require -exact - set lib_diversion_name [lindex $line1 3] - #check not an exact match - but is a -nocase match - i.e differs in case only - if {($lib_diversion_name ne $libfound) && [string match -nocase $lib_diversion_name $libfound]} { - if {[lindex $line1 4] eq $ver} { - set is_package_require_self_recased 1 - } - } - } else { - #may be package require - #or package require ??... - set lib_diversion_name [lindex $line1 2] - #check not an exact match - but is a -nocase match - i.e differs in case only - if {($lib_diversion_name ne $libfound) && [string match -nocase $lib_diversion_name $libfound]} { - set requiredversions [lrange $line1 3 end] - if {$ver in $requiredversions} { - set is_package_require_self_recased 1 - } - } - } - } - } - } - - if {$is_package_require_self_recased && [string length $lib_diversion_name]} { - #we only follow one level of package require redirection - seems unlikely/imprudent to follow arbitrarily in a while loop(?) - set libfound $lib_diversion_name - set loadinfo [package ifneeded $libfound $ver] - set loadinfo [string map [list \r\n \n] $loadinfo] - set loadinfo_lines [split $loadinfo \n] - if {[catch {llength $loadinfo}]} { - set loadinfo_is_listshaped 0 - } else { - set loadinfo_is_listshaped 1 - } - - - } else { - if {$is_package_require_diversion} { - #single - #for now - we'll abort and tell the user to run again with specified pkg/version - #We could automate - but it seems likely to be surprising. - puts stderr "Loadinfo for $libfound seems to be diverting to another pkg/version: $loadinfo_lines" - puts stderr "Review and consider trying with the pkg/version described in the result above." - return - } - } - - - if {$loadinfo_is_listshaped && ([llength $loadinfo] == 2 && [lindex $loadinfo 0] eq "source")} { - set source_file [lindex $loadinfo 1] - } elseif {[string match "*source*" $loadinfo]} { - set parts [list] - foreach ln $loadinfo_lines { - if {![string length $ln]} {continue} - lappend parts {*}[split $ln ";"] - } - set sources_found [list] - set loads_found [list] - set dependencies [list] - set incomplete_lines [list] - foreach p $parts { - set p [string trim $p] - if {![string length $p]} { - continue ;#empty line or trailing colon - } - if {[string match "*tclPkgSetup*" $p]} { - puts stderr "Unable to process load script for library $libfound" - puts stderr "The library appears to use the deprecated tcl library support utility 'tclPkgSetup'" - return false - } - if {![::info complete $p]} { - # - #probably a perfectly valid script - but slightly more complicated than we can handle - #better to defer to manual processing - lappend incomplete_lines $p - continue - } - if {[lindex $p 0] eq "source"} { - #may have args.. e.g -encoding utf-8 - lappend sources_found [lindex $p end] - } - if {[lindex $p 0] eq "load"} { - lappend loads_found [lrange $p 1 end] - } - if {[lrange $p 0 1] eq "package require"} { - lappend dependencies [lrange $p 2 end] - } - } - if {[llength $incomplete_lines]} { - puts stderr "unable to interpret load script for library $libfound" - puts stderr "Load info: $loadinfo" - return false - } - if {[llength $loads_found]} { - puts stderr "package $libfound appears to have binary components" - foreach l $loads_found { - puts stderr " binary - $l" - } - foreach s $sources_found { - puts stderr " script - $s" - } - puts stderr "Unable to automatically copy binary libraries to your module folder." - return false - } - - if {[llength $sources_found] != 1} { - puts stderr "sorry - unable to interpret source library location" - puts stderr "Only 1 source supported for now" - puts stderr "Load info: $loadinfo" - return false - } - if {[llength $dependencies]} { - #todo - check/ignore if dependency is Tcl ? - puts stderr "WARNING the package appears to depend on at least one other. Review and copy dependencies if required." - foreach d $dependencies { - puts stderr " - $d" - } - } - - set source_file [lindex $sources_found 0] - } else { - puts stderr "sorry - unable to interpret source library location" - puts stderr "Load info: $loadinfo" - return false - } - - # -- --------------------------------------- - #Analyse source file - if {![file exists $source_file]} { - error "Unable to verify source file existence at: $source_file" - } - set source_data [fcat -translation binary $source_file] - if {![string match "*package provide*" $source_data]} { - puts stderr "Sorry - unable to verify source file contains 'package provide' statement of some sort - copy manually" - return false - } else { - if {![string match "*$libfound*" $source_data]} { - # as an exception - look for the specific 'package provide $pkg $version' as occurs in the auto-name auto-version modules - #e.g anyname-0.1.tm example - if {![string match "*package provide \$pkg \$version*" $source_data]} { - puts stderr "Sorry - unable to verify source file contains 'package provide' and '$libfound' - copy manually" - return false - } - } - } - - - if {[string match "*lappend ::auto_path*" $source_data] || [string match "*lappend auto_path*" $source_data] || [string match "*set ::auto_path*" $source_data]} { - puts stderr "Sorry - '$libfound' source file '$source_file' appears to rely on ::auto_path and can't be automatically copied as a .tm module" - puts stderr "Copy the library across to a lib folder instead" - return false - } - # -- --------------------------------------- - - set moduleprefix [punk::ns::nsprefix $libfound] - if {[string length $moduleprefix]} { - set moduleprefix_parts [punk::ns::nsparts $moduleprefix] - set relative_path [file join {*}$moduleprefix_parts] - } else { - set relative_path "" - } - set pkgtail [punk::ns::nstail $libfound] - set target_path [file join $modulefolder_path $relative_path ${pkgtail}-${ver}.tm] - - if {$opt_askme} { - puts stdout "WARNING - you should check that there aren't extra required files for the library/modules" - puts stdout "" - puts stdout "This is not intended for binary modules - use at own risk and check results" - puts stdout "" - puts stdout "Base module path: $modulefolder_path" - puts stdout "Target path : $target_path" - puts stdout "results of 'package ifneeded $libfound'" - puts stdout "---" - puts stdout "$loadinfo" - puts stdout "---" - puts stdout "Proceed to create ${pkgtail}-${ver}.tm module? Y|N" - set stdin_state [fconfigure stdin] - fconfigure stdin -blocking 1 - set answer [string tolower [gets stdin]] - fconfigure stdin -blocking [dict get $stdin_state -blocking] - if {$answer ne "y"} { - puts stderr "mix libcopy.asmodule aborting due to user response '$answer' (required Y|y to proceed) use -askme 0 to avoid prompts." - return - } - } - - if {![file exists $modulefolder_path]} { - puts stdout "Creating module base folder at $modulefolder_path" - file mkdir $modulefolder_path - } - if {![file exists [file dirname $target_path]]} { - puts stdout "Creating relative folder at [file dirname $target_path]" - file mkdir [file dirname $target_path] - } - - if {[file exists $target_path]} { - puts stdout "WARNING - module already exists at $target_path" - if {$opt_askme} { - puts stdout "Copy anyway? Y|N" - set stdin_state [fconfigure stdin] - fconfigure stdin -blocking 1 - set answer [string tolower [gets stdin]] - fconfigure stdin -blocking [dict get $stdin_state -blocking] - if {$answer ne "y"} { - puts stderr "mix libcopy.asmodule aborting due to user response '$answer' (required Y|y to proceed) use -askme 0 to avoid prompts." - return - } - } - } - - file copy -force $source_file $target_path - - return $target_path - } - - - -} - - - - - - - - - - - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::mix::commandset::loadedlib [namespace eval punk::mix::commandset::loadedlib { - variable version - set version 0.1.0 -}] -return diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/module-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/module-0.1.0.tm deleted file mode 100644 index 19fbadb3..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/module-0.1.0.tm +++ /dev/null @@ -1,419 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::mix::commandset::module 0.1.0 -# Meta platform tcl -# Meta license -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::mix::commandset::module { - namespace export * - - proc paths {} { - set roots [punk::repo::find_repos ""] - set project [lindex [dict get $roots project] 0] - if {$project ne ""} { - set is_project 1 - set searchbase $project - } else { - set is_project 0 - set searchbase [pwd] - } - - if {[catch { - set source_module_folderlist [punk::mix::cli::lib::find_source_module_paths $searchbase] - } errMsg]} { - set source_module_folderlist [list] - } - - set tm_folders [tcl::tm::list] - package require overtype - - set result "" - if {$is_project} { - append result "Project module source paths:" \n - foreach f $source_module_folderlist { - append result "$f" \n - } - } - append result \n - append result "tcl::tm::list" \n - foreach f $tm_folders { - if {$is_project} { - if {[punk::mix::cli::lib::path_a_aboveorat_b $project $f]} { - set pinfo "(within project)" - } else { - set pinfo "" - } - } else { - set pinfo "" - } - set warning "" - if {![file isdirectory $f]} { - set warning "(PATH NOT FOUND)" - } - append result "$f $pinfo $warning" \n - } - - - return $result - } - #require current dir when calling to be the projectdir, or - proc templates {args} { - set tdict [templates_dict {*}$args] - - package require overtype - set paths [dict values $tdict] - set names [dict keys $tdict] - - set title1 "Path" - set widest1 [tcl::mathfunc::max {*}[lmap v [concat [list $title1] $paths] {punk::strlen $v}]] - set col1 [string repeat " " $widest1] - - set title2 "Template Name" - set widest2 [tcl::mathfunc::max {*}[lmap v [concat [list $title2] $names] {punk::strlen $v}]] - set col2 [string repeat " " $widest2] - - set tablewidth [expr {$widest1 + 1 + $widest2}] - set table "" - append table [string repeat - $tablewidth] \n - append table "[overtype::left $col1 $title1] [overtype::left $col2 $title2]" \n - append table [string repeat - $tablewidth] \n - - foreach p $paths n $names { - append table "[overtype::left $col1 $p] [overtype::left $col2 $n]" \n - } - - return $table - } - #return all module templates with repeated ones suffixed with .2 .3 etc - proc templates_dict {args} { - tailcall lib::templates_dict {*}$args - } - proc new {module args} { - set year [clock format [clock seconds] -format %Y] - set defaults [list\ - -project \uFFFF\ - -version \uFFFF\ - -license \ - -template module-0.0.1.tm\ - -type \uFFFF\ - -force 0\ - ] - set opts [dict merge $defaults $args] - - #todo - review compatibility between -template and -type - #-type is the wrapping technology e.g 'plain' for none or tarjar/zipkit etc (consider also snappy/snappy-tcl) - #-template may be a folder - but only if the selected -type suports it - - - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - # option -version - # we need this value before looking at the named argument - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_version_supplied [dict get $opts -version] - if {$opt_version_supplied eq "\uFFFF"} { - set opt_version "0.1.0" - } else { - set opt_version $opt_version_supplied - if {![util::is_valid_tm_version $opt_version]} { - error "pmix module.new error - supplied -version $opt_version doesn't appear to be a valid Tcl module version" - } - } - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - #named argument - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - set mversion_supplied "" ;#version supplied directly in module argument - if {[string first - $module]> 0} { - #if it has a dash then version is required to be valid - lassign [punk::mix::cli::lib::split_modulename_version $module] modulename mversion - if {![util::is_valid_tm_version $mversion]} { - error "pmix module.new error - unable to determine modulename-version from supplied value '$module'" - } - set mversion_supplied $mversion ;#record as may need to compare to version from templatefile name - set vcompare_is_mversion_bigger [package vcompare $mversion $opt_version] - if {$vcompare_is_mversion_bigger > 0} { - set opt_version $mversion; #module parameter has higher value than -version - set vmsg "from module argument: $module" - } else { - set vmsg "from -version option: $opt_version_supplied" - } - if {$opt_version_supplied ne "\uFFFF"} { - if {$vcompare_is_mversion_bigger != 0} { - #is bigger or smaller - puts stderr "module.new WARNING: version supplied in module argument as well as -version option. Using the higher version number $vmsg" - } - } - } else { - set modulename $module - } - punk::mix::cli::lib::validate_modulename $modulename -name_description "mix module.new name" - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - #options - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_project [dict get $opts -project] - set testdir [pwd] - if {![string length [set projectdir [punk::repo::find_project $testdir]]]} { - if {![string length [set projectdir [punk::repo::find_candidate $testdir]]]} { - set msg [punkc::repo::is_candidate_root_requirements_msg] - error "module.new unable to create module in projectdir:$projectdir - directory doesn't appear to meet basic standards $msg" - } - } - if {$opt_project == "\uFFFF"} { - set projectname [file tail $projectdir] - } else { - set projectname $opt_project - if {$projectname ne [file tail $projectdir]} { - error "module.new -project '$opt_project' doesn't match detected projectname '$projectname' at path: $projectdir" - } - } - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_license [dict get $opts -license] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_template [dict get $opts -template] - - set templates_dict [templates_dict] ;#possibly suffixed with .2 .3 etc - #todo - allow versionless name - pick latest which isn't suffixed with .2 etc - if {![dict exists $templates_dict $opt_template]} { - error "module.new unable to find template '$opt_template'. Known templates: [dict keys $templates_dict]" - } - set templatefile [dict get $templates_dict $opt_template] - set tpldir [file dirname $templatefile] ;#use same folder for modulename_buildversion.txt, modulename_description.txt if they exist - - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_type [dict get $opts -type] - if {$opt_type eq "\uFFFF"} { - set opt_type [lindex [punk::mix::cli::lib::module_types] 0] ;#default to plain - } - if {$opt_type ni [punk::mix::cli::lib::module_types]} { - error "module.new - error - unknown -type '$opt_type' known-types: [punk::mix::cli::lib::module_types]" - } - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - - - - - - set subpath [punk::mix::cli::lib::module_subpath $modulename] ;#commonly empty string for simple modulename e.g "mymodule" but x::mymodule has subpath 'x' and x::y::mymodule has subpath 'x/y' - if {![string length $subpath]} { - set modulefolder $projectdir/src/modules - } else { - set modulefolder $projectdir/src/modules/$subpath - } - file mkdir $modulefolder - - set moduletail [namespace tail $modulename] - set magicversion [punk::mix::util::magic_tm_version] ;#deliberately large so given load-preference when testing - - - - - set template_tail [file tail $templatefile] ;#convert template_xxx-version.tm.x to {xxx version} - set template_tail [string range $template_tail [string length template_] end] - set ext [string tolower [file extension $template_tail]] - if {$ext eq ".tm"} { - set template_modulename_part [file rootname $template_tail] - } elseif {[string is integer -strict [string range $ext 1 end]]} { - #something like modulename-0.0.1.tm.2 - #strip of last 2 dotted parts - set shortened [file rootname $template_tail] - if {![string equal -nocase [file extension $shortened] ".tm"]} { - error "module.new error: Unable to interpret filename components of template file '$templatefile' (expected .tm as second-last or last component)" - } - set template_modulename_part [file rootname $shortened] - } else { - error "module.new error: Unable to interpret filename components of template file '$templatefile'" - } - lassign [punk::mix::cli::lib::split_modulename_version $template_modulename_part] t_mname t_version - #t_version may be empty string if template is unversioned e.g template_whatever.tm - - set fd [open $templatefile r]; set template_filedata [read $fd]; close $fd - if {[string match "*$magicversion*" $template_filedata]} { - set use_magic 1 - set build_version $opt_version - set infile_version $magicversion - } else { - set use_magic 0 - if {$opt_version_supplied ne "\uFFFF"} { - set build_version $opt_version - } else { - if {[util::is_valid_tm_version $t_version]} { - if {$mversion_supplied eq ""} { - set build_version $t_version - } else { - #we have a version from the named argument 'module' - if {[package vcompare $mversion_supplied $t_version] > 0} { - set build_version $mversion_supplied - } else { - set build_version $t_version - } - } - } else { - #probably an unversioned module template - #use opt_version default from above - set build_version $opt_version - } - } - set infile_version $build_version - } - - set template_filedata [string map [list %project% $projectname %pkg% $modulename %year% $year %license% $opt_license %version% $infile_version] $template_filedata] - - set modulefile $modulefolder/${moduletail}-$infile_version.tm - if {[file exists $modulefile]} { - set errmsg "module.new error: module file $modulefile already exists - aborting" - if {[string match "*$magicversion*" $modulefile]} { - append errmsg \n "If you are attempting to create a module file with a specific version in the source-file name - you will need to use a template that doesn't contain the string '$magicversion' e.g the provided template moduleexactversion-0.0.1.tm" - } - error $errmsg - } - - - if {[file exists $tpldir/modulename_buildversion.txt]} { - set fd [open $tpldir/modulename_buildversion.txt r]; set buildversion_filedata [read $fd]; close $fd - } else { - #mix_templates_dir warns of deprecation - review - set lib_tpldir [file join [punk::mix::cli::lib::mix_templates_dir] modules];#fallback for modulename_buildversion.txt, modulename_description.txt - set fd [open $lib_tpldir/modulename_buildversion.txt r]; set buildversion_filedata [read $fd]; close $fd - } - set buildversionfile [file join $modulefolder ${moduletail}-buildversion.txt] - set existing_build_version "" - if {[file exists $buildversionfile]} { - set buildversiondata [punk::mix::util::fcat $buildversionfile] - set lines [split $buildversiondata \n] - set existing_build_version [string trim [lindex $lines 0]] - if {[package vcompare $existing_build_version $build_version] >= 0} { - #existing version in -buildversion.txt file is lower than the module version we are creating - error "module.new error: there is an existing buildversion file $buildversionfile with version $existing_build_version equal to or higher than $build_version - unable to continue" - } - } - - set existing_versions [glob -nocomplain -dir $modulefolder -tails ${moduletail}-*.tm] - #it shouldn't be possible to overmatch with the glob - because '-' is not valid in a Tcl module name - if {[llength $existing_versions]} { - set name_version_pairs [list] - lappend name_version_pairs [list $moduletail $infile_version] - foreach existing $existing_versions { - lappend name_version_pairs [punk::mix::cli::lib::split_modulename_version $existing] ;# .tm is stripped and ignored - } - set name_version_pairs [lsort -command {package vcompare} -index 1 $name_version_pairs] ;#while plain lsort will often work with versions - it can get order wrong with some - so use package vcompare - if {[lindex $name_version_pairs end] ne [list $moduletail $infile_version]} { - set thisposn [lsearch -index 1 $name_version_pairs $infile_version] - set name_version_pairs [lreplace $name_version_pairs $thisposn $thisposn] - set other_versions [lsearch -all -inline -index 1 -subindices $name_version_pairs *] - set errmsg "module.new error: There are existing modules in the target folder with higher versions than $infile_version." - append errmsg \n "Other versions found: $other_versions" - if {$magicversion in $other_versions} { - append errmsg \n "Existing build version for special source file name: '$magicversion' is: '$existing_build_version'" - append errmsg \n "If '$magicversion' file doesn't represent the latest source it should be removed or the filename and contents adjusted to be a specific version" - } - error $errmsg - } else { - puts stderr "module.new WARNING: There are existing modules in the target folder with lower versions than $infile_version - manual review recommended" - puts stderr "Other versions found: [lsearch -all -inline -index 1 -subindices [lrange $name_version_pairs 0 end-1] *]" - } - } - - - set fd [open $modulefile w] - fconfigure $fd -translation binary - puts -nonewline $fd $template_filedata - close $fd - - - set buildversion_filedata [string map [list %Major.Minor.Level% $build_version] $buildversion_filedata] - set fd [open $buildversionfile w] - fconfigure $fd -translation binary - puts -nonewline $fd $buildversion_filedata - close $fd - - return [list file $modulefile version $build_version] - } - - namespace eval lib { - proc templates_dict {args} { - set defaults [list -scriptpath ""] - set opts [dict merge $defaults $args] - set opt_scriptpath [dict get $opts -scriptpath] - - set module_template_bases [list] - set tbasedict [punk::mix::base::lib::get_template_basefolders $opt_scriptpath] - dict for {tbase folderinfo} $tbasedict { - lappend module_template_bases [file join $tbase modules] - } - - - - set template_files [list] - foreach basefld $module_template_bases { - set matched_files [glob -nocomplain -dir $basefld -type f template_*] - foreach tf $matched_files { - if {[string match ignore* $tf]} { - continue - } - set ext [file extension $tf] - if {$ext in [list ".tm"]} { - lappend template_files $tf - } - } - } - - set tdict [dict create] - set seen_dict [dict create] - foreach fullpath $template_files { - set ftail [file tail $fullpath] - set tname [string range $ftail [string length template_] end] - if {![dict exists $seen_dict $tname]} { - dict set seen_dict $tname 1 - dict set tdict $tname $fullpath ; #first seen of filename gets no number - } else { - set n [dict get $seen_dict $tname] - incr n - dict incr seen_dict $tname - dict set tdict ${tname}.$n $fullpath - } - } - return $tdict - } - } - - - -} - - - - - - - - - - - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::mix::commandset::module [namespace eval punk::mix::commandset::module { - variable version - set version 0.1.0 -}] -return diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/project-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/project-0.1.0.tm deleted file mode 100644 index d8da3a47..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/project-0.1.0.tm +++ /dev/null @@ -1,983 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::mix::commandset::project 0.1.0 -# Meta platform tcl -# Meta license -# @@ Meta End - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# doctools header -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -#*** !doctools -#[manpage_begin punkshell_module_punk::mix::commandset::project 0 0.1.0] -#[copyright "2023"] -#[titledesc {pmix commandset - project}] [comment {-- Name section and table of contents description --}] -#[moddesc {pmix CLI commandset - project}] [comment {-- Description at end of page heading --}] -#[require punk::mix::commandset::project] -#[description] - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ - -#*** !doctools -#[section Overview] -#[para] overview of punk::mix::commandset::project -#[para]Import into an ensemble namespace similarly to the way it is done with punk::mix::cli e.g -#[example { -# namespace eval myproject::cli { -# namespace export * -# namespace ensemble create -# package require punk::overlay -# -# package require punk::mix::commandset::project -# punk::overlay::import_commandset project . ::punk::mix::commandset::project -# punk::overlay::import_commandset projects . ::punk::mix::commandset::project::collection -# } -#}] -#[para] Where the . in the above example is the prefix/command separator -#[para]The prefix ('project' in the above example) can be any string desired to disambiguate commands imported from other commandsets. -#[para]The above results in the availability of the ensemble command: ::myproject::cli project.new, which is implemented in ::punk::mix::commandset::project::new -#[para]Similarly, procs under ::punk::mix::commandset::project::collection will be available as subcommands of the ensemble as projects. -#[para] -#[subsection Concepts] -#[para] see punk::overlay - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ - -#*** !doctools -#[subsection dependencies] -#[para] packages used by punk::mix::commandset::project -#[list_begin itemized] - -package require Tcl 8.6 -#*** !doctools -#[item] [package {Tcl 8.6}] -#[item] [package punk::ns] -#[item] [package sqlite3] (binary) -#[item] [package overtype] -#[item] [package textutil] (tcllib) - - -# #package require frobz -# #*** !doctools -# #[item] [package {frobz}] - -#*** !doctools -#[list_end] - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ - -#*** !doctools -#[section API] - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::mix::commandset::project { - namespace export * - #*** !doctools - #[subsection {Namespace punk::mix::commandset::project}] - #[para] core commandset functions for punk::mix::commandset::project - #[list_begin definitions] - - proc _default {} { - package require punk::ns - set dispatched_to [lindex [info level 2] 0] ;#e.g ::punk::mix::cli::project - set dispatch_tail [punk::ns::nstail $dispatched_to] - set dispatch_ensemble [punk::ns::nsprefix $dispatched_to] ;#e.g ::punk::mix::cli - set sibling_commands [namespace eval $dispatch_ensemble {namespace export}] - #todo - get separator? - set sep "." - set result [list] - foreach sib $sibling_commands { - if {[string match ${dispatch_tail}${sep}* $sib]} { - lappend result $sib - } - } - return [lsort $result] - } - - - - - proc new {newprojectpath_or_name args} { - #*** !doctools - # [call [fun new] [arg newprojectpath_or_name] [opt args]] - #new project structure - may be dedicated to one module, or contain many. - #create minimal folder structure only by specifying in args: -modules {} - if {[file pathtype $newprojectpath_or_name] eq "absolute"} { - set projectfullpath [file normalize $newprojectpath_or_name] - set projectname [file tail $projectfullpath] - set projectparentdir [file dirname $newprojectpath_or_name] - } else { - set projectfullpath [file join [pwd] $newprojectpath_or_name] - set projectname [file tail $projectfullpath] - set projectparentdir [file dirname $projectfullpath] - } - if {[file type $projectparentdir] ne "directory"} { - error "punk::mix::cli::new error: unable to determine containing folder for '$newprojectpath_or_name'" - } - - punk::mix::cli::lib::validate_projectname $projectname -name_description "punk mix project.new" - - - set defaults [list\ - -type plain\ - -empty 0\ - -force 0\ - -update 0\ - -confirm 1\ - -modules \uFFFF\ - -layout project - ] ;#todo - set known_opts [dict keys $defaults] - foreach {k v} $args { - if {$k ni $known_opts} { - error "project.new error: option '$k' not known. Known options: $known_opts" - } - } - set opts [dict merge $defaults $args] - # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_type [dict get $opts -type] - if {$opt_type ni [punk::mix::cli::lib::module_types]} { - error "pmix new error - unknown type '$opt_type' known types: [punk::mix::cli::lib::module_types]" - } - # -- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_force [dict get $opts -force] - set opt_confirm [string tolower [dict get $opts -confirm]] - # -- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_modules [dict get $opts -modules] - if {[llength $opt_modules] == 1 && [lindex $opt_modules 0] eq "\uFFFF"} { - #if not specified - add a single module matching project name - set opt_modules [list $projectname] - } - # -- --- --- --- --- --- --- --- --- --- --- --- --- - set opt_layout [dict get $opts -layout] - set opt_update [dict get $opts -update] - # -- --- --- --- --- --- --- --- --- --- --- --- --- - - - set fossil_prog [auto_execok fossil] - if {![string length $fossil_prog]} { - puts stderr "The fossil program was not found. A fossil executable is required to use most pmix features." - if {[string length [set scoop_prog [auto_execok scoop]]]} { - #restrict to windows? - set answer [util::askuser "scoop detected. Would you like pmix to install fossil now using scoop? Y|N"] - if {[string tolower $answer] ne "y"} { - puts stderr "mix new aborting due to user response '$answer' (required Y|y to proceed) use -confirm 0 to avoid prompts." - return - } - #we don't assume 'unknown' is configured to run shell commands - if {[string length [package provide shellrun]]} { - set exitinfo [run {*}$scoop_prog install fossil] - #scoop tends to return successful exitcode (0) even when packages not found etc. - so exitinfo not much use. - puts stdout "scoop install fossil ran with result: $exitinfo" - } else { - puts stdout "Please wait while scoop runs - there may be a slight delay and then scoop output will be shown. (use punk shellrun package for )" - set result [exec {*}$scoop_prog install fossil] - puts stdout $result - } - catch {::auto_reset} ;#can be missing (unsure under what circumstances - but I've seen it raise error 'invalid command name "auto_reset"') - if {![string length [auto_execok fossil]]} { - puts stderr "Fossil still not detected. If it was successfully installed, try restarting your punk/tcl shell." - return - } - #todo - ask user if they want to configure fosssil first.. - set answer [util::askuser "Fossil command now appears to be available. You may wish to answer N to exit and customize it - but default config may be ok. Type the word 'continue' to proceed with default configuration."] - if {[string tolower $answer] ne "continue"} { - return - } - - } else { - puts stdout "See: https://fossil-scm.org/home/uv/download.html" - if {"windows" eq $::tcl_platform(platform)} { - puts stdout "Consider using a package manager such as scoop: https://scoop.sh" - puts stdout "(Then: scoop install fossil)" - } - return - } - } - set startdir [pwd] - if {[set in_project [punk::repo::find_project $startdir]] ne ""} { - # use this project as source of templates - puts stdout "-------------------------------------------" - puts stdout "Currently in a project directory '$in_project'" - puts stdout "This project will be searched for templates" - puts stdout "-------------------------------------------" - } - set template_base_dict [punk::mix::base::lib::get_template_basefolders] - set template_bases_containing_layout [list] - dict for {tbase folderinfo} $template_base_dict { - if {[file exists $tbase/layouts/$opt_layout]} { - lappend template_bases_containing_layout $tbase - } - } - if {![llength $template_bases_containing_layout]} { - puts stderr "layout '$opt_layout' was not found in template dirs" - puts stderr "searched [dict size $template_base_dict] template folders" - dict for {tbase folderinfo} $template_base_dict { - puts stderr " - $tbase $folderinfo" - } - return - } - #review: silently use last entry which had the layout (?) - set templatebase [lindex $template_bases_containing_layout end] - - - - #todo - detect whether inside cwd-project or inside a different project - set projectdir $projectparentdir/$projectname - if {[set target_in_project [punk::repo::find_project $projectparentdir]] ne ""} { - puts stderr "Target location for new project is already within a project: $target_in_project" - error "Nested projects not yet supported aborting" - } - - - - if {[punk::repo::is_git $projectparentdir]} { - puts stderr "mix new WARNING: target project location is within a git repo based at [punk::repo::find_git $projectparentdir]" - puts stderr "The new project will create a fossil repository (which you are free to ignore - but but will be used to confirm project base)" - puts stderr "If you intend to use both git and fossil in the same project space - you should research and understand the details and any possible interactions/issues" - set answer [util::askuser "Do you want to proceed to create a project based at: $projectdir? Y|N"] - if {[string tolower $answer] ne "y"} { - puts stderr "mix new aborting due to user response '$answer' (required Y|y to proceed) use -confirm 0 to avoid prompts." - return - } - } - set is_nested_fossil 0 ;#default assumption - if {[punk::repo::is_fossil $projectparentdir]} { - puts stderr "mix new WARNING: target project location is within an open fossil repo based at [punk::repo::find_fossil $projectparentdir] NESTED fossil repository" - if {$opt_confirm ni [list 0 no false]} { - puts stderr "If you proceed - the new project's fossil repo will be created using the --nested flag" - set answer [util::askuser "Do you want to proceed to create a NESTED project based at: $projectdir? Y|N"] - if {[string tolower $answer] ne "y"} { - puts stderr "mix new aborting due to user response '$answer' (required Y|y to proceed) use -confirm 0 to avoid prompts." - return - } - set is_nested_fossil 1 - } - } - - - set project_dir_exists [file exists $projectdir] - if {$project_dir_exists && !($opt_force || $opt_update)} { - puts stderr "Unable to create new project at $projectdir - file/folder already exists use -update 1 to fill in missing items from template use -force 1 to overwrite from template" - return - } elseif {$project_dir_exists && $opt_force} { - puts stderr "mix new WARNING: -force 1 was supplied. Will copy layout $templatebase/layouts/$opt_layout using -force option to overwrite from template" - if {$opt_confirm ni [list 0 no false]} { - set answer [util::askuser "Do you want to proceed to possibly overwrite existing files in $projectdir? Y|N"] - if {[string tolower $answer] ne "y"} { - puts stderr "mix new aborting due to user response '$answer' (required Y|y to proceed) use -confirm 0 to avoid prompts." - return - } - } - } elseif {$project_dir_exists && $opt_update} { - puts stderr "mix new WARNING: -update 1 was supplied. Will copy layout $templatebase/layouts/$opt_layout using -update option to add missing items" - } - - set fossil_repo_file "" - set is_fossil_root 0 - if {$project_dir_exists && [punk::repo::is_fossil_root $projectdir]} { - set is_fossil_root 1 - set fossil_repo_file [punk::repo::fossil_get_repository_file $projectdir] - if {$fossil_repo_file ne ""} { - set repodb_folder [file dirname $fossil_repo_file] - } - } - - if {$fossil_repo_file eq ""} { - set repodb_folder [punk::repo::fossil_get_repository_folder_for_project $projectname -parentfolder $startdir] - if {![string length $repodb_folder]} { - puts stderr "No usable repository database folder selected for $projectname.fossil file" - return - } - } - if {[file exists $repodb_folder/$projectname.fossil]} { - puts stdout "NOTICE: $repodb_folder/$projectname.fossil already exists" - if {!($opt_force || $opt_update)} { - puts stderr "-force 1 or -update 1 not specified - aborting" - return - } - } - - if {$fossil_repo_file eq ""} { - puts stdout "Initialising fossil repo: $repodb_folder/$projectname.fossil" - set fossilinit [runx -n {*}$fossil_prog init $repodb_folder/$projectname.fossil -project-name $projectname] - if {[dict get $fossilinit exitcode] != 0} { - puts stderr "fossil init failed:" - puts stderr [dict get $fossilinit stderr] - return - } else { - puts stdout "fossil init result:" - puts stdout [dict get $fossilinit stdout] - } - } - - file mkdir $projectdir - - set layout_dir $templatebase/layouts/$opt_layout - puts stdout ">>> about to call punkcheck::install $layout_dir $projectdir" - set resultdict [dict create] - set antipaths [list\ - src/doc/*\ - src/doc/include/*\ - ] - - #default antiglob_dir_core will stop .fossil* from being updated - which is generally desirable as these are likely to be customized - if {$opt_force} { - puts stdout "copying layout files - with force applied - overwrite all-targets" - set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -overwrite ALL-TARGETS -antiglob_paths $antipaths] - #file copy -force $layout_dir $projectdir - } else { - puts stdout "copying layout files - (if source file changed)" - set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -overwrite installedsourcechanged-targets -antiglob_paths $antipaths] - } - puts stdout [punkcheck::summarize_install_resultdict $resultdict] - - puts stdout "copying layout src/doc files (if target missing)" - set resultdict [punkcheck::install $layout_dir/src/doc $projectdir/src/doc -punkcheck_folder $projectdir -installer project.new -overwrite SYNCED-TARGETS] - puts stdout [punkcheck::summarize_install_resultdict $resultdict] - - #target folders .fossil-custom and .fossil-settings may not exist. use -createdir 1 to ensure existence. - #In this case we need to override the default dir antiglob - as .fossil-xxx folders need to be installed from template if missing, or if target is uncustomized. - ## default_antiglob_dir_core [list "#*" "_aside" ".git" ".fossil*"] - set override_antiglob_dir_core [list #* _aside .git] - puts stdout "copying layout src/.fossil-custom files (if target missing or uncustomised)" - set resultdict [punkcheck::install $layout_dir/.fossil-custom $projectdir/.fossil-custom -createdir 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS] - puts stdout [punkcheck::summarize_install_resultdict $resultdict] - - puts stdout "copying layout src/.fossil-settings files (if target missing or uncustomised)" - set resultdict [punkcheck::install $layout_dir/.fossil-settings $projectdir/.fossil-settings -createdir 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS] - puts stdout [punkcheck::summarize_install_resultdict $resultdict] - - - - - #lappend substfiles $projectdir/README.md - #lappend substfiles $projectdir/src/README.md - #lappend substfiles $projectdir/src/doc/main.man - #expect this in all templates? - todo make these substitutions independent of specific paths and filenames? - #scan all files in template - # - #TODO - pmix command to substitute templates? - set templatefiles [punk::mix::commandset::layout::lib::layout_scan_for_template_files $opt_layout] - set stripprefix [file normalize $layout_dir] - - foreach templatefullpath $templatefiles { - set templatetail [punk::repo::path_strip_alreadynormalized_prefixdepth $templatefullpath $stripprefix] - - set fpath [file join $projectdir $templatetail] - if {[file exists $fpath]} { - set fd [open $fpath r]; fconfigure $fd -translation binary; set data [read $fd]; close $fd - set data2 [string map [list [lib::template_tag project] $projectname] $data] - if {$data2 ne $data} { - puts stdout "updated template file: $fpath" - set fdout [open $fpath w]; fconfigure $fdout -translation binary; puts -nonewline $fdout $data2; close $fdout - } - } else { - puts stderr "warning: Missing template file $fpath" - } - } - #todo - tag substitutions in src/doc tree - - - ::cd $projectdir - - if {[file exists $projectdir/src/modules]} { - foreach m $opt_modules { - if {![file exists $projectdir/src/modules/$m-[punk::mix::util::magic_tm_version].tm]} { - punk::mix::commandset::module::new $m -project $projectname -type $opt_type - } else { - if {$opt_force} { - punk::mix::commandset::module::new $m -project $projectname -type $opt_type -force 1 - } - } - } - } else { - puts stderr "project.new WARNING template hasn't created src/modules - skipping creation of new module(s) for project" - } - - #generate www/man/md output in 'embedded' folder which should be checked into repo for online documentation - if {[file exists $projectdir/src]} { - ::cd $projectdir/src - #---------- - set installer [punkcheck::installtrack new project.new $projectdir/src/.punkcheck] - $installer set_source_target $projectdir/src/doc $projectdir/src/embedded - set event [$installer start_event {-install_step kettledoc}] - $event targetset_init VIRTUAL kettle_build_doc ;#VIRTUAL - since there is no specific target file - and we don't know all the files that will be generated - $event targetset_addsource $projectdir/src/doc ;#whole doc tree is considered the source - #---------- - if {\ - [llength [dict get [$event targetset_source_changes] changed]]\ - } { - $event targetset_started - # -- --- --- --- --- --- - puts stdout "BUILDING DOCS at src/embedded from src/doc" - if {[catch { - - punk::mix::cli::lib::kettle_call lib doc - #Kettle doc - - } errM]} { - $event targetset_end FAILED -note "kettle_build_doc failed: $errM" - } else { - $event targetset_end OK - } - # -- --- --- --- --- --- - } else { - puts stderr "No change detected in src/doc" - $event targetset_end SKIPPED - } - $event end - $event destroy - $installer destroy - } - - ::cd $projectdir - - if {![punk::repo::is_fossil_root $projectdir]} { - set first_fossil 1 - #-k = keep. (only modify the manifest file(s)) - if {$is_nested_fossil} { - set fossilopen [runx -n {*}$fossil_prog open --nested $repodb_folder/$projectname.fossil -k --workdir $projectdir] - } else { - set fossilopen [runx -n {*}$fossil_prog open $repodb_folder/$projectname.fossil -k --workdir $projectdir] - } - if {[file exists $projectdir/_FOSSIL_] && ![file exists $projectdir/.fslckout]} { - file rename $projectdir/_FOSSIL_ $projectdir/.fslckout - } - if {[dict get $fossilopen exitcode] != 0} { - puts stderr "fossil open in project workdir '$projectdir' FAILED:" - puts stderr [dict get $fossilopen stderr] - return - } else { - puts stdout "fossil open in project workdir '$projectdir' OK:" - puts stdout [dict get $fossilopen stdout] - } - } else { - set first_fossil 0 - } - set fossiladd [runx -n {*}$fossil_prog add --dotfiles $projectdir] - if {[dict get $fossiladd exitcode] != 0} { - puts stderr "fossil add workfiles in workdir '$projectdir' FAILED:" - puts stderr [dict get $fossiladd stderr] - return - } else { - puts stdout "fossil add workfiles in workdir '$projectdir' OK:" - puts stdout [dict get $fossiladd stdout] - } - if {$first_fossil} { - #fossil commit may prompt user for input.. runx runout etc will pause with no prompts - util::do_in_path $projectdir { - set fossilcommit [run -n {*}$fossil_prog commit -m "initial project commit"] - } - if {[dict get $fossilcommit exitcode] != 0} { - puts stderr "fossil commit in workdir '$projectdir' FAILED" - return - } else { - puts stdout "fossil commit in workdir '$projectdir' OK" - } - } - - puts stdout "-done- project:$projectname projectdir: $projectdir" - } - - #*** !doctools - #[list_end] [comment {--- end definitions namespace punk::mix::commandset::project ---}] - - namespace eval collection { - #*** !doctools - #[subsection {Namespace punk::mix::commandset::project::collection}] - #[para] commandset functions for operating with multiple projects. - #[para] It would usually be imported with the prefix "projects" and separator "." to result in commands such as: projects.detail - #[list_begin definitions] - namespace export * - namespace path [namespace parent] - - #e.g imported as 'projects' - proc _default {{glob {}} args} { - #*** !doctools - #[call [fun _default] [arg glob] [opt {option value...}]] - #[para]List projects under fossil management, showing fossil db location and number of checkouts - #[para]The glob argument is optional unless option/value pairs are also supplied, in which case * should be explicitly supplied - #[para]glob restricts output based on the name of the fossil db file e.g s* for all projects beginning with s - #[para]The _default function is made available in the ensemble by the name of the prefix used when importing the commandset. - #[para]e.g - #[para] punk::overlay::import_commandset projects . ::punk::mix::commandset::project::collection - #[para]Will result in the command being available as projects - package require overtype - set db_projects [lib::get_projects $glob] - set col1items [lsearch -all -inline -index 0 -subindices $db_projects *] - set col2items [lsearch -all -inline -index 1 -subindices $db_projects *] - set checkouts [lsearch -all -inline -index 2 -subindices $db_projects *] - set col3items [lmap v $checkouts {llength $v}] - - set title1 "Fossil Repo DB" - set widest1 [tcl::mathfunc::max {*}[lmap v [concat [list $title1] $col1items] {punk::strlen $v}]] - set col1 [string repeat " " $widest1] - set title2 "File Name" - set widest2 [tcl::mathfunc::max {*}[lmap v [concat [list $title2] $col2items] {punk::strlen $v}]] - set col2 [string repeat " " $widest2] - set title3 "Checkouts" - set widest3 [tcl::mathfunc::max {*}[lmap v [concat [list $title3] $col3items] {punk::strlen $v}]] - set col3 [string repeat " " $widest3] - set tablewidth [expr {$widest1 + 1 + $widest2 + 1 + $widest3}] - - - append msg "[overtype::left $col1 $title1] [overtype::left $col2 $title2] [overtype::left $col3 $title3]" \n - append msg [string repeat "=" $tablewidth] \n - foreach p $col1items n $col2items c $col3items { - append msg "[overtype::left $col1 $p] [overtype::left $col2 $n] [overtype::right $col3 $c]" \n - } - return $msg - #return [list_as_lines [lib::get_projects $glob]] - } - proc detail {{glob {}} args} { - package require overtype - package require textutil - set defaults [dict create\ - -description 0\ - ] - set opts [dict merge $defaults $args] - # -- --- --- --- --- --- --- - set opt_description [dict get $opts -description] - # -- --- --- --- --- --- --- - - - set db_projects [lib::get_projects $glob] - set col1_dbfiles [lsearch -all -inline -index 0 -subindices $db_projects *] - set col2items [lsearch -all -inline -index 1 -subindices $db_projects *] - set checkouts [lsearch -all -inline -index 2 -subindices $db_projects *] - set col3items [lmap v $checkouts {llength $v}] - - set col4_pnames [list] - set col5_pcodes [list] - set col6_dupids [list] - set col7_pdescs [list] - set codes [dict create] - foreach dbfile $col1_dbfiles { - set project_name "" - set project_code "" - set project_desc "" - sqlite3 dbp $dbfile - dbp eval {select name,value from config where name like 'project-%';} r { - if {$r(name) eq "project-name"} { - set project_name $r(value) - } elseif {$r(name) eq "project-code"} { - set project_code $r(value) - } elseif {$r(name) eq "project-description"} { - set project_desc $r(value) - } - } - dbp close - lappend col4_pnames $project_name - lappend col5_pcodes $project_code - dict lappend codes $project_code $dbfile - lappend col7_pdescs $project_desc - } - - set setid 1 - set codeset [dict create] - dict for {code dbs} $codes { - if {[llength $dbs]>1} { - dict set codeset $code setid $setid - dict set codeset $code count [llength $dbs] - dict set codeset $code seen 0 - incr setid - } - } - set dupid 1 - foreach pc $col5_pcodes { - if {[dict exists $codeset $pc]} { - set seen [dict get $codeset $pc seen] - set this_seen [expr {$seen + 1}] - dict set codeset $pc seen $this_seen - lappend col6_dupids "[dict get $codeset $pc setid].${this_seen}/[dict get $codeset $pc count]" - } else { - lappend col6_dupids "" - } - } - - set title1 "Fossil Repo DB" - set widest1 [tcl::mathfunc::max {*}[lmap v [concat [list $title1] $col1_dbfiles] {punk::strlen $v}]] - set col1 [string repeat " " $widest1] - set title2 "File Name" - set widest2 [tcl::mathfunc::max {*}[lmap v [concat [list $title2] $col2items] {punk::strlen $v}]] - set col2 [string repeat " " $widest2] - set title3 "Checkouts" - set widest3 [tcl::mathfunc::max {*}[lmap v [concat [list $title3] $col3items] {punk::strlen $v}]] - set col3 [string repeat " " $widest3] - set title4 "Project Name" - set widest4 [tcl::mathfunc::max {*}[lmap v [concat [list $title4] $col4_pnames] {punk::strlen $v}]] - set col4 [string repeat " " $widest4] - set title5 "Project Code" - set widest5 [tcl::mathfunc::max {*}[lmap v [concat [list $title5] $col5_pcodes] {punk::strlen $v}]] - set col5 [string repeat " " $widest5] - set title6 "Dup" - set widest6 [tcl::mathfunc::max {*}[lmap v [concat [list $title6] $col6_dupids] {punk::strlen $v}]] - set col6 [string repeat " " $widest6] - set title7 "Description" - #set widest7 [tcl::mathfunc::max {*}[lmap v [concat [list $title4] $col7_pdescs] {punk::strlen $v}]] - set widest7 35 - set col7 [string repeat " " $widest7] - - - set tablewidth [expr {$widest1 + 1 + $widest2 + 1 + $widest3 +1 + $widest4 + 1 + $widest5 + 1 + $widest6}] - - append msg "[overtype::left $col1 $title1] [overtype::left $col2 $title2] [overtype::left $col3 $title3]\ - [overtype::left $col4 $title4] [overtype::left $col5 $title5] [overtype::left $col6 $title6]" - if {!$opt_description} { - append msg \n - } else { - append msg "[overtype::left $col7 $title7]" \n - set tablewidth [expr {$tablewidth + 1 + $widest7}] - } - - append msg [string repeat "=" $tablewidth] \n - foreach p $col1_dbfiles n $col2items c $col3items pn $col4_pnames pc $col5_pcodes dup $col6_dupids desc $col7_pdescs { - set desclines [split [textutil::adjust $desc -length $widest7] \n] - set desc1 [lindex $desclines 0] - append msg "[overtype::left $col1 $p] [overtype::left $col2 $n] [overtype::right $col3 $c]\ - [overtype::left $col4 $pn] [overtype::left $col5 $pc] [overtype::left $col6 $dup]" - if {!$opt_description} { - append msg \n - } else { - append msg " [overtype::left $col7 $desc1]" \n - foreach dline [lrange $desclines 1 end] { - append msg "$col1 $col2 $col3 $col4 $col5 $col6 [overtype::left $col7 $dline]" \n - } - } - } - return $msg - #return [list_as_lines [lib::get_projects $glob]] - } - proc cd {{glob {}} args} { - dict set args -cd 1 - work $glob {*}$args - } - proc work {{glob {}} args} { - package require sqlite3 - set db_projects [lib::get_projects $glob] - if {[llength $db_projects] == 0} { - puts stderr "::punk::mix::commandset::project::work No Repo DB name matches found for '$glob'" - return "" - } - #list of lists of the form: - #{fosdb fname workdirlist} - set defaults [dict create\ - -cd 0\ - -detail "\uFFFF"\ - ] - set opts [dict merge $defaults $args] - # -- --- --- --- --- --- --- - set opt_cd [dict get $opts -cd] - # -- --- --- --- --- --- --- - set opt_detail [dict get $opts -detail] - set opt_detail_explicit_zero 1 ;#default assumption only - if {$opt_detail eq "\uFFFF"} { - set opt_detail_explicit_zero 0 - set opt_detail 0; #default - } - # -- --- --- --- --- --- --- - set workdir_dict [dict create] - set all_workdirs [list] - foreach pinfo $db_projects { - lassign $pinfo fosdb name workdirs - foreach wdir $workdirs { - dict set workdir_dict $wdir $pinfo - lappend all_workdirs $wdir - } - } - set col_rowids [list] - set workdirs [lsort -index 0 $all_workdirs] - set col_dupids [list] - set col_fnames [list] - set col_pnames [list] - set col_pcodes [list] - set col_dupids [list] - - set fosdb_count [dict create] - set fosdb_dupset [dict create] - set fosdb_cache [dict create] - set dupset 0 - set rowid 1 - foreach wd $workdirs { - set wdinfo [dict get $workdir_dict $wd] - lassign $wdinfo fosdb nm siblingworkdirs - dict incr fosdb_count $fosdb - set dbcount [dict get $fosdb_count $fosdb] - if {[llength $siblingworkdirs] > 1} { - if {![dict exists $fosdb_dupset $fosdb]} { - #first time this multi-checkout fosdb seen - dict set fosdb_dupset $fosdb [incr dupset] - } - set dupid "[dict get $fosdb_dupset $fosdb].$dbcount/[llength $siblingworkdirs]" - } else { - set dupid "" - } - if {$dbcount == 1} { - set pname "" - set pcode "" - if {[file exists $fosdb]} { - if {[catch { - sqlite3 fdb $fosdb - set pname [lindex [fdb eval {select value from config where name = 'project-name'}] 0] - set pcode [lindex [fdb eval {select value from config where name = 'project-code'}] 0] - fdb close - dict set fosdb_cache $fosdb [list name $pname code $pcode] - } errM]} { - puts stderr "!!! problem with fossil db: $fosdb when examining workdir $wd" - puts stderr "!!! error: $errM" - } - } else { - puts stderr "!!! missing fossil db $fosdb" - } - } else { - set info [dict get $fosdb_cache $fosdb] - lassign $info _name pname _code pcode - } - lappend col_rowids $rowid - lappend col_fnames $nm - lappend col_dupids $dupid - lappend col_pnames $pname - lappend col_pcodes [string range $pcode 0 9] - incr rowid - } - - set col_states [list] - set state_title "" - #if only one set of fossil checkouts in the resultset and opt_detail is 0 and not explicit - retrieve workingdir state for each co - if {([llength [dict keys $fosdb_cache]] == 1)} { - if {!$opt_detail_explicit_zero} { - set opt_detail 1 - } - puts stderr "Result is from a single repo db [dict keys $fosdb_cache]" - } - if {$opt_detail} { - puts stderr "Gathering file state for [llength $workdirs] checkout folder(s). Use -detail 0 to omit file state" - set c_rev [list] - set c_rev_iso [list] - set c_unchanged [list] - set c_changed [list] - set c_new [list] - set c_missing [list] - set c_extra [list] - foreach wd $workdirs { - set wd_state [punk::repo::workingdir_state $wd] - set state_dict [punk::repo::workingdir_state_summary_dict $wd_state] - lappend c_rev [string range [dict get $state_dict revision] 0 9] - lappend c_rev_iso [dict get $state_dict revision_iso8601] - lappend c_unchanged [dict get $state_dict unchanged] - lappend c_changed [dict get $state_dict changed] - lappend c_new [dict get $state_dict new] - lappend c_missing [dict get $state_dict missing] - lappend c_extra [dict get $state_dict extra] - puts -nonewline stderr "." - } - puts -nonewline stderr \n - set t0 "Revision" - set w0 [tcl::mathfunc::max {*}[lmap v [concat [list $t0] $c_rev] {string length $v}]] - set c0 [string repeat " " $w0] - set t0b "Revision iso8601" - set w0b [tcl::mathfunc::max {*}[lmap v [concat [list $t0] $c_rev_iso] {string length $v}]] - set c0b [string repeat " " $w0b] - set t1 "Unch" - set w1 [tcl::mathfunc::max {*}[lmap v [concat [list $t1] $c_unchanged] {string length $v}]] - set c1 [string repeat " " $w1] - set t2 "Chgd" - set w2 [tcl::mathfunc::max {*}[lmap v [concat [list $t2] $c_changed] {string length $v}]] - set c2 [string repeat " " $w2] - set t3 "New" - set w3 [tcl::mathfunc::max {*}[lmap v [concat [list $t3] $c_new] {string length $v}]] - set c3 [string repeat " " $w3] - set t4 "Miss" - set w4 [tcl::mathfunc::max {*}[lmap v [concat [list $t4] $c_missing] {string length $v}]] - set c4 [string repeat " " $w4] - set t5 "Extr" - set w5 [tcl::mathfunc::max {*}[lmap v [concat [list $t5] $c_extra] {string length $v}]] - set c5 [string repeat " " $w5] - - set state_title "[overtype::left $c0 $t0] [overtype::left $c0b $t0b] [overtype::right $c1 $t1] [overtype::right $c2 $t2] [overtype::right $c3 $t3] [overtype::right $c4 $t4] [overtype::right $c5 $t5]" - foreach r $c_rev iso $c_rev_iso u $c_unchanged c $c_changed n $c_new m $c_missing e $c_extra { - lappend col_states "[overtype::left $c0 $r] [overtype::left $c0b $iso] [overtype::right $c1 $u] [overtype::right $c2 $c] [overtype::right $c3 $n] [overtype::right $c4 $m] [overtype::right $c5 $e]" - } - } - - set msg "" - if {$opt_cd} { - set title0 "CD" - } else { - set title0 "" - } - set widest0 [tcl::mathfunc::max {*}[lmap v [concat [list $title0] $col_rowids] {punk::strlen $v}]] - set col0 [string repeat " " $widest0] - set title1 "Checkout dir" - set widest1 [tcl::mathfunc::max {*}[lmap v [concat [list $title1] $workdirs] {punk::strlen $v}]] - set col1 [string repeat " " $widest1] - set title2 "Repo DB name" - set widest2 [tcl::mathfunc::max {*}[lmap v [concat [list $title2] $col_fnames] {string length $v}]] - set col2 [string repeat " " $widest2] - set title3 "CO dup" - set widest3 [tcl::mathfunc::max {*}[lmap v [concat [list $title3] $col_dupids] {string length $v}]] - set col3 [string repeat " " $widest3] - set title4 "Project Name" - set widest4 [tcl::mathfunc::max {*}[lmap v [concat [list $title4] $col_pnames] {string length $v}]] - set col4 [string repeat " " $widest4] - set title5 "Project Code" - set widest5 [tcl::mathfunc::max {*}[lmap v [concat [list $title5] $col_pcodes] {string length $v}]] - set col5 [string repeat " " $widest5] - - set tablewidth [expr {$widest0 + 1 + $widest1 + 1 + $widest2 + 1 + $widest3 +1 + $widest4 + 1 + $widest5}] - append msg "[overtype::right $col0 $title0] [overtype::left $col1 $title1] [overtype::left $col2 $title2] [overtype::left $col3 $title3] [overtype::left $col4 $title4] [overtype::left $col5 $title5]" - - if {[llength $col_states]} { - set title6 $state_title - set widest6 [tcl::mathfunc::max {*}[lmap v [concat [list $title6] $col_states] {string length $v}]] - set col6 [string repeat " " $widest6] - incr tablewidth [expr {$widest6 + 1}] - append msg " [overtype::left $col6 $title6]" \n - } else { - append msg \n - } - append msg [string repeat "=" $tablewidth] \n - - if {[llength $col_states]} { - foreach row $col_rowids wd $workdirs db $col_fnames dup $col_dupids pname $col_pnames pcode $col_pcodes s $col_states { - append msg "[overtype::right $col0 $row] [overtype::left $col1 $wd] [overtype::left $col2 $db] [overtype::right $col3 $dup] [overtype::left $col4 $pname] [overtype::left $col5 $pcode] [overtype::left $col6 $s]" \n - } - } else { - foreach row $col_rowids wd $workdirs db $col_fnames dup $col_dupids pname $col_pnames pcode $col_pcodes { - append msg "[overtype::right $col0 $row] [overtype::left $col1 $wd] [overtype::left $col2 $db] [overtype::right $col3 $dup] [overtype::left $col4 $pname] [overtype::left $col5 $pcode]" \n - } - } - set numrows [llength $col_rowids] - if {$opt_cd && $numrows >= 1} { - puts stdout $msg - if {$numrows == 1} { - set workingdir [lindex $workdirs 0] - puts stdout "1 result. Changing dir to $workingdir" - if {[file exists $workingdir]} { - ::cd $workingdir - return $workingdir - } else { - puts stderr "path $workingdir doesn't appear to exist" - return [pwd] - } - } else { - set answer [util::askuser "Change directory to working folder - select a number from 1 to [llength $col_rowids] or any other key to cancel."] - if {[string trim $answer] in $col_rowids} { - set index [expr {$answer - 1}] - set workingdir [lindex $workdirs $index] - ::cd $workingdir - puts stdout [pmix stat] - return $workingdir - } - } - } - return $msg - } - #*** !doctools - #[list_end] [comment {-- end collection namespace definitions --}] - } - - namespace eval lib { - proc template_tag {tagname} { - #todo - support different tagwrappers - it shouldn't be so likely to collide with common code idioms etc. - #we need to detect presence of tags intended for punk::mix system - #consider using punk::cap to enable multiple template-substitution providers with their own set of tagnames and/or tag wrappers, where substitution providers are all run - return [string cat % $tagname %] - } - #get project info only by opening the central confg-db - #(will not have proper project-name etc) - proc get_projects {{globlist {}} args} { - if {![llength $globlist]} { - set globlist [list *] - } - set fossil_prog [auto_execok fossil] - - set fossilinfo [exec {*}$fossil_prog info] ;#will give us the necessary config-db info whether in a project folder or not - set matching_lines [punk::repo::grep {config-db:*} $fossilinfo] - if {[llength $matching_lines] != 1} { - puts stderr "Unable to find config-db info from fossil. Check your fossil installation." - puts stderr "Fossil output was:" - puts stderr "-------------" - puts stderr "$fossilinfo" - puts stderr "-------------" - puts stderr "config-db info:" - puts stderr "$matching_lines" - return - } - set ln [lindex $matching_lines 0] - set configdb [string trim [string range $ln [string length "config-db: "] end]] - if {![file exists $configdb]} { - error "config-db not found at path $configdb" - } - package require sqlite3 - ::sqlite3 fosconf $configdb - #set testresult [fosconf eval {select name,value from global_config;}] - #puts stderr $testresult - set project_repos [fosconf eval {select name from global_config where name like 'repo:%';}] - set paths_and_names [list] - foreach pr $project_repos { - set path [string trim [string range $pr 5 end]] - set nm [file rootname [file tail $path]] - set ckouts [fosconf eval {select name from global_config where value = $path;}] - set checkout_paths [list] - #strip "ckout:" - foreach ck $ckouts { - lappend checkout_paths [string trim [string range $ck 6 end]] - } - lappend paths_and_names [list $path $nm $checkout_paths] - } - set filtered_list [list] - foreach glob $globlist { - set matches [lsearch -all -inline -index 1 $paths_and_names $glob] - foreach m $matches { - if {$m ni $filtered_list} { - lappend filtered_list $m - } - } - } - set projects [lsort -index 1 $filtered_list] - return $projects - } - - } - - - - - -} - - -#*** !doctools -#[manpage_end] - - - - - - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::mix::commandset::project [namespace eval punk::mix::commandset::project { - variable version - set version 0.1.0 -}] -return diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/repo-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/repo-0.1.0.tm deleted file mode 100644 index abfb0e55..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/repo-0.1.0.tm +++ /dev/null @@ -1,92 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::mix::commandset::repo 0.1.0 -# Meta platform tcl -# Meta license -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::mix::commandset::repo { - namespace export * - proc tickets {{project ""}} { - set result "" - if {[string length $project]} { - puts stderr "project status unimplemented" - return - } - set active_dir [pwd] - append result "Retrieving top 10 tickets only (for more, use fossil timeline -n -t t)" \n - append result [exec fossil timeline -n 10 -t t] - - return $result - } - - proc fossilize { args} { - #check if project already managed by fossil.. initialise and check in if not. - puts stderr "unimplemented" - } - - proc unfossilize {projectname args} { - #remove/archive .fossil - puts stderr "unimplemented" - } - proc state {} { - set result "" - set repopaths [punk::repo::find_repos [pwd]] - set repos [dict get $repopaths repos] - if {![llength $repos]} { - append result [dict get $repopaths warnings] - } else { - append result [dict get $repopaths warnings] - lassign [lindex $repos 0] repopath repotypes - if {"fossil" in $repotypes} { - append result \n "Fossil repo based at $repopath" - set repostate [punk::repo::workingdir_state $repopath -repopaths $repopaths -repotypes fossil] - append result \n [punk::repo::workingdir_state_summary $repostate] - } - if {"git" in $repotypes} { - append result \n "Git repo based at $repopath" - set repostate [punk::repo::workingdir_state $repopath -repopaths $repopaths -repotypes git] - append result \n [punk::repo::workingdir_state_summary $repostate] - } - } - return $result - } -} - - - - - - - - - - - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::mix::commandset::repo [namespace eval punk::mix::commandset::repo { - variable version - set version 0.1.0 -}] -return diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/scriptwrap-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/scriptwrap-0.1.0.tm deleted file mode 100644 index e40bc899..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/scriptwrap-0.1.0.tm +++ /dev/null @@ -1,681 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::mix::commandset::scriptwrap 0.1.0 -# Meta platform tcl -# Meta license -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz - -package require punk::mix -package require punk::mix::base - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::mix::commandset::scriptwrap { - namespace export * - - - #scriptpath allows templates command to use same custom template set as when multishell pointed to a filepath - #it may or may not be within a project - #by using the same folder or path, the same project root will be discovered. REVIEW. - proc templates_dict {args} { - set defaults [list -scriptpath ""] - set opts [dict merge $defaults $args] - set opt_scriptpath [dict get $opts -scriptpath] - - set wrapper_folders [lib::get_wrapper_folders $opt_scriptpath] - - set wrapper_templates [list] - foreach fld $wrapper_folders { - set templates [glob -nocomplain -dir $fld -type f *] - foreach tf $templates { - if {[string match ignore* $tf]} { - continue - } - set ext [file extension $tf] - if {$ext in [list "" ".bat" ".cmd" ".sh"]} { - lappend wrapper_templates $tf - } - } - } - - set tdict [dict create] - set seen_dict [dict create] - foreach fullpath $wrapper_templates { - set ftail [file tail $fullpath] - if {![dict exists $seen_dict $ftail]} { - dict set seen_dict $ftail 1 - dict set tdict $ftail $fullpath ; #first seen of filename gets no number - } else { - set n [dict get $seen_dict $ftail] - incr n - dict incr seen_dict $ftail - dict set tdict ${ftail}.$n $fullpath - } - } - return $tdict - } - proc templates {args} { - package require overtype - set tdict [templates_dict {*}$args] - - - set paths [dict values $tdict] - set names [dict keys $tdict] - - set title1 "Path" - set widest1 [tcl::mathfunc::max {*}[lmap v [concat [list $title1] $paths] {punk::strlen $v}]] - set col1 [string repeat " " $widest1] - - set title2 "Template Name" - set widest2 [tcl::mathfunc::max {*}[lmap v [concat [list $title2] $names] {punk::strlen $v}]] - set col2 [string repeat " " $widest2] - - set tablewidth [expr {$widest1 + 1 + $widest2}] - set table "" - append table [string repeat - $tablewidth] \n - append table "[overtype::left $col1 $title1] [overtype::left $col2 $title2]" \n - append table [string repeat - $tablewidth] \n - - foreach p $paths n $names { - append table "[overtype::left $col1 $p] [overtype::left $col2 $n]" \n - } - - - return $table - } - #specific filepath to just wrap one script at the tcl-payload or xxx-payload-pre-tcl site - #scriptset name to substiture multiple scriptset.xxx files at the default locations - or as specified in scriptset.wrapconf - proc multishell {filepath_or_scriptset args} { - set defaults [dict create\ - -askme 1\ - -outputfolder "\uFFFF"\ - -template "\uFFFF"\ - ] - set known_opts [dict keys $defaults] - dict for {k v} $args { - if {$k ni $known_opts} { - - error "punk::mix::commandset::scriptwrap error. Unrecognized option '$k'. Known-options: $known_opts" - } - } - set usage "" - append usage "Use directly with the script file to wrap, or supply the name of a scriptset" \n - append usage "The scriptset name will be used to search for yourname.sh|tcl|ps1 or names as you specify in yourname.wrapconfig if it exists" \n - append usage "If no template is specified in a .wrapconfig and no -template argument is supplied, it will default to punk-multishell.cmd" \n - if {![string length $filepath_or_scriptset]} { - puts stderr "No filepath_or_scriptset specified" - puts stderr $usage - return false - } - set opts [dict merge $defaults $args] - # -- --- --- --- --- --- --- --- --- --- --- --- - set opt_askme [dict get $opts -askme] - set opt_template [dict get $opts -template] - set opt_outputfolder [dict get $opts -outputfolder] - # -- --- --- --- --- --- --- --- --- --- --- --- - - - set ext [file extension $filepath_or_scriptset] - set startdir [pwd] - - - - #first check if relative or absolute path matches a file - if {[file pathtype $filepath_or_scriptset] eq "absolute"} { - set specified_path $filepath_or_scriptset - } else { - set specified_path [file join $startdir $filepath_or_scriptset] - } - set ext [string trim [file extension $filepath_or_scriptset] .] - set allowed_extensions [list wrapconfig tcl ps1 sh bash] - #set allowed_extensions [list tcl] - set found_script 0 - if {[file exists $specified_path]} { - set found_script 1 - } else { - foreach e $allowed_extensions { - if {[file exists $filepath_or_scriptset.$e]} { - set found_script 1 - break - } - } - } - - #TODO! - use get_wrapper_folders - multishell should use same available templates as the 'templates' function - set scriptset [file rootname [file tail $specified_path]] - if {$found_script} { - if {[file type $specified_path] eq "file"} { - set specified_root [file dirname $specified_path] - set pathinfo [punk::repo::find_repos [file dirname $specified_path]] - set projectroot [dict get $pathinfo closest] - if {[string length $projectroot]} { - #use the specified files folder - but use the main scriptapps/wrappers folder if specified one has no wrappers subfolder - set scriptroot [file dirname $specified_path] - if {[file exists $scriptroot/wrappers]} { - set customwrapper_folder $scriptroot/wrappers - } else { - set customwrapper_folder $projectroot/src/scriptapps/wrappers - } - } else { - #outside of any project - set scriptroot [file dirname $specified_path] - if {[file exists $scriptroot/wrappers]} { - set customwrapper_folder $scriptroot/wrappers - } else { - #no customwrapper folder available - set customwrapper_folder "" - } - } - } else { - puts stderr "wrap_in_multishell doesn't currently support a directory as the path." - puts stderr $usage - return false - } - } else { - set pathinfo [punk::repo::find_repos $startdir] - set projectroot [dict get $pathinfo closest] - if {[string length $projectroot]} { - if {[llength [file split $filepath_or_scriptset]] > 1} { - puts stderr "filepath_or_scriptset looks like a path - but doesn't seem to point to a file" - puts stderr "Ensure you are within a project and use just the name of the scriptset, or pass in the full correct path or relative path to current directory" - puts stderr $usage - return false - } else { - #we've already ruled out empty string - so must have a single element representing scriptset - possibly with file extension - set scriptroot $projectroot/src/scriptapps - set customwrapper_folder $projectroot/src/scriptapps/wrappers - #check something matches the scriptset.. - set something_found "" - if {[file exists $scriptroot/$scriptset]} { - set found_script 1 - set something_found $scriptroot/$scriptset ;#extensionless file - that's ok too - } else { - foreach e $allowed_extensions { - if {[file exists $scriptroot/$scriptset.$e]} { - set found_script 1 - set something_found $scriptroot/$scriptset.$e - break - } - } - } - if {!$found_script} { - puts stderr "Searched within $scriptroot" - puts stderr "Unable to find a file matching $scriptset or one of the extensions: $allowed_extensions" - puts stderr $usage - return false - } else { - if {[file type $something_found] ne "file"} { - puts stderr "Found '$something_found'" - puts stderr "wrap_in_multishell doesn't currently support a directory as the path." - puts stderr $usage - return false - } - } - } - - } else { - puts stderr "filepath_or_scriptset parameter doesn't seem to refer to a file, and you are not within a directory where projectroot and src/scriptapps/wrappers can be determined" - puts stderr $usage - return false - } - } - #assert - customwrapper_folder var exists - but might be empty - - - if {[string length $ext]} { - #If there was an explicitly supplied extension - then that file should exist - if {![file exists $scriptroot/$scriptset.$ext]} { - puts stderr "Explicit extension .$ext was supplied - but matching file not found." - puts stderr $usage - return false - } else { - if {$ext eq "wrapconfig"} { - set process_extensions ALLFOUNDORCONFIGURED - } else { - set process_extensions $ext - } - } - } else { - #no explicit extension - process all for scriptset - set process_extensions ALLFOUNDORCONFIGURED - } - #process_extensions - either a single one - or all found or as per .wrapconfig - - if {$opt_template eq "\uFFFF"} { - set templatename punk-multishell.cmd - } else { - set templatename $opt_template - } - - - - set template_base_dict [punk::mix::base::lib::get_template_basefolders] - set tpldirs [list] - dict for {tdir tsourceinfo} $template_base_dict { - if {[file exists $tdir/utility/scriptappwrappers/$templatename]} { - lappend tpldirs $tdir - } - } - - if {[string length $customwrapper_folder] && [file exists [file join $customwrapper_folder $templatename] ]} { - set wrapper_template [file join $customwrapper_folder $templatename] - } else { - if {![llength $tpldirs]} { - set msg "No template named '$templatename' found in src/scriptapps/wrappers or in template dirs from packages" - append msg \n "Searched [dict size $template_base_dict] template dirs" - error $msg - } - - #last pkg with templates cap which was loaded has highest precedence - set wrapper_template "" - foreach tdir [lreverse $tpldirs] { - set ftest [file join $tdir utility scriptappwrappers $templatename] - if {[file exists $ftest]} { - set wrapper_template $ftest - break - } - } - } - - if {$wrapper_template eq "" || ![file exists $wrapper_template]} { - error "wrap_in_multishell: unable to find multishell template $templatename in template folders [concat $tpldirs $customwrapper_folder]" - } - - - if {$opt_outputfolder eq "\uFFFF"} { - #outputfolder not explicitly specified by caller - if {[string length $projectroot]} { - set output_folder [file join $projectroot/bin] - } else { - set output_folder $startdir - } - } else { - if {[file pathtype $opt_outputfolder] eq "relative"} { - if {[string length $projectroot]} { - set output_folder [file join $projectroot $opt_outputfolder] - } else { - set output_folder [file join $startdir $opt_outputfolder] - } - } else { - set output_folder $opt_outputfolder - } - } - if {![file isdirectory $output_folder]} { - error "wrap_in_multishell: output folder '$output_folder' not found. Please ensure target directory exists" - } - - - #todo - #output_file extension may also depend on the template being used.. and/or the .wrapconfig - if {$::tcl_platform(platform) eq "windows"} { - set output_extension cmd - } else { - set output_extension sh - } - set output_file [file join $output_folder $scriptset.$output_extension] - if {[file exists $output_file]} { - error "wrap_in_multishell: target file $output_file already exists.. aborting" - } - - - set fdt [open $wrapper_template r] - fconfigure $fdt -translation binary - set template_data [read $fdt] - close $fdt - puts stdout "Read [string length $template_data] bytes of template data.." - set template_lines [split $template_data \n] - puts stdout "Displaying first 3 lines of template between dashed lines..." - puts stdout "-----------------------------------------------" - foreach ln [lrange $template_lines 0 3] { - puts stdout $ln - } - puts stdout "-----------------------------------------------\n" - #foreach ln $template_lines { - #} - - set list_input_files [list] - if {$process_extensions eq "ALLFOUNDORCONFIGURED"} { - #todo - look for .wrapconfig or all extensions for the scriptset - puts stderr "Sorry - only single input file supported. Supply a file extension or use a .wrapconfig with a single input file for now - implementation incomplete" - return false - } else { - lappend list_input_files $scriptroot/$scriptset.$ext - } - - #todo - split template at each etc marker and build a dict of parts - - - #hack - process one input - set filepath [lindex $list_input_files 0] - - set fdscript [open $filepath r] - fconfigure $fdscript -translation binary - set script_data [read $fdscript] - close $fdscript - puts stdout "Read [string length $script_data] bytes of template data.." - set script_lines [split $script_data \n] - puts stdout "Displaying first 3 lines of your script between dashed lines..." - puts stdout "-----------------------------------------------" - foreach ln [lrange $script_lines 0 3] { - puts stdout $ln - } - puts stdout "-----------------------------------------------\n" - puts stdout "Target for above data is '$output_file'" - if {$opt_askme} { - set answer [util::askuser "Does this look correct? Y|N"] - if {[string tolower $answer] ne "y"} { - puts stderr "mix new aborting due to user response '$answer' (required Y or y to proceed) use -askme 0 to avoid prompts." - return - } - } - - set start_idx 0 - set end_idx 0 - set line_idx 0 - set existing_payload [list] - foreach ln $template_lines { - - if {[string match "#*" $ln]} { - set start_idx $line_idx - } elseif {[string match "#*" $ln]} { - set end_idx $line_idx - break - } elseif {$start_idx > 0} { - if {$end_idx > 0} { - lappend existing_payload [string trim $ln] - } - } else { - - } - incr line_idx - } - if {($start_idx == 0) || ($end_idx == 0)} { - error "wrap_in_multishell was unable to find payload area in template marked with # and # on separate lines" - } - set existing_string [join $existing_payload \n] - if {[string length [string trim $existing_string]]} { - puts stdout "EXISTING PAYLOAD!!" - puts stdout "-----------------------------------------------\n" - puts stdout $existing_string - puts stdout "-----------------------------------------------\n" - error "wrap_in_multishell found existing payload.. aborting." - #todo - allow overwrite only in files outside of punkshell distribution? - if 0 { - puts stderr "Found existing payload.. overwrite?" - if {$opt_askme} { - set answer [util::askuser "Are you sure you want to replace the tcl payload shown above? Y|N"] - if {[string tolower $answer] ne "y"} { - puts stderr "mix new aborting due to user response '$answer' (required Y|y to proceed) use -askme 0 to avoid prompts." - return - } - } - } - } - - set tpl_head_lines [lrange $template_lines 0 $start_idx] ;#include tag line - set tpl_tail_lines [lrange $template_lines $end_idx end] - set newscript [join $tpl_head_lines \n]\n[join $script_lines \n]\n[join $tpl_tail_lines \n] - puts stdout "New script is [string length $newscript] bytes" - puts stdout $newscript - set fdtarget [open $output_file w] - fconfigure $fdtarget -translation binary - puts -nonewline $fdtarget $newscript - close $fdtarget - puts stdout "Wrote script file at $output_file" - - #even though chmod might exist on windows - we will leave permissions alone - if {$::tcl_platform(platform) ne "windows"} { - catch {exec chmod +x $output_file} - } - puts stdout "-done-" - return $output_file - } - - namespace eval lib { - - #get_wrapper_folders - # scriptpath - file or folder - # It represents the base point from which to search for /wrapper folders either directly above the scriptpath or in the containing project if any - # The cwd will also be searched for /wrapper folder and project - but with lower precedence in the resultset (later in list) - proc get_wrapper_folders {{scriptpath ""}} { - set wrapper_folders [list] - if {$scriptpath ne ""} { - if {[file type $scriptpath] eq "file"} { - set searchbase [file dirname $scriptpath] - } else { - set searchbase $scriptpath - } - if {[file isdirectory [file join $searchbase wrappers]]} { - lappend wrapper_folders [file join $searchbase wrappers] - } - set pathinfo [punk::repo::find_repos $searchbase] - set scriptpath_projectroot [dict get $pathinfo closest] - if {$scriptpath_projectroot ne ""} { - set fld [file join $scriptpath_projectroot src/scriptapps/wrappers] - if {[file isdirectory $fld]} { - if {$fld ni $wrapper_folders} { - lappend wrapper_folders $fld - } - } - } - } - set searchbase [pwd] - set fld [file join $searchbase wrappers] - if {[file isdirectory $fld]} { - if {$fld ni $wrapper_folders} { - lappend wrapper_folders $fld - } - } - set pathinfo [punk::repo::find_repos $searchbase] - set pwd_projectroot [dict get $pathinfo closest] - if {$pwd_projectroot ne ""} { - set fld [file join $pwd_projectroot src/scriptapps/wrappers] - if {[file isdirectory $fld]} { - if {$fld ni $wrapper_folders} { - lappend wrapper_folders $fld - } - } - } - - set template_base_dict [punk::mix::base::lib::get_template_basefolders] - set tpldirs [list] - dict for {tdir tsourceinfo} $template_base_dict { - if {[file exists $tdir/utility/scriptappwrappers]} { - lappend tpldirs $tdir - } - } - foreach tpldir $tpldirs { - set fld [file join $tpldir utility scriptappwrappers] - if {[file isdirectory $fld]} { - if {$fld ni $wrapper_folders} { - lappend wrapper_folders $fld - } - } - } - return $wrapper_folders - } - proc _scriptapp_tag_from_line {line} { - set result [list istag 0 raw ""] ;#default assumption. All - #---- - set startc [string first "#" $line] ;#tags must be commented - #todo - review. next line is valid - note # doesn't have to be the only one before - # @REM # etc < blah # etc - #--- - #fix - we should use a regexp on at least and only catch tagname without whitespace - regexp {(\s*).*} $line _ln indent ;#will match on empty line, whitespace only line - or anything really. - set indent [string map [list \t " "] $indent] ;#opinionated I guess - but need to normalize to something. The spec is that spaces should be used anyway. - dict set result indent [string length $indent] - set starttag [string first "<" $line] - set pretag [string range $line $startc $starttag-1] - if {[string match "*>*" $pretag]} { - return [list istag 0 raw $line reason pretag_contents] - } - set closetag [string first ">" $line] - set inelement [string range $line $starttag+1 $closetag-1] - if {[string match "*<*" $inelement]} { - return [list istag 0 raw $line reason tag_malformed_angles] - } - set elementchars [split $inelement ""] - set numslashes [llength [lsearch -all $elementchars "/"]] - if {$numslashes == 0} { - dict set result type "open" - } elseif {$numslashes == 1} { - if {[lindex $elementchars 0] eq "/"} { - dict set result type "close" - } elseif {[lindex $elementchars end] eq "/"} { - dict set result type "openclose" - } else { - return [list istag 0 raw $line reason tag_malformed_slashes] - } - } else { - return [list istag 0 raw $line reason tag_malformed_extraslashes] - } - if {[dict get $result type] eq "open"} { - dict set result name $inelement - } elseif {[dict get $result type] eq "close"} { - dict set result name [string range $inelement 1 end] - } else { - dict set result name [string range $inelement 0 end-1] - } - dict set result istag 1 - dict set result raw $line - return $result - } - - #get all \n#\n ...\n# data - where number of intervening newlines is at least one (and whitespace and/or other data can precede #) - #we don't verify 'something' against known tags - as custom templates can have own tags - #An openclose tag # is used to substitute a specific line in its entirety - but the tag *must* remain in the line - # - #e.g for the line: - # @set "nextshell=pwsh" & :: # - #The .wrapconfig might contain - # tag line {@set "nextshell=tclsh" & :: @} - # - proc scriptapp_wrapper_get_tags {wrapperdata} { - set wrapperdata [string map [list \r\n \n] $wrapperdata] - set lines [split $wrapperdata \n] - #set tags_in_data [dict create];#active tags - list of lines accumulating. supports nested tags - set status 0 - set tags [dict create] - set errors [list] - set errortags [dict create] ;#mark names invalid on first error so that more than 2 tags can't obscure config problem - set linenum 1 ;#editors and other utils use 1-based indexing when referencing files - we should too to avoid confusion, despite it being less natural for lindex operations on the result. - foreach ln $lines { - set lntrim [string trim $ln] - if {![string length $lntrim]} { - incr linenum - continue - } - if {[string match "*#*<*>*" $lntrim]} { - set taginfo [_scriptapp_tag_from_line $ln] ;#use untrimmed line - to get indent - if {[dict get $taginfo istag]} { - set nm [dict get $taginfo name] - if {[dict exists $errortags $nm]} { - #tag is already in error condition - - } else { - set tp [dict get $taginfo type] ;# type singular - related to just one line - #set raw [dict get $taginfo raw] #equivalent to $ln - if {[dict exists $tags $nm]} { - #already seen tag name - #tags dict has types key *plural* - need to track whether we have type open and type close (or openclose for self-closing tags) - if {[dict get $tags $nm types] ne "open"} { - lappend errors "line: $linenum tag $nm encountered type $tp after existing type [dict get $tags $nm types]" - dict incr errortags $nm - } else { - #we already have open - expect only close - if {$tp ne "close"} { - lappend errors "line: $linenum tag $nm encountered type $tp after existing type [dict get $tags $nm types]" - dict incr errortags $nm - } else { - #close after open - dict set tags $nm types [list open close] - dict set tags $nm end $linenum - set taglines [dict get $tags $nm taglines] - if {[llength $taglines] != 1} { - error "Unexpected result when closing tag $nm. Existing taglines length not 1." - } - dict set tags $nm taglines [concat $taglines $ln] - } - } - } else { - #first seen of tag name - if {$tp eq "close"} { - lappend errors "line: $linenum tag $nm encountered type $p close first" - dict incr errortags $nm - } else { - dict set tags $nm types $tp - dict set tags $nm indent [dict get $taginfo indent] - if {$tp eq "open"} { - dict set tags $nm start $linenum - dict set tags $nm taglines [list $ln] ;#first entry - another will be added on encountering matching closing tag - } elseif {$tp eq "openclose"} { - dict set tags $nm start $linenum - dict set tags $nm end $linenum - dict set tags $nm taglines [list $ln] ;#single entry is final result for self-closing tag - } - } - } - } - } else { - #looks like it should be a tag.. but failed to even parse for some reason.. just add to errorlist - lappend errors "line: $linenum tag parse failure reason: [dict get $taginfo reason] raw line: [dict get $taginfo raw]" - } - } - #whether the line is tag or not append to any tags_in_data - #foreach t [dict keys $tags_in_data] { - # dict lappend tags_in_data $t $ln ;#accumulate raw lines - written to the tag entry in tags only on encountering a closing tag, then removed from tags_in_data - #} - incr linenum - } - #assert [expr {$linenum -1 == [llength $lines]}] - if {[llength $errors]} { - set status 0 - } else { - set status 1 - } - if {$linenum == 0} { - - } - return [dict create ok $status linecount [llength $lines] data $tags errors $errors] - } - - - } - - -} - - - - - - - - - - - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::mix::commandset::scriptwrap [namespace eval punk::mix::commandset::scriptwrap { - variable version - set version 0.1.0 -}] -return diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/templates-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/templates-0.1.0.tm deleted file mode 100644 index 46065bda..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/templates-0.1.0.tm +++ /dev/null @@ -1,84 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::mix::templates 0.1.0 -# Meta platform tcl -# Meta license BSD -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz -package require punk::cap - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::mix::templates { - variable pkg punk::mix::templates - variable cap_provider - - #punk::cap::register_package punk::mix::templates [list\ - # {punk.templates {relpath ../templates}}\ - #] - - namespace eval capsystem { - if {[info commands capprovider.registration] eq ""} { - punk::cap::class::interface_capprovider.registration create capprovider.registration - oo::objdefine capprovider.registration { - method get_declarations {} { - set decls [list] - lappend decls [list punk.templates {relpath ../templates}] - lappend decls [list punk.templates {relpath ../templates2}] - lappend decls [list punk.test {something blah}] - return $decls - } - } - } - } - - if {[info commands provider] eq ""} { - punk::cap::class::interface_capprovider.provider create provider punk::mix::templates - oo::objdefine provider { - method register {{capabilityname_glob *}} { - #puts registering punk::mix::templates $capabilityname - next - } - method capabilities {} { - next - } - } - } - - # -- --- - #provider api - # -- --- - #none - declarations only -} - - - - - - - - - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::mix::templates [namespace eval punk::mix::templates { - variable version - set version 0.1.0 -}] -return \ No newline at end of file diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/util-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/util-0.1.0.tm deleted file mode 100644 index 5622bc02..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/util-0.1.0.tm +++ /dev/null @@ -1,350 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::mix::util 0.1.0 -# Meta platform tcl -# Meta license -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz - -namespace eval punk::mix::util { - variable has_winpath 0 -} - -if {"windows" eq $::tcl_platform(platform)} { - if {![catch {package require punk::winpath}]} { - set punk::mix::util::has_winpath 1 - } -} - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::mix::util { - variable tmpfile_counter 0 ;#additional tmpfile collision avoidance - - namespace export * - - - proc fcat {args} { - variable has_winpath - - if {$::tcl_platform(platform) ne "windows"} { - return [fileutil::cat {*}$args] - } - - set knownopts [list -eofchar -translation -encoding --] - set last_opt 0 - for {set i 0} {$i < [llength $args]} {incr i} { - set ival [lindex $args $i] - #puts stdout "i:$i a: $ival known: [expr {$ival in $knownopts}]" - if {$ival eq "--"} { - set last_opt $i - break - } else { - if {$ival in $knownopts} { - #puts ">known at $i : [lindex $args $i]" - if {($i % 2) != 0} { - error "unexpected option at index $i. known options: $knownopts must come in -opt val pairs." - } - incr i - set last_opt $i - } else { - set last_opt [expr {$i - 1}] - break - } - } - } - set first_non_opt [expr {$last_opt + 1}] - - #puts stderr "first_non_opt: $first_non_opt" - set opts [lrange $args -1 $first_non_opt-1] - set paths [lrange $args $first_non_opt end] - if {![llength $paths]} { - error "Unable to find file in the supplied arguments: $args. Ensure options are all -opt val pairs and that file name(s) follow" - } - #puts stderr "opts: $opts paths: $paths" - set finalpaths [list] - foreach p $paths { - if {$has_winpath && [punk::winpath::illegalname_test $p]} { - lappend finalpaths [punk::winpath::illegalname_fix $p] - } else { - lappend finalpaths $p - } - } - fileutil::cat {*}$opts {*}$finalpaths - } - - #---------------------------------------- - namespace eval internal { - proc path_common_prefix_pop {varname} { - upvar 1 $varname var - set var [lassign $var head] - return $head - } - } - proc path_common_prefix {args} { - set dirs $args - set parts [file split [internal::path_common_prefix_pop dirs]] - while {[llength $dirs]} { - set r {} - foreach cmp $parts elt [file split [internal::path_common_prefix_pop dirs]] { - if {$cmp ne $elt} break - lappend r $cmp - } - set parts $r - } - if {[llength $parts]} { - return [file join {*}$parts] - } else { - return "" - } - } - - #retains case from first argument only - caseless comparison - proc path_common_prefix_nocase {args} { - set dirs $args - set parts [file split [internal::path_common_prefix_pop dirs]] - while {[llength $dirs]} { - set r {} - foreach cmp $parts elt [file split [internal::path_common_prefix_pop dirs]] { - if {![string equal -nocase $cmp $elt]} break - lappend r $cmp - } - set parts $r - } - if {[llength $parts]} { - return [file join {*}$parts] - } else { - return "" - } - } - #---------------------------------------- - - #namespace import ::punk::ns::nsimport_noclobber - - proc namespace_import_pattern_to_namespace_noclobber {pattern ns} { - set source_ns [namespace qualifiers $pattern] - if {![namespace exists $source_ns]} { - error "namespace_import_pattern_to_namespace_noclobber error namespace $source_ns not found" - } - if {![string match ::* $ns]} { - set nscaller [uplevel 1 {namespace current}] - set ns [punk::nsjoin $nscaller $ns] - } - set a_export_patterns [namespace eval $source_ns {namespace export}] - set a_commands [info commands $pattern] - set a_tails [lmap v $a_commands {namespace tail $v}] - set a_exported_tails [list] - foreach pattern $a_export_patterns { - set matches [lsearch -all -inline $a_tails $pattern] - foreach m $matches { - if {$m ni $a_exported_tails} { - lappend a_exported_tails $m - } - } - } - set imported_commands [list] - foreach e $a_exported_tails { - set imported [namespace eval $ns [string map [list $e $source_ns] { - set cmd "" - if {![catch {namespace import ::}]} { - set cmd - } - set cmd - }]] - if {[string length $imported]} { - lappend imported_commands $imported - } - } - return $imported_commands - } - - proc askuser {question} { - puts stdout $question - flush stdout - set stdin_state [fconfigure stdin] - fconfigure stdin -blocking 1 - set answer [gets stdin] - fconfigure stdin -blocking [dict get $stdin_state -blocking] - return $answer - } - - proc do_in_path {path script} { - #from ::kettle::path::in - set here [pwd] - try { - cd $path - uplevel 1 $script - } finally { - cd $here - } - } - - proc foreach-file {path script_pathvariable script} { - upvar 1 $script_pathvariable thepath - - set known {} - lappend waiting $path - while {[llength $waiting]} { - set pending $waiting - set waiting {} - set at 0 - while {$at < [llength $pending]} { - set current [lindex $pending $at] - incr at - - # Do not follow into parent. - if {[string match *.. $current]} continue - - # Ignore what we have visited already. - set c [file dirname [file normalize $current/___]] - if {[dict exists $known $c]} continue - dict set known $c . - - if {[file tail $c] eq ".git"} { - continue - } - - # Expand directories. - if {[file isdirectory $c]} { - lappend waiting {*}[lsort -unique [glob -directory $c * .*]] - continue - } - - # Handle files as per the user's will. - set thepath $current - switch -exact -- [catch { uplevel 1 $script } result] { - 0 - 4 { - # ok, continue - nothing - } - 2 { - # return, abort, rethrow - return -code return - } - 3 { - # break, abort - return - } - 1 - default { - # error, any thing else - rethrow - return -code error $result - } - } - } - } - return - } - - proc is_valid_tm_version {versionpart} { - #Needs to be suitable for use with Tcl's 'package vcompare' - if {![catch [list package vcompare $versionpart $versionpart]]} { - return 1 - } else { - return 0 - } - } - #Note that semver only has a small overlap with tcl tm versions. - #todo - work out what overlap and whether it's even useful - #see also TIP #439: Semantic Versioning (tcl 9??) - proc semver {versionstring} { - set re {^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$} - } - #todo - semver conversion/validation for other systems? - proc magic_tm_version {} { - set magicbase 999999 ;#deliberately large so given load-preference when testing! - #we split the literal to avoid the literal appearing here - reduce risk of accidentally converting to a release version - return ${magicbase}.0a1.0 - } - - - - proc tmpfile {{prefix tmp_}} { - #note risk of collision if pregenerating a list of tmpfile names - #we will maintain an icrementing id so the caller doesn't have to bear that in mind - variable tmpfile_counter - global tcl_platform - return .punkutil_$prefix[pid]_[clock microseconds]_[incr tmpfile_counter]_[info hostname]_$tcl_platform(user) - } - - proc tmpdir {} { - # Taken from tcllib fileutil. - global tcl_platform env - - set attempdirs [list] - set problems {} - - foreach tmp {TEMP TMP TMPDIR} { - if { [info exists env($tmp)] } { - lappend attempdirs $env($tmp) - } else { - lappend problems "No environment variable $tmp" - } - } - - switch $tcl_platform(platform) { - windows { - lappend attempdirs "C:\\TEMP" "C:\\TMP" "\\TEMP" "\\TMP" - } - macintosh { - lappend attempdirs $env(TRASH_FOLDER) ;# a better place? - } - default { - lappend attempdirs \ - [file join / tmp] \ - [file join / var tmp] \ - [file join / usr tmp] - } - } - - lappend attempdirs [pwd] - - foreach tmp $attempdirs { - if { [file isdirectory $tmp] && - [file writable $tmp] } { - return [file normalize $tmp] - } elseif { ![file isdirectory $tmp] } { - lappend problems "Not a directory: $tmp" - } else { - lappend problems "Not writable: $tmp" - } - } - - # Fail if nothing worked. - return -code error "Unable to determine a proper directory for temporary files\n[join $problems \n]" - } - - - -} - - - - - - - - - - - - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Ready -package provide punk::mix::util [namespace eval punk::mix::util { - variable version - set version 0.1.0 -}] -return diff --git a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/ns-0.1.0.tm b/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/ns-0.1.0.tm deleted file mode 100644 index 156d51d1..00000000 --- a/src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/ns-0.1.0.tm +++ /dev/null @@ -1,1698 +0,0 @@ -# -*- tcl -*- -# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from -buildversion.txt -# -# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem. -# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository. -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -# (C) 2023 -# -# @@ Meta Begin -# Application punk::ns 0.1.0 -# Meta platform tcl -# Meta license -# @@ Meta End - - - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -## Requirements -##e.g package require frobz - - -namespace eval ::punk_dynamic::ns { - -} - -# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++ -namespace eval punk::ns { - variable ns_current "::" - variable ns_re_cache [dict create] ;#cache regular expressions used in globmatchns - namespace export nsjoin nsprefix nstail nsparts nseval nsimport_noclobber corp - - #leading colon makes it hard (impossible?) to call directly if not within the namespace - proc ns/ {v {ns_or_glob ""} args} { - variable ns_current ;#change active ns of repl by setting ns_current - - set ns_caller [uplevel 1 {::namespace current}] - #puts stderr "ns_cur:$ns_current ns_call:$ns_caller" - - - set types [list all] - set nspathcommands 0 - if {$v eq "/"} { - set types [list children] - } - if {$v eq "///"} { - set nspathcommands 1 - } - - #todo - cooperate with repl? - set out "" - if {$ns_or_glob eq ""} { - set is_absolute 1 - set ns_queried $ns_current - set out [nslist [nsjoin $ns_current *] -types $types -nspathcommands $nspathcommands] - } else { - set is_absolute [string match ::* $ns_or_glob] - set has_globchars [regexp {[*?]} $ns_or_glob] - if {$is_absolute} { - if {!$has_globchars} { - if {![namespace exists $ns_or_glob]} { - error "cannot change to namespace $ns_or_glob" - } - set ns_current $ns_or_glob - set ns_queried $ns_current - tailcall ns/ $v "" - } else { - set ns_queried $ns_or_glob - set out [nslist $ns_or_glob -types $types -nspathcommands $nspathcommands] - } - } else { - if {!$has_globchars} { - set nsnext [nsjoin $ns_current $ns_or_glob] - if {![namespace exists $nsnext]} { - error "cannot change to namespace $ns_or_glob" - } - set ns_current $nsnext - set ns_queried $nsnext - set out [nslist [nsjoin $nsnext *] -types $types -nspathcommands $nspathcommands] - } else { - set ns_queried [nsjoin $ns_current $ns_or_glob] - set out [nslist [nsjoin $ns_current $ns_or_glob] -types $types -nspathcommands $nspathcommands] - } - } - } - set ns_display "\n$ns_queried" - if {$ns_current eq $ns_queried} { - if {$ns_current in [info commands $ns_current] } { - if {![catch [list namespace ensemble configure $ns_current] ensemble_info]} { - if {[llength $ensemble_info] > 0} { - #this namespace happens to match ensemble command. - #todo - keep cache of encountered ensembles from commands.. and examine namespace in the configure info. - set ns_display "\n[a+ yellow bold]$ns_current (ensemble)[a+]" - } - } - } - } - append out $ns_display - return $out - - - } - - - #create possibly nested namespace structure - but only if not already existant - proc n/new {args} { - variable ns_current - if {![llength $args]} { - error "usage: :/new \[ ...\]" - } - set a1 [lindex $args 0] - set is_absolute [string match ::* $a1] - if {$is_absolute} { - set nspath [nsjoinall {*}$args] - } else { - if {[string match :* $a1]} { - puts stderr "n/new WARNING namespace with leading colon '$a1' is likely to have unexpected results" - } - set nspath [nsjoinall $ns_current {*}$args] - } - - set ns_exists [nseval [nsprefix $nspath] [list ::namespace exists [nstail $nspath] ]] - - if {$ns_exists} { - error "Namespace $nspath already exists" - } - #namespace eval [nsprefix $nspath] [list namespace eval [nstail $nspath] {}] - nseval [nsprefix $nspath] [list ::namespace eval [nstail $nspath] {}] - n/ $nspath - } - - - #nn/ ::/ nsup/ - back up one namespace level - proc nsup/ {v args} { - variable ns_current - if {$ns_current eq "::"} { - puts stderr "Already at global namespace '::'" - } else { - set out "" - set nsq [nsprefix $ns_current] - if {$v eq "/"} { - set out [get_nslist -match [nsjoin $nsq *] -types [list children]] - } else { - set out [get_nslist -match [nsjoin $nsq *] -types [list all]] - } - #set out [nslist [nsjoin $nsq *]] - set ns_current $nsq - append out "\n$ns_current" - return $out - } - } - - #todo - walk up each ns - testing for possibly weirdly named namespaces - #review - do we even need it. - proc nsexists {nspath} { - error "unimplemented" - } - - #recursive nseval - for introspection of weird namespace trees - #approx 10x slower than normal namespace eval - but still only a few microseconds.. fine for repl introspection - proc nseval_script {location} { - set parts [nsparts $location] - if {[lindex $parts 0] eq ""} { - lset parts 0 :: - } - if {[lindex $parts end] eq ""} { - set parts [lrange $parts 0 end-1] - } - - set body "" - set i 0 - set tails [lrepeat [llength $parts] ""] - foreach ns $parts { - set cmdlist [list ::namespace eval $ns] - set t "" - if {$i > 0} { - append body " " - } - append body $cmdlist - if {$i == ([llength $parts] -1)} { - append body "