make.tcl fixes, commandline script.tcl execution fixes, layout template updates

.fossil-settings/ignore-glob

@@ -1,4 +1,5 @@
 .git
+CVS
 bin
 lib
 #The directory for compiled/built Tcl modules

src/bootsupport/modules/oolib-0.1.tm

@@ -0,0 +1,195 @@
+#JMN - api should be kept in sync with package patternlib where possible
+#
+package provide oolib [namespace eval oolib {
+    variable version
+    set version 0.1
+}]
+
+namespace eval oolib {
+    oo::class create collection {
+        variable o_data ;#dict
+        variable o_alias
+        constructor {} {
+            set o_data [dict create]
+        }
+        method info {} {
+            return [dict info $o_data]
+        }
+        method count {} {
+            return [dict size $o_data]
+        }
+        method isEmpty {} {
+            expr {[dict size $o_data] == 0}
+        }
+        method names {{globOrIdx {}}} {
+            if {[llength $globOrIdx]} {
+                if {[string is integer -strict $globOrIdx]} {
+                    if {$idx < 0} {
+                        set idx "end-[expr {abs($idx + 1)}]"
+                    }
+                    if {[catch {lindex [dict keys $o_data] $idx} result]} {
+                        error "[self object] no such index : '$idx'"
+                    } else {
+                        return $result
+                    }
+                } else {
+                    #glob
+                    return [lsearch -glob -all -inline [dict keys $o_data] $globOrIdx]
+                }
+            } else {
+                return [dict keys $o_data]
+            }
+        }
+        #like names but without globbing
+        method keys {} {
+            dict keys $o_data
+        }
+        method key {{posn 0}} {
+            if {$posn < 0} {
+                set posn "end-[expr {abs($posn + 1)}]"
+            }
+            if {[catch {lindex [dict keys $o_data] $posn} result]} {
+                error "[self object] no such index : '$posn'"
+            } else {
+                return $result
+            }
+        }
+        method hasKey {key} {
+            dict exists $o_data $key
+        }
+        method get {} {
+            return $o_data
+        }
+        method items {} {
+            return [dict values $o_data]
+        }
+        method item {key} {
+            if {[string is integer -strict $key]} {
+                if {$key > 0} {
+                    set valposn [expr {(2*$key) +1}]
+                    return [lindex $o_data $valposn]
+                } else {
+                    set key "end-[expr {abs($key + 1)}]"
+                    return [lindex [dict keys $o_data] $key]
+                }
+            }
+            if {[dict exists $o_data $key]} {
+                return [dict get $o_data $key]
+            }
+        }
+        #inverse lookup
+        method itemKeys {value} {
+            set value_indices [lsearch -all [dict values $o_data] $value]
+            set keylist [list]
+            foreach i $value_indices {
+                set idx [expr {(($i + 1) *2) -2}]
+                lappend keylist [lindex $o_data $idx]
+            }
+            return $keylist
+        }
+        method search {value args} {
+            set matches [lsearch {*}$args [dict values $o_data] $value]
+            if {"-inline" in $args} {
+                return $matches
+            } else {
+                set keylist [list]
+                foreach i $matches {
+                    set idx [expr {(($i + 1) *2) -2}]
+                    lappend keylist [lindex $o_data $idx]
+                }
+                return $keylist
+            }
+        }
+        #review - see patternlib. Is the intention for aliases to be configurable independent of whether the target exists?
+        method alias {newAlias existingKeyOrAlias} {
+            if {[string is integer -strict $newAlias]} {
+                error "[self object] collection key alias cannot be integer"
+            }
+            if {[string length $existingKeyOrAlias]} {
+                set o_alias($newAlias) $existingKeyOrAlias
+            } else {
+                unset o_alias($newAlias)
+            }
+        }
+        method aliases {{key ""}} {
+            if {[string length $key]} {
+                set result [list]
+                foreach {n v} [array get o_alias] {
+                    if {$v eq $key} {
+                        lappend result $n $v
+                    }
+                }		
+                return $result
+            } else {
+                return [array get o_alias]
+            }
+        }
+        #if the supplied index is an alias, return the underlying key; else return the index supplied.
+        method realKey {idx} {
+            if {[catch {set o_alias($idx)} key]} {
+                return $idx
+            } else {
+                return $key
+            }
+        }
+        method add {value key} {
+            if {[string is integer -strict $key]} {
+                error "[self object] collection key must not be an integer. Use another structure if integer keys required"
+            }
+            if {[dict exists $o_data $key]} {
+                error "[self object] col_processors object error: key '$key' already exists in collection"
+            }
+            dict set o_data $key $value
+            return [expr {[dict size $o_data] - 1}] ;#return index of item
+        }
+        method remove {idx {endRange ""}} {
+            if {[string length $endRange]} {
+                error "[self object] collection error: ranged removal not yet implemented.. remove one item at a time"
+            }
+            if {[string is integer -strict $idx]} {
+                if {$idx < 0} {
+                    set idx "end-[expr {abs($idx+1)}]"
+                }
+                set key [lindex [dict keys $o_data] $idx]
+                set posn $idx
+            } else {
+                set key $idx
+                set posn [lsearch -exact [dict keys $o_data] $key]
+                if {$posn < 0} {
+                    error "[self object] no such index: '$idx' in this collection"
+                }
+            }
+            dict unset o_data $key
+            return
+        }
+        method clear {} {
+            set o_data [dict create]
+            return
+        }
+        method reverse {} {
+            set dictnew [dict create]
+            foreach k [lreverse [dict keys $o_data]] {
+                dict set dictnew $k [dict get $o_data $k]
+            }
+            set o_data $dictnew
+            return
+        }
+        #review - cmd as list vs cmd as script?
+        method map {cmd} {
+            set seed [list]
+            dict for {k v} $o_data {
+                lappend seed [uplevel #0 [list {*}$cmd $v]]
+            }
+            return $seed
+        }
+        method objectmap {cmd} {
+            set seed [list]
+            dict for {k v} $o_data {
+                lappend seed [uplevel #0 [list $v {*}$cmd]]
+            }
+            return $seed
+        }
+    }
+
+}
+

src/bootsupport/modules/punk/du-0.1.0.tm

@@ -9,7 +9,7 @@
 # @@ Meta Begin
 # Application punk::du 0.1.0
 # Meta platform     tcl
-# Meta license      <unspecified>
+# Meta license      BSD 
 # @@ Meta End
 
 
@@ -17,6 +17,8 @@
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 ##  Requirements
 ##e.g package require frobz
+package require punk::mix::base
+
 
 namespace eval punk::du {
     variable has_twapi 0
@@ -29,7 +31,7 @@ if {"windows" eq $::tcl_platform(platform)} {
     #} else {
     #    set punk::du::has_twapi 1
     #}
-    package require punk::winpath
+    #package require punk::winpath
 }
 
 
@@ -37,7 +39,14 @@ if {"windows" eq $::tcl_platform(platform)} {
 namespace eval punk::du {
     
 
-    proc dirlisting {{folderpath {}}} {
+    proc dirlisting {folderpath args} {
+        set defaults [dict create\
+            -glob *\
+        ]
+        set opts [dict merge $defaults $args]
+        # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+        set opt_glob            [dict get $opts -glob]
+        # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
         if {[lib::pathcharacterlen $folderpath] == 0} {
            set folderpath [pwd]
         } elseif {[file pathtype $folderpath] ne "absolute"} {
@@ -46,7 +55,7 @@ namespace eval punk::du {
             
         }
         #run whichever of du_dirlisting_twapi, du_dirlisting_generic, du_dirlisting_unix has been activated
-        set dirinfo [active::du_dirlisting $folderpath]
+        set dirinfo [active::du_dirlisting $folderpath {*}$opts]
     }
 
     
@@ -151,6 +160,8 @@ namespace eval punk::du {
             set opt_extra 1
         }
         set opt_vfs 0
+        #This configures whether to enter a vfsmount point 
+        #It will have no effect if cwd already with a vfs mount point - as then opt_vfs will be set to 1 automatically anyway.
         if {"--vfs" in $lc_opts} {
             set opt_vfs 1
         }
@@ -213,10 +224,18 @@ namespace eval punk::du {
                 #e.g tcl glob based dirlisting doesn't support gathering file sizes at the same time
               
                 set in_vfs 0
-                if {$opt_vfs} {
+                if {[package provide vfs] ne ""} {
                     foreach vfsmount [vfs::filesystem info] {
-                        if {[punk::repo::path_a_atorbelow_b $folderpath $vfsmount]} {
+                        if {[file pathtype $folderpath] ne "absolute"} {
+                            set testpath [file normalize $folderpath]
+                        } else {
+                            set testpath $folderpath
+                        }
+                            
+                        if {[punk::mix::base::lib::path_a_atorbelow_b $testpath $vfsmount]} {
                             set in_vfs 1
+                            #if already descended to or below a vfs mount point - set opt_vfs true
+                            set opt_vfs 1
                             break
                         }
                     }
@@ -233,6 +252,26 @@ namespace eval punk::du {
                 set dirs [dict get $du_info dirs]
                 set files [dict get $du_info files]
                 set filesizes [dict get $du_info filesizes]
+                set vfsmounts [dict get $du_info vfsmounts]
+                #puts "---> vfsmounts $vfsmounts "
+                if {$opt_vfs} {
+                    foreach vm $vfsmounts {
+                        #puts stderr "vm: $vm"
+                        #check if vfs is mounted over a file or a dir
+                        if {$vm in $files} {
+                            puts stderr "vfs mounted over file $vm"
+                            set mposn [lsearch $files $vm]
+                            set files [lreplace $files $mposn $mposn]
+                            if {[llength $filesizes]} {
+                                set filesizes [lreplace $filesizes $mposn $mposn]
+                            }
+                        }
+                        if {$vm ni $dirs} {
+                            puts stderr "treating $vm as dir"
+                            lappend dirs $vm
+                        }
+                    }
+                }
 
 
                 incr leveldirs  [llength $dirs]
@@ -361,7 +400,7 @@ namespace eval punk::du {
         variable functions_known [dict create]
 
         #known functions from lib namespace
-        dict set functions_known du_dirlisting [list du_dirlisting_twapi du_dirlisting_generic du_dirlisting_unix]
+        dict set functions_known du_dirlisting [list du_dirlisting_twapi du_dirlisting_generic du_dirlisting_unix du_dirlisting_undecided]
 
         proc show_functions {} {
             variable functions
@@ -443,10 +482,11 @@ namespace eval punk::du {
                     dict set result -shortname [dict get $iteminfo altname]
                     dict set result -rawflags $attrinfo
                     set extras [list]
-                    foreach prop {ctime atime mtime size} {
+                    #foreach prop {ctime atime mtime size} {
-                        lappend extras $prop [dict get $iteminfo $prop]
+                    #    lappend extras $prop [dict get $iteminfo $prop]
-                    }
+                    #}
-                    dict set result -extras $extras
+                    #dict set result -extras $extras
+                    dict set result -raw $iteminfo
                     return $result
                 } else {
                     error "could not read attributes for $path"
@@ -455,29 +495,101 @@ namespace eval punk::du {
                 catch {twapi::find_file_close $iterator}
             }
         }
-        namespace export du_dirlisting_twapi du_dirlisting_generic du_dirlisting_unix
+
+        #todo - review 'errors' key. We have errors relating to containing folder and args vs per child-item errors - additional key needed?
+        namespace export du_dirlisting_twapi du_dirlisting_generic du_dirlisting_unix du_dirlisting_undecided
         # get listing without using unix-tools (may not be installed on the windows system)
         # this dirlisting is customised for du - so only retrieves dirs,files,filesizes (minimum work needed to perform du function)
-        proc du_dirlisting_twapi {folderpath} {
+        # This also preserves path rep for elements in the dirs/folders keys etc - which can make a big difference in performance 
+        proc du_dirlisting_twapi {folderpath args} {
+            set defaults [dict create\
+                -glob *\
+                -with_sizes 1\
+                -with_times 1\
+            ]
+            set opts [dict merge $defaults $args]
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+            set opt_glob            [dict get $opts -glob]
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+            set opt_with_sizes      [dict get $opts -with_sizes]
+            set ftypes [list f d l]
+            if {"$opt_with_sizes" in {0 1}} {
+                #don't use string is boolean - (f false vs f file!)
+                #only accept 0|1 
+                if {$opt_with_sizes} {
+                    set sized_types $ftypes
+                } else {
+                    set sized_types [list]
+                }
+            } else {
+                set sized_types $opt_with_sizes
+            }
+            if {[llength $sized_types]} {
+                foreach st $sized_types {
+                    if {$st ni $ftypes} {
+                        error "du_dirlisting_twapi unrecognized element in -with_sizes '$st'"
+                    }
+                }
+            }
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+            set opt_with_times      [dict get $opts -with_times]
+            if {"$opt_with_times" in {0 1}} {
+                if {$opt_with_times} {
+                    set timed_types $ftypes
+                } else {
+                    set timed_types [list]
+                }
+            } else {
+                set timed_types $opt_with_times
+            }
+            if {[llength $timed_types]} {
+                foreach item $timed_types {
+                    if {$item ni $ftypes} {
+                        error "du_dirlisting_twapi unrecognised element in -with-times '$item'"
+                    }
+                }
+            }
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+
+            set errors [dict create]
             set altname "" ;#possible we have to use a different name e.g short windows name or dos-device path //?/
             # return it so it can be stored and tried as an alternative for problem paths
+            #puts stderr ">>> glob: $opt_glob"
+            #REVIEW! windows api pattern matchttps://www.red-gate.com/simple-talk/blogs/the-unexpected-behaviour-of-directoryinfo-getfiles-with-three-letter-extensions/hing is .. weird. partly due to 8.3 filenames
+            #https://www.red-gate.com/simple-talk/blogs/the-unexpected-behaviour-of-directoryinfo-getfiles-with-three-letter-extensions/
+            #we will certainly need to check the resulting listing with our supplied glob.. but maybe we will have to change the glob passed to find_file_open too.
+            # using * all the time may be inefficient - so we might be able to avoid that in some cases.
             try {
-                set iterator [twapi::find_file_open [file join $folderpath *] -detail basic] ;# -detail full only adds data to the altname field
+                #glob of * will return dotfiles too on windows
+                set iterator [twapi::find_file_open [file join $folderpath $opt_glob] -detail basic] ;# -detail full only adds data to the altname field
             } on error args {
                 try {
                     if {[string match "*denied*" $args]} {
                         #output similar format as unixy du
                         puts stderr "[file dirname $folderpath] '[file tail $folderpath]':$args"
-                        return [list dirs {} vfsmounts {} links {} files {} filesizes {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {}]
+                        dict lappend errors $folderpath $::errorCode
+                        return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors]
                     }
                     if {[string match "*TWAPI_WIN32 59*" $::errorCode]} {
                         puts stderr "[file dirname $folderpath] '[file tail $folderpath]':$args (possibly blocked by permissions or share config e.g follow symlinks = no on samba)"
                         puts stderr "  (errorcode: $::errorCode)\n"
-                        return [list dirs {} vfsmounts {} links {} files {} filesizes {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {}]
+                        dict lappend errors $folderpath $::errorCode
+                        return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors]
                     }
 
 
-
+                    #errorcode TWAPI_WIN32 2 {The system cannot find the file specified.}
+                    #This can be a perfectly normal failure to match the glob.. which means we shouldn't really warn or error
+                    #The find-all glob * won't get here because it returns . & .. 
+                    #so we should return immediately only if the glob has globchars ? or *  but isn't equal to just "*" ? (review)
+                    #Note that windows glob ? seems to return more than just single char results - it includes .. - which differs to tcl glob 
+                    #also ???? seems to returns items 4 or less - not just items exactly 4 long (review - where is this documented?)
+                    if {$opt_glob ne "*" && [regexp {[?*]} $opt_glob]} {
+                        if {[string match "*TWAPI_WIN32 2 *" $::errorCode]} {
+                            #looks like an ordinary no results for chosen glob
+                            return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors]
+                        }
+                    }
 
 
                     if {[set plen [pathcharacterlen $folderpath]] >= 250} {
@@ -503,15 +615,17 @@ namespace eval punk::du {
                         set errmsg "error reading folder: $folderpath (len:$plen)\n"
                         append errmsg "error: $args" \n
                         append errmsg "errorcode: $::errorCode" \n
+                        set tmp_errors [list $::errorCode]
                         #possibly an illegal windows filename - easily happens on a machine with WSL or with drive mapped to unix share
                         #we can use //?/path  dos device path - but not with tcl functions
                         #unfortunately we can't call find_file_open directly on the problem name - we have to call the parent folder and iterate through again..
                         #this gets problematic as we go deeper unless we rewrite the .. but we can get at least one level further here
                        
+                        set fixedtail ""
+
                         set parent [file dirname $folderpath]
                         set badtail [file tail $folderpath]
                         set iterator [twapi::find_file_open [file join $parent *] -detail full] ;#retrieve with altnames
-                        set fixedtail ""
                         while {[twapi::find_file_next $iterator iteminfo]} {
                             set nm [dict get $iteminfo name]
                             if {$nm eq $badtail} {
@@ -521,9 +635,11 @@ namespace eval punk::du {
                         }
 
                         if {![string length $fixedtail]} {
+                            dict lappend errors $folderpath {*}$tmp_errors
                             puts stderr "[file dirname $folderpath] '[file tail $folderpath]':$args (Unable to retrieve altname to progress further with path - returning no contents for this folder)"
-                            return [list dirs {} vfsmounts {} links {} files {} filesizes {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {}]
+                            return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors]
                         }
+
                         #twapi as at 2023-08 doesn't seem to support //?/ dos device paths.. 
                         #Tcl can test only get as far as testing existence of illegal name by prefixing with //?/ - but can't glob inside it
                         #we can call file attributes on it - but we get no shortname (but we could get shortname for parent that way)
@@ -537,21 +653,35 @@ namespace eval punk::du {
                         
                     }
 
-
+                    if {[catch {
                         set iterator [twapi::find_file_open $fixedpath/* -detail basic]
+                    } errMsg]} {
+                        puts stderr "[file dirname $folderpath] '[file tail $folderpath]':$args (failed to read even with fixedpath:'$fixedpath')"
+                        puts stderr "  (errorcode: $::errorCode)\n"
+                        puts stderr "$errMsg"   
+                        dict lappend errors $folderpath $::errorCode
+                        return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors]
+                    }
 
 
                 } on error args {
                         set errmsg "error reading folder: $folderpath\n"
-                        append errmsg "error: $args"
+                        append errmsg "error: $args" \n
-                        append errmsg "aborting.."
+                        append errmsg "errorInfo: $::errorInfo" \n
-                        error $errmsg
+                        puts stderr "$errmsg"
+                        puts stderr "FAILED to collect info for folder '$folderpath'"
+                        #append errmsg "aborting.."
+                        #error $errmsg
+                        return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors]
                     
                 }
             }
             set dirs [list]
             set files [list]
             set filesizes [list]
+            set allsizes [dict create]
+            set alltimes [dict create]
+
             set links [list]
             set flaggedhidden [list]
             set flaggedsystem [list]
@@ -559,13 +689,19 @@ namespace eval punk::du {
 
             while {[twapi::find_file_next $iterator iteminfo]} {
                 set nm [dict get $iteminfo name]
+                #recheck glob
+                #review!
+                if {![string match $opt_glob $nm]} {
+                    continue
+                }
                 set tail_altname [dict get $iteminfo altname] ;#altname of tail - not whole path
                 set attrinfo [decode_win_attributes [dict get $iteminfo attrs]]
                 #puts stderr "$iteminfo"
                 #puts stderr "$nm -> [dict get $iteminfo attrs] -> $attrinfo"
-
+                set ftype ""
                 #attributes applicable to any classification
                 set fullname [file_join_one $folderpath $nm]
+                
                 if {"hidden" in $attrinfo} {
                     lappend flaggedhidden $fullname 
                 }
@@ -578,6 +714,13 @@ namespace eval punk::du {
 
                 #main classification
                 if {"reparse_point" in $attrinfo} {
+                    #this concept doesn't correspond 1-to-1 with unix links
+                    #https://learn.microsoft.com/en-us/windows/win32/fileio/reparse-points
+                    #review - and see which if any actually belong in the links key of our return
+
+
+                    #One thing it could be, is a 'mounted folder' https://learn.microsoft.com/en-us/windows/win32/fileio/determining-whether-a-directory-is-a-volume-mount-point
+                    #
                     #we will treat as  zero sized for du purposes.. review - option -L  for symlinks like BSD du?
                     #Note 'file readlink' can fail on windows - reporting 'invalid argument' - according to tcl docs, 'On systems that don't support symbolic links this option is undefined'
                     #The link may be viewable ok in windows explorer, and cmd.exe /c dir and unix tools such as ls
@@ -593,36 +736,73 @@ namespace eval punk::du {
                     #Note also - a shortcut created in explorer with drag and drop to an existant folder is a different animal to a symlink (file with .lnk extension) even though it looks the same in explorer window.
                     #
                     #links are techically files too, whether they point to a file/dir or nothing.
+
                     lappend links $fullname
+                    set ftype "l"
                 } elseif {"directory" in $attrinfo} {
                     if {$nm in {. ..}} {
                         continue
                     }
                     lappend dirs $fullname
+                    set ftype "d"
                 } else {
                     
                     #review - is anything that isn't a reparse_point or a directory, some sort of 'file' in this context? What about the 'device' attribute? Can that occur in a directory listing of some sort?
                     lappend files $fullname
+                    if {"f" in $sized_types} {
                         lappend filesizes   [dict get $iteminfo size]
                     }
+                    set ftype "f"
+                }
+                if {$ftype in $sized_types} {
+                    dict set allsizes $fullname [dict create bytes [dict get $iteminfo size]]
+                }
+                if {$ftype in $timed_types} {
+                    #convert time from windows (100ns units since jan 1, 1601) to Tcl time (seconds since Jan 1, 1970)
+                    #We lose some precision by not passing the boolean to the large_system_time_to_secs_since_1970 function which returns fractional seconds
+                    #but we need to maintain compatibility with other platforms and other tcl functions so if we want to return more precise times we will need another flag and/or result dict
+                    dict set alltimes $fullname [dict create\
+                        c [twapi::large_system_time_to_secs_since_1970 [dict get $iteminfo ctime]]\
+                        a [twapi::large_system_time_to_secs_since_1970 [dict get $iteminfo atime]]\
+                        m [twapi::large_system_time_to_secs_since_1970 [dict get $iteminfo mtime]]\
+                    ]
+                }
             }
             twapi::find_file_close $iterator
             set vfsmounts [get_vfsmounts_in_folder $folderpath]
+
+            set effective_opts $opts
+            dict set effective_opts -with_times $timed_types
+            dict set effective_opts -with_sizes $sized_types
+
             #also determine whether vfs.  file system x  is *much* faster than file attributes
             #whether or not there is a corresponding file/dir add any applicable mountpoints for the containing folder
-            return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes $filesizes flaggedhidden $flaggedhidden flaggedsystem $flaggedsystem flaggedreadonly $flaggedreadonly altname $altname]
+            return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes $filesizes sizes $allsizes times $alltimes flaggedhidden $flaggedhidden flaggedsystem $flaggedsystem flaggedreadonly $flaggedreadonly altname $altname opts $effective_opts errors $errors]
         }
         proc get_vfsmounts_in_folder {folderpath} {
             set vfsmounts [list]
+            if {![llength [package provide vfs]]} {
+               return [list] 
+            }
+            set fpath [punk::objclone $folderpath]
+            set is_rel 0
+            if {[file pathtype $fpath] ne "absolute"} {
+                set fpath [file normalize $fpath]
+                set is_rel 1
+            }
             set known_vfs_mounts [vfs::filesystem info]
             foreach mount $known_vfs_mounts {
-                if {[punk::repo::path_a_above_b $folderpath $mount]} {
+                if {[punk::mix::base::lib::path_a_above_b $fpath $mount]} {
-                    if {([llength [file split $mount]] - [llength [file split $folderpath]]) == 1} {
+                    if {([llength [file split $mount]] - [llength [file split $fpath]]) == 1} {
                         #the mount is in this folder
+                        if {$is_rel} {
+                            lappend vfsmounts [file join $folderpath [file tail $mount]]
+                        } else {
                             lappend vfsmounts $mount
                         }
                     }
                 }
+            }
             return $vfsmounts
         }
         #work around the horrible tilde-expansion thing (not needed for tcl 9+)
@@ -636,7 +816,66 @@ namespace eval punk::du {
 
         #this is the cross-platform pure-tcl version - which calls glob multiple times to make sure it gets everythign it needs and can ignore everything it needs to.
         #These repeated calls to glob will be a killer for performance  - especially on a network share or when walking a large directory structure
-        proc du_dirlisting_generic {folderpath} {
+        proc du_dirlisting_generic {folderpath args} {
+            set defaults [dict create\
+                -glob *\
+                -with_sizes 0\
+                -with_times 0\
+            ]
+            set errors [dict create]
+            set known_opts [dict keys $defaults] 
+            foreach k [dict keys $args] {
+                if {$k ni $known_opts} {
+                    error  "du_dirlisting_generic unknown-option $k"
+                }
+            }
+            set opts [dict merge $defaults $args]
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+            set opt_glob            [dict get $opts -glob]
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+            set opt_with_sizes      [dict get $opts -with_sizes]
+            set ftypes [list f d l]
+            if {"$opt_with_sizes" in {0 1}} {
+                #dn't use string is boolean (false vs f problem)
+                if {$opt_with_sizes} {
+                    set sized_types $ftypes
+                } else {
+                    set sized_types [list]
+                }
+            } else {
+                set sized_types $opt_with_sizes
+            }
+            if {[llength $sized_types]} {
+                foreach st $sized_types {
+                    if {$st ni $ftypes} {
+                        error "du_dirlisting_generic unrecognized element in -with_sizes '$st'"
+                    }
+                }
+            }
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+            set opt_with_times      [dict get $opts -with_times]
+            if {"$opt_with_times" in {0 1}} {
+                if {$opt_with_times} {
+                    set timed_types $ftypes
+                } else {
+                    set timed_types [list]
+                }
+            } else {
+                set timed_types $opt_with_times
+            }
+            if {[llength $timed_types]} {
+                foreach item $timed_types {
+                    if {$item ni $ftypes} {
+                        error "du_dirlisting_generic unrecognised element in -with-times '$item'"
+                    }
+                }
+            }
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+            # The repeated globs are a source of slowness for this function.
+            #TODO - we could minimize the number of globs if we know we need to do a file stat and/or file attributes on each entry anyway
+            #For the case where we don't need times,sizes or other metadata - it is faster to do multiple globs
+            #This all makes this function complicated to gather the required data efficiently.
+            
             #note platform differences between what is considered hidden make this tricky.
             # on windows 'glob .*' will not return some hidden dot items but will return . ..   and glob -types hidden .* will not return some dotted items
             # glob -types hidden *  on windows will not necessarily return all dot files/folders
@@ -644,6 +883,10 @@ namespace eval punk::du {
             #  we need to process * and .* in the same glob calls and remove duplicates 
             #  if we do * and .* in separate iterations of this loop we lose the ability to filter duplicates easily
 
+            #note - with this design, we can't glob for all except dotfiles - this is for cross-platform consistency and efficiency (Review).
+            #dotfiles aren't considered hidden on all platforms
+            #some sort of antiglob is a possible enhancement and more consistent with the view that leading dot should be treated as any other filename character in this context.
+            if {$opt_glob eq "*"} {
                 #Note - glob operations seem to be faster than looped tests like 'file isdirectory' & 'file readlink' 
                 #set parent [lindex $folders $folderidx]
                 set hdirs  [glob -nocomplain -dir $folderpath -types {hidden d}  * .*]
@@ -659,13 +902,68 @@ namespace eval punk::du {
                 #set hfiles {}
                 set files  [glob -nocomplain -dir $folderpath -types f           * .*]
                 #set files  {}
+            } else {
+                set hdirs  [glob -nocomplain -dir $folderpath -types {hidden d}   $opt_glob]
+                set dirs   [glob -nocomplain -dir $folderpath -types d            $opt_glob]
+
+                set hlinks [glob -nocomplain -dir $folderpath -types {hidden l}   $opt_glob]
+                set links  [glob -nocomplain -dir $folderpath -types l            $opt_glob] ;#links may have dupes - we don't care. struct::set difference will remove
+
+                set hfiles [glob -nocomplain -dir $folderpath -types {hidden f}   $opt_glob]
+                set files  [glob -nocomplain -dir $folderpath -types f            $opt_glob]
+            }
 
             #note struct::set difference produces unordered result
             #struct::set difference removes duplicates
             #remove links and . .. from directories, remove links from files
-            set dirs  [struct::set difference [concat $hdirs $dirs[unset dirs]]    [concat $links [list [file join $folderpath .]  [file join $folderpath ..] ]]]
             set files [struct::set difference [concat $hfiles $files[unset files]] $links]
+            set dirs  [struct::set difference [concat $hdirs $dirs[unset dirs]]    [concat $links [list [file join $folderpath .]  [file join $folderpath ..] ]]]
             set links [lsort -unique [concat $links $hlinks]]
+
+            set meta_dict [dict create]
+            set meta_types [concat $sized_types $timed_types]
+            #known tcl stat keys 2023 - review
+            set empty_stat_dict [dict create atime {} ctime {} dev {} gid {} ino {} mode {} mtime {} nlink {} size {} type {} uid {}]
+            #make sure we call file stat only once per item
+            set statkeys [list]
+            if {[llength $meta_types]} {
+                foreach ft {f d l} lvar {files dirs links} {
+                    if {"$ft" in $meta_types} {
+                        foreach path [set $lvar] {
+                            #caller may have read perm on the containing folder - but not on child item - so file stat could raise an error
+                            if {![catch {file stat $path arrstat} errM]} {
+                                dict set meta_dict $path [dict create shorttype $ft {*}[array get arrstat]]
+                            } else {
+                                dict lappend errors $path "file stat error: $errM"
+                                dict set meta_dict $path [dict create shorttype $ft {*}$empty_stat_dict]
+                            }
+                        }
+                    }
+                }
+            }
+            set fsizes [list]
+            set allsizes [dict create]
+            set alltimes [dict create]
+            #review birthtime field of stat? cross-platform differences ctime etc?
+            dict for {path pathinfo} $meta_dict {
+                set ft [dict get $pathinfo shorttype]
+                if {$ft in $sized_types} {
+                    dict set allsizes $path [dict create bytes [dict get $pathinfo size]]
+                    if {$ft eq "f"} {
+                        lappend fsizes [dict get $pathinfo size]
+                    }
+                }
+                if {$ft in $timed_types} {
+                    dict set alltimes $path [dict create c [dict get $pathinfo ctime] a [dict get $pathinfo atime] m [dict get $pathinfo mtime]]
+                }
+            }
+            if {"f" in $sized_types} {
+                if {[llength $fsizes] ne [llength $files]} {
+                    dict lappend errors $folderpath "failed to retrieve all file sizes"
+                }
+            }
+
+
             if {"windows" eq $::tcl_platform(platform)} {
                 set flaggedhidden [concat $hdirs $hfiles $hlinks]
             } else {
@@ -676,32 +974,202 @@ namespace eval punk::du {
 
             set vfsmounts [get_vfsmounts_in_folder $folderpath]
 
-            set filesizes [list]; #not available in listing-call - as opposed to twapi which can do it as it goes
+            set effective_opts $opts
-            return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {}]
+            dict set effective_opts -with_times $timed_types
+            dict set effective_opts -with_sizes $sized_types
+
+            return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes $fsizes sizes $allsizes times $alltimes flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $effective_opts errors $errors]
+        }
+
+        proc du_dirlisting_tclvfs {folderpath args} {
+            set defaults [dict create\
+                -glob *\
+                -with_sizes 0\
+                -with_times 0\
+            ]
+            set opts [dict merge $defaults $args]
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+            set opt_glob            [dict get $opts -glob]
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+            set opt_with_sizes      [dict get $opts -with_sizes]
+            set ftypes [list f d l]
+            if {"$opt_with_sizes" in {0 1}} {
+                #dn't use string is boolean (false vs f problem)
+                if {$opt_with_sizes} {
+                    set sized_types $ftypes
+                } else {
+                    set sized_types [list]
+                }
+            } else {
+                set sized_types $opt_with_sizes
+            }
+            if {[llength $sized_types]} {
+                foreach st $sized_types {
+                    if {$st ni $ftypes} {
+                        error "du_dirlisting_generic unrecognized element in -with_sizes '$st'"
+                    }
+                }
+            }
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+            set opt_with_times      [dict get $opts -with_times]
+            if {"$opt_with_times" in {0 1}} {
+                if {$opt_with_times} {
+                    set timed_types $ftypes
+                } else {
+                    set timed_types [list]
+                }
+            } else {
+                set timed_types $opt_with_times
+            }
+            if {[llength $timed_types]} {
+                foreach item $timed_types {
+                    if {$item ni $ftypes} {
+                        error "du_dirlisting_generic unrecognised element in -with-times '$item'"
                     }
+                }
+            }
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
 
-        #we can halve the number of round trips on unix-like systems, where 'hidden' always corresponds to dotted files
+            set errors [dict create]
-        proc du_dirlisting_unix {folderpath} {
+            if {$opt_glob eq "*"} {
                 set dirs   [glob -nocomplain -dir $folderpath -types d           * .*] ;# also returns links to dirs
+                #review - how are links handled in vfs? presumably if the vfs is a full implementation they should work at least within the vfs? 
                 set links  [glob -nocomplain -dir $folderpath -types l           * .*]  ;# links may have dupes - we don't care. struct::set difference will remove
                 set files  [glob -nocomplain -dir $folderpath -types f           * .*] ;# also returns links to files
+            } else {
+                set dirs   [glob -nocomplain -dir $folderpath -types d           $opt_glob]
+                set links  [glob -nocomplain -dir $folderpath -types l           $opt_glob]
+                set files  [glob -nocomplain -dir $folderpath -types f           $opt_glob]
+            }
             #remove any links from our dirs and files collections
             set dirs  [struct::set difference $dirs[unset dirs]    [concat $links [list [file join $folderpath .] [file join $folderpath ..] ]]]
             set files [struct::set difference $files[unset files] $links]
+            #nested vfs mount.. REVIEW - does anything need special handling?
             set vfsmounts [get_vfsmounts_in_folder $folderpath]
-            return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {}]
+
+            set meta_dict [dict create]
+            set meta_types [concat $sized_types $timed_types]
+            #known tcl stat keys 2023 - review
+            set empty_stat_dict [dict create atime {} ctime {} dev {} gid {} ino {} mode {} mtime {} nlink {} size {} type {} uid {}]
+            #make sure we call file stat only once per item
+            set statkeys [list]
+            if {[llength $meta_types]} {
+                foreach ft {f d l} lvar {files dirs links} {
+                    if {"$ft" in $meta_types} {
+                        foreach path [set $lvar] {
+                            #caller may have read perm on the containing folder - but not on child item - so file stat could raise an error
+                            if {![catch {file stat $path arrstat} errM]} {
+                                dict set meta_dict $path [dict create shorttype $ft {*}[array get arrstat]]
+                            } else {
+                                dict lappend errors $path "file stat error: $errM"
+                                dict set meta_dict $path [dict create shorttype $ft {*}$empty_stat_dict]
+                            }
+                        }
+                    }
+                }
+            }
+            set fsizes [list]
+            set allsizes [dict create]
+            set alltimes [dict create]
+            #review birthtime field of stat? cross-platform differences ctime etc?
+            dict for {path pathinfo} $meta_dict {
+                set ft [dict get $pathinfo shorttype]
+                if {$ft in $sized_types} {
+                    dict set allsizes $path [dict create bytes [dict get $pathinfo size]]
+                    if {$ft eq "f"} {
+                        lappend fsizes [dict get $pathinfo size]
+                    }
+                }
+                if {$ft in $timed_types} {
+                    dict set alltimes $path [dict create c [dict get $pathinfo ctime] a [dict get $pathinfo atime] m [dict get $pathinfo mtime]]
+                }
+            }
+            if {"f" in $sized_types} {
+                if {[llength $fsizes] ne [llength $files]} {
+                    dict lappend errors $folderpath "failed to retrieve all file sizes"
+                }
+            }
+
+
+            set effective_opts $opts
+            dict set effective_opts -with_times $timed_types
+            dict set effective_opts -with_sizes $sized_types
+
+            return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes $fsizes sizes $allsizes times $alltimes flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $effective_opts errors $errors]
+        }
+
+        #we can halve the number of round trips on unix-like systems, where 'hidden' always corresponds to dotted files
+        proc du_dirlisting_unix {folderpath args} {
+            set defaults [dict create\
+                -glob *\
+                -with_sizes 0\
+                -with_times 0\
+            ]
+            set errors [dict create]
+            dict lappend errors $folderpath "metdata support incomplete - prefer du_dirlisting_generic"
+            set opts [dict merge $defaults $args]
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+            set opt_glob            [dict get $opts -glob]
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+            set opt_with_sizes      [dict get $opts -with_sizes]
+            set ftypes [list f d l]
+            if {"$opt_with_sizes" in {0 1}} {
+                #dn't use string is boolean (false vs f problem)
+                if {$opt_with_sizes} {
+                    set sized_types $ftypes
+                } else {
+                    set sized_types [list]
+                }
+            } else {
+                set sized_types $opt_with_sizes
+            }
+            if {[llength $sized_types]} {
+                foreach st $sized_types {
+                    if {$st ni $ftypes} {
+                        error "du_dirlisting_generic unrecognized element in -with_sizes '$st'"
+                    }
+                }
             }
-        proc du_dirlisting_tclvfs {folderpath} {
+            # -- --- --- --- --- --- --- --- --- --- --- --- --- ---
+            set opt_with_times      [dict get $opts -with_times]
+            if {"$opt_with_times" in {0 1}} {
+                if {$opt_with_times} {
+                    set timed_types $ftypes
+                } else {
+                    set timed_types [list]
+                }
+            } else {
+                set timed_types $opt_with_times
+            }
+            if {[llength $timed_types]} {
+                foreach item $timed_types {
+                    if {$item ni $ftypes} {
+                        error "du_dirlisting_generic unrecognised element in -with-times '$item'"
+                    }
+                }
+            }
+            
+            #this is not consistent with standard way * works for unix folders - but it is consistent with behaviour of this facility on windows 
+            if {$opt_glob eq "*"} {
                 set dirs   [glob -nocomplain -dir $folderpath -types d           * .*] ;# also returns links to dirs
-            #review - how are links handled in vfs? presumably if the vfs is a full implementation they should work at least within the vfs? 
                 set links  [glob -nocomplain -dir $folderpath -types l           * .*]  ;# links may have dupes - we don't care. struct::set difference will remove
                 set files  [glob -nocomplain -dir $folderpath -types f           * .*] ;# also returns links to files
+            } else {
+                set dirs   [glob -nocomplain -dir $folderpath -types d           $opt_glob]
+                set links  [glob -nocomplain -dir $folderpath -types l           $opt_glob]
+                set files  [glob -nocomplain -dir $folderpath -types f           $opt_glob]
+            }
             #remove any links from our dirs and files collections
             set dirs  [struct::set difference $dirs[unset dirs]    [concat $links [list [file join $folderpath .] [file join $folderpath ..] ]]]
             set files [struct::set difference $files[unset files] $links]
-            #nested vfs mount.. REVIEW - does anything need special handling?
             set vfsmounts [get_vfsmounts_in_folder $folderpath]
-            return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {}]
+
+            set effective_opts $opts
+            dict set effective_opts -with_times $timed_types
+            dict set effective_opts -with_sizes $sized_types
+
+
+            return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors]
         }
 
 
@@ -740,9 +1208,15 @@ namespace eval punk::du {
             return $newlist
         }
 
-        #just an experiment
+        #same implementation as punk::strlen
         #get length of path which has internal rep of path - maintaining path/list rep without shimmering to string representation.
         proc pathcharacterlen {pathrep} {
+            append str2 $pathrep {}
+            string length $str2
+        }
+        #just an experiment
+        proc pathcharacterlen1 {pathrep} {
+            #This works - but is unnecessarily complex
             set l 0
             set parts [file split $pathrep]
             if {[llength $parts] < 2} {
@@ -769,6 +1243,26 @@ namespace eval punk::du {
             }
         }
 
+        proc du_dirlisting_undecided {folderpath args} {
+            if {"windows" eq $::tcl_platform(platform)} {
+                set loadstate [zzzload::pkg_require twapi]
+                if {$loadstate ni [list loading failed]} {
+                    package require twapi ;#should be fast once twapi dll loaded in zzzload thread
+                    set ::punk::du::has_twapi 1
+                    punk::du::active::set_active_function du_dirlisting du_dirlisting_twapi
+                    tailcall du_dirlisting_twapi $folderpath {*}$args
+                } else {
+                    if {$loadstate eq "failed"} {
+                        puts stderr "punk::du defaulting to du_dirlisting_generic because twapi load failed"
+                        punk::du::active::set_active_function du_dirlisting du_dirlisting_generic
+                    }
+                    tailcall du_dirlisting_generic $folderpath {*}$args                        
+                }
+            } else {
+                punk::du::active::set_active_function du_dirlisting du_dirlisting_unix
+                tailcall du_dirlisting_unix $folderpath {*}$args
+            }
+        }
 
 
     }
@@ -791,15 +1285,8 @@ namespace eval punk::du {
     variable functions_kown
     upvar ::punk::du::has_twapi has_twapi
 
-    if {"windows" eq $::tcl_platform(platform)} {
+    set_active_function du_dirlisting du_dirlisting_undecided
-        if {$has_twapi} {
+
-            set_active_function du_dirlisting du_dirlisting_twapi
-        } else {
-            set_active_function du_dirlisting du_dirlisting_generic
-        }
-    } else {
-        set_active_function du_dirlisting du_dirlisting_unix
-    }
 }
 
 

src/bootsupport/modules/punk/repo-0.1.1.tm

@@ -767,6 +767,30 @@ namespace eval punk::repo {
 
         return $root_dict
     }
+    proc fossil_get_repository_file {{path {}}} {
+        if {$path eq {}} { set path [pwd] }
+        set fossilcmd [auto_execok fossil]
+        if {[llength $fossilcmd]} {
+            do_in_path $path {
+                set fossilinfo [::exec {*}$fossilcmd info]
+            }
+            set matching_lines [punk::repo::grep {repository:*} $fossilinfo]
+            if {![llength $matching_lines]} {
+                return ""
+            }
+            set trimmedline [string trim [lindex $matching_lines 0]]
+            set firstcolon [string first : $trimmedline]
+            set repofile_path [string trim [string range $trimmedline $firstcolon+1 end]]
+            if {![file exists $repofile_path]} {
+                puts stderr "Repository file pointed to by fossil configdb doesn't exist: $repofile_path"
+                return ""
+            }
+            return $repofile_path
+        } else {
+            puts stderr "fossil_get_repository_file: fossil command unavailable"
+            return "" 
+        }
+    }
     proc fossil_get_repository_folder_for_project {projectname args} {
 
         set defaults [list -parentfolder \uFFFF -extrachoice \uFFFF] 
@@ -1040,7 +1064,7 @@ namespace eval punk::repo {
             do_in_path $path {
                 set info [::exec {*}$fossilcmd remote ls]
             }
-            return [string trim $v]
+            return [string trim $info]
         } else {
             return Unknown
         }

src/bootsupport/modules/punk/winpath-0.1.0.tm

@@ -25,111 +25,18 @@
 namespace eval punk::winpath {
     namespace export winpath windir cdwin cdwindir illegalname_fix illegalname_test
 
-    #review - is this intended to be useful/callable on non-windows platforms?
-    #it should in theory be useable from another platform that wants to create a path for use on windows.
-    #In this case - we shouldn't examine what volumes exist (assume A: .. Z: are valid)
-    #review zipfs://  other uri schemes?
-    proc winpath {path} {
-        #NOTE: tcl file exists gives different answers on windows for paths like  /c depending on cwd (presumably based on file pathtype of volumerelative)
-        #This is add odds with attempting to navigate on a windows system which has cygwin, wsl etc... It also makes it difficult for functions intended to operate independent of CWD.
-        #e.g there is potential confusion when there is a c folder on c: drive  (c:/c)
-        #I will attempt to provide a coherent operation for winpath ./ ../ etc  , but it may disallow for example; change to /something  or /x  where these don't match a driveletter or /mnt
-        #whereas tcl may allow cd to /something  if a something folder happens to exist on the current volume based on cwd.
-        #I think it's preferable to require an explicit driveletter /x or /mnt  when using unix-like paths on windows - but practical considerations may prove me wrong.. 
-        #It's possible that this function should also ignore the current set of driveletters - and operate completely independent of whether a path actually exists
-        #This makes it hard to use things like 'file normalize' - which also looks at things like current volume.
-        #
-        #Note for example the results of 'which' grep on windows can produce a path like /c/Users/somewhere/bin/grep 
-        #which tcl's file normalize may change to C:/c/Users or X:/c/Users  - based on current volumen.  Given that C:/c might exist - this can be problematic in a couple of ways.
-        #The mixing of unix-like and windows commands on the same machine is a large part of the problem.. but this mix is now common
-        #
-        #convert /c/etc to C:/etc 
-        set re_slash_x_slash {^/([[:alpha:]]){1}/.*}
-        set re_slash_else {^/([[:alpha:]]*)(.*)}
-        set volumes [file volumes]
-        #exclude things like //zipfs:/
-        set driveletters [list]
-        foreach v $volumes {
-            if {[regexp {^([[:alpha:]]){1}:/$} $v _ letter]} {
-                lappend driveletters $letter
-            }
-        }
-        #puts stderr "->$driveletters"
-
-        if {[regexp $re_slash_x_slash $path _ letter]} {
-            #upper case appears to be windows canonical form
-            set path [string toupper $letter]:/[string range $path 3 end] 
-        } elseif {[regexp {^/mnt/([[:alpha:]]){1}/.*} [string tolower $path] _ letter]} {
-            set path [string toupper $letter]:/[string range $path 7 end] 
-        } elseif {[regexp {^/mnt/([[:alpha:]]){1}$} [string tolower $path] _ letter]} {
-            set path [string toupper $letter]:/
-        } elseif {[regexp $re_slash_else $path _ firstpart remainder]} {
-            #could be for example /c   or /something/users
-            if {[string length $firstpart] == 1} {
-                set letter $firstpart
-                set path [string toupper $letter]:/
-            } else {
-                #attempt to use cygpath helper 
-                if {![catch  {
-                    set cygpath [runout -n cygpath -w $path] ;#!
-                    set ::punk::last_run_display [list]   ;#hack - review shouldn't really be necessary.. but because we call winpath from ./  - the repl looks for last_run_display
-                } errM]} {
-                    set path [string map [list "\\" "/"] $cygpath]
-                } else {
-                    error "Path '$path' does not appear to be in a standard form. For unix-like paths on windows such as /x, x must correspond to a drive letter. Consider installing cygwin's cygpath tool to see if that helps."
-                }
-            }
-        }
-        #puts stderr "=> $path"
-        #things like 'which' seem to return a path minus the .exe - so we'll just test the containing folder
-        #
-        #By now file normalize shouldn't do too many shannanigans related to cwd..
-        #We want it to look at cwd for relative paths.. but we don't consider things like /c/Users to be relative even on windows
-        if {![file exists [file dirname $path]]} {
-            set path [file normalize $path]
-            #may still not exist.. that's ok.
-        }
-        #file normalize may change backslashes to forward slashes.. including things like the special \\?\ prefix which is intended to stop windows api from parsing a name
-        #2023 - this is ok as //?/ also seems to work.. but it is unclear if that is because Tcl is re-converting to backslashes 
-        if {[illegalname_test $path]} {
-            set path [illegalname_fix $path]
-        }
 
-        return $path
-    }
-
-    proc windir {path} {
-        if {$path eq "~"} {
-            #as the tilde hasn't been normalized.. we can't assume we're running on the actual platform 
-            return ~/..
-        }
-        return [file dirname [winpath $path]]
-    }
 
-    #REVIEW high-coupling
-    proc cdwin {path} {
-        set path [winpath $path]
-        if {$::repl::running} {
-            repl::term::set_console_title $path
-        }
-        cd $path
-    }
-    proc cdwindir {path} {
-        set path [winpath $path]
-        if {$::repl::running} {
-            repl::term::set_console_title $path
-        }
-        cd [file dirname $path]
-    }
 
     #\\servername\share etc  or \\?\UNC\servername\share  etc.
     proc is_unc_path {path} {
-        set path [string map [list \\ /] $path] ;#normalize to forward slashes for testing purposes (and forward slashes seem to be auto-translated by windows anyway)
+        set strcopy_path [punk::objclone $path]
-        if {[string first "//" $path] == 0} {
+        set strcopy_path [string map [list \\ /] $strcopy_path] ;#normalize to forward slashes for testing purposes (and forward slashes seem to be auto-translated by windows anyway)
+        if {[string first "//" $strcopy_path] == 0} {
             #check for "Dos device path" syntax
-            if {[string range $path 0 3] in [list "//?/" "//./"]} {
+            if {[string range $strcopy_path 0 3] in [list "//?/" "//./"]} {
                 #Note that //./ doesn't appear to be supported in Tcl as at 2023-08 - but //?/ works  (except for //?/UNC/Server/share)
-                if {[string range $path 4 6] eq "UNC"} {
+                if {[string range $strcopy_path 4 6] eq "UNC"} {
                     return 1
                 } else {
                     #some other Dos device path. Could be a drive which is mapped to a UNC path - but the path itself isn't a unc path
@@ -146,7 +53,7 @@ namespace eval punk::winpath {
     #ordinary \\Servername   or \\servername\share or \\servername\share\path  (or forward-slash equivalent) with no dos device syntax //?/ //./ etc.
     proc is_unc_path_plain {path} {
         if {[is_unc_path $path]} {
-            if {![is_dos_device_path]} {
+            if {![is_dos_device_path $path]} {
                 return 1
             } else {
                 return 0
@@ -156,9 +63,9 @@ namespace eval punk::winpath {
         }
     }
 
-    #'file attributes', and therefor this operation, is expensive (on windows at least)
+    #int-rep path preserved - but 'file attributes', and therefor this operation, is expensive (on windows at least)
     proc pwdshortname {{path {}}} {
-        if {![string length $path]} {
+        if {$path eq ""} {
            set path [pwd]
         } else {
             if {[file pathtype $path] eq "relative"} {
@@ -170,8 +77,9 @@ namespace eval punk::winpath {
     #dos device path syntax allows windows api to acces extended-length paths and filenames with illegal path characters such as trailing dots or whitespace
     #(can exist on server shares and on NTFS - but standard apps can't access without dos device syntax)
     proc is_dos_device_path {path} {
-        set path [string map [list \\ /] $path] ;#normalize to forward slashes for testing purposes (and forward slashes seem to be auto-translated by windows anyway)
+        set strcopy_path [punk::objclone $path]
-        if {[string range $path 0 3] in [list "//?/" "//./"]} {
+        set strcopy_path [string map [list \\ /] $strcopy_path] ;#normalize to forward slashes for testing purposes (and forward slashes seem to be auto-translated by windows anyway)
+        if {[string range $strcopy_path 0 3] in [list "//?/" "//./"]} {
             return 1
         } else {
             return 0
@@ -192,17 +100,35 @@ namespace eval punk::winpath {
     proc strip_unc_path_prefix {path} {
         if {[is_unc_path $path]} {
             #//?/UNC/server/etc
-            return [string range $path 7 end]
+            set strcopy_path [punk::objclone $path]
+            set trimmedpath [string range $strcopy_path 7 end]
+            file pathtype $trimmedpath ;#shimmer it to path rep
+            return $trimmedpath
         } elseif {is_unc_path_plain $path} {
             #plain unc //server
-            return [string range $path 2 end]
+            set strcopy_path [punk::objclone $path]
+            set trimmedpath [string range $strcopy_path 2 end]
+            file pathtype $trimmedpath
+            return $trimmedpath
         } else {
             return $path
         }
     }
     #we don't validate that path is actually illegal because we don't know the full range of such names.
     #The caller can apply this to any path.
-    #don't test for platform here - needs to be callable from any platform for potential passing to windows
+    #don't test for platform here - needs to be callable from any platform for potential passing to windows (what usecase? 8.3 name is not always calculable independently)
+    #The utility of this is questionable. prepending a dos-device path won't make a filename with illegal characters readable by windows.
+    #It will need the 'shortname' at least for the illegal segment - if not the whole path
+    #Whilst the 8.3 name algorithm - including undocumented hash function has been reverse engineered 
+    #- it depends on the content of the directory - as collisions cause a different name (e.g incremented number) 
+    #- it also depends on the history of the folder
+    #- you can't take the current dir contents and a particular *existing* longname and determine the shortname algorithmically...
+    #- the shortname may have been generated during a different directory state.
+    #- It is then stored on disk (where?) - so access to reading the existing shortname is required.
+    #- An implementation of the 8.3 algorithm would only be potentially useful in determining the name that will result from adding a new file 
+    #  and would be subject to potential collisions if there are race-conditions in file creation
+    #- Using an 8.3 algorithm externally would be dangerous in that it could appear to work a lot of the time - but return a different file entirely sometimes.
+    #- Conclusion is that the 8.3 name must be retrieved rathern than calclated
     proc illegalname_fix {path} {
         #don't add extra dos device path syntax protection-prefix if already done
         if {[is_unc_path $path]} {
@@ -214,6 +140,7 @@ namespace eval punk::winpath {
         }
 
 
+
         #\\servername\share theoretically maps to: \\?\UNC\servername\share  in protected form. https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats
         #NOTE: 2023-08 on windows 10 at least  \\?\UNC\Server\share  doesn't work - ie we can't use illegalname_fix on UNC paths such as \\Server\share 
         #(but mapped drive to same path will work)
@@ -225,6 +152,10 @@ namespace eval punk::winpath {
             append err \n   " - because //?/UNC/Servername/share is not supported in Tcl (and only minimally even in powershell) as at 2023. (on windows use mapped drive instead)"
             error $err
         }
+
+        set strcopy_path [punk::objclone $path]
+
+
         #Note: path could still have leading double slash if it is a Dos device path: e.g. //?/c:/etc
         if {[file pathtype $path] eq "absolute"} {
             if {$path eq "~"} {
@@ -239,10 +170,10 @@ namespace eval punk::winpath {
         } else {
             #set fullpath [file normalize $path] ;#very slow on windows
             #set fullpath [pwd]/$path ;#will keep ./ in middle of path - not valid for dos-device paths
-            if {[string range $path 0 1] eq "./"} {
+            if {[string range $strcopy_path 0 1] eq "./"} {
-                set path [string range $path 2 end]
+                set strcopy_path [string range $strcopy_path 2 end]
             }
-            set fullpath [file join [pwd] $path]
+            set fullpath [file join [pwd] $strcopy_path]
         }
         #For file I/O, the "\\?\" prefix to a path string tells the Windows APIs to disable all string parsing
         # and to send the string that follows it straight to the file system.
@@ -252,16 +183,21 @@ namespace eval punk::winpath {
 
 
         #choose //?/  as normalized version - since likely 'file normalize' will do it anyway, and experimentall, the windows API accepts both  REVIEW
-        return ${protect2}$fullpath
+        set result ${protect2}$fullpath
+        file pathtype $result ;#make it return a path rep
+        return $result
     }
 
     #don't test for platform here - needs to be callable from any platform for potential passing to windows
     #we can create files with windows illegal names by using //?/ dos device path syntax - but we need to detect when that is required.
+    #
+    # path int-rep preserving
     proc illegalname_test {path} {
-        #first test if already protected - we return false even if the file would be illegal without the protection!
+        #https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file
-        if {[is_dos_device_path $path]} {
+        #according to the above: Use any character in the current code page for a name, including Unicode characters and characters in the extended character set (128–255), except for the following:
-            return 0
+        set reserved [list  < > : \" / \\ | ? *]
-        }
+
+
         #we need to exclude things like path/..   path/.
         foreach seg [file split $path] {
             if {$seg in [list . ..]} {
@@ -289,14 +225,23 @@ namespace eval punk::winpath {
         return 0
     }
 
+    proc test_ntfs_tunneling {f1 f2 args} {
+        file mkdir $f1
+        puts stderr  "waiting 15secs..."
+        after 5000 {puts -nonewline stderr .}
+        after 5000 {puts -nonewline stderr .}
+        after 5000 {puts -nonewline stderr .}
+        after 500 {puts stderr \n}
+        file mkdir $f2
+        puts stdout "$f1   [file stat $f1]"
+        puts stdout "$f2   [file stat $f2]"
+        file delete $f1
+        puts stdout "renaming $f2 to $f1"
+        file rename $f2 $f1
+        puts stdout "$f1   [file stat $f1]"
+
+    }
 
-    #----------------------------------------------
-    #leave the winpath related aliases available on all platforms
-    interp alias {} cdwin {} punk::winpath::cdwin
-    interp alias {} cdwindir {} punk::winpath::cdwindir
-    interp alias {} winpath {} punk::winpath::winpath
-    interp alias {} windir {} punk::winpath::windir
-    #----------------------------------------------
 }
 
 

src/doc/include/general.inc

@@ -2,5 +2,5 @@
 [comment {- -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---}]
 [moddesc   {punkshell - a Tcl }]
 [category  {shell}]
-[keywords  {shell repl}]
+[keywords  shell repl punk]
 [require Tcl 8.6]

src/make.tcl

@@ -2,8 +2,6 @@
 #
 #make any tclkits and modules in src and place them and associated data files/scripts in the parent folder of src.
 #e.g in 'bin' and 'modules' folders at same level as 'src' folder.
-#It is assumed the src folder has been placed somewhere where appropriate 
-#(e.g not in /usr  or c:/  - unless you intend it to directly make and place folders and files in those locations)
 
 set hashline "# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###"
 puts $hashline
@@ -25,13 +23,18 @@ if {"::try" ni [info commands ::try]} {
 #------------------------------------------------------------------------------
 #Module loading from /src/bootsupport or src/*.vfs if script is within a .vfs folder
 #------------------------------------------------------------------------------
-#If the there is a folder directly under the current directory /src/bootsupport/modules which contains .tm files when the starts
+#If the there is a folder  under the current directory /src/bootsupport/modules which contains .tm files 
 # - then it will attempt to preference these modules
-# This allows a source update via 'fossil update' 'git pull' etc to pull in support modules for the make script
+# This allows a source update via 'fossil update' 'git pull' etc to pull in a minimal set of support modules for the make script
-# and load these in preference to ones that may have been in the interps tcl::tm::list or auto_path due to environment variables
+# and load these in preference to ones that may have been in the interp's tcl::tm::list or auto_path due to environment variables
 set startdir [pwd]
+if {[file exists [file join $startdir src bootsupport]]} {
     set bootsupport_mod [file join $startdir src bootsupport modules]
     set bootsupport_lib [file join $startdir src bootsupport lib]
+} else {
+    set bootsupport_mod [file join $startdir bootsupport modules]
+    set bootsupport_lib [file join $startdir bootsupport lib]
+}
 if {[file exists $bootsupport_mod] || [file exists $bootsupport_lib]} {
 
     set original_tm_list [tcl::tm::list]
@@ -60,31 +63,16 @@ if {[file exists $bootsupport_mod] || [file exists $bootsupport_lib]} {
     }
 
 
-    #todo - review usecase
-    if {[string match "*.vfs/*" [info script]]} {
-        #src/xxx.vfs/lib/app-punk/repl.tcl
-        #we assume if calling directly into .vfs that the user would prefer to use src/modules - so go up 4 levels
-        set modulefolder [file dirname [file dirname [file dirname [file dirname [info script]]]]]/modules
-
-    } else {
-        #  .../bin/punkXX.exe  look for ../modules  (i.e modules folder at same level as bin folder)
-        set modulefolder [file dirname [file dirname [info nameofexecutable]]]/modules
-    }
-
-    if {[file exists $modulefolder]} {
-        tcl::tm::add $modulefolder
-    } else {
-        puts stderr "Warning unable to find module folder at: $modulefolder"
-    }
     if {[file exists [pwd]/modules]} {
         tcl::tm::add [pwd]/modules
     }
 
     #package require Thread
-    #These are strong dependencies 
+    # - the full repl requires Threading and punk,shellfilter,shellrun to call and display properly.
-    # - the repl requires Threading and punk,shellfilter,shellrun to call and display properly.
-    # tm list already indexed - need 'package forget' to find modules based on current tcl::tm::list
     
+    
+    # tm list already indexed - need 'package forget' to find modules based on current tcl::tm::list
+    #These are strong dependencies 
     package forget  punk::mix
     package require punk::mix
     package forget  punk::repo 
@@ -144,6 +132,8 @@ proc punkmake_gethelp {args} {
     append h "  $scriptname project ?-k?" \n
     append h "  - this is the literal word project - and confirms you want to run the project build" \n
     append h "  - the optional -k flag will terminate processes running as the executable being built (if applicable)" \n \n
+    append h "  $scriptname bootsupport"  \n
+    append h "  - update the src/bootsupport modules as well as the mixtemplates/layouts/<layoutname>/src/bootsupport modules if the folder exists" \n \n
     append h "  $scriptname get-project-info" \n
     append h "  - show the name and base folder of the project to be built" \n
     append h "" \n
@@ -189,7 +179,7 @@ if {[llength $commands_found] != 1 } {
 }
 if {$do_help} {
     puts stderr [punkmake_gethelp] 
-    exit 1
+    exit 0
 }
 
 set ::punkmake::command [lindex $commands_found 0]
@@ -224,6 +214,8 @@ if {![string length [set projectroot [punk::repo::find_project $scriptfolder]]]}
     
 }
 
+set sourcefolder $projectroot/src
+
 if {$::punkmake::command eq "get-project-info"} {
     puts stdout "- -- --- --- --- --- --- --- --- --- ---"
     puts stdout "- -- get-project-info -- -"
@@ -260,9 +252,128 @@ if {$::punkmake::command eq "shell"} {
 }
 
 if {$::punkmake::command eq "bootsupport"} {
+    puts "projectroot: $projectroot"
+    puts "script: [info script]"
+    #puts "-- [tcl::tm::list] --"
+    puts stdout "Updating bootsupport from local files"
+
+    proc bootsupport_localupdate {projectroot} {
+        set bootsupport_modules [list]
+        set bootsupport_config  $projectroot/src/bootsupport/include_modules.config ;#
+        if {[file exists $bootsupport_config]} {
+            source $bootsupport_config ;#populate $bootsupport_modules with project-specific list
+            if {![llength $bootsupport_modules]} {
+                puts stderr "No local bootsupport modules configured for updating"
+                return
+            }
+            set targetroot $projectroot/src/bootsupport/modules
 
+            if {[catch {
+                #----------
+                set boot_installer [punkcheck::installtrack new make.tcl $projectroot/src/bootsupport/.punkcheck]
+                $boot_installer set_source_target $projectroot $projectroot/src/bootsupport
+                set boot_event [$boot_installer start_event {-make_step bootsupport}]
+                #----------
+            } errM]} {
+                puts stderr "Unable to use punkcheck for bootsupport error: $errM"
+                set boot_event ""
+            }
+
+            foreach {relpath module} $bootsupport_modules {
+                set module [string trim $module :]
+                set module_subpath [string map [list :: /] [namespace qualifiers $module]]
+                set srclocation [file join $projectroot $relpath $module_subpath]
+                #puts stdout "$relpath  $module $module_subpath $srclocation"
+                set pkgmatches [glob -nocomplain -dir $srclocation -tail [namespace tail $module]-*]
+                #lsort won't sort version numbers properly e.g with -dictionary 0.1.1 comes before 0.1
+                if {![llength $pkgmatches]} {
+                    puts stderr "Missing source for bootsupport module $module - not found in $srclocation"
+                    continue
+                }
+                set latestfile [lindex $pkgmatches 0]
+                set latestver [lindex [split [file rootname $latestfile] -] 1]
+                foreach m $pkgmatches {
+                    lassign [split [file rootname $m] -] _pkg ver
+                    #puts "comparing $ver vs $latestver"
+                    if {[package vcompare $ver $latestver] == 1} {
+                        set latestver $ver
+                        set latestfile $m
+                    } 
+                }
+                set srcfile [file join $srclocation $latestfile]
+                set tgtfile [file join $targetroot $module_subpath $latestfile]
+                if {$boot_event ne ""} {
+                    #----------
+                    $boot_event targetset_init INSTALL   $tgtfile
+                    $boot_event targetset_addsource      $srcfile
+                    #----------
+                    if {\
+                            [llength [dict get [$boot_event targetset_source_changes] changed]]\
+                        || [llength [$boot_event get_targets_exist]] < [llength [$boot_event get_targets]]\
+                        } {
+                        $boot_event targetset_started
+                        # -- --- --- --- --- ---
+                        puts "BOOTSUPPORT update: $srcfile -> $tgtfile"
+                        if {[catch {
+                            file copy -force $srcfile  $tgtfile
+                        } errM]} {
+                            $boot_event targetset_end FAILED
+                        } else {
+                            $boot_event targetset_end OK
+                        }
+                        # -- --- --- --- --- ---
+                    } else {
+                        puts -nonewline stderr "."
+                        $boot_event targetset_end SKIPPED
+                    }
+                    $boot_event end
+                } else {
+                    file copy -force $srcfile  $tgtfile
+                }
+            }
+            if {$boot_event ne ""} {
+                puts \n
+                $boot_event destroy
+                $boot_installer destroy
+            }
+        }
+    }
 
-    exit 1
+    bootsupport_localupdate $projectroot
+
+    #/modules/punk/mix/templates/layouts only applies if the project has it's own copy of the punk/mix modules. Generally this should only apply to the punkshell project itself.
+    set layout_bases [list\
+        $sourcefolder/mixtemplates/layouts\
+        $sourcefolder/modules/punk/mix/templates/layouts\
+    ]
+    foreach project_layout_base $layout_bases {
+        if {[file exists $project_layout_base]} {
+            set project_layouts [glob -nocomplain -dir $project_layout_base -type d -tail *]
+            foreach layoutname $project_layouts {
+                if {[file exists  [file join $project_layout_base $layoutname/src/bootsupport]]} {
+                    set unpublish [list\
+                        README.md\
+                    ]
+                    set sourcemodules $projectroot/src/bootsupport/modules
+                    set targetroot [file join $project_layout_base $layoutname/src/bootsupport/modules]
+                    file mkdir $targetroot
+
+                    puts stdout "BOOTSUPPORT layouts/$layoutname: copying from $sourcemodules to $targetroot (if source file changed)"
+                    set resultdict [punkcheck::install $sourcemodules $targetroot -overwrite installedsourcechanged-targets -unpublish_paths $unpublish]
+                    puts stdout [punkcheck::summarize_install_resultdict $resultdict]
+                    flush stdout
+                }
+            }
+        } else {
+            puts stderr "No layout base at $project_layout_base"
+        }
+    }
+    puts stdout " bootsupport done "
+    flush stderr
+    flush stdout
+    #punk86 can hang if calling make.tcl via 'run' without this 'after' delay. punk87 unaffected. cause unknown.
+    #after 500
+    ::exit 0
 }
 
 
@@ -273,7 +384,6 @@ if {$::punkmake::command ne "project"} {
 }
 
 
-set sourcefolder $projectroot/src
 
 #only a single consolidated /modules folder used for target
 set target_modules_base $projectroot/modules
@@ -290,37 +400,21 @@ if {[file exists $sourcefolder/vendorlib]} {
         README.md\
     ]
 
+    puts stdout "VENDORLIB: copying from $sourcefolder/vendorlib to $projectroot/lib (if source file changed)"
     set resultdict [punkcheck::install $sourcefolder/vendorlib $projectroot/lib -overwrite installedsourcechanged-targets -unpublish_paths $unpublish]
-    set copied [dict get $resultdict files_copied]
+    puts stdout [punkcheck::summarize_install_resultdict $resultdict]
-    set sources_unchanged [dict get $resultdict sources_unchanged]
+
-    puts stdout "--------------------------"
-    flush stdout
-    puts stderr "Copied [llength $copied] vendor lib files from src/vendorlib to $projectroot/lib"
-    foreach f $copied {
-        puts stdout "COPIED $f"
-    }
-    puts stdout "[llength $sources_unchanged] unchanged source files"
-    puts stdout "--------------------------"
 } else {
-    puts stderr "NOTE: No src/vendorlib folder found."
+    puts stderr "VENDORLIB: No src/vendorlib folder found."
 }
 
-
 if {[file exists $sourcefolder/vendormodules]} {
     #install .tm *and other files*
+    puts stdout "VENDORMODULES: copying from $sourcefolder/vendormodules to $target_modules_base (if source file changed)"
     set resultdict [punkcheck::install $sourcefolder/vendormodules  $target_modules_base -installer make.tcl -overwrite installedsourcechanged-targets -unpublish_paths {README.md}]
-    set copied [dict get $resultdict files_copied]
+    puts stdout [punkcheck::summarize_install_resultdict $resultdict]
-    set sources_unchanged [dict get $resultdict sources_unchanged]
-    puts stdout "--------------------------"
-    flush stdout
-    puts stderr "Copied [llength $copied] vendor module files from src/vendormodules to $target_modules_base"
-    foreach f $copied {
-        puts stdout "COPIED $f"
-    }
-    puts stdout "[llength $sources_unchanged] unchanged source files"
-    puts stdout "--------------------------"
 } else {
-    puts stderr "NOTE: No src/vendormodules folder found."
+    puts stderr "VENDORMODULES: No src/vendormodules folder found."
 }
 
 ########################################################
@@ -354,6 +448,7 @@ foreach layoutinfo $layout_update_list {
 
     foreach filepair $pairs {
         lassign $filepair srcfile tgtfile
+        file mkdir [file dirname $tgtfile]
         #----------
         $tpl_event targetset_init INSTALL   $tgtfile
         $tpl_event targetset_addsource      $srcfile
@@ -386,8 +481,6 @@ foreach layoutinfo $layout_update_list {
 ########################################################
 
 
-
-
 #default source module folder is at projectroot/src/modules
 #There may be multiple other src module folders at same level (e.g folder not being other special-purpose folder and not matching name vendor* that contains at least one .tm file in its root) 
 set source_module_folderlist [punk::mix::cli::lib::find_source_module_paths $projectroot]
@@ -403,19 +496,54 @@ foreach src_module_dir $source_module_folderlist {
 
     set overwrite "installedsourcechanged-targets"
     #set overwrite "ALL-TARGETS"
+    puts stdout "MODULEFOLDER non_tm_files $src_module_dir - copying to $target_modules_base (if source file changed)"
     set resultdict [punkcheck::install_non_tm_files $src_module_dir $target_modules_base -installer make.tcl -overwrite $overwrite -unpublish_paths {README.md}]
-    set copied                  [dict get $resultdict files_copied]
+    puts stdout [punkcheck::summarize_install_resultdict $resultdict]
-    set sources_unchanged       [dict get $resultdict sources_unchanged]
-    puts stdout "--------------------------"
-    flush stdout
-    puts stderr "Copied [llength $copied] non-tm source files from $src_module_dir to $target_modules_base"
-    puts stderr "[llength $sources_unchanged] unchanged source files"
-    flush stderr
-    puts stdout "--------------------------"
 }
 
+set installername "make.tcl"
+
 # ----------------------------------------
+if {[punk::repo::is_fossil_root $projectroot]} {
+    set config [dict create\
+        -make-step configure_fossil\
+    ]
+    #----------
+    set installer [punkcheck::installtrack new $installername $projectroot/.punkcheck]
+    $installer set_source_target $projectroot $projectroot
+
+    set event [$installer start_event $config]
+    $event targetset_init VIRTUAL fossil_settings_mainmenu ;#VIRTUAL - since there is no actual target file
+    set menufile $projectroot/.fossil-custom/mainmenu
+    $event targetset_addsource $menufile
+    #----------
 
+    if {\
+            [llength [dict get [$event targetset_source_changes] changed]]\
+        } {
+        $event targetset_started
+        # -- --- --- --- --- ---
+        puts stdout "Configuring fossil setting: mainmenu from: $menufile"
+        if {[catch {
+            set fd [open $menufile r]
+            fconfigure $fd -translation binary
+            set data [read $fd]
+            close $fd
+            exec fossil settings mainmenu $data
+        } errM]} {
+            $event targetset_end FAILED -note "fossil update failed: $errM"
+        } else {
+            $event targetset_end OK
+        }
+        # -- --- --- --- --- ---
+    } else {
+        puts stderr "."
+        $event targetset_end SKIPPED
+    }
+    $event end
+    $event destroy
+    $installer destroy
+}
 
 set buildfolder [punk::mix::cli::lib::get_build_workdir $sourcefolder]
 if {$buildfolder ne "$sourcefolder/_build"} {
@@ -430,7 +558,8 @@ set rtfolder $sourcefolder/runtime
 set runtimes [glob -nocomplain -dir $rtfolder -types {f x} -tail *]
 if {![llength $runtimes]} {
     puts stderr "No executable runtimes found in $rtfolder - unable to build any .vfs folders into executables."
-    exit 2
+    puts stderr "Add runtimes to $sourcefolder/runtime if required"
+    exit 0
 }
 
 if {[catch {exec sdx help} errM]} {
@@ -465,6 +594,7 @@ if {[file exists $mapfile]} {
             #drop windows .exe suffix so same config can work cross platform - extension will be re-added if necessary later
             set runtime [string range $runtime 0 end-4]
         }
+        if {$runtime ne "-"} {
             set runtime_test $runtime
             if {"windows" eq $::tcl_platform(platform)} {
                 set runtime_test $runtime.exe
@@ -473,6 +603,7 @@ if {[file exists $mapfile]} {
                 puts stderr "WARNING: Missing runtime file $rtfolder/$runtime_test (line in mapvfs.config: $ln)"
                 lappend missing $runtime
             }
+        }
         foreach vfs $vfspaths {
             if {![file isdirectory [file join $sourcefolder $vfs]]} {
                 puts stderr "WARNNING: Missing vfs folder [file join $sourcefolder $vfs] specified in mapvfs.config for runtime $runtime"
@@ -511,7 +642,6 @@ if {![llength $vfs_folders]} {
 
 set vfs_folder_changes [dict create] ;#cache whether each .vfs folder has changes so we don't re-run tests if building from same .vfs with multiple runtime executables
 
-set installername "make.tcl"
 # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
 #set runtimefile [lindex $runtimes 0]
 foreach  runtimefile $runtimes {
@@ -593,7 +723,7 @@ foreach vfs $vfs_folders {
             set runtimes_raw $runtimes
             set runtimes [list]
             foreach rt $runtimes_raw {
-                if {![string match *.exe $rt]} {
+                if {![string match *.exe $rt] && $rt ne "-"} {
                     set rt $rt.exe
                 }
                 lappend runtimes $rt
@@ -615,27 +745,36 @@ foreach vfs $vfs_folders {
     #assert $runtimes is a list of executable names suffixed with .exe if on windows - whether or not specified with .exe in the mapvfs.config
 
 
+    #todo - non kit based - zipkit?
+    # $runtimes may now include a dash entry "-" (from mapvfs.config file)
     foreach rtname $runtimes {
+        #rtname of "-" indicates build a kit without a runtime
 
         #first configured runtime will be the one to use the same name as .vfs folder for output executable. Additional runtimes on this .vfs will need to suffix the runtime name to disambiguate.
         #review: This mechanism may not be great for multiplatform builds ? We may be better off consistently combining vfsname and rtname and letting a later platform-specific step choose ones to install in bin with simpler names.
+        if {$rtname eq "-"} {
+            set targetkit $vfsname.kit
+        } else {
             if {$::tcl_platform(platform) eq "windows"} {
-            set targetexe ${vfsname}.exe
+                set targetkit ${vfsname}.exe
             } else {
-            set targetexe $vfsname
+                set targetkit $vfsname
             }
-        if {$targetexe in $exe_names_seen} {
+            if {$targetkit in $exe_names_seen} {
                 #more than one runtime for this .vfs 
-            set targetexe ${vfsname}_$rtname
+                set targetkit ${vfsname}_$rtname
             }
-        lappend exe_names_seen $targetexe
+        }
+        lappend exe_names_seen $targetkit
         # -- ----------
         set vfs_installer [punkcheck::installtrack new $installername $basedir/.punkcheck]
         $vfs_installer set_source_target $sourcefolder $buildfolder
         set vfs_event [$vfs_installer start_event {-make-step build_vfs}]
-        $vfs_event targetset_init INSTALL $buildfolder/$targetexe
+        $vfs_event targetset_init INSTALL $buildfolder/$targetkit
         $vfs_event targetset_addsource $sourcefolder/$vfs
+        if {$rtname ne "-"} {
             $vfs_event targetset_addsource $buildfolder/build_$rtname
+        }
         # -- ----------
 
         set changed_unchanged [$vfs_event targetset_source_changes]
@@ -655,9 +794,17 @@ foreach vfs $vfs_folders {
 
 
             if {[catch {
+                if {$rtname ne "-"} {
                     exec sdx wrap $buildfolder/$vfsname.new -vfs $sourcefolder/$vfs -runtime $buildfolder/build_$rtname -verbose
+                } else {
+                    exec sdx wrap $buildfolder/$vfsname.new -vfs $sourcefolder/$vfs  -verbose
+                }
                 } result]} {
+                if {$rtname ne "-"} {
                     puts stderr "sdx wrap $buildfolder/$vfsname.new -vfs $sourcefolder/$vfs -runtime $buildfolder/build_$rtname -verbose failed with msg: $result"
+                } else {
+                    puts stderr "sdx wrap $buildfolder/$vfsname.new -vfs $sourcefolder/$vfs  -verbose failed with msg: $result"
+                }
             } else {
                 puts stdout "ok - finished sdx"
                 set separator [string repeat = 40]
@@ -679,6 +826,8 @@ foreach vfs $vfs_folders {
                 set pscmd "ps"
             }
 
+            #killing process doesn't apply to .kit build 
+            if {$rtname ne "-"} {
                 if {![catch {
                         exec $pscmd | grep $vfsname
                     } still_running]} {
@@ -710,8 +859,14 @@ foreach vfs $vfs_folders {
                                 } errMsg]} {
                             puts stderr "$killcmd returned an error:"
                             puts stderr $errMsg
+                            if {!$forcekill} {
                                 puts stderr "(try '[info script] -k' option to force kill)"
+                            }
+                            #avoid exiting if the kill failure was because the task has already exited
+                            #review - *no running instance* works with windows taskkill - "*No such process*" works with kill -9 on FreeBSD and linux - other platforms?
+                            if {![string match "*no running instance*" $errMsg] && ![string match "*No such process*" $errMsg]} {
                                 exit 4
+                            }
                         } else {
                             puts stderr "$killcmd ran without error"
                             incr count_killed
@@ -724,19 +879,20 @@ foreach vfs $vfs_folders {
                 } else {
                     puts stderr "Ok.. no running '$vfsname' processes found"
                 }
+            }
 
-            if {[file exists $buildfolder/$targetexe]} {
+            if {[file exists $buildfolder/$targetkit]} {
-                puts stderr "deleting existing $buildfolder/$targetexe"
+                puts stderr "deleting existing $buildfolder/$targetkit"
                 if {[catch {
-                    file delete $buildfolder/$targetexe
+                    file delete $buildfolder/$targetkit
                 } msg]} {
-                    puts stderr "Failed to delete $buildfolder/$targetexe"
+                    puts stderr "Failed to delete $buildfolder/$targetkit"
                     exit 4
                 }
             }
-            #WINDOWS filesystem 'tunneling' (file replacement within 15secs) could cause targetexe to copy ctime & shortname metadata from previous file!
+            #WINDOWS filesystem 'tunneling' (file replacement within 15secs) could cause targetkit to copy ctime & shortname metadata from previous file!
             #This is probably harmless - but worth being aware of.
-            file rename $buildfolder/$vfsname.new $buildfolder/$targetexe
+            file rename $buildfolder/$vfsname.new $buildfolder/$targetkit
             # -- --- --- --- --- ---
             $vfs_event targetset_end OK
 
@@ -748,36 +904,39 @@ foreach vfs $vfs_folders {
             # -- ----------
             set bin_installer [punkcheck::installtrack new "make.tcl" $deployment_folder/.punkcheck]
             $bin_installer set_source_target $buildfolder $deployment_folder
-            set bin_event [$bin_installer start_event {-make-step final_exe_install}]
+            set bin_event [$bin_installer start_event {-make-step final_kit_install}]
-            $bin_event targetset_init INSTALL $deployment_folder/$targetexe
+            $bin_event targetset_init INSTALL $deployment_folder/$targetkit
-            $bin_event targetset_addsource $buildfolder/$targetexe
+            #todo - move final deployment step outside of the build vfs loop? (final deployment can fail and then isn't rerun even though _build and deployed versions differ, unless .vfs modified again)
+            #set last_completion [$bin_event targetset_last_complete]
+
+            $bin_event targetset_addsource $buildfolder/$targetkit
             $bin_event targetset_started
             # -- ----------
 
 
             set delete_failed 0
-            if {[file exists $deployment_folder/$targetexe]} {
+            if {[file exists $deployment_folder/$targetkit]} {
-                puts stderr "deleting existing deployed at $deployment_folder/$targetexe"
+                puts stderr "deleting existing deployed at $deployment_folder/$targetkit"
                 if {[catch {
-                    file delete $deployment_folder/$targetexe
+                    file delete $deployment_folder/$targetkit
                     } errMsg]} {
-                    puts stderr "deletion of deployed version at $deployment_folder/$targetexe failed: $errMsg"
+                    puts stderr "deletion of deployed version at $deployment_folder/$targetkit failed: $errMsg"
-                    #exit 5
                     set delete_failed 1
                 }
             }
             if {!$delete_failed} {
                 puts stdout "copying.."
-                puts stdout "$buildfolder/$targetexe"
+                puts stdout "$buildfolder/$targetkit"
                 puts stdout "to:"
-                puts stdout "$deployment_folder/$targetexe"
+                puts stdout "$deployment_folder/$targetkit"
                 after 300
-                file copy $buildfolder/$targetexe $deployment_folder/$targetexe
+                file copy $buildfolder/$targetkit $deployment_folder/$targetkit
                 # -- ----------
                 $bin_event targetset_end OK
                 # -- ----------
             } else {
-                $bin_event targetset_end FAILED -note "could not delete  
+                $bin_event targetset_end FAILED -note "could not delete"
+                exit 5
             }
             $bin_event destroy
             $bin_installer destroy
@@ -785,7 +944,7 @@ foreach vfs $vfs_folders {
         } else {
             set skipped_vfs_build 1
             puts stderr "."
-            puts stdout "Skipping build for vfs $vfs - no change detected"
+            puts stdout "Skipping build for vfs $vfs with runtime $rtname - no change detected"
             $vfs_event targetset_end SKIPPED
         }
         $vfs_event destroy

src/mixtemplates/module/template_unversioned.tm

@@ -1,50 +0,0 @@
-# -*- tcl -*-
-#
-# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem.
-# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository.
-# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
-# (C) %year%
-#
-# @@ Meta Begin
-# Application %pkg% %version%
-# Meta platform     tcl
-# Meta license      %license%
-# @@ Meta End
-
-
-
-# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
-##  Requirements
-##e.g package require frobz
-
-
-
-
-# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
-namespace eval %pkg% {
-
-
-
-
-}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
-##  Ready   
-package provide %pkg% [namespace eval %pkg% {
-    variable version
-    set version %version% 
-}]
-return

src/modules/platform-1.0.17.tm

@@ -1,428 +0,0 @@
-# -*- tcl -*-
-# ### ### ### ######### ######### #########
-## Overview
-
-# Heuristics to assemble a platform identifier from publicly available
-# information. The identifier describes the platform of the currently
-# running tcl shell. This is a mixture of the runtime environment and
-# of build-time properties of the executable itself.
-#
-# Examples:
-# <1> A tcl shell executing on a x86_64 processor, but having a
-#   wordsize of 4 was compiled for the x86 environment, i.e. 32
-#   bit, and loaded packages have to match that, and not the
-#   actual cpu.
-#
-# <2> The hp/solaris 32/64 bit builds of the core cannot be
-#   distinguished by looking at tcl_platform. As packages have to
-#   match the 32/64 information we have to look in more places. In
-#   this case we inspect the executable itself (magic numbers,
-#   i.e. fileutil::magic::filetype).
-#
-# The basic information used comes out of the 'os' and 'machine'
-# entries of the 'tcl_platform' array. A number of general and
-# os/machine specific transformation are applied to get a canonical
-# result.
-#
-# General
-# Only the first element of 'os' is used - we don't care whether we
-# are on "Windows NT" or "Windows XP" or whatever.
-#
-# Machine specific
-# % amd64  -> x86_64
-# % arm*   -> arm
-# % sun4*  -> sparc
-# % ia32*  -> ix86
-# % intel  -> ix86
-# % i*86*  -> ix86
-# % Power* -> powerpc
-# % x86_64 + wordSize 4 => x86 code
-#
-# OS specific
-# % AIX are always powerpc machines
-# % HP-UX 9000/800 etc means parisc
-# % linux has to take glibc version into account
-# % sunos -> solaris, and keep version number
-#
-# NOTE: A platform like linux glibc 2.3, which can use glibc 2.2 stuff
-# has to provide all possible allowed platform identifiers when
-# searching search. Ditto a solaris 2.8 platform can use solaris 2.6
-# packages. Etc. This is handled by the other procedure, see below.
-
-# ### ### ### ######### ######### #########
-## Requirements
-
-namespace eval ::platform {}
-
-# ### ### ### ######### ######### #########
-## Implementation
-
-# -- platform::generic
-#
-# Assembles an identifier for the generic platform. It leaves out
-# details like kernel version, libc version, etc.
-
-proc ::platform::generic {} {
-    global tcl_platform
-
-    set plat [string tolower [lindex $tcl_platform(os) 0]]
-    set cpu  $tcl_platform(machine)
-
-    switch -glob -- $cpu {
-	sun4* {
-	    set cpu sparc
-	}
-	intel -
-	ia32* -
-	i*86* {
-	    set cpu ix86
-	}
-	x86_64 {
-	    if {$tcl_platform(wordSize) == 4} {
-		# See Example <1> at the top of this file.
-		set cpu ix86
-	    }
-	}
-	ppc -
-	"Power*" {
-	    set cpu powerpc
-	}
-	"arm*" {
-	    set cpu arm
-	}
-	ia64 {
-	    if {$tcl_platform(wordSize) == 4} {
-		append cpu _32
-	    }
-	}
-    }
-
-    switch -glob -- $plat {
-	windows {
-	    if {$tcl_platform(platform) == "unix"} {
-		set plat cygwin
-	    } else {
-		set plat win32
-	    }
-	    if {$cpu eq "amd64"} {
-		# Do not check wordSize, win32-x64 is an IL32P64 platform.
-		set cpu x86_64
-	    }
-	}
-	sunos {
-	    set plat solaris
-	    if {[string match "ix86" $cpu]} {
-		if {$tcl_platform(wordSize) == 8} {
-		    set cpu x86_64
-		}
-	    } elseif {![string match "ia64*" $cpu]} {
-		# sparc
-		if {$tcl_platform(wordSize) == 8} {
-		    append cpu 64
-		}
-	    }
-	}
-	darwin {
-	    set plat macosx
-	    # Correctly identify the cpu when running as a 64bit
-	    # process on a machine with a 32bit kernel
-	    if {$cpu eq "ix86"} {
-		if {$tcl_platform(wordSize) == 8} {
-		    set cpu x86_64
-		}
-	    }
-	}
-	aix {
-	    set cpu powerpc
-	    if {$tcl_platform(wordSize) == 8} {
-		append cpu 64
-	    }
-	}
-	hp-ux {
-	    set plat hpux
-	    if {![string match "ia64*" $cpu]} {
-		set cpu parisc
-		if {$tcl_platform(wordSize) == 8} {
-		    append cpu 64
-		}
-	    }
-	}
-	osf1 {
-	    set plat tru64
-	}
-	default {
-	    set plat [lindex [split $plat _-] 0]
-	}
-    }
-
-    return "${plat}-${cpu}"
-}
-
-# -- platform::identify
-#
-# Assembles an identifier for the exact platform, by extending the
-# generic identifier. I.e. it adds in details like kernel version,
-# libc version, etc., if they are relevant for the loading of
-# packages on the platform.
-
-proc ::platform::identify {} {
-    global tcl_platform
-
-    set id [generic]
-    regexp {^([^-]+)-([^-]+)$} $id -> plat cpu
-
-    switch -- $plat {
-	solaris {
-	    regsub {^5} $tcl_platform(osVersion) 2 text
-	    append plat $text
-	    return "${plat}-${cpu}"
-	}
-	macosx {
-	    set major [lindex [split $tcl_platform(osVersion) .] 0]
-	    if {$major > 19} {
-		set minor [lindex [split $tcl_platform(osVersion) .] 1]
-		incr major -9
-		append plat $major.[expr {$minor - 1}]
-	    } else {
-		incr major -4
-		append plat 10.$major
-		return "${plat}-${cpu}"
-	    }
-	    return "${plat}-${cpu}"
-	}
-	linux {
-	    # Look for the libc*.so and determine its version
-	    # (libc5/6, libc6 further glibc 2.X)
-
-	    set v unknown
-
-	    # Determine in which directory to look. /lib, or /lib64.
-	    # For that we use the tcl_platform(wordSize).
-	    #
-	    # We could use the 'cpu' info, per the equivalence below,
-	    # that however would be restricted to intel. And this may
-	    # be a arm, mips, etc. system. The wordsize is more
-	    # fundamental.
-	    #
-	    # ix86   <=> (wordSize == 4) <=> 32 bit ==> /lib
-	    # x86_64 <=> (wordSize == 8) <=> 64 bit ==> /lib64
-	    #
-	    # Do not look into /lib64 even if present, if the cpu
-	    # doesn't fit.
-
-	    # TODO: Determine the prefixes (i386, x86_64, ...) for
-	    # other cpus.  The path after the generic one is utterly
-	    # specific to intel right now.  Ok, on Ubuntu, possibly
-	    # other Debian systems we may apparently be able to query
-	    # the necessary CPU code. If we can't we simply use the
-	    # hardwired fallback.
-
-	    switch -exact -- $tcl_platform(wordSize) {
-		4 {
-		    lappend bases /lib
-		    if {[catch {
-			exec dpkg-architecture -qDEB_HOST_MULTIARCH
-		    } res]} {
-			lappend bases /lib/i386-linux-gnu
-		    } else {
-			# dpkg-arch returns the full tripled, not just cpu.
-			lappend bases /lib/$res
-		    }
-		}
-		8 {
-		    lappend bases /lib64
-		    if {[catch {
-			exec dpkg-architecture -qDEB_HOST_MULTIARCH
-		    } res]} {
-			lappend bases /lib/x86_64-linux-gnu
-		    } else {
-			# dpkg-arch returns the full tripled, not just cpu.
-			lappend bases /lib/$res
-		    }
-		}
-		default {
-		    return -code error "Bad wordSize $tcl_platform(wordSize), expected 4 or 8"
-		}
-	    }
-
-	    foreach base $bases {
-		if {[LibcVersion $base -> v]} break
-	    }
-
-	    append plat -$v
-	    return "${plat}-${cpu}"
-	}
-    }
-
-    return $id
-}
-
-proc ::platform::LibcVersion {base _->_ vv} {
-    upvar 1 $vv v
-    set libclist [lsort [glob -nocomplain -directory $base libc*]]
-
-    if {![llength $libclist]} { return 0 }
-
-    set libc [lindex $libclist 0]
-
-    # Try executing the library first. This should suceed
-    # for a glibc library, and return the version
-    # information.
-
-    if {![catch {
-	set vdata [lindex [split [exec $libc] \n] 0]
-    }]} {
-	regexp {version ([0-9]+(\.[0-9]+)*)} $vdata -> v
-	foreach {major minor} [split $v .] break
-	set v glibc${major}.${minor}
-	return 1
-    } else {
-	# We had trouble executing the library. We are now
-	# inspecting its name to determine the version
-	# number. This code by Larry McVoy.
-
-	if {[regexp -- {libc-([0-9]+)\.([0-9]+)} $libc -> major minor]} {
-	    set v glibc${major}.${minor}
-	    return 1
-	}
-    }
-    return 0
-}
-
-# -- platform::patterns
-#
-# Given an exact platform identifier, i.e. _not_ the generic
-# identifier it assembles a list of exact platform identifier
-# describing platform which should be compatible with the
-# input.
-#
-# I.e. packages for all platforms in the result list should be
-# loadable on the specified platform.
-
-# << Should we add the generic identifier to the list as well ? In
-#    general it is not compatible I believe. So better not. In many
-#    cases the exact identifier is identical to the generic one
-#    anyway.
-# >>
-
-proc ::platform::patterns {id} {
-    set res [list $id]
-    if {$id eq "tcl"} {return $res}
-
-    switch -glob --  $id {
-	solaris*-* {
-	    if {[regexp {solaris([^-]*)-(.*)} $id -> v cpu]} {
-		if {$v eq ""} {return $id}
-		foreach {major minor} [split $v .] break
-		incr minor -1
-		for {set j $minor} {$j >= 6} {incr j -1} {
-		    lappend res solaris${major}.${j}-${cpu}
-		}
-	    }
-	}
-	linux*-* {
-	    if {[regexp {linux-glibc([^-]*)-(.*)} $id -> v cpu]} {
-		foreach {major minor} [split $v .] break
-		incr minor -1
-		for {set j $minor} {$j >= 0} {incr j -1} {
-		    lappend res linux-glibc${major}.${j}-${cpu}
-		}
-	    }
-	}
-	macosx-powerpc {
-	    lappend res macosx-universal
-	}
-	macosx-x86_64 {
-	    lappend res macosx-i386-x86_64
-	}
-	macosx-ix86 {
-	    lappend res macosx-universal macosx-i386-x86_64
-	}
-	macosx*-*    {
-	    # 10.5+,11.0+
-	    if {[regexp {macosx([^-]*)-(.*)} $id -> v cpu]} {
-
-		switch -exact -- $cpu {
-		    ix86    {
-			lappend alt i386-x86_64
-			lappend alt universal
-		    }
-		    x86_64  {
-			if {[lindex [split $::tcl_platform(osVersion) .] 0] < 19} {
-			    set alt i386-x86_64
-			} else {
-			    set alt {}
-			}
-		    }
-		    arm  {
-			lappend alt x86_64
-		    }
-		    default { set alt {} }
-		}
-
-		if {$v ne ""} {
-		    foreach {major minor} [split $v .] break
-
-		    set res {}
-		    if {$major eq 11} {
-			# Add 11.0 to 11.minor to patterns.
-			for {set j $minor} {$j >= 0} {incr j -1} {
-			    lappend res macosx${major}.${j}-${cpu}
-			    foreach a $alt {
-				lappend res macosx${major}.${j}-$a
-			    }
-			}
-			set major 10
-			set minor 15
-		    }
-		    # Add 10.5 to 10.minor to patterns.
-		    for {set j $minor} {$j >= 5} {incr j -1} {
-			if {$cpu ne "arm"} {
-			    lappend res macosx${major}.${j}-${cpu}
-			}
-			foreach a $alt {
-			    lappend res macosx${major}.${j}-$a
-			}
-		    }
-
-		    # Add unversioned patterns for 10.3/10.4 builds.
-		    lappend res macosx-${cpu}
-		    foreach a $alt {
-			lappend res macosx-$a
-		    }
-		} else {
-		    # No version, just do unversioned patterns.
-		    foreach a $alt {
-			lappend res macosx-$a
-		    }
-		}
-	    } else {
-		# no v, no cpu ... nothing
-	    }
-	}
-    }
-    lappend res tcl ; # Pure tcl packages are always compatible.
-    return $res
-}
-
-
-# ### ### ### ######### ######### #########
-## Ready
-
-package provide platform 1.0.17
-
-# ### ### ### ######### ######### #########
-## Demo application
-
-if {[info exists argv0] && ($argv0 eq [info script])} {
-    puts ====================================
-    parray tcl_platform
-    puts ====================================
-    puts Generic\ identification:\ [::platform::generic]
-    puts Exact\ identification:\ \ \ [::platform::identify]
-    puts ====================================
-    puts Search\ patterns:
-    puts *\ [join [::platform::patterns [::platform::identify]] \n*\ ]
-    puts ====================================
-    exit 0
-}

src/modules/platform/shell-1.1.4.tm

@@ -1,241 +0,0 @@
-
-# -*- tcl -*-
-# ### ### ### ######### ######### #########
-## Overview
-
-# Higher-level commands which invoke the functionality of this package
-# for an arbitrary tcl shell (tclsh, wish, ...). This is required by a
-# repository as while the tcl shell executing packages uses the same
-# platform in general as a repository application there can be
-# differences in detail (i.e. 32/64 bit builds).
-
-# ### ### ### ######### ######### #########
-## Requirements
-
-package require platform
-namespace eval ::platform::shell {}
-
-# ### ### ### ######### ######### #########
-## Implementation
-
-# -- platform::shell::generic
-
-proc ::platform::shell::generic {shell} {
-    # Argument is the path to a tcl shell.
-
-    CHECK $shell
-    LOCATE base out
-
-    set     code {}
-    # Forget any pre-existing platform package, it might be in
-    # conflict with this one.
-    lappend code {package forget platform}
-    # Inject our platform package
-    lappend code [list source $base]
-    # Query and print the architecture
-    lappend code {puts [platform::generic]}
-    # And done
-    lappend code {exit 0}
-
-    set arch [RUN $shell [join $code \n]]
-
-    if {$out} {file delete -force $base}
-    return $arch
-}
-
-# -- platform::shell::identify
-
-proc ::platform::shell::identify {shell} {
-    # Argument is the path to a tcl shell.
-
-    CHECK $shell
-    LOCATE base out
-
-    set     code {}
-    # Forget any pre-existing platform package, it might be in
-    # conflict with this one.
-    lappend code {package forget platform}
-    # Inject our platform package
-    lappend code [list source $base]
-    # Query and print the architecture
-    lappend code {puts [platform::identify]}
-    # And done
-    lappend code {exit 0}
-
-    set arch [RUN $shell [join $code \n]]
-
-    if {$out} {file delete -force $base}
-    return $arch
-}
-
-# -- platform::shell::platform
-
-proc ::platform::shell::platform {shell} {
-    # Argument is the path to a tcl shell.
-
-    CHECK $shell
-
-    set     code {}
-    lappend code {puts $tcl_platform(platform)}
-    lappend code {exit 0}
-
-    return [RUN $shell [join $code \n]]
-}
-
-# ### ### ### ######### ######### #########
-## Internal helper commands.
-
-proc ::platform::shell::CHECK {shell} {
-    if {![file exists $shell]} {
-	return -code error "Shell \"$shell\" does not exist"
-    }
-    if {![file executable $shell]} {
-	return -code error "Shell \"$shell\" is not executable (permissions)"
-    }
-    return
-}
-
-proc ::platform::shell::LOCATE {bv ov} {
-    upvar 1 $bv base $ov out
-
-    # Locate the platform package for injection into the specified
-    # shell. We are using package management to find it, whereever it
-    # is, instead of using hardwired relative paths. This allows us to
-    # install the two packages as TMs without breaking the code
-    # here. If the found package is wrapped we copy the code somewhere
-    # where the spawned shell will be able to read it.
-
-    # This code is brittle, it needs has to adapt to whatever changes
-    # are made to the TM code, i.e. the provide statement generated by
-    # tm.tcl
-
-    set pl [package ifneeded platform [package require platform]]
-    set base [lindex $pl end]
-
-    set out 0
-    if {[lindex [file system $base]] ne "native"} {
-	set temp [TEMP]
-	file copy -force $base $temp
-	set base $temp
-	set out 1
-    }
-    return
-}
-
-proc ::platform::shell::RUN {shell code} {
-    set     c [TEMP]
-    set    cc [open $c w]
-    puts  $cc $code
-    close $cc
-
-    set e [TEMP]
-
-    set code [catch {
-        exec $shell $c 2> $e
-    } res]
-
-    file delete $c
-
-    if {$code} {
-	append res \n[read [set chan [open $e r]]][close $chan]
-	file delete $e
-	return -code error "Shell \"$shell\" is not executable ($res)"
-    }
-
-    file delete $e
-    return $res
-}
-
-proc ::platform::shell::TEMP {} {
-    set prefix platform
-
-    # This code is copied out of Tcllib's fileutil package.
-    # (TempFile/tempfile)
-
-    set tmpdir [DIR]
-
-    set chars "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
-    set nrand_chars 10
-    set maxtries 10
-    set access [list RDWR CREAT EXCL TRUNC]
-    set permission 0600
-    set channel ""
-    set checked_dir_writable 0
-    set mypid [pid]
-    for {set i 0} {$i < $maxtries} {incr i} {
- 	set newname $prefix
- 	for {set j 0} {$j < $nrand_chars} {incr j} {
- 	    append newname [string index $chars \
-		    [expr {int(rand()*62)}]]
- 	}
-	set newname [file join $tmpdir $newname]
- 	if {[file exists $newname]} {
- 	    after 1
- 	} else {
- 	    if {[catch {open $newname $access $permission} channel]} {
- 		if {!$checked_dir_writable} {
- 		    set dirname [file dirname $newname]
- 		    if {![file writable $dirname]} {
- 			return -code error "Directory $dirname is not writable"
- 		    }
- 		    set checked_dir_writable 1
- 		}
- 	    } else {
- 		# Success
-		close $channel
- 		return [file normalize $newname]
- 	    }
- 	}
-    }
-    if {$channel ne ""} {
- 	return -code error "Failed to open a temporary file: $channel"
-    } else {
- 	return -code error "Failed to find an unused temporary file name"
-    }
-}
-
-proc ::platform::shell::DIR {} {
-    # This code is copied out of Tcllib's fileutil package.
-    # (TempDir/tempdir)
-
-    global tcl_platform env
-
-    set attempdirs [list]
-
-    foreach tmp {TMPDIR TEMP TMP} {
-	if { [info exists env($tmp)] } {
-	    lappend attempdirs $env($tmp)
-	}
-    }
-
-    switch $tcl_platform(platform) {
-	windows {
-	    lappend attempdirs "C:\\TEMP" "C:\\TMP" "\\TEMP" "\\TMP"
-	}
-	macintosh {
-	    set tmpdir $env(TRASH_FOLDER)  ;# a better place?
-	}
-	default {
-	    lappend attempdirs \
-		[file join / tmp] \
-		[file join / var tmp] \
-		[file join / usr tmp]
-	}
-    }
-
-    lappend attempdirs [pwd]
-
-    foreach tmp $attempdirs {
-	if { [file isdirectory $tmp] && [file writable $tmp] } {
-	    return [file normalize $tmp]
-	}
-    }
-
-    # Fail if nothing worked.
-    return -code error "Unable to determine a proper directory for temporary files"
-}
-
-# ### ### ### ######### ######### #########
-## Ready
-
-package provide platform::shell 1.1.4

src/modules/punk-0.1.tm

@@ -3,8 +3,13 @@
 
 
 namespace eval punk {
+    proc lazyload {pkg} {
         package require zzzload
-    zzzload::pkg_require twapi
+        if {[package provide $pkg] eq ""} {
+            zzzload::pkg_require $pkg 
+        } 
+    }
+    #lazyload twapi
 
     catch {package require vfs} ;#attempt load now so we can use faster 'package provide' to test existence later
 }
@@ -165,17 +170,49 @@ namespace eval punk {
 
     proc ::punk::K {x y} { return $x}
 
+    proc stacktrace {} {
+        set stack "Stack trace:\n"
+        for {set i 1} {$i < [info level]} {incr i} {
+            set lvl [info level -$i]
+            set pname [lindex $lvl 0]
+            append stack [string repeat " " $i]$pname
+
+            if {![catch {info args $pname} pargs]} {
+                foreach value [lrange $lvl 1 end] arg $pargs {
+
+                    if {$value eq ""} {
+                        if {$arg != 0} {
+                            info default $pname $arg value
+                        }
+                    }
+                    append stack " $arg='$value'"
+                }
+            } else {
+                append stack " !unknown vars for $pname"
+            }
+
+            append stack \n
+        }
+        return $stack
+    }
     proc ::punk::uuid {} {
         set has_twapi 0
+        if 0 {
             if {"windows" eq $::tcl_platform(platform)} {
+                if {![catch {
                     set loader [zzzload::pkg_wait twapi]
+                } errM]} {
                     if {$loader in [list failed loading]} {
                         puts stderr "Unexpected problem during thread-load of pkg twapi - zzload::pkg_wait returned $loader"
                     }
+                } else {
+                    package require twapi
+                }
                 if {[package provide twapi] ne ""} {
                     set has_twapi 1
                 }
             }
+        }
         if {!$has_twapi} {
             if {[catch {package require uuid} errM]} {
                 error "Unable to load a package for uuid on this platform. Try tcllib's uuid (any platform) - twapi for windows"

src/modules/punk/ansi-999999.0a1.0.tm

@@ -158,7 +158,7 @@ namespace eval punk::ansi {
         overline    53  nooverline      55  frame           51  framecircle 52  noframe     54
     }
     variable SGR_colour_map {
-        black       30  red             31  green           32  yellow      33  blue        4       purple 35   cyan        36      white 37
+        black       30  red             31  green           32  yellow      33  blue        34       purple 35   cyan        36      white 37
         Black       40  Red             41  Green           42  Yellow      43  Blue        44      Purple 45   Cyan        46      White 47
         BLACK       100 RED             101 GREEN           102 YELLOW      103 BLUE        104     PURPLE 105  CYAN        106     WHITE 107
     }

src/modules/punk/console-999999.0a1.0.tm

@@ -21,8 +21,8 @@ package require punk::ansi
 
 
 if {"windows" eq $::tcl_platform(platform)} {
-    package require zzzload
+    #package require zzzload
-    zzzload::pkg_require twapi
+    #zzzload::pkg_require twapi
 }
 
 #see https://learn.microsoft.com/en-us/windows/console/classic-vs-vt  
@@ -129,8 +129,12 @@ namespace eval punk::console {
             }
         }
         proc define_windows_procs {} {
+            package require zzzload
             set loadstate [zzzload::pkg_require twapi]
-            if {$loadstate ni [list loading failed]} {
+            if {$loadstate ni [list failed]} {
+                #review zzzload usage
+                #puts stdout "=========== console loading twapi ============="
+                zzzload::pkg_wait twapi
                 package require twapi ;#should be fast once twapi dll loaded in zzzload thread
                 set ::punk::console::has_twapi 1
 

src/modules/punk/mix/base-0.1.tm

@@ -284,7 +284,7 @@ namespace eval punk::mix::base {
     #    puts stdout "base doingstuff-with-args:'$args'-in-namespace:'[namespace current]'"
     #}
     namespace eval lib {
-
+        variable sha3_implementation "" ;#set by cksum_algorithms (which is called by cksum_path) It looks for fossil or sqlite3. Todo - add proper Tcl implementation.
         namespace export *
 
         #-----------------------------------------------------
@@ -445,9 +445,28 @@ namespace eval punk::mix::base {
         #adler32 is fastest for some larger files of a few MB but slower on small files (possibly due to Tcl-based file load?)
         #sha1 as at 2023 seems a good default 
         proc cksum_algorithms {} {
+            variable sha3_implementation
             #sha2 is an alias for sha256
-            #2023 - no sha3 available in tcllib
+            #2023 - no sha3 available in tcllib - we can exec fossil for now - which will be very slow
-            return [list md5 sha1 sha2 sha256 cksum adler32]
+            set algs [list md5 sha1 sha2 sha256 cksum adler32]
+            set sha3_algs [list sha3 sha3-224 sha3-256 sha3-384 sha3-512]
+            if {[auto_execok sqlite3] ne ""} {
+                lappend algs {*}$sha3_algs 
+                set sha3_implementation sqlite3_sha3
+            } else {
+                if {[auto_execok fossil] ne ""} {
+                    lappend algs {*}$sha3_algs 
+                    set sha3_implementation fossil_sha3
+                }
+            }
+            return $algs
+        }
+
+        proc sqlite3_sha3 {bits filename} {
+            return [exec sqlite3 :memory: "select lower(hex(sha3(readfile('$filename'),$bits)))"]
+        }
+        proc fossil_sha3 {bits filename} {
+            return [lindex [exec fossil sha3sum -$bits $filename] 0]
         }
 
         #adler32 via file-slurp
@@ -465,6 +484,7 @@ namespace eval punk::mix::base {
         #-noperms only available on extraction - so that doesn't help
         #Needs to operate on non-existant paths and return empty string in cksum field
         proc cksum_path {path args} {
+            variable sha3_implementation
             if {$path eq {}} { set path [pwd] }
             if {[file pathtype $path] eq "relative"} {
                 set path [file normalize $path]
@@ -473,6 +493,13 @@ namespace eval punk::mix::base {
             set startdir [pwd]
             
             set defaults [cksum_default_opts]
+            set known_opts [dict keys $defaults]
+            foreach {k v} $args {
+                if {$k ni $known_opts} {
+                    error "cksum_path unknown option '$k' known_options: $known_opts"
+                }
+            }
+
             set opts [dict merge $defaults $args]
             set opts_actual $opts ;#default  - auto updated to 0 or 1 later
 
@@ -582,6 +609,14 @@ namespace eval punk::mix::base {
                 set cksum_command [list crc::cksum -format 0x%X -file]
             } elseif {$opt_cksum_algorithm eq "adler32"} {
                 set cksum_command [list cksum_adler32_file]
+            } elseif {$opt_cksum_algorithm in [list "sha3" "sha3-256"]} {
+                #todo - replace with something that doesn't call another process
+                #set cksum_command [list apply {{file} {lindex [exec fossil sha3sum -256 $file] 0}}]
+                set cksum_command [list $sha3_implementation 256]
+            } elseif {$opt_cksum_algorithm in [list "sha3-224" "sha3-384" "sha3-512"]} {
+                set bits [lindex [split $opt_cksum_algorithm -] 1]
+                #set cksum_command [list apply {{bits file} {lindex [exec fossil sha3sum -$bits $file] 0}} $bits]
+                set cksum_command [list $sha3_implementation $bits]
             }
 
             set cksum ""

src/modules/punk/mix/cli-0.3.tm

@@ -41,12 +41,12 @@ namespace eval punk::mix::cli {
     catch {
         package require punk::mix::commandset::project
         punk::overlay::import_commandset project    .       ::punk::mix::commandset::project
-        punk::overlay::import_commandset ""             ::punk::mix::commandset::project::collection 
+        punk::overlay::import_commandset projects   .       ::punk::mix::commandset::project::collection 
     }
     if {[catch {
         package require punk::mix::commandset::layout
         punk::overlay::import_commandset project.layout       . ::punk::mix::commandset::layout
-        punk::overlay::import_commandset "project."      ::punk::mix::commandset::layout::collection
+        punk::overlay::import_commandset project.layouts      . ::punk::mix::commandset::layout::collection
     } errM]} {
         puts stderr "error loading punk::mix::commandset::layout"
         puts stderr $errM
@@ -54,12 +54,20 @@ namespace eval punk::mix::cli {
     if {[catch {
         package require punk::mix::commandset::buildsuite
         punk::overlay::import_commandset buildsuite     .   ::punk::mix::commandset::buildsuite
-        punk::overlay::import_commandset ""      ::punk::mix::commandset::buildsuite::collection
+        punk::overlay::import_commandset buildsuites    .   ::punk::mix::commandset::buildsuite::collection
     } errM]} {
         puts stderr "error loading punk::mix::commandset::buildsuite"
         puts stderr $errM
     }
     punk::overlay::import_commandset scriptwrap  . ::punk::mix::commandset::scriptwrap
+    if {[catch {
+        package require punk::mix::commandset::doc
+        punk::overlay::import_commandset doc . ::punk::mix::commandset::doc
+        punk::overlay::import_commandset ""  ""   ::punk::mix::commandset::doc::collection
+    } errM]} {
+        puts stderr "error loading punk::mix::commandset::doc"
+        puts stderr $errM
+    }
 
 
     proc help {args} {
@@ -148,7 +156,7 @@ namespace eval punk::mix::cli {
         set lc_proj_bin [string tolower $project_base/bin]
         set lc_build_bin [string tolower $project_base/src/_build]
 
-
+        if {"project" in $args} {
             set is_own_exe 0
             if {[string match "${lc_proj_bin}*" $lc_this_exe] || [string match "${lc_build_bin}" $lc_this_exe]} {
                 set is_own_exe 1
@@ -159,10 +167,17 @@ namespace eval punk::mix::cli {
                     return
                 }
             }
+        }
         cd $sourcefolder
         #use run so that stdout visible as it goes
-        set exitinfo [run [info nameofexecutable] $sourcefolder/make.tcl project]
+        if {![catch {run --timeout=5000 -debug [info nameofexecutable] $sourcefolder/make.tcl {*}$args} exitinfo]} {
+            puts stderr "exitinfo: $exitinfo"
             set exitcode [dict get $exitinfo exitcode]
+        } else {
+            puts stderr "Error unable to determine exitcode. err: $exitinfo"
+            cd $startdir
+            return false
+        }
 
         cd $startdir
         if {$exitcode != 0} {
@@ -388,7 +403,7 @@ namespace eval punk::mix::cli {
                 -call-depth-internal 0\
                 -max_depth 1000\
                 -subdirlist {}\
-                -punkcheck_eventid "\uFFFF"\
+                -punkcheck_eventobj "\uFFFF"\
                 -glob *.tm\
                 ]
             set opts [dict merge $defaults $args]
@@ -405,7 +420,7 @@ namespace eval punk::mix::cli {
                 error "build_modules_from_source_to_base -glob '$fileglob' doesn't seem to target tcl modules."
             }
             # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
-            set opt_punkcheck_eventid                   [dict get $opts -punkcheck_eventid]
+            set opt_punkcheck_eventobj                   [dict get $opts -punkcheck_eventobj]
 
             set magicversion [punk::mix::util::magic_tm_version] ;#deliberately large so given load-preference when testing
             set module_list [list]
@@ -443,10 +458,15 @@ namespace eval punk::mix::cli {
                     -glob $fileglob\
                     -max_depth 0\
                 ]
-                lassign [punkcheck::start_installer_event $punkcheck_file $installername $srcdir $basedir $config] _eventid punkcheck_eventid _recordset record_list
+                #lassign [punkcheck::start_installer_event $punkcheck_file $installername $srcdir $basedir $config] _eventid punkcheck_eventid _recordset record_list
+                # -- ---
+                set installer [punkcheck::installtrack new $installername $punkcheck_file]
+                $installer set_source_target $srcdir $basedir
+                set event [$installer start_event $config]
+                # -- ---
 
             } else {
-                set punkcheck_eventid $opt_punkcheck_eventid
+                set event $opt_punkcheck_eventobj
             }
             #----------------------------------------
 
@@ -482,6 +502,7 @@ namespace eval punk::mix::cli {
 
 
                     if {[file exists $current_source_dir/#tarjar-$basename-$magicversion]} {
+                        #TODO
                         file mkdir $buildfolder
 
                         if {[file exists $current_source_dir/#tarjar-$basename-$magicversion/DESCRIPTION.txt]} {
@@ -519,24 +540,31 @@ namespace eval punk::mix::cli {
 
                         #------------------------------
                         #
-                        set target_relpath [punkcheck::lib::path_relative $basedir $target_module_dir/$basename-$module_build_version.tm]
+                        #set target_relpath [punkcheck::lib::path_relative $basedir $target_module_dir/$basename-$module_build_version.tm]
-
+                        #set file_record [punkcheck::installfile_begin $basedir $target_relpath $installername -eventid $punkcheck_eventid]
-                        set file_record [punkcheck::installfile_begin $basedir $target_relpath $installername -eventid $punkcheck_eventid]
+                        $event targetset_init INSTALL $target_module_dir/$basename-$module_build_version.tm 
-
+                        $event targetset_addsource $versionfile
-                        set changed_list [list]
+                        $event targetset_addsource $current_source_dir/$m
-                        # -- --- --- --- --- ---
+
-                        set source_relpath [punkcheck::lib::path_relative $basedir $versionfile]
+                        #set changed_list [list]
-                        set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record]
+                        ## -- --- --- --- --- ---
-                        # -- --- --- --- --- ---
+                        #set source_relpath [punkcheck::lib::path_relative $basedir $versionfile]
-                        set source_relpath [punkcheck::lib::path_relative $basedir $current_source_dir/$m]
+                        #set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record]
-                        set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record]
+                        ## -- --- --- --- --- ---
-                        # -- --- --- --- --- ---
+                        #set source_relpath [punkcheck::lib::path_relative $basedir $current_source_dir/$m]
-                        set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]]
+                        #set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record]
-                        set changed_list [dict get $changed_unchanged changed]
+                        ## -- --- --- --- --- ---
-
+                        #set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]]
-
+                        #set changed_list [dict get $changed_unchanged changed]
-                        if {[llength $changed_list]} {
+
-                            set file_record [punkcheck::installfile_started_install $basedir $file_record]
+
+                        if {\
+                                [llength [dict get [$event targetset_source_changes] changed]]\
+                            || [llength [$event get_targets_exist]] < [llength [$event get_targets]]\
+                            } {
+
+                            #set file_record [punkcheck::installfile_started_install $basedir $file_record]
+                            $event targetset_started
                             # -- --- --- --- --- ---
                             set target $target_module_dir/$basename-$module_build_version.tm
                             if {$did_skip} {set did_skip 0; puts -nonewline stdout \n}
@@ -550,12 +578,14 @@ namespace eval punk::mix::cli {
                             #file copy -force $srcdir/$m $target
                             lappend module_list $target
                             # -- --- --- --- --- ---
-                            set file_record [punkcheck::installfile_finished_install $basedir $file_record]
+                            #set file_record [punkcheck::installfile_finished_install $basedir $file_record]
+                            $event targetset_end OK
                         } else {
                             #puts stdout "skipping module $current_source_dir/$m - no change in sources detected"
                             puts -nonewline stderr "."
                             set did_skip 1
-                            set file_record [punkcheck::installfile_skipped_install $basedir $file_record]
+                            #set file_record [punkcheck::installfile_skipped_install $basedir $file_record]
+                            $event targetset_end SKIPPED
                         }
 
                         #------------------------------
@@ -572,33 +602,41 @@ namespace eval punk::mix::cli {
                     exit 1
                 }
 
-                #------------------------------
+                ##------------------------------
-                #
+                ##
-                set target_relpath [punkcheck::lib::path_relative $basedir $target_module_dir/$m]
+                #set target_relpath [punkcheck::lib::path_relative $basedir $target_module_dir/$m]
-
+                #set file_record [punkcheck::installfile_begin $basedir $target_relpath $installername -eventid $punkcheck_eventid]
-                set file_record [punkcheck::installfile_begin $basedir $target_relpath $installername -eventid $punkcheck_eventid]
+                #set changed_list [list]
-
+                ## -- --- --- --- --- ---
-                set changed_list [list]
+                #set source_relpath [punkcheck::lib::path_relative $basedir $current_source_dir/$m]
-                # -- --- --- --- --- ---
+                #set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record]
-                set source_relpath [punkcheck::lib::path_relative $basedir $current_source_dir/$m]
+                ## -- --- --- --- --- ---
-                set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record]
+                #set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]]
-                # -- --- --- --- --- ---
+                #set changed_list [dict get $changed_unchanged changed]
-                set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]]
+
-                set changed_list [dict get $changed_unchanged changed]
+                #----------
-
+                $event targetset_init INSTALL $target_module_dir/$m 
-                if {[llength $changed_list]} {
+                $event targetset_addsource $current_source_dir/$m
-                    set file_record [punkcheck::installfile_started_install $basedir $file_record]
+                if {\
+                        [llength [dict get [$event targetset_source_changes] changed]]\
+                    || [llength [$event get_targets_exist]] < [llength [$event get_targets]]\
+                    } {
+
+                    #set file_record [punkcheck::installfile_started_install $basedir $file_record]
+                    $event targetset_started
                     # -- --- --- --- --- ---
                     if {$did_skip} {set did_skip 0; puts -nonewline stdout \n}
                     puts stderr "Copied already versioned module $current_source_dir/$m to $target_module_dir"
                     lappend module_list $current_source_dir/$m
                     file copy -force $current_source_dir/$m $target_module_dir
                     # -- --- --- --- --- ---
-                    set file_record [punkcheck::installfile_finished_install $basedir $file_record]
+                    #set file_record [punkcheck::installfile_finished_install $basedir $file_record]
+                    $event targetset_end OK -note "already versioned module"
                 } else {
                     puts -nonewline stderr "."
                     set did_skip 1
-                    set file_record [punkcheck::installfile_skipped_install $basedir $file_record]
+                    #set file_record [punkcheck::installfile_skipped_install $basedir $file_record]
+                    $event targetset_end SKIPPED
                 }
 
             }
@@ -625,17 +663,90 @@ namespace eval punk::mix::cli {
                 lappend module_list {*}[build_modules_from_source_to_base $srcdir $basedir\
                     -call-depth-internal [expr {$CALLDEPTH +1}]\
                     -subdirlist [list {*}$subdirlist $d]\
-                    -punkcheck_eventid $punkcheck_eventid\
+                    -punkcheck_eventobj $event\
                     -glob $fileglob\
                 ]
             }
             if {$did_skip} {
                 puts -nonewline stdout \n
             }
+            if {$CALLDEPTH == 0} {
+                $event destroy
+                $installer destroy 
+            }
             return $module_list
         }
 
+        variable kettle_reset_bodies [dict create]
+        variable kettle_reset_args [dict create]
+        #We are abusing kettle to run in-process.
+        # when we change to another project we need recipes to be reloaded.
+        # Kettle rewrites some of it's own procs - stopping reloading of recipes when we change folders
+        #kettle_init stores the original proc bodies & args
+        proc kettle_init {} {
+            variable kettle_reset_bodies ;#dict
+            variable kettle_reset_args
+            set reset_procs [list\
+                ::kettle::benchmarks\
+                ::kettle::doc\
+                ::kettle::figures\
+                ::kettle::meta::scan\
+                ::kettle::testsuite\
+            ]
+            foreach p $reset_procs {
+                set b [info body $p]
+                if {[string match "*Overwrite self*" $b]} {
+                    dict set kettle_reset_bodies $p $b
+                    set argnames [info args $p]
+                    set arglist [list]
+                    foreach a $argnames {
+                        if {[info default $p $a dval]} {
+                            lappend arglist [list $a $dval]
+                        } else {
+                            lappend arglist $a
+                        }
+                    }
+                    dict set kettle_reset_args $p $arglist
+                }
+            } 
+
+        }
+        #call kettle_reinit to ensure recipes point to current project
+        proc kettle_reinit {} {
+            variable kettle_reset_bodies
+            variable kettle_reset_args
+            foreach p [dict keys $kettle_reset_bodies] {
+                set b [dict get $kettle_reset_bodies $p]
+                set argl [dict get $kettle_reset_args $p]
+                uplevel 1 [list ::proc $p $argl $b]
+            }
+            #todo - determine standard recipes by examining standard.tcl instead of hard coding?
+            set standard_recipes [list\
+                null\
+                forever\
+                list-recipes\
+                help-recipes\
+                help-dump\
+                help-recipes\
+                help\
+                list\
+                list-options\
+                help-options\
+                show-configuration\
+                show-state\
+                show\
+                meta-status\
+                gui\
+            ]
+            #set ::kettle::recipe::recipe [dict create]
+            foreach r [dict keys $::kettle::recipe::recipe] {
+                if {$r ni $standard_recipes} {
+                    dict unset ::kettle::recipe::recipe $r
+                }
+            }
+        }
         proc kettle_call {calltype args} {
+            variable kettle_reset_bodies
             if {$calltype ni [list lib shell]} {
                 error "pmix kettle_call 1st argument must be one of: 'lib' for direct use of kettle module or 'shell' to call as separate process"
             }
@@ -659,9 +770,18 @@ namespace eval punk::mix::cli {
             if {![file exists $startdir/build.tcl]} {
                 error "pmix kettle must be run from a folder containing build.tcl (cwd: [pwd])"
             }
-            if {[catch {package present kettle}]} {
+            if {[package provide kettle] eq ""} {
                 puts stdout "Loading kettle package - may be delay on first load ..."
                 package require kettle
+                kettle_init ;#store original procs for those kettle procs that rewrite themselves
+            } else {
+                if {[dict size $kettle_reset_bodies] == 0} {
+                    #presumably package require kettle was called without calling our kettle_init hack.
+                    kettle_init
+                } else {
+                    #undo proc rewrites
+                    kettle_reinit
+                }
             }
             set first [lindex $args 0] 
             if {[string match @* $first]} {

src/modules/punk/mix/commandset/buildsuite-999999.0a1.0.tm

@@ -107,7 +107,7 @@ namespace eval punk::mix::commandset::buildsuite {
 
     namespace eval collection {
         namespace export *
-        proc buildsuites {{glob {}}} {
+        proc _default {{glob {}}} {
             if {![string length $glob]} {
                 set glob *
             }

src/modules/punk/mix/commandset/doc-999999.0a1.0.tm

@@ -0,0 +1,181 @@
+# -*- tcl -*-
+# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from <pkg>-buildversion.txt
+#
+# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem.
+# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository.
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+# (C) 2023
+#
+# @@ Meta Begin
+# Application punk::mix::commandset::doc 999999.0a1.0
+# Meta platform     tcl
+# Meta license      <unspecified>
+# @@ Meta End
+
+
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+##  Requirements
+##e.g package require frobz
+
+
+
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+namespace eval punk::mix::commandset::doc {
+    namespace export *
+
+    proc _default {} {
+        puts "documentation subsystem"
+        puts "commands: doc.build"
+        puts "        build documentation from src/doc to src/embedded using the kettle build tool"
+    }
+
+    proc build {} {
+        puts "build docs"
+        set projectdir [punk::repo::find_project]
+        if {$projectdir eq ""} {
+            puts stderr "No current project dir - unable to build docs"
+            return
+        }
+        if {[file exists $projectdir/src/doc]} {
+            set original_wd [pwd]
+            cd $projectdir/src
+            #----------
+            set installer [punkcheck::installtrack new project.new $projectdir/src/.punkcheck]
+            $installer set_source_target $projectdir/src/doc $projectdir/src/embedded
+            set event [$installer start_event {-install_step kettledoc}]
+            #use same virtual id "kettle_build_doc" as project.new - review best way to keep identifiers like this in sync. 
+            $event targetset_init VIRTUAL kettle_build_doc ;#VIRTUAL - since there is no specific target file - and we don't know all the files that will be generated
+            $event targetset_addsource $projectdir/src/doc  ;#whole doc tree is considered the source 
+            #----------
+            if {\
+                    [llength [dict get [$event targetset_source_changes] changed]]\
+                } {
+                $event targetset_started
+                # -- --- --- --- --- ---
+                puts stdout "BUILDING DOCS at $projectdir/src/embedded from src/doc"
+                if {[catch {
+
+                    punk::mix::cli::lib::kettle_call lib doc
+                    #Kettle doc
+
+                } errM]} {
+                    $event targetset_end FAILED -note "kettle_build_doc failed: $errM"
+                } else {
+                    $event targetset_end OK
+                }
+                # -- --- --- --- --- ---
+            } else {
+                puts stderr "No change detected in src/doc"
+                $event targetset_end SKIPPED
+            }
+            $event end
+            $event destroy
+            $installer destroy
+            cd $original_wd
+        } else {
+            puts stderr "No doc folder found at $projectdir/src/doc"
+        }
+    }
+    proc status {} {
+        set projectdir [punk::repo::find_project]
+        if {$projectdir eq ""} {
+            puts stderr "No current project dir - unable to check doc status"
+            return
+        }
+        if {![file exists $projectdir/src/doc]} {
+            set result "No documentation source found. Expected .man files in doctools format at $projectdir/src/doc"
+            return $result
+        }
+        set original_wd [pwd]
+        cd $projectdir/src
+        puts stdout "Testing status of doctools source location $projectdir/src/doc ..."
+        flush stdout
+        #----------
+        set installer [punkcheck::installtrack new project.new $projectdir/src/.punkcheck]
+        $installer set_source_target $projectdir/src/doc $projectdir/src/embedded
+        set event [$installer start_event {-install_step kettledoc}]
+        #use same virtual id "kettle_build_doc" as project.new - review best way to keep identifiers like this in sync. 
+        $event targetset_init QUERY kettle_build_doc ;#usually VIRTUAL - since there is no specific target file - and we don't know all the files that will be generated - but here we use QUERY to ensure no writes to .punkcheck
+        set last_completion [$event targetset_last_complete]
+
+        if {[llength $last_completion]} {
+            #adding a source causes it to be checksummed
+            $event targetset_addsource $projectdir/src/doc  ;#whole doc tree is considered the source 
+            #----------
+            set changeinfo [$event targetset_source_changes]
+            if {\
+                    [llength [dict get $changeinfo changed]]\
+                } {
+                    puts stdout "changed"
+                    puts stdout $changeinfo
+            } else {
+                puts stdout "No changes detected in $projectdir/src/doc tree"
+            }
+        } else {
+            #no previous completion-record for this target - must assume changed - no need to trigger checksumming
+            puts stdout "No existing record of doc build in .punkcheck. Assume it needs to be rebuilt."
+        }
+
+
+        $event destroy
+        $installer destroy
+
+        cd $original_wd
+    }
+    proc validate {} {
+        set projectdir [punk::repo::find_project]
+        if {$projectdir eq ""} {
+            puts stderr "No current project dir - unable to check doc status"
+            return
+        }
+        if {![file exists $projectdir/src/doc]} {
+            set result "No documentation source found. Expected .man files in doctools format at $projectdir/src/doc"
+            return $result
+        }
+        set original_wd [pwd]
+        cd $projectdir/src
+
+        punk::mix::cli::lib::kettle_call lib validate-doc
+
+        cd $original_wd
+    }
+
+    namespace eval collection {
+        variable pkg
+        set pkg punk::mix::commandset::doc
+
+        namespace export *
+        namespace path [namespace parent]
+
+    }
+
+    namespace eval lib {
+        variable pkg
+        set pkg punk::mix::commandset::doc
+
+    }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+##  Ready   
+package provide punk::mix::commandset::doc [namespace eval punk::mix::commandset::doc {
+    variable pkg punk::mix::commandset::doc
+    variable version
+    set version 999999.0a1.0 
+}]
+return

src/modules/punk/mix/commandset/doc-buildversion.txt

@@ -0,0 +1,3 @@
+0.1.0
+#First line must be a semantic version number
+#all other lines are ignored.

src/modules/punk/mix/commandset/layout-999999.0a1.0.tm

@@ -68,7 +68,7 @@ namespace eval punk::mix::commandset::layout {
     #layout collection functions - to be imported with punk::overlay::import_commandset separately
     namespace eval collection {
         namespace export *
-        proc layouts {{glob {}}} {
+        proc _default {{glob {}}} {
             if {![string length $glob]} {
                 set glob *
             }

src/modules/punk/mix/commandset/module-999999.0a1.0.tm

@@ -102,6 +102,7 @@ namespace eval punk::mix::commandset::module {
 
         return $table
     }
+    #return all module templates with repeated ones suffixed with .2 .3 etc
     proc templates_dict {args} {
         tailcall lib::templates_dict {*}$args
     }
@@ -186,8 +187,8 @@ namespace eval punk::mix::commandset::module {
         set opt_license                             [dict get $opts -license]
         # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
         set opt_template                            [dict get $opts -template]
-        set lib_tpldir [file join [punk::mix::cli::lib::mix_templates_dir] module];#fallback for modulename_buildversion.txt, modulename_description.txt
+
-        set templates_dict [templates_dict]
+        set templates_dict [templates_dict] ;#possibly suffixed with .2 .3 etc
         #todo - allow versionless name - pick latest which isn't suffixed with .2 etc
         if {![dict exists $templates_dict $opt_template]} {
             error "module.new unable to find template '$opt_template'. Known templates: [dict keys $templates_dict]"
@@ -283,9 +284,12 @@ namespace eval punk::mix::commandset::module {
             error $errmsg
         }
 
+
         if {[file exists $tpldir/modulename_buildversion.txt]} {
             set fd [open $tpldir/modulename_buildversion.txt r]; set buildversion_filedata [read $fd]; close $fd
         } else {
+            #mix_templates_dir warns of deprecation - review
+            set lib_tpldir [file join [punk::mix::cli::lib::mix_templates_dir] modules];#fallback for modulename_buildversion.txt, modulename_description.txt
             set fd [open $lib_tpldir/modulename_buildversion.txt r]; set buildversion_filedata [read $fd]; close $fd
         }
         set buildversionfile [file join $modulefolder ${moduletail}-buildversion.txt]

src/modules/punk/mix/commandset/project-999999.0a1.0.tm

@@ -242,44 +242,40 @@ namespace eval punk::mix::commandset::project {
         set layout_dir $templatebase/layouts/$opt_layout
         puts stdout ">>> about to call punkcheck::install $layout_dir $projectdir" 
         set resultdict [dict create]
-        #In this case we need to override the default dir antiglob - as .fossil- folders need to be installed from template
-        ## default_antiglob_dir_core  [list "#*" "_aside" ".git" ".fossil*"]
-        set override_antiglob_dir_core [list #* _aside .git] 
         set unpublish [list\
             src/doc/*\
             src/doc/include/*\
         ]
+
+        #default antiglob_dir_core will stop .fossil* from being updated - which is generally desirable as these are likely to be customized
         if {$opt_force} {
-            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite ALL-TARGETS -unpublish_paths $unpublish]
+            puts stdout "copying layout files - with force applied - overwrite all-targets" 
+            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new  -overwrite ALL-TARGETS -unpublish_paths $unpublish]
             #file copy -force $layout_dir $projectdir
         } else {
-            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite installedsourcechanged-targets -unpublish_paths $unpublish]
+            puts stdout "copying layout files - (if source file changed)" 
-        }
+            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -overwrite installedsourcechanged-targets -unpublish_paths $unpublish]
-        if {[dict size $resultdict]} {
-            set copied [dict get $resultdict files_copied]
-            set sources_unchanged [dict get $resultdict sources_unchanged]
-            puts stdout "--------------------------"
-            flush stdout
-            puts stderr "Copied [llength $copied] files from $layout_dir to $projectdir"
-            foreach f $copied {
-                puts stdout "COPIED $f"
-            }
-            puts stdout "[llength $sources_unchanged] unchanged source files"
-            puts stdout "--------------------------"
-        }
-        set resultdict [punkcheck::install $layout_dir/src/doc $projectdir/src/doc -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite NO-TARGETS]
-        if {[dict size $resultdict]} {
-            set copied [dict get $resultdict files_copied]
-            set files_skipped [dict get $resultdict files_skipped]
-            puts stdout "--------------------------"
-            flush stdout
-            puts stderr "Copied [llength $copied] doc files from $layout_dir/src/doc to $projectdir/src/doc"
-            foreach f $copied {
-                puts stdout "COPIED $f"
-            }
-            puts stdout "[llength $files_skipped] skipped files"
-            puts stdout "--------------------------"
         }
+        puts stdout [punkcheck::summarize_install_resultdict $resultdict]
+
+        puts stdout "copying layout src/doc files (if target missing)"
+        set resultdict [punkcheck::install $layout_dir/src/doc $projectdir/src/doc -punkcheck_folder $projectdir -installer project.new  -overwrite SYNCED-TARGETS]
+        puts stdout [punkcheck::summarize_install_resultdict $resultdict]
+
+        #target folders .fossil-custom and .fossil-settings may not exist. use -createdir 1 to ensure existence. 
+        #In this case we need to override the default dir antiglob - as .fossil-xxx folders need to be installed from template if missing, or if target is uncustomized.
+        ## default_antiglob_dir_core  [list "#*" "_aside" ".git" ".fossil*"]
+        set override_antiglob_dir_core [list #* _aside .git] 
+        puts stdout "copying layout src/.fossil-custom files (if target missing or uncustomised)"
+        set resultdict [punkcheck::install $layout_dir/.fossil-custom $projectdir/.fossil-custom -createdir 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS]
+        puts stdout [punkcheck::summarize_install_resultdict $resultdict]
+
+        puts stdout "copying layout src/.fossil-settings files (if target missing or uncustomised)"
+        set resultdict [punkcheck::install $layout_dir/.fossil-settings $projectdir/.fossil-settings -createdir 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS]
+        puts stdout [punkcheck::summarize_install_resultdict $resultdict]
+
+
+
 
         #lappend substfiles $projectdir/README.md
         #lappend substfiles $projectdir/src/README.md
@@ -328,8 +324,37 @@ namespace eval punk::mix::commandset::project {
         #generate www/man/md output in 'embedded' folder which should be checked into repo for online documentation
         if {[file exists $projectdir/src]} {
             cd $projectdir/src
+            #----------
+            set installer [punkcheck::installtrack new project.new $projectdir/src/.punkcheck]
+            $installer set_source_target $projectdir/src/doc $projectdir/src/embedded
+            set event [$installer start_event {-install_step kettledoc}]
+            $event targetset_init VIRTUAL kettle_build_doc ;#VIRTUAL - since there is no specific target file - and we don't know all the files that will be generated
+            $event targetset_addsource $projectdir/src/doc  ;#whole doc tree is considered the source 
+            #----------
+            if {\
+                    [llength [dict get [$event targetset_source_changes] changed]]\
+                } {
+                $event targetset_started
+                # -- --- --- --- --- ---
+                puts stdout "BUILDING DOCS at src/embedded from src/doc"
+                if {[catch {
+
                     punk::mix::cli::lib::kettle_call lib doc
                     #Kettle doc
+
+                } errM]} {
+                    $event targetset_end FAILED -note "kettle_build_doc failed: $errM"
+                } else {
+                    $event targetset_end OK
+                }
+                # -- --- --- --- --- ---
+            } else {
+                puts stderr "No change detected in src/doc"
+                $event targetset_end SKIPPED
+            }
+            $event end
+            $event destroy
+            $installer destroy
         }
 
         cd $projectdir
@@ -386,7 +411,8 @@ namespace eval punk::mix::commandset::project {
         namespace export *
         namespace path [namespace parent]
 
-        proc projects {{glob {}} args} {
+        #e.g imported as 'projects'
+        proc _default {{glob {}} args} {
             package require overtype
             set db_projects [lib::get_projects $glob] 
             set col1items [lsearch -all -inline -index 0 -subindices $db_projects *]
@@ -414,7 +440,7 @@ namespace eval punk::mix::commandset::project {
             return $msg
             #return [list_as_lines [lib::get_projects $glob]] 
         }
-        proc projects.detail {{glob {}} args} {
+        proc detail {{glob {}} args} {
             package require overtype
             package require textutil
             set defaults [dict create\
@@ -533,11 +559,11 @@ namespace eval punk::mix::commandset::project {
             return $msg            
             #return [list_as_lines [lib::get_projects $glob]] 
         }
-        proc projects.cd {{glob {}} args} {
+        proc cd {{glob {}} args} {
             dict set args -cd 1
-            projects.work $glob {*}$args 
+            work $glob {*}$args 
         }
-        proc projects.work {{glob {}} args} {
+        proc work {{glob {}} args} {
             package require sqlite3
             set db_projects [lib::get_projects $glob] 
             #list of lists of the form:
@@ -586,11 +612,22 @@ namespace eval punk::mix::commandset::project {
                     set dupid "" 
                 }
                 if {$dbcount == 1} {
+                    set pname ""
+                    set pcode ""
+                    if {[file exists $fosdb]} {
+                        if {[catch {
                             sqlite3 fdb $fosdb
                             set pname [lindex [fdb eval {select value from config where name = 'project-name'}] 0]
                             set pcode [lindex [fdb eval {select value from config where name = 'project-code'}] 0]
                             fdb close
                             dict set fosdb_cache $fosdb [list name $pname code $pcode]
+                        }  errM]} {
+                            puts stderr "!!! problem with fossil db: $fosdb when examining workdir $wd"
+                            puts stderr "!!! error: $errM"
+                        }
+                    } else {
+                        puts stderr "!!! missing fossil db $fosdb" 
+                    }
                 } else {
                     set info [dict get $fosdb_cache $fosdb]
                     lassign $info _name pname _code pcode
@@ -704,8 +741,13 @@ namespace eval punk::mix::commandset::project {
                 if {$numrows == 1} {
                     set workingdir [lindex $workdirs 0]
                     puts stdout "1 result. Changing dir to $workingdir"
+                    if {[file exists $workingdir]} {
                         cd $workingdir
                         return $workingdir
+                    } else {
+                        puts stderr "path $workingdir doesn't appear to exist" 
+                        return [pwd]
+                    }
                 } else {
                     set answer [util::askuser "Change directory to working folder - select a number from 1 to [llength $col_rowids] or any other key to cancel."]
                     if {[string trim $answer] in $col_rowids} {

src/modules/punk/mix/templates/layouts/project/.gitignore

@@ -35,6 +35,7 @@ _FOSSIL_
 #miscellaneous editor files etc 
 *.swp
 
+.punkcheck
 
 todo.txt
 

src/modules/punk/mix/templates/layouts/project/src/bootsupport/include_modules.config

@@ -0,0 +1,6 @@
+## e.g
+#set bootsupport_modules [list\
+#    src/vendormodules cksum\
+#    modules punkcheck\
+#]
+

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/cksum-1.1.4.tm

@@ -0,0 +1,200 @@
+# cksum.tcl - Copyright (C) 2002 Pat Thoyts <patthoyts@users.sourceforge.net>
+#
+# Provides a Tcl only implementation of the unix cksum(1) command. This is
+# similar to the sum(1) command but the algorithm is better defined and
+# standardized across multiple platforms by POSIX 1003.2/D11.2
+#
+# This command has been verified against the cksum command from the GNU
+# textutils package version 2.0
+#
+# -------------------------------------------------------------------------
+# See the file "license.terms" for information on usage and redistribution
+# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
+# -------------------------------------------------------------------------
+
+package require Tcl 8.5-;                # tcl minimum version
+
+namespace eval ::crc {
+    namespace export cksum
+
+    variable cksum_tbl [list 0x0 \
+           0x04C11DB7 0x09823B6E 0x0D4326D9 0x130476DC 0x17C56B6B \
+           0x1A864DB2 0x1E475005 0x2608EDB8 0x22C9F00F 0x2F8AD6D6 \
+           0x2B4BCB61 0x350C9B64 0x31CD86D3 0x3C8EA00A 0x384FBDBD \
+           0x4C11DB70 0x48D0C6C7 0x4593E01E 0x4152FDA9 0x5F15ADAC \
+           0x5BD4B01B 0x569796C2 0x52568B75 0x6A1936C8 0x6ED82B7F \
+           0x639B0DA6 0x675A1011 0x791D4014 0x7DDC5DA3 0x709F7B7A \
+           0x745E66CD 0x9823B6E0 0x9CE2AB57 0x91A18D8E 0x95609039 \
+           0x8B27C03C 0x8FE6DD8B 0x82A5FB52 0x8664E6E5 0xBE2B5B58 \
+           0xBAEA46EF 0xB7A96036 0xB3687D81 0xAD2F2D84 0xA9EE3033 \
+           0xA4AD16EA 0xA06C0B5D 0xD4326D90 0xD0F37027 0xDDB056FE \
+           0xD9714B49 0xC7361B4C 0xC3F706FB 0xCEB42022 0xCA753D95 \
+           0xF23A8028 0xF6FB9D9F 0xFBB8BB46 0xFF79A6F1 0xE13EF6F4 \
+           0xE5FFEB43 0xE8BCCD9A 0xEC7DD02D 0x34867077 0x30476DC0 \
+           0x3D044B19 0x39C556AE 0x278206AB 0x23431B1C 0x2E003DC5 \
+           0x2AC12072 0x128E9DCF 0x164F8078 0x1B0CA6A1 0x1FCDBB16 \
+           0x018AEB13 0x054BF6A4 0x0808D07D 0x0CC9CDCA 0x7897AB07 \
+           0x7C56B6B0 0x71159069 0x75D48DDE 0x6B93DDDB 0x6F52C06C \
+           0x6211E6B5 0x66D0FB02 0x5E9F46BF 0x5A5E5B08 0x571D7DD1 \
+           0x53DC6066 0x4D9B3063 0x495A2DD4 0x44190B0D 0x40D816BA \
+           0xACA5C697 0xA864DB20 0xA527FDF9 0xA1E6E04E 0xBFA1B04B \
+           0xBB60ADFC 0xB6238B25 0xB2E29692 0x8AAD2B2F 0x8E6C3698 \
+           0x832F1041 0x87EE0DF6 0x99A95DF3 0x9D684044 0x902B669D \
+           0x94EA7B2A 0xE0B41DE7 0xE4750050 0xE9362689 0xEDF73B3E \
+           0xF3B06B3B 0xF771768C 0xFA325055 0xFEF34DE2 0xC6BCF05F \
+           0xC27DEDE8 0xCF3ECB31 0xCBFFD686 0xD5B88683 0xD1799B34 \
+           0xDC3ABDED 0xD8FBA05A 0x690CE0EE 0x6DCDFD59 0x608EDB80 \
+           0x644FC637 0x7A089632 0x7EC98B85 0x738AAD5C 0x774BB0EB \
+           0x4F040D56 0x4BC510E1 0x46863638 0x42472B8F 0x5C007B8A \
+           0x58C1663D 0x558240E4 0x51435D53 0x251D3B9E 0x21DC2629 \
+           0x2C9F00F0 0x285E1D47 0x36194D42 0x32D850F5 0x3F9B762C \
+           0x3B5A6B9B 0x0315D626 0x07D4CB91 0x0A97ED48 0x0E56F0FF \
+           0x1011A0FA 0x14D0BD4D 0x19939B94 0x1D528623 0xF12F560E \
+           0xF5EE4BB9 0xF8AD6D60 0xFC6C70D7 0xE22B20D2 0xE6EA3D65 \
+           0xEBA91BBC 0xEF68060B 0xD727BBB6 0xD3E6A601 0xDEA580D8 \
+           0xDA649D6F 0xC423CD6A 0xC0E2D0DD 0xCDA1F604 0xC960EBB3 \
+           0xBD3E8D7E 0xB9FF90C9 0xB4BCB610 0xB07DABA7 0xAE3AFBA2 \
+           0xAAFBE615 0xA7B8C0CC 0xA379DD7B 0x9B3660C6 0x9FF77D71 \
+           0x92B45BA8 0x9675461F 0x8832161A 0x8CF30BAD 0x81B02D74 \
+           0x857130C3 0x5D8A9099 0x594B8D2E 0x5408ABF7 0x50C9B640 \
+           0x4E8EE645 0x4A4FFBF2 0x470CDD2B 0x43CDC09C 0x7B827D21 \
+           0x7F436096 0x7200464F 0x76C15BF8 0x68860BFD 0x6C47164A \
+           0x61043093 0x65C52D24 0x119B4BE9 0x155A565E 0x18197087 \
+           0x1CD86D30 0x029F3D35 0x065E2082 0x0B1D065B 0x0FDC1BEC \
+           0x3793A651 0x3352BBE6 0x3E119D3F 0x3AD08088 0x2497D08D \
+           0x2056CD3A 0x2D15EBE3 0x29D4F654 0xC5A92679 0xC1683BCE \
+           0xCC2B1D17 0xC8EA00A0 0xD6AD50A5 0xD26C4D12 0xDF2F6BCB \
+           0xDBEE767C 0xE3A1CBC1 0xE760D676 0xEA23F0AF 0xEEE2ED18 \
+           0xF0A5BD1D 0xF464A0AA 0xF9278673 0xFDE69BC4 0x89B8FD09 \
+           0x8D79E0BE 0x803AC667 0x84FBDBD0 0x9ABC8BD5 0x9E7D9662 \
+           0x933EB0BB 0x97FFAD0C 0xAFB010B1 0xAB710D06 0xA6322BDF \
+           0xA2F33668 0xBCB4666D 0xB8757BDA 0xB5365D03 0xB1F740B4 ]
+
+    variable uid
+    if {![info exists uid]} {set uid 0}
+}
+
+# crc::CksumInit -- 
+#
+#	Create and initialize a cksum context. This is cleaned up when we
+#	call CksumFinal to obtain the result.
+#
+proc ::crc::CksumInit {} {
+    variable uid
+    set token [namespace current]::[incr uid]
+    upvar #0 $token state
+    array set state {t 0 l 0}
+    return $token
+}
+
+proc ::crc::CksumUpdate {token data} {
+    variable cksum_tbl
+    upvar #0 $token state
+    set t $state(t)
+    binary scan $data c* r
+    foreach {n} $r {
+        set index [expr { (($t >> 24) ^ ($n & 0xFF)) & 0xFF }]
+        # Since the introduction of built-in bigInt support with Tcl
+        # 8.5, bit-shifting $t to the left no longer overflows,
+        # keeping it 32 bits long.  The value grows bigger and bigger
+        # instead - a severe hit on performance.  For this reason we
+        # do a bitwise AND against 0xFFFFFFFF at each step to keep the
+        # value within limits.
+        set t [expr {0xFFFFFFFF & (($t << 8) ^ [lindex $cksum_tbl $index])}]
+        incr state(l)
+    }
+    set state(t) $t
+    return
+}
+
+proc ::crc::CksumFinal {token} {
+    variable cksum_tbl
+    upvar #0 $token state
+    set t $state(t)
+    for {set i $state(l)} {$i > 0} {set i [expr {$i>>8}]} {
+        set index [expr {(($t >> 24) ^ $i) & 0xFF}]
+        set t [expr {0xFFFFFFFF & (($t << 8) ^ [lindex $cksum_tbl $index])}]
+    }
+    unset state
+    return [expr {~$t & 0xFFFFFFFF}]
+}
+
+# crc::Pop --
+#
+#	Pop the nth element off a list. Used in options processing.
+#
+proc ::crc::Pop {varname {nth 0}} {
+    upvar $varname args
+    set r [lindex $args $nth]
+    set args [lreplace $args $nth $nth]
+    return $r
+}
+
+# Description:
+#  Provide a Tcl equivalent of the unix cksum(1) command.
+# Options:
+#  -filename name  - return a checksum for the specified file.
+#  -format string  - return the checksum using this format string.
+#  -chunksize size - set the chunking read size
+#
+proc ::crc::cksum {args} {
+    array set opts [list -filename {} -channel {} -chunksize 4096 \
+                        -format %u -command {}]
+    while {[string match -* [set option [lindex $args 0]]]} {
+        switch -glob -- $option {
+            -file*   { set opts(-filename) [Pop args 1] }
+            -chan*   { set opts(-channel) [Pop args 1] }
+            -chunk*  { set opts(-chunksize) [Pop args 1] }
+            -for*    { set opts(-format)   [Pop args 1] }
+            -command { set opts(-command)  [Pop args 1] }
+            default {
+                if {[llength $args] == 1} { break }
+                if {[string compare $option "--"] == 0} { Pop args ; break }
+                set err [join [lsort [array names opts -*]] ", "]
+                return -code error "bad option \"option\": must be $err"
+            }
+        }
+        Pop args
+    }
+
+    if {$opts(-filename) != {}} {
+        set opts(-channel) [open $opts(-filename) r]
+        fconfigure $opts(-channel) -translation binary
+    }
+
+    if {$opts(-channel) == {}} {
+
+        if {[llength $args] != 1} {
+            return -code error "wrong # args: should be\
+                cksum ?-format string?\
+                -channel chan | -filename file | string"
+        }
+        set tok [CksumInit]
+        CksumUpdate $tok [lindex $args 0]
+        set r [CksumFinal $tok]
+    
+    } else {
+
+        set tok [CksumInit]
+        while {![eof $opts(-channel)]} {
+            CksumUpdate $tok [read $opts(-channel) $opts(-chunksize)]
+        }
+        set r [CksumFinal $tok]
+
+        if {$opts(-filename) != {}} {
+            close $opts(-channel)
+        }
+    }
+
+    return [format $opts(-format) $r]
+}
+
+# -------------------------------------------------------------------------
+
+package provide cksum 1.1.4
+
+# -------------------------------------------------------------------------
+# Local variables:
+#   mode: tcl
+#   indent-tabs-mode: nil
+# End:

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/cmdline-1.5.2.tm

@@ -0,0 +1,933 @@
+# cmdline.tcl --
+#
+#	This package provides a utility for parsing command line
+#	arguments that are processed by our various applications.
+#	It also includes a utility routine to determine the
+#	application name for use in command line errors.
+#
+# Copyright (c) 1998-2000 by Ajuba Solutions.
+# Copyright (c) 2001-2015 by Andreas Kupries <andreas_kupries@users.sf.net>.
+# Copyright (c) 2003      by David N. Welton  <davidw@dedasys.com>
+# See the file "license.terms" for information on usage and redistribution
+# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
+
+package require Tcl 8.5-
+package provide cmdline 1.5.2
+
+namespace eval ::cmdline {
+    namespace export getArgv0 getopt getKnownOpt getfiles getoptions \
+	    getKnownOptions usage
+}
+
+# ::cmdline::getopt --
+#
+#	The cmdline::getopt works in a fashion like the standard
+#	C based getopt function.  Given an option string and a
+#	pointer to an array or args this command will process the
+#	first argument and return info on how to proceed.
+#
+# Arguments:
+#	argvVar		Name of the argv list that you
+#			want to process.  If options are found the
+#			arg list is modified and the processed arguments
+#			are removed from the start of the list.
+#	optstring	A list of command options that the application
+#			will accept.  If the option ends in ".arg" the
+#			getopt routine will use the next argument as
+#			an argument to the option.  Otherwise the option
+#			is a boolean that is set to 1 if present.
+#	optVar		The variable pointed to by optVar
+#			contains the option that was found (without the
+#			leading '-' and without the .arg extension).
+#	valVar		Upon success, the variable pointed to by valVar
+#			contains the value for the specified option.
+#			This value comes from the command line for .arg
+#			options, otherwise the value is 1.
+#			If getopt fails, the valVar is filled with an
+#			error message.
+#
+# Results:
+# 	The getopt function returns 1 if an option was found, 0 if no more
+# 	options were found, and -1 if an error occurred.
+
+proc ::cmdline::getopt {argvVar optstring optVar valVar} {
+    upvar 1 $argvVar argsList
+    upvar 1 $optVar option
+    upvar 1 $valVar value
+
+    set result [getKnownOpt argsList $optstring option value]
+
+    if {$result < 0} {
+        # Collapse unknown-option error into any-other-error result.
+        set result -1
+    }
+    return $result
+}
+
+# ::cmdline::getKnownOpt --
+#
+#	The cmdline::getKnownOpt works in a fashion like the standard
+#	C based getopt function.  Given an option string and a
+#	pointer to an array or args this command will process the
+#	first argument and return info on how to proceed.
+#
+# Arguments:
+#	argvVar		Name of the argv list that you
+#			want to process.  If options are found the
+#			arg list is modified and the processed arguments
+#			are removed from the start of the list.  Note that
+#			unknown options and the args that follow them are
+#			left in this list.
+#	optstring	A list of command options that the application
+#			will accept.  If the option ends in ".arg" the
+#			getopt routine will use the next argument as
+#			an argument to the option.  Otherwise the option
+#			is a boolean that is set to 1 if present.
+#	optVar		The variable pointed to by optVar
+#			contains the option that was found (without the
+#			leading '-' and without the .arg extension).
+#	valVar		Upon success, the variable pointed to by valVar
+#			contains the value for the specified option.
+#			This value comes from the command line for .arg
+#			options, otherwise the value is 1.
+#			If getopt fails, the valVar is filled with an
+#			error message.
+#
+# Results:
+# 	The getKnownOpt function returns 1 if an option was found,
+#	0 if no more options were found, -1 if an unknown option was
+#	encountered, and -2 if any other error occurred.
+
+proc ::cmdline::getKnownOpt {argvVar optstring optVar valVar} {
+    upvar 1 $argvVar argsList
+    upvar 1 $optVar  option
+    upvar 1 $valVar  value
+
+    # default settings for a normal return
+    set value ""
+    set option ""
+    set result 0
+
+    # check if we're past the end of the args list
+    if {[llength $argsList] != 0} {
+
+	# if we got -- or an option that doesn't begin with -, return (skipping
+	# the --).  otherwise process the option arg.
+	switch -glob -- [set arg [lindex $argsList 0]] {
+	    "--" {
+		set argsList [lrange $argsList 1 end]
+	    }
+	    "--*" -
+	    "-*" {
+		set option [string range $arg 1 end]
+		if {[string equal [string range $option 0 0] "-"]} {
+		    set option [string range $arg 2 end]
+		}
+
+		# support for format: [-]-option=value
+		set idx [string first "=" $option 1]
+		if {$idx != -1} {
+		    set _val   [string range $option [expr {$idx+1}] end]
+		    set option [string range $option 0   [expr {$idx-1}]]
+		}
+
+		if {[lsearch -exact $optstring $option] != -1} {
+		    # Booleans are set to 1 when present
+		    set value 1
+		    set result 1
+		    set argsList [lrange $argsList 1 end]
+		} elseif {[lsearch -exact $optstring "$option.arg"] != -1} {
+		    set result 1
+		    set argsList [lrange $argsList 1 end]
+
+		    if {[info exists _val]} {
+			set value $_val
+		    } elseif {[llength $argsList]} {
+			set value [lindex $argsList 0]
+			set argsList [lrange $argsList 1 end]
+		    } else {
+			set value "Option \"$option\" requires an argument"
+			set result -2
+		    }
+		} else {
+		    # Unknown option.
+		    set value "Illegal option \"-$option\""
+		    set result -1
+		}
+	    }
+	    default {
+		# Skip ahead
+	    }
+	}
+    }
+
+    return $result
+}
+
+# ::cmdline::getoptions --
+#
+#	Process a set of command line options, filling in defaults
+#	for those not specified.  This also generates an error message
+#	that lists the allowed flags if an incorrect flag is specified.
+#
+# Arguments:
+#	argvVar		The name of the argument list, typically argv.
+#			We remove all known options and their args from it.
+#                       In other words, after the call to this command the
+#                       referenced variable contains only the non-options,
+#                       and unknown options.
+#	optlist		A list-of-lists where each element specifies an option
+#			in the form:
+#				(where flag takes no argument)
+#					flag comment
+#
+#				(or where flag takes an argument)
+#					flag default comment
+#
+#			If flag ends in ".arg" then the value is taken from the
+#			command line. Otherwise it is a boolean and appears in
+#			the result if present on the command line. If flag ends
+#			in ".secret", it will not be displayed in the usage.
+#	usage		Text to include in the usage display. Defaults to
+#			"options:"
+#
+# Results
+#	Name value pairs suitable for using with array set.
+#       A modified `argvVar`.
+
+proc ::cmdline::getoptions {argvVar optlist {usage options:}} {
+    upvar 1 $argvVar argv
+
+    set opts [GetOptionDefaults $optlist result]
+
+    set argc [llength $argv]
+    while {[set err [getopt argv $opts opt arg]]} {
+	if {$err < 0} {
+            set result(?) ""
+            break
+	}
+	set result($opt) $arg
+    }
+    if {[info exist result(?)] || [info exists result(help)]} {
+	Error [usage $optlist $usage] USAGE
+    }
+    return [array get result]
+}
+
+# ::cmdline::getKnownOptions --
+#
+#	Process a set of command line options, filling in defaults
+#	for those not specified.  This ignores unknown flags, but generates
+#	an error message that lists the correct usage if a known option
+#	is used incorrectly.
+#
+# Arguments:
+#	argvVar		The name of the argument list, typically argv.  This
+#			We remove all known options and their args from it.
+#                       In other words, after the call to this command the
+#                       referenced variable contains only the non-options,
+#                       and unknown options.
+#	optlist		A list-of-lists where each element specifies an option
+#			in the form:
+#				flag default comment
+#			If flag ends in ".arg" then the value is taken from the
+#			command line. Otherwise it is a boolean and appears in
+#			the result if present on the command line. If flag ends
+#			in ".secret", it will not be displayed in the usage.
+#	usage		Text to include in the usage display. Defaults to
+#			"options:"
+#
+# Results
+#	Name value pairs suitable for using with array set.
+#       A modified `argvVar`.
+
+proc ::cmdline::getKnownOptions {argvVar optlist {usage options:}} {
+    upvar 1 $argvVar argv
+
+    set opts [GetOptionDefaults $optlist result]
+
+    # As we encounter them, keep the unknown options and their
+    # arguments in this list.  Before we return from this procedure,
+    # we'll prepend these args to the argList so that the application
+    # doesn't lose them.
+
+    set unknownOptions [list]
+
+    set argc [llength $argv]
+    while {[set err [getKnownOpt argv $opts opt arg]]} {
+	if {$err == -1} {
+            # Unknown option.
+
+            # Skip over any non-option items that follow it.
+            # For now, add them to the list of unknownOptions.
+            lappend unknownOptions [lindex $argv 0]
+            set argv [lrange $argv 1 end]
+            while {([llength $argv] != 0) \
+                    && ![string match "-*" [lindex $argv 0]]} {
+                lappend unknownOptions [lindex $argv 0]
+                set argv [lrange $argv 1 end]
+            }
+	} elseif {$err == -2} {
+            set result(?) ""
+            break
+        } else {
+            set result($opt) $arg
+        }
+    }
+
+    # Before returning, prepend the any unknown args back onto the
+    # argList so that the application doesn't lose them.
+    set argv [concat $unknownOptions $argv]
+
+    if {[info exist result(?)] || [info exists result(help)]} {
+	Error [usage $optlist $usage] USAGE
+    }
+    return [array get result]
+}
+
+# ::cmdline::GetOptionDefaults --
+#
+#	This internal procedure processes the option list (that was passed to
+#	the getopt or getKnownOpt procedure).  The defaultArray gets an index
+#	for each option in the option list, the value of which is the option's
+#	default value.
+#
+# Arguments:
+#	optlist		A list-of-lists where each element specifies an option
+#			in the form:
+#				flag default comment
+#			If flag ends in ".arg" then the value is taken from the
+#			command line. Otherwise it is a boolean and appears in
+#			the result if present on the command line. If flag ends
+#			in ".secret", it will not be displayed in the usage.
+#	defaultArrayVar	The name of the array in which to put argument defaults.
+#
+# Results
+#	Name value pairs suitable for using with array set.
+
+proc ::cmdline::GetOptionDefaults {optlist defaultArrayVar} {
+    upvar 1 $defaultArrayVar result
+
+    set opts {? help}
+    foreach opt $optlist {
+	set name [lindex $opt 0]
+	if {[regsub -- {\.secret$} $name {} name] == 1} {
+	    # Need to hide this from the usage display and getopt
+	}
+	lappend opts $name
+	if {[regsub -- {\.arg$} $name {} name] == 1} {
+
+	    # Set defaults for those that take values.
+
+	    set default [lindex $opt 1]
+	    set result($name) $default
+	} else {
+	    # The default for booleans is false
+	    set result($name) 0
+	}
+    }
+    return $opts
+}
+
+# ::cmdline::usage --
+#
+#	Generate an error message that lists the allowed flags.
+#
+# Arguments:
+#	optlist		As for cmdline::getoptions
+#	usage		Text to include in the usage display. Defaults to
+#			"options:"
+#
+# Results
+#	A formatted usage message
+
+proc ::cmdline::usage {optlist {usage {options:}}} {
+    set str "[getArgv0] $usage\n"
+    set longest 20
+    set lines {}
+    foreach opt [concat $optlist \
+	     {{- "Forcibly stop option processing"} {help "Print this message"} {? "Print this message"}}] {
+	set name "-[lindex $opt 0]"
+	if {[regsub -- {\.secret$} $name {} name] == 1} {
+	    # Hidden option
+	    continue
+	}
+	if {[regsub -- {\.arg$} $name {} name] == 1} {
+	    append name " value"
+	    set desc "[lindex $opt 2] <[lindex $opt 1]>"
+	} else {
+	    set desc "[lindex $opt 1]"
+	}
+	set n [string length $name]
+	if {$n > $longest} { set longest $n }
+	# max not available before 8.5 - set longest [expr {max($longest, )}]
+	lappend lines $name $desc
+    }
+    foreach {name desc} $lines {
+	append str "[string trimright [format " %-*s %s" $longest $name $desc]]\n"
+    }
+
+    return $str
+}
+
+# ::cmdline::getfiles --
+#
+#	Given a list of file arguments from the command line, compute
+#	the set of valid files.  On windows, file globbing is performed
+#	on each argument.  On Unix, only file existence is tested.  If
+#	a file argument produces no valid files, a warning is optionally
+#	generated.
+#
+#	This code also uses the full path for each file.  If not
+#	given it prepends [pwd] to the filename.  This ensures that
+#	these files will never conflict with files in our zip file.
+#
+# Arguments:
+#	patterns	The file patterns specified by the user.
+#	quiet		If this flag is set, no warnings will be generated.
+#
+# Results:
+#	Returns the list of files that match the input patterns.
+
+proc ::cmdline::getfiles {patterns quiet} {
+    set result {}
+    if {$::tcl_platform(platform) == "windows"} {
+	foreach pattern $patterns {
+	    set pat [file join $pattern]
+	    set files [glob -nocomplain -- $pat]
+	    if {$files == {}} {
+		if {! $quiet} {
+		    puts stdout "warning: no files match \"$pattern\""
+		}
+	    } else {
+		foreach file $files {
+		    lappend result $file
+		}
+	    }
+	}
+    } else {
+	set result $patterns
+    }
+    set files {}
+    foreach file $result {
+	# Make file an absolute path so that we will never conflict
+	# with files that might be contained in our zip file.
+	set fullPath [file join [pwd] $file]
+
+	if {[file isfile $fullPath]} {
+	    lappend files $fullPath
+	} elseif {! $quiet} {
+	    puts stdout "warning: no files match \"$file\""
+	}
+    }
+    return $files
+}
+
+# ::cmdline::getArgv0 --
+#
+#	This command returns the "sanitized" version of argv0.  It will strip
+#	off the leading path and remove the ".bin" extensions that our apps
+#	use because they must be wrapped by a shell script.
+#
+# Arguments:
+#	None.
+#
+# Results:
+#	The application name that can be used in error messages.
+
+proc ::cmdline::getArgv0 {} {
+    global argv0
+
+    set name [file tail $argv0]
+    return [file rootname $name]
+}
+
+##
+# ### ### ### ######### ######### #########
+##
+# Now the typed versions of the above commands.
+##
+# ### ### ### ######### ######### #########
+##
+
+# typedCmdline.tcl --
+#
+#    This package provides a utility for parsing typed command
+#    line arguments that may be processed by various applications.
+#
+# Copyright (c) 2000 by Ross Palmer Mohn.
+# See the file "license.terms" for information on usage and redistribution
+# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
+#
+# RCS: @(#) $Id: cmdline.tcl,v 1.28 2011/02/23 17:41:52 andreas_kupries Exp $
+
+namespace eval ::cmdline {
+    namespace export typedGetopt typedGetoptions typedUsage
+
+    # variable cmdline::charclasses --
+    #
+    #    Create regexp list of allowable character classes
+    #    from "string is" error message.
+    #
+    # Results:
+    #    String of character class names separated by "|" characters.
+
+    variable charclasses
+    #checker exclude badKey
+    catch {string is . .} charclasses
+    variable dummy
+    regexp      -- {must be (.+)$} $charclasses dummy charclasses
+    regsub -all -- {, (or )?}      $charclasses {|}   charclasses
+    unset dummy
+}
+
+# ::cmdline::typedGetopt --
+#
+#	The cmdline::typedGetopt works in a fashion like the standard
+#	C based getopt function.  Given an option string and a
+#	pointer to a list of args this command will process the
+#	first argument and return info on how to proceed. In addition,
+#	you may specify a type for the argument to each option.
+#
+# Arguments:
+#	argvVar		Name of the argv list that you want to process.
+#			If options are found, the arg list is modified
+#			and the processed arguments are removed from the
+#			start of the list.
+#
+#	optstring	A list of command options that the application
+#			will accept.  If the option ends in ".xxx", where
+#			xxx is any valid character class to the tcl
+#			command "string is", then typedGetopt routine will
+#			use the next argument as a typed argument to the
+#			option. The argument must match the specified
+#			character classes (e.g. integer, double, boolean,
+#			xdigit, etc.). Alternatively, you may specify
+#			".arg" for an untyped argument.
+#
+#	optVar		Upon success, the variable pointed to by optVar
+#			contains the option that was found (without the
+#			leading '-' and without the .xxx extension).  If
+#			typedGetopt fails the variable is set to the empty
+#			string. SOMETIMES! Different for each -value!
+#
+#	argVar		Upon success, the variable pointed to by argVar
+#			contains the argument for the specified option.
+#			If typedGetopt fails, the variable is filled with
+#			an error message.
+#
+# Argument type syntax:
+#	Option that takes no argument.
+#		foo
+#
+#	Option that takes a typeless argument.
+#		foo.arg
+#
+#	Option that takes a typed argument. Allowable types are all
+#	valid character classes to the tcl command "string is".
+#	Currently must be one of alnum, alpha, ascii, control,
+#	boolean, digit, double, false, graph, integer, lower, print,
+#	punct, space, true, upper, wordchar, or xdigit.
+#		foo.double
+#
+#	Option that takes an argument from a list.
+#		foo.(bar|blat)
+#
+# Argument quantifier syntax:
+#	Option that takes an optional argument.
+#		foo.arg?
+#
+#	Option that takes a list of arguments terminated by "--".
+#		foo.arg+
+#
+#	Option that takes an optional list of arguments terminated by "--".
+#		foo.arg*
+#
+#	Argument quantifiers work on all argument types, so, for
+#	example, the following is a valid option specification.
+#		foo.(bar|blat|blah)?
+#
+# Argument syntax miscellany:
+#	Options may be specified on the command line using a unique,
+#	shortened version of the option name. Given that program foo
+#	has an option list of {bar.alpha blah.arg blat.double},
+#	"foo -b fob" returns an error, but "foo -ba fob"
+#	successfully returns {bar fob}
+#
+# Results:
+#	The typedGetopt function returns one of the following:
+#	 1	a valid option was found
+#	 0	no more options found to process
+#	-1	invalid option
+#	-2	missing argument to a valid option
+#	-3	argument to a valid option does not match type
+#
+# Known Bugs:
+#	When using options which include special glob characters,
+#	you must use the exact option. Abbreviating it can cause
+#	an error in the "cmdline::prefixSearch" procedure.
+
+proc ::cmdline::typedGetopt {argvVar optstring optVar argVar} {
+    variable charclasses
+
+    upvar $argvVar argsList
+
+    upvar $optVar retvar
+    upvar $argVar optarg
+
+    # default settings for a normal return
+    set optarg ""
+    set retvar ""
+    set retval 0
+
+    # check if we're past the end of the args list
+    if {[llength $argsList] != 0} {
+
+        # if we got -- or an option that doesn't begin with -, return (skipping
+        # the --).  otherwise process the option arg.
+        switch -glob -- [set arg [lindex $argsList 0]] {
+            "--" {
+                set argsList [lrange $argsList 1 end]
+            }
+
+            "-*" {
+                # Create list of options without their argument extensions
+
+                set optstr ""
+                foreach str $optstring {
+                    lappend optstr [file rootname $str]
+                }
+
+                set _opt [string range $arg 1 end]
+
+                set i [prefixSearch $optstr [file rootname $_opt]]
+                if {$i != -1} {
+                    set opt [lindex $optstring $i]
+
+                    set quantifier "none"
+                    if {[regexp -- {\.[^.]+([?+*])$} $opt dummy quantifier]} {
+                        set opt [string range $opt 0 end-1]
+                    }
+
+                    if {[string first . $opt] == -1} {
+                        set retval 1
+                        set retvar $opt
+                        set argsList [lrange $argsList 1 end]
+
+                    } elseif {[regexp -- "\\.(arg|$charclasses)\$" $opt dummy charclass]
+                            || [regexp -- {\.\(([^)]+)\)} $opt dummy charclass]} {
+				if {[string equal arg $charclass]} {
+                            set type arg
+			} elseif {[regexp -- "^($charclasses)\$" $charclass]} {
+                            set type class
+                        } else {
+                            set type oneof
+                        }
+
+                        set argsList [lrange $argsList 1 end]
+                        set opt [file rootname $opt]
+
+                        while {1} {
+                            if {[llength $argsList] == 0
+                                    || [string equal "--" [lindex $argsList 0]]} {
+                                if {[string equal "--" [lindex $argsList 0]]} {
+                                    set argsList [lrange $argsList 1 end]
+                                }
+
+                                set oneof ""
+                                if {$type == "arg"} {
+                                    set charclass an
+                                } elseif {$type == "oneof"} {
+                                    set oneof ", one of $charclass"
+                                    set charclass an
+                                }
+
+                                if {$quantifier == "?"} {
+                                    set retval 1
+                                    set retvar $opt
+                                    set optarg ""
+                                } elseif {$quantifier == "+"} {
+                                    set retvar $opt
+                                    if {[llength $optarg] < 1} {
+                                        set retval -2
+                                        set optarg "Option requires at least one $charclass argument$oneof -- $opt"
+                                    } else {
+                                        set retval 1
+                                    }
+                                } elseif {$quantifier == "*"} {
+                                    set retval 1
+                                    set retvar $opt
+                                } else {
+                                    set optarg "Option requires $charclass argument$oneof -- $opt"
+                                    set retvar $opt
+                                    set retval -2
+                                }
+                                set quantifier ""
+                            } elseif {($type == "arg")
+                                    || (($type == "oneof")
+                                    && [string first "|[lindex $argsList 0]|" "|$charclass|"] != -1)
+                                    || (($type == "class")
+                                    && [string is $charclass [lindex $argsList 0]])} {
+                                set retval 1
+                                set retvar $opt
+                                lappend optarg [lindex $argsList 0]
+                                set argsList [lrange $argsList 1 end]
+                            } else {
+                                set oneof ""
+                                if {$type == "arg"} {
+                                    set charclass an
+                                } elseif {$type == "oneof"} {
+                                    set oneof ", one of $charclass"
+                                    set charclass an
+                                }
+                                set optarg "Option requires $charclass argument$oneof -- $opt"
+                                set retvar $opt
+                                set retval -3
+
+                                if {$quantifier == "?"} {
+                                    set retval 1
+                                    set optarg ""
+                                }
+                                set quantifier ""
+                            }
+                             if {![regexp -- {[+*]} $quantifier]} {
+                                break;
+                            }
+                        }
+                    } else {
+                        Error \
+			    "Illegal option type specification: must be one of $charclasses" \
+			    BAD OPTION TYPE
+                    }
+                } else {
+                    set optarg "Illegal option -- $_opt"
+                    set retvar $_opt
+                    set retval -1
+                }
+            }
+	    default {
+		# Skip ahead
+	    }
+        }
+    }
+
+    return $retval
+}
+
+# ::cmdline::typedGetoptions --
+#
+#	Process a set of command line options, filling in defaults
+#	for those not specified. This also generates an error message
+#	that lists the allowed options if an incorrect option is
+#	specified.
+#
+# Arguments:
+#	argvVar		The name of the argument list, typically argv
+#	optlist		A list-of-lists where each element specifies an option
+#			in the form:
+#
+#				option default comment
+#
+#			Options formatting is as described for the optstring
+#			argument of typedGetopt. Default is for optionally
+#			specifying a default value. Comment is for optionally
+#			specifying a comment for the usage display. The
+#			options "--", "-help", and "-?" are automatically included
+#			in optlist.
+#
+# Argument syntax miscellany:
+#	Options formatting and syntax is as described in typedGetopt.
+#	There are two additional suffixes that may be applied when
+#	passing options to typedGetoptions.
+#
+#	You may add ".multi" as a suffix to any option. For options
+#	that take an argument, this means that the option may be used
+#	more than once on the command line and that each additional
+#	argument will be appended to a list, which is then returned
+#	to the application.
+#		foo.double.multi
+#
+#	If a non-argument option is specified as ".multi", it is
+#	toggled on and off for each time it is used on the command
+#	line.
+#		foo.multi
+#
+#	If an option specification does not contain the ".multi"
+#	suffix, it is not an error to use an option more than once.
+#	In this case, the behavior for options with arguments is that
+#	the last argument is the one that will be returned. For
+#	options that do not take arguments, using them more than once
+#	has no additional effect.
+#
+#	Options may also be hidden from the usage display by
+#	appending the suffix ".secret" to any option specification.
+#	Please note that the ".secret" suffix must be the last suffix,
+#	after any argument type specification and ".multi" suffix.
+#		foo.xdigit.multi.secret
+#
+# Results
+#	Name value pairs suitable for using with array set.
+
+proc ::cmdline::typedGetoptions {argvVar optlist {usage options:}} {
+    variable charclasses
+
+    upvar 1 $argvVar argv
+
+    set opts {? help}
+    foreach opt $optlist {
+        set name [lindex $opt 0]
+        if {[regsub -- {\.secret$} $name {} name] == 1} {
+            # Remove this extension before passing to typedGetopt.
+        }
+        if {[regsub -- {\.multi$} $name {} name] == 1} {
+            # Remove this extension before passing to typedGetopt.
+
+            regsub -- {\..*$} $name {} temp
+            set multi($temp) 1
+        }
+        lappend opts $name
+        if {[regsub -- "\\.(arg|$charclasses|\\(.+).?\$" $name {} name] == 1} {
+            # Set defaults for those that take values.
+            # Booleans are set just by being present, or not
+
+            set dflt [lindex $opt 1]
+            if {$dflt != {}} {
+                set defaults($name) $dflt
+            }
+        }
+    }
+    set argc [llength $argv]
+    while {[set err [typedGetopt argv $opts opt arg]]} {
+        if {$err == 1} {
+            if {[info exists result($opt)]
+                    && [info exists multi($opt)]} {
+                # Toggle boolean options or append new arguments
+
+                if {$arg == ""} {
+                    unset result($opt)
+                } else {
+                    set result($opt) "$result($opt) $arg"
+                }
+            } else {
+                set result($opt) "$arg"
+            }
+        } elseif {($err == -1) || ($err == -3)} {
+            Error [typedUsage $optlist $usage] USAGE
+        } elseif {$err == -2 && ![info exists defaults($opt)]} {
+            Error [typedUsage $optlist $usage] USAGE
+        }
+    }
+    if {[info exists result(?)] || [info exists result(help)]} {
+        Error [typedUsage $optlist $usage] USAGE
+    }
+    foreach {opt dflt} [array get defaults] {
+        if {![info exists result($opt)]} {
+            set result($opt) $dflt
+        }
+    }
+    return [array get result]
+}
+
+# ::cmdline::typedUsage --
+#
+#	Generate an error message that lists the allowed flags,
+#	type of argument taken (if any), default value (if any),
+#	and an optional description.
+#
+# Arguments:
+#	optlist		As for cmdline::typedGetoptions
+#
+# Results
+#	A formatted usage message
+
+proc ::cmdline::typedUsage {optlist {usage {options:}}} {
+    variable charclasses
+
+    set str "[getArgv0] $usage\n"
+    set longest 20
+    set lines {}
+    foreach opt [concat $optlist \
+            {{help "Print this message"} {? "Print this message"}}] {
+        set name "-[lindex $opt 0]"
+        if {[regsub -- {\.secret$} $name {} name] == 1} {
+            # Hidden option
+	    continue
+	}
+
+	if {[regsub -- {\.multi$} $name {} name] == 1} {
+	    # Display something about multiple options
+	}
+
+	if {[regexp -- "\\.(arg|$charclasses)\$" $name dummy charclass] ||
+	    [regexp -- {\.\(([^)]+)\)} $opt dummy charclass]
+	} {
+	    regsub -- "\\..+\$" $name {} name
+	    append name " $charclass"
+	    set desc [lindex $opt 2]
+	    set default [lindex $opt 1]
+	    if {$default != ""} {
+		append desc " <$default>"
+	    }
+	} else {
+	    set desc [lindex $opt 1]
+	}
+	lappend accum $name $desc
+	set n [string length $name]
+	if {$n > $longest} { set longest $n }
+	# max not available before 8.5 - set longest [expr {max($longest, [string length $name])}]
+    }
+    foreach {name desc} $accum {
+	append str "[string trimright [format " %-*s %s" $longest $name $desc]]\n"
+    }
+    return $str
+}
+
+# ::cmdline::prefixSearch --
+#
+#	Search a Tcl list for a pattern; searches first for an exact match,
+#	and if that fails, for a unique prefix that matches the pattern
+#	(i.e, first "lsearch -exact", then "lsearch -glob $pattern*"
+#
+# Arguments:
+#	list		list of words
+#	pattern		word to search for
+#
+# Results:
+#	Index of found word is returned. If no exact match or
+#	unique short version is found then -1 is returned.
+
+proc ::cmdline::prefixSearch {list pattern} {
+    # Check for an exact match
+
+    if {[set pos [::lsearch -exact $list $pattern]] > -1} {
+        return $pos
+    }
+
+    # Check for a unique short version
+
+    set slist [lsort $list]
+    if {[set pos [::lsearch -glob $slist $pattern*]] > -1} {
+        # What if there is nothing for the check variable?
+
+        set check [lindex $slist [expr {$pos + 1}]]
+        if {[string first $pattern $check] != 0} {
+            return [::lsearch -exact $list [lindex $slist $pos]]
+        }
+    }
+    return -1
+}
+# ::cmdline::Error --
+#
+#	Internal helper to throw errors with a proper error-code attached.
+#
+# Arguments:
+#	message		text of the error message to throw.
+#	args		additional parts of the error code to use,
+#                       with CMDLINE as basic prefix added by this command.
+#
+# Results:
+#	An error is thrown, always.
+
+proc ::cmdline::Error {message args} {
+    return -code error -errorcode [linsert $args 0 CMDLINE] $message
+}

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/oolib-0.1.tm

@@ -0,0 +1,195 @@
+#JMN - api should be kept in sync with package patternlib where possible
+#
+package provide oolib [namespace eval oolib {
+    variable version
+    set version 0.1
+}]
+
+namespace eval oolib {
+    oo::class create collection {
+        variable o_data ;#dict
+        variable o_alias
+        constructor {} {
+            set o_data [dict create]
+        }
+        method info {} {
+            return [dict info $o_data]
+        }
+        method count {} {
+            return [dict size $o_data]
+        }
+        method isEmpty {} {
+            expr {[dict size $o_data] == 0}
+        }
+        method names {{globOrIdx {}}} {
+            if {[llength $globOrIdx]} {
+                if {[string is integer -strict $globOrIdx]} {
+                    if {$idx < 0} {
+                        set idx "end-[expr {abs($idx + 1)}]"
+                    }
+                    if {[catch {lindex [dict keys $o_data] $idx} result]} {
+                        error "[self object] no such index : '$idx'"
+                    } else {
+                        return $result
+                    }
+                } else {
+                    #glob
+                    return [lsearch -glob -all -inline [dict keys $o_data] $globOrIdx]
+                }
+            } else {
+                return [dict keys $o_data]
+            }
+        }
+        #like names but without globbing
+        method keys {} {
+            dict keys $o_data
+        }
+        method key {{posn 0}} {
+            if {$posn < 0} {
+                set posn "end-[expr {abs($posn + 1)}]"
+            }
+            if {[catch {lindex [dict keys $o_data] $posn} result]} {
+                error "[self object] no such index : '$posn'"
+            } else {
+                return $result
+            }
+        }
+        method hasKey {key} {
+            dict exists $o_data $key
+        }
+        method get {} {
+            return $o_data
+        }
+        method items {} {
+            return [dict values $o_data]
+        }
+        method item {key} {
+            if {[string is integer -strict $key]} {
+                if {$key > 0} {
+                    set valposn [expr {(2*$key) +1}]
+                    return [lindex $o_data $valposn]
+                } else {
+                    set key "end-[expr {abs($key + 1)}]"
+                    return [lindex [dict keys $o_data] $key]
+                }
+            }
+            if {[dict exists $o_data $key]} {
+                return [dict get $o_data $key]
+            }
+        }
+        #inverse lookup
+        method itemKeys {value} {
+            set value_indices [lsearch -all [dict values $o_data] $value]
+            set keylist [list]
+            foreach i $value_indices {
+                set idx [expr {(($i + 1) *2) -2}]
+                lappend keylist [lindex $o_data $idx]
+            }
+            return $keylist
+        }
+        method search {value args} {
+            set matches [lsearch {*}$args [dict values $o_data] $value]
+            if {"-inline" in $args} {
+                return $matches
+            } else {
+                set keylist [list]
+                foreach i $matches {
+                    set idx [expr {(($i + 1) *2) -2}]
+                    lappend keylist [lindex $o_data $idx]
+                }
+                return $keylist
+            }
+        }
+        #review - see patternlib. Is the intention for aliases to be configurable independent of whether the target exists?
+        method alias {newAlias existingKeyOrAlias} {
+            if {[string is integer -strict $newAlias]} {
+                error "[self object] collection key alias cannot be integer"
+            }
+            if {[string length $existingKeyOrAlias]} {
+                set o_alias($newAlias) $existingKeyOrAlias
+            } else {
+                unset o_alias($newAlias)
+            }
+        }
+        method aliases {{key ""}} {
+            if {[string length $key]} {
+                set result [list]
+                foreach {n v} [array get o_alias] {
+                    if {$v eq $key} {
+                        lappend result $n $v
+                    }
+                }		
+                return $result
+            } else {
+                return [array get o_alias]
+            }
+        }
+        #if the supplied index is an alias, return the underlying key; else return the index supplied.
+        method realKey {idx} {
+            if {[catch {set o_alias($idx)} key]} {
+                return $idx
+            } else {
+                return $key
+            }
+        }
+        method add {value key} {
+            if {[string is integer -strict $key]} {
+                error "[self object] collection key must not be an integer. Use another structure if integer keys required"
+            }
+            if {[dict exists $o_data $key]} {
+                error "[self object] col_processors object error: key '$key' already exists in collection"
+            }
+            dict set o_data $key $value
+            return [expr {[dict size $o_data] - 1}] ;#return index of item
+        }
+        method remove {idx {endRange ""}} {
+            if {[string length $endRange]} {
+                error "[self object] collection error: ranged removal not yet implemented.. remove one item at a time"
+            }
+            if {[string is integer -strict $idx]} {
+                if {$idx < 0} {
+                    set idx "end-[expr {abs($idx+1)}]"
+                }
+                set key [lindex [dict keys $o_data] $idx]
+                set posn $idx
+            } else {
+                set key $idx
+                set posn [lsearch -exact [dict keys $o_data] $key]
+                if {$posn < 0} {
+                    error "[self object] no such index: '$idx' in this collection"
+                }
+            }
+            dict unset o_data $key
+            return
+        }
+        method clear {} {
+            set o_data [dict create]
+            return
+        }
+        method reverse {} {
+            set dictnew [dict create]
+            foreach k [lreverse [dict keys $o_data]] {
+                dict set dictnew $k [dict get $o_data $k]
+            }
+            set o_data $dictnew
+            return
+        }
+        #review - cmd as list vs cmd as script?
+        method map {cmd} {
+            set seed [list]
+            dict for {k v} $o_data {
+                lappend seed [uplevel #0 [list {*}$cmd $v]]
+            }
+            return $seed
+        }
+        method objectmap {cmd} {
+            set seed [list]
+            dict for {k v} $o_data {
+                lappend seed [uplevel #0 [list $v {*}$cmd]]
+            }
+            return $seed
+        }
+    }
+
+}
+

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/mix-0.2.tm

@@ -0,0 +1,15 @@
+
+package require punk::cap
+package require punk::mix::templates ;#registers 'templates' capability with punk::cap
+package require punk::mix::base
+package require punk::mix::cli
+
+namespace eval punk::mix {
+    
+}
+
+package provide punk::mix [namespace eval punk::mix {
+    variable version
+    set version 0.2
+
+}]

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/winpath-0.1.0.tm

@@ -0,0 +1,266 @@
+# -*- tcl -*-
+# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from <pkg>-buildversion.txt
+#
+# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem.
+# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository.
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+# (C) 2023
+#
+# @@ Meta Begin
+# Application punk::winpath 0.1.0
+# Meta platform     tcl
+# Meta license      BSD 
+# @@ Meta End
+
+
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+##  Requirements
+##e.g package require frobz
+
+
+
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+namespace eval punk::winpath {
+    namespace export winpath windir cdwin cdwindir illegalname_fix illegalname_test
+
+
+
+
+    #\\servername\share etc  or \\?\UNC\servername\share  etc.
+    proc is_unc_path {path} {
+        set strcopy_path [punk::objclone $path]
+        set strcopy_path [string map [list \\ /] $strcopy_path] ;#normalize to forward slashes for testing purposes (and forward slashes seem to be auto-translated by windows anyway)
+        if {[string first "//" $strcopy_path] == 0} {
+            #check for "Dos device path" syntax
+            if {[string range $strcopy_path 0 3] in [list "//?/" "//./"]} {
+                #Note that //./ doesn't appear to be supported in Tcl as at 2023-08 - but //?/ works  (except for //?/UNC/Server/share)
+                if {[string range $strcopy_path 4 6] eq "UNC"} {
+                    return 1
+                } else {
+                    #some other Dos device path. Could be a drive which is mapped to a UNC path - but the path itself isn't a unc path
+                    return 0
+                }
+            } else {
+                #leading double slash and not dos device path syntax
+                return 1
+            }
+        }
+        return 0
+    }
+
+    #ordinary \\Servername   or \\servername\share or \\servername\share\path  (or forward-slash equivalent) with no dos device syntax //?/ //./ etc.
+    proc is_unc_path_plain {path} {
+        if {[is_unc_path $path]} {
+            if {![is_dos_device_path $path]} {
+                return 1
+            } else {
+                return 0
+            }
+        } else {
+            return 0
+        }
+    }
+
+    #int-rep path preserved - but 'file attributes', and therefor this operation, is expensive (on windows at least)
+    proc pwdshortname {{path {}}} {
+        if {$path eq ""} {
+           set path [pwd]
+        } else {
+            if {[file pathtype $path] eq "relative"} {
+                set path [file normalize $path]
+            }
+        }
+        return [dict get [file attributes $path] -shortname]
+    }
+    #dos device path syntax allows windows api to acces extended-length paths and filenames with illegal path characters such as trailing dots or whitespace
+    #(can exist on server shares and on NTFS - but standard apps can't access without dos device syntax)
+    proc is_dos_device_path {path} {
+        set strcopy_path [punk::objclone $path]
+        set strcopy_path [string map [list \\ /] $strcopy_path] ;#normalize to forward slashes for testing purposes (and forward slashes seem to be auto-translated by windows anyway)
+        if {[string range $strcopy_path 0 3] in [list "//?/" "//./"]} {
+            return 1
+        } else {
+            return 0
+        }
+    }
+    proc strip_dos_device_prefix {path} {
+        #it's unlikely to be valid to strip only //?/ from a //?/UNC  path so check for it here and diver to strip that.
+        #(review.. or raise error because a //?/UNC path isn't *strictly* a UNC path? )
+        if {[is_unc_path $path]} {
+            return [strip_unc_path_prefix $path]
+        }
+        if {[is_dos_device_path $path]} {
+            return [string range $path 4 end]
+        } else {
+            return $path
+        }
+    }
+    proc strip_unc_path_prefix {path} {
+        if {[is_unc_path $path]} {
+            #//?/UNC/server/etc
+            set strcopy_path [punk::objclone $path]
+            set trimmedpath [string range $strcopy_path 7 end]
+            file pathtype $trimmedpath ;#shimmer it to path rep
+            return $trimmedpath
+        } elseif {is_unc_path_plain $path} {
+            #plain unc //server
+            set strcopy_path [punk::objclone $path]
+            set trimmedpath [string range $strcopy_path 2 end]
+            file pathtype $trimmedpath
+            return $trimmedpath
+        } else {
+            return $path
+        }
+    }
+    #we don't validate that path is actually illegal because we don't know the full range of such names.
+    #The caller can apply this to any path.
+    #don't test for platform here - needs to be callable from any platform for potential passing to windows (what usecase? 8.3 name is not always calculable independently)
+    #The utility of this is questionable. prepending a dos-device path won't make a filename with illegal characters readable by windows.
+    #It will need the 'shortname' at least for the illegal segment - if not the whole path
+    #Whilst the 8.3 name algorithm - including undocumented hash function has been reverse engineered 
+    #- it depends on the content of the directory - as collisions cause a different name (e.g incremented number) 
+    #- it also depends on the history of the folder
+    #- you can't take the current dir contents and a particular *existing* longname and determine the shortname algorithmically...
+    #- the shortname may have been generated during a different directory state.
+    #- It is then stored on disk (where?) - so access to reading the existing shortname is required.
+    #- An implementation of the 8.3 algorithm would only be potentially useful in determining the name that will result from adding a new file 
+    #  and would be subject to potential collisions if there are race-conditions in file creation
+    #- Using an 8.3 algorithm externally would be dangerous in that it could appear to work a lot of the time - but return a different file entirely sometimes.
+    #- Conclusion is that the 8.3 name must be retrieved rathern than calclated
+    proc illegalname_fix {path} {
+        #don't add extra dos device path syntax protection-prefix if already done
+        if {[is_unc_path $path]} {
+            error "illegalname_fix called on UNC path $path - unable to process"
+        }
+        if {[is_dos_device_path $path]} {
+            #we may have appended 
+            return $path
+        }
+
+
+
+        #\\servername\share theoretically maps to: \\?\UNC\servername\share  in protected form. https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats
+        #NOTE: 2023-08 on windows 10 at least  \\?\UNC\Server\share  doesn't work - ie we can't use illegalname_fix on UNC paths such as \\Server\share 
+        #(but mapped drive to same path will work)
+        #Note that test-path cmdlet in powershell is also flaky with regards to \\?\UNC\Server paths.
+        #It seems prudent for now to disallow \\?\ protection for UNC paths such as \\server\etc
+        if {[is_unc_path $path]} {
+            set err ""
+            append err      "illegalname_fix doesn't currently support UNC paths (non dos device leading double slash or //?/UNC/...)"
+            append err \n   " - because //?/UNC/Servername/share is not supported in Tcl (and only minimally even in powershell) as at 2023. (on windows use mapped drive instead)"
+            error $err
+        }
+
+        set strcopy_path [punk::objclone $path]
+
+
+        #Note: path could still have leading double slash if it is a Dos device path: e.g. //?/c:/etc
+        if {[file pathtype $path] eq "absolute"} {
+            if {$path eq "~"} {
+                # non-normalized ~ is classified as absolute
+                # tilde special meaning is a bit of a nuisance..  but as it's the entire path in this case.. presumably it should be kept that way
+                # leave for caller to interpret it - but it's not an illegal name whether it's interpreted with special meaning or not
+                # unlikely this fix will be called on a plain tilde anyway
+                return $path
+            } else {
+                set fullpath $path
+            }
+        } else {
+            #set fullpath [file normalize $path] ;#very slow on windows
+            #set fullpath [pwd]/$path ;#will keep ./ in middle of path - not valid for dos-device paths
+            if {[string range $strcopy_path 0 1] eq "./"} {
+                set strcopy_path [string range $strcopy_path 2 end]
+            }
+            set fullpath [file join [pwd] $strcopy_path]
+        }
+        #For file I/O, the "\\?\" prefix to a path string tells the Windows APIs to disable all string parsing
+        # and to send the string that follows it straight to the file system.
+        set protect "\\\\?\\"  ;# value is: \\?\ prefix   
+        set protect2 "//?/" ;#file normalize may do this - it still works 
+        #don't use "//./" - not currently supported in Tcl - seems to work in powershell though.
+
+
+        #choose //?/  as normalized version - since likely 'file normalize' will do it anyway, and experimentall, the windows API accepts both  REVIEW
+        set result ${protect2}$fullpath
+        file pathtype $result ;#make it return a path rep
+        return $result
+    }
+
+    #don't test for platform here - needs to be callable from any platform for potential passing to windows
+    #we can create files with windows illegal names by using //?/ dos device path syntax - but we need to detect when that is required.
+    #
+    # path int-rep preserving
+    proc illegalname_test {path} {
+        #https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+        #according to the above: Use any character in the current code page for a name, including Unicode characters and characters in the extended character set (128–255), except for the following:
+        set reserved [list  < > : \" / \\ | ? *]
+
+
+        #we need to exclude things like path/..   path/.
+        foreach seg [file split $path] {
+            if {$seg in [list . ..]} {
+                #review - what if there is a folder or file that actually has a name such as . or .. ? 
+                #unlikely in normal use - but could done deliberately for bad reasons?
+                #We are unable to check for it here anyway - as this command is intended for checking the path string - not the actual path on a filesystem.
+                #
+                #/./  /../  segments don't require protection - keep checking.
+                continue
+            }
+
+            #only check for actual space as other whitespace seems to work without being stripped
+            #trailing tab and trailing \n or \r seem to be creatable in windows with Tcl - map to some glyph
+            if {[string index $seg end] in [list " " "."]} {
+                #windows API doesn't handle trailing dots or spaces (silently strips) - even though such files can be created on NTFS systems (or seen via samba etc)
+                return 1
+            }
+        }
+        #glob chars '* ?' are probably illegal.. but although x*y.txt and x?y.txt don't display properly (* ? replaced with some other glyph)
+        #- they seem to be readable from cmd and tclsh as is.
+        # pipe symbol also has glyph substitution and behaves the same  e.g a|b.txt
+        #(at least with encoding system utf-8)
+
+        #todo - determine what else constitutes an illegal name according to windows APIs and requires protection with dos device syntax
+        return 0
+    }
+
+    proc test_ntfs_tunneling {f1 f2 args} {
+        file mkdir $f1
+        puts stderr  "waiting 15secs..."
+        after 5000 {puts -nonewline stderr .}
+        after 5000 {puts -nonewline stderr .}
+        after 5000 {puts -nonewline stderr .}
+        after 500 {puts stderr \n}
+        file mkdir $f2
+        puts stdout "$f1   [file stat $f1]"
+        puts stdout "$f2   [file stat $f2]"
+        file delete $f1
+        puts stdout "renaming $f2 to $f1"
+        file rename $f2 $f1
+        puts stdout "$f1   [file stat $f1]"
+
+    }
+
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+##  Ready   
+package provide punk::winpath [namespace eval punk::winpath {
+    variable version
+    set version 0.1.0 
+}]
+return

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punkcheck-0.1.0.tm

@@ -234,10 +234,12 @@ namespace eval punkcheck {
                 #related - installfile_begin
                 #call init before we know if we are going to run the operation vs skip
                 method targetset_init {operation targetset} {
-                    set known_ops [list INSTALL MODIFY DELETE VIRTUAL]
+                    set known_ops [list QUERY INSTALL MODIFY DELETE VIRTUAL]
                     if {[string toupper $operation] ni $known_ops} {
                         error "[self] add_target unknown operation '$operation'. Known operations $known_ops"
                     }
+                    set o_operation [string toupper $operation]
+
                     if {$o_operation_start_ts ne ""} {
                         error "[self] targetset_tart $o_operation operation already in progress. Use targetset_finished or targetset_complete to finish."
                     }
@@ -245,18 +247,24 @@ namespace eval punkcheck {
                     set seconds [expr {$o_operation_start_ts / 1000000}]
                     set tsiso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"]
                     set punkcheck_file [$o_installer get_checkfile]
+                    set punkcheck_folder [file dirname $punkcheck_file]
 
                     set relativepath_targetset [list]
+                    if {$o_operation eq "VIRTUAL"} {
+                        foreach p $targetset {
+                            lappend relativepath_targetset $p
+                        }
+                    } else {
                         foreach p $targetset {
                             if {[file pathtype $p] eq "absolute"} {
-                            lappend relativepath_targetset [punkcheck::lib::path_relative [file dirname $punkcheck_file] $p]
+                                lappend relativepath_targetset [punkcheck::lib::path_relative $punkcheck_folder $p]
                             } else {
                                 lappend relativepath_targetset $p
                             }
                         }
+                    }
 
 
-                    set o_operation $operation
                     set fields [list\
                         -tsiso $tsiso\
                         -ts $o_operation_start_ts\
@@ -280,7 +288,7 @@ namespace eval punkcheck {
                     #-installer and -eventid keys are added here
                     set new_inprogress_record [dict create tag [string toupper $operation]-INPROGRESS {*}$fields -tempcontext [my as_record] body {}]
                     #set existing_body [dict_getwithdefault $o_fileset_record body [list]]
-                    #todo - look for existing "-INPROGRESS" records - mark as failed?
+                    #todo - look for existing "-INPROGRESS" records - mark as failed or incomplete?
                     dict lappend o_fileset_record body $new_inprogress_record
 
                     if {$isnew} {
@@ -288,15 +296,36 @@ namespace eval punkcheck {
                     } else {
                         set record_list [linsert $record_list[unset record_list] $oldposition $o_fileset_record]
                     }
-
+                    if {$o_operation ne "QUERY"} {
                         punkcheck::save_records_to_file $record_list $punkcheck_file
+                    }
                     return $o_fileset_record
 
                 }
                 #operation has been started
+                #todo - upgrade .punkcheck format to hold more than just list of SOURCE entries in each record.
+                # - allow arbitrary targetset_startphase <name> targetset_endphase <name> calls to store timestamps and calculate elapsed time
                 method targetset_started {} {
                     set punkcheck_folder [file dirname [$o_installer get_checkfile]]
-                    set o_fileset_record [punkcheck::installfile_started_install $punkcheck_folder $o_fileset_record]
+                    if {$o_operation eq "QUERY"} {
+                        set fileinfo_body [dict get $o_fileset_record body] ;#body of FILEINFO record
+                        set installing_record [lindex $fileinfo_body end]
+
+                        set ts_start [dict get $installing_record -ts]
+                        set ts_now [clock microseconds] 
+                        set metadata_us [expr {$ts_now - $ts_start}]
+
+                        dict set installing_record -metadata_us $metadata_us
+                        dict set installing_record -ts_start_transfer $ts_now
+
+                        lset fileinfo_body end $installing_record
+                        
+                        return [dict set o_fileset_record body $fileinfo_body]
+                    } else {
+                        #legacy call
+                        #saves to .punkcheck file
+                        return [set o_fileset_record [punkcheck::installfile_started_install $punkcheck_folder $o_fileset_record]]
+                    }
                 }
                 method targetset_end {status args} {
                     set defaults [dict create\
@@ -311,7 +340,6 @@ namespace eval punkcheck {
                         dict unset opts -note
                     }
 
-
                     set status [string toupper $status]
                     set statusdict [dict create OK RECORD SKIPPED SKIPPED FAILED FAILED] 
                     if {$o_operation_start_ts eq ""} {
@@ -332,6 +360,7 @@ namespace eval punkcheck {
                     set file_record_body [dict get $o_fileset_record body]
                     set installing_record [lindex $file_record_body end]
                     set punkcheck_file [$o_installer get_checkfile]
+                    set punkcheck_folder [file dirname $punkcheck_file]
                     set record_list [punkcheck::load_records_from_file $punkcheck_file]
                     if {[dict exists $installing_record -ts_start_transfer]} {
                         set ts_start_transfer [dict get $installing_record -ts_start_transfer]
@@ -345,6 +374,23 @@ namespace eval punkcheck {
                     dict set installing_record -elapsed_us $elapsed_us
                     dict unset installing_record -tempcontext
                     dict set installing_record tag "${o_operation}-[dict get $statusdict $status]" ;# e.g INSTALL-RECORD, INSTALL-SKIPPED
+                    if {$o_operation in [list INSTALL MODIFY] && [dict get $statusdict $status] eq "RECORD"} {
+                        #only calculate and store post operation target cksums on successful INSTALL or MODIFY, doesn't make sense for DELETE or VIRTUAL operations
+                        set new_targets_cksums [list] ;#ordered list of cksums matching targetset order
+                        set cksum_all_opts "" ;#same cksum opts for each target so we store it once
+                        set ts_begin_cksum [clock microseconds]
+                        foreach p $o_targets {
+                            set tgt_cksum_info [punk::mix::base::lib::cksum_path [file join $punkcheck_folder $p]]
+                            lappend new_targets_cksums [dict get $tgt_cksum_info cksum]
+                            if {$cksum_all_opts eq ""} {
+                                set cksum_all_opts [dict get $tgt_cksum_info opts]
+                            }
+                        }
+                        set cksum_us [expr {[clock microseconds] - $ts_begin_cksum}]
+                        dict set installing_record -targets_cksums $new_targets_cksums 
+                        dict set installing_record -cksum_all_opts $cksum_all_opts
+                        dict set installing_record -cksum_us $cksum_us
+                    }
                     lset file_record_body end $installing_record
                     dict set o_fileset_record body $file_record_body 
                     set o_fileset_record [punkcheck::recordlist::file_record_prune $o_fileset_record]
@@ -356,7 +402,9 @@ namespace eval punkcheck {
                     } else {
                         lset record_list $old_posn $o_fileset_record
                     }
+                    if {$o_operation ne "QUERY"} {
                         punkcheck::save_records_to_file $record_list $punkcheck_file
+                    }
                     set o_operation_start_ts ""
                     set o_operation ""
                     return $o_fileset_record
@@ -372,6 +420,20 @@ namespace eval punkcheck {
 
                     set o_fileset_record [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $rel_source_path $o_fileset_record]
 
+                }
+                method targetset_last_complete {} {
+                    #retrieve last completed record for the fileset ie exclude SKIPPED,INSTALL-INPROGRESS,DELETE-INPROGRESS,MODIFY-INPROGRESS
+                    set body [punkcheck::dict_getwithdefault $o_fileset_record body [list]]
+                    set previous_records [lrange $body 0 end]
+                    #get last that is tagged INSTALL-RECORD,MODIFY-RECORD,DELETE-RECORD
+                    set revlist [lreverse $previous_records]
+                    foreach rec $revlist {
+                        if {[dict get $rec tag] in [list "INSTALL-RECORD" "MODIFY-RECORD" "DELETE-RECORD" "VIRTUAL-RECORD"]} {
+                            return $rec
+                        }
+                    }
+                    return [list]
+
                 }
                 method targetset_source_changes {} {
                     punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $o_fileset_record body] end]
@@ -551,16 +613,6 @@ namespace eval punkcheck {
                 method get_event {} {
                     return $o_active_event
                 }
-                if 0 {
-                method unknown {args} {
-                    puts "[self] unknown called with args:$args"
-                    if {[llength $args]} {
-
-                    } else {
-
-                    }
-                }
-                }
             }
         }
         proc start_installer_event {punkcheck_file installername from_fullpath to_fullpath config} {
@@ -722,6 +774,10 @@ namespace eval punkcheck {
             } else {
                 set cksum_opts ""
             }
+
+            #todo - accept argument of cached source cksum info (for client calling multiple targets with same source in quick succession e.g when building .vfs kits with multiple runtimes)
+            #if same cksum_opts - then use cached data instead of checksumming here.
+
             #allow nonexistant as a source 
             set fpath [file join $punkcheck_folder $source_relpath]
             if {![file exists $fpath]} {
@@ -857,7 +913,7 @@ namespace eval punkcheck {
             set tsnow [clock microseconds]
             set elapsed_us [expr {$tsnow - $ts_start}]
             dict set installing_record -elapsed_us $elapsed_us
-            dict set installing_record tag "SKIPPED" 
+            dict set installing_record tag "INSTALL-SKIPPED" 
              
             lset file_record_body end $installing_record
             dict set file_record body $file_record_body
@@ -879,13 +935,14 @@ namespace eval punkcheck {
         #then: file_record_add_installrecord
 
         namespace eval lib {
+            set pkg punkcheck
             namespace path ::punkcheck
             proc is_file_record_inprogress {file_record} {
                 if {[dict get $file_record tag] ne "FILEINFO"} {
                     return 0
                 }
                 set installing_record [lindex [dict_getwithdefault $file_record body [list]] end]
-                if {[dict_getwithdefault $installing_record tag [list]] ni [list INSTALL-INPROGRESS MODIFY-INPROGRESS DELETE-INPROGRESS VIRTUAL-INPROGRESS]} {
+                if {[dict_getwithdefault $installing_record tag [list]] ni [list QUERY-INPROGRESS INSTALL-INPROGRESS MODIFY-INPROGRESS DELETE-INPROGRESS VIRTUAL-INPROGRESS]} {
                     return 0
                 }
                 return 1
@@ -1021,9 +1078,16 @@ namespace eval punkcheck {
         }
         proc install_non_tm_files {srcdir basedir args} {
             #set keys [dict keys $args]
+            #adjust the default anti_glob_dir_core entries so that .fossil-custom, .fossil-settings are copied
+            set antiglob_dir_core [punkcheck::default_antiglob_dir_core]
+            set posn [lsearch $antiglob_dir_core ".fossil*"]
+            if {$posn >=0} {
+                set antiglob_dir_core [lreplace $antiglob_dir_core $posn $posn]
+            }
             set defaults [list\
                 -glob *\
                 -antiglob_file [list "*.tm" "*-buildversion.txt" "*.exe"]\
+                -antiglob_dir_core $antiglob_dir_core\
                 -installer punkcheck::install_non_tm_files\
                 ]
             set opts [dict merge $defaults $args]
@@ -1070,7 +1134,8 @@ namespace eval punkcheck {
         #           -overwrite newer-targets will copy files with older source timestamp over newer target timestamp and those missing at the target (a form of 'restore' operation)
         #           -overwrite older-targets will copy files with newer source timestamp over older target timestamp and those missing at the target
         #           -overwrite all-targets will copy regardless of timestamp at target
-        #           -overwrite installedsourcechanged-targets 
+        #           -overwrite installedsourcechanged-targets will copy if the target doesn't exist or the source changed 
+        #           -overwrite synced-targets will copy if the target doesn't exist or the source changed and the target cksum is the same as the last INSTALL-RECORD targets_cksums entry
         #           review - timestamps unreliable
         #            - what about slightly mismatched system clocks and mounted filesystems? caller responsibility to verify first? 
         #           if timestamp exactly equal -  should we check content-hash? This is presumably only likely to occur deliberately(maliciously?) 
@@ -1091,11 +1156,13 @@ namespace eval punkcheck {
         #  -source_checksum  compare|store|comparestore|false|true  where true == comparestore
         #  -punkcheck_folder target|source|project|<absolutepath>  target is default and is generally recommended
         #  -punkcheck_records empty string | parsed TDL records ie {tag xxx k v}  structure
+        # install creates FILEINFO records with a single entry in the -targets field (it is legitimate to have a list of targets for an installation operation - the oo interface supports this)
         proc install {srcdir tgtdir args} {
             set defaults [list\
                 -call-depth-internal 0\
                 -max_depth 1000\
                 -subdirlist {}\
+                -createdir 0\
                 -glob *\
                 -antiglob_file_core "\uFFFF"\
                 -antiglob_file "" \
@@ -1130,11 +1197,15 @@ namespace eval punkcheck {
             set max_depth                               [dict get $opts -max_depth]
             set subdirlist                              [dict get $opts -subdirlist] ;# generally should be same length as CALLDEPTH - but user could prefill
             set fileglob                                [dict get $opts -glob]
+            set createdir                               [dict get $opts -createdir] ;#defaults to zero to help avoid mistakes with initial target dir - required target subdirs are created regardless of this setting
 
             if {$CALLDEPTH == 0} {
                 #expensive to normalize  but we need to do it at least once
                 set srcdir [file normalize $srcdir]
                 set tgtdir [file normalize $tgtdir]
+                if {$createdir} {
+                    file mkdir $tgtdir
+                }
                 #now the values we build from these will be properly cased
             }
             # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
@@ -1157,7 +1228,7 @@ namespace eval punkcheck {
             set opt_unpublish_paths                     [dict get $opts -unpublish_paths] ;#todo - combine with config file in source tree .punkcheckpublish (?)
             set unpublish_paths_matched [list]
             # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
-            set known_whats [list no-targets newer-targets older-targets all-targets installedsourcechanged-targets]
+            set known_whats [list no-targets newer-targets older-targets all-targets installedsourcechanged-targets synced-targets]
             set overwrite_what                          [string tolower [dict get $opts -overwrite]]; #accept any case for value to allow emphasis by caller e.g -overwrite NEWER-TARGETS
             if {$overwrite_what ni $known_whats} {
                 error "punkcheck::install received unrecognised value for -overwrite. Received value '$overwrite_what' vs known values '$known_whats'"
@@ -1246,9 +1317,9 @@ namespace eval punkcheck {
             }
 
             if {[string match *store* $opt_source_checksum]} {
-                set store_cksums 1
+                set store_source_cksums 1
             } else {
-                set store_cksums 0
+                set store_source_cksums 0
             }
             
 
@@ -1280,7 +1351,7 @@ namespace eval punkcheck {
                 #puts "testing folder - globmatchpath $unpub $relative_source_dir"
                 if {[globmatchpath $unpub $relative_source_dir]} {
                     lappend unpublish_paths_matched $current_source_dir
-                    return [list files_copied {} files_skipped {} sources_unchanged {} punkcheck_records $punkcheck_records unpublish_paths_matched $unpublish_paths_matched]
+                    return [list files_copied {} files_skipped {} sources_unchanged {} punkcheck_records $punkcheck_records unpublish_paths_matched $unpublish_paths_matched srcdir $srcdir tgtdir $tgtdir punkcheck_folder $punkcheck_folder]
                 }
             }
 
@@ -1343,6 +1414,7 @@ namespace eval punkcheck {
 
             #puts stdout     "Current target dir: $current_target_dir"
             foreach m $match_list {
+                set new_tgt_cksum_info [list]
                 set relative_target_path [file join $relative_target_dir $m]
                 set relative_source_path [file join $relative_source_dir $m]
                 set punkcheck_target_relpath [file join $target_relative_to_punkcheck_dir $m]
@@ -1401,9 +1473,13 @@ namespace eval punkcheck {
                 #It doesn't save to .punkcheck (the only punkcheck::installfile_ method which doesn't)
                 set filerec [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $relative_source_path $filerec]
 
+
+
+                #changeinfo comes from last record in body - which is the record we are working on and so will always exist
                 set changeinfo [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $filerec body] end]]
                 set changed [dict get $changeinfo changed]
                 set unchanged [dict get $changeinfo unchanged]
+
                 if {[llength $unchanged]} {
                     lappend sources_unchanged $current_source_dir/$m
                 }
@@ -1415,6 +1491,7 @@ namespace eval punkcheck {
                 } else {
                     if {![file exists $current_target_dir/$m]} {
                         file copy $current_source_dir/$m $current_target_dir
+                        set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
                         lappend files_copied $current_source_dir/$m
                         incr filecount_new
                     } else {
@@ -1422,7 +1499,40 @@ namespace eval punkcheck {
                             if {[llength $changed]} {
                                 #An unrecorded installation is considered a source change (from unknown/unrecorded source to recorded)
                                 file copy -force $current_source_dir/$m $current_target_dir
+                                set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
+                                lappend files_copied $current_source_dir/$m
+                            } else {
+                                set is_skip 1
+                                lappend files_skipped $current_source_dir/$m
+                            }
+                        } elseif {$overwrite_what eq "synced-targets"} {
+                            if {[llength $changed]} {
+                                #only overwrite if the target checksum equals the last installed checksum (ie target is in sync with source and so hasn't been customized)
+                                set existing_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
+                                set is_target_unmodified_since_install 0
+                                set target_cksum_compare "unknown"
+                                set latest_install_record [punkcheck::recordlist::file_record_latest_installrecord $filerec] ;#may be no such record - in which case we get an empty list
+                                if {[dict exists $latest_install_record -targets_cksums]} {
+                                set last_install_cksum [dict get $latest_install_record -targets_cksums] ;#in this case we know there is only one as 'install' always uses targetset size of 1. (FILEINFO record per file in source folder)
+                                    if {[dict get $existing_tgt_cksum_info cksum] eq $last_install_cksum} {
+                                        set is_target_unmodified_since_install 1
+                                        set target_cksum_compare "match"
+                                    } else {
+                                        set target_cksum_compare "nomatch"
+                                    }
+                                } else {
+                                    set target_cksum_compare "norecord"
+                                }
+                                if {$is_target_unmodified_since_install} {
+                                    file copy -force $current_source_dir/$m $current_target_dir
+                                    set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
                                     lappend files_copied $current_source_dir/$m
+                                } else {
+                                    #either cksum is different or we were unable to verify the record. Either way we can't know if the target is in sync so we must skip it
+                                    set is_skip 1
+                                    puts stderr "Skipping file copy $m target $current_target_dir/$m - require synced_target to overwrite - current target cksum compared to previous install: $target_cksum_compare" 
+                                    lappend files_skipped $current_source_dir/$m
+                                }
                             } else {
                                 set is_skip 1
                                 lappend files_skipped $current_source_dir/$m
@@ -1430,7 +1540,7 @@ namespace eval punkcheck {
                         } else {
                             set is_skip 1
                             puts stderr "Skipping file copy $m target $current_target_dir/$m already exists (use -overwrite all-targets  to overwrite)"
-                            #TODO! implement newer-targets older-targets
+                            #TODO! implement newer-targets older-targets? (note ctimes/mtimes are unreliable - may not be worth implementing)
                             lappend files_skipped $current_source_dir/$m
                         }
                     }
@@ -1440,7 +1550,8 @@ namespace eval punkcheck {
                 set ts_now    [clock microseconds]
                 set elapsed_us [expr {$ts_now - $ts_start}]
 
-                if {$store_cksums} {
+                #if {$store_source_cksums} {
+                #}
 
                 set install_records [dict get $filerec body] 
                 set current_install_record [lindex $install_records end]
@@ -1452,6 +1563,10 @@ namespace eval punkcheck {
                 }
                 dict set current_install_record tag $tag
                 dict set current_install_record -elapsed_us $elapsed_us
+                if {[llength $new_tgt_cksum_info]} {
+                    dict set current_install_record -targets_cksums [list [dict get $new_tgt_cksum_info cksum]]
+                    dict set current_install_record -cksum_all_opts [dict get $new_tgt_cksum_info opts]
+                }
                 lset install_records end $current_install_record
                 dict set filerec body $install_records
                 set filerec [punkcheck::recordlist::file_record_prune $filerec] ;#prune now that tag is finalized
@@ -1461,7 +1576,6 @@ namespace eval punkcheck {
                 } else {
                     lset punkcheck_records $existing_filerec_posn $filerec
                 }
-                }
 
             }
 
@@ -1536,11 +1650,10 @@ namespace eval punkcheck {
                 #puts "subdirlist: $subdirlist"
                 if {$CALLDEPTH == 0} {
                     if {[llength $files_copied] || [llength $files_skipped]} {
-                        puts stdout ">>>>>>>>>>>>>>>>>>>"
+                        #puts stdout ">>>>>>>>>>>>>>>>>>>"
                         set saveresult [punkcheck::save_records_to_file $punkcheck_records $punkcheck_file]
-                        puts stdout "[dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file"
+                        puts stdout "punkcheck::install [dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file copied: [llength $files_copied] skipped: [llength $files_skipped]"
-                        puts stdout "copied: [llength $files_copied] skipped: [llength $files_skipped]"
+                        #puts stdout ">>>>>>>>>>>>>>>>>>>"
-                        puts stdout ">>>>>>>>>>>>>>>>>>>"
                     } else {
                         #todo - write db INSTALLER record if -debug true
                         
@@ -1551,10 +1664,30 @@ namespace eval punkcheck {
                 }
             }
 
-            return [list files_copied $files_copied files_skipped $files_skipped sources_unchanged $sources_unchanged unpublish_paths_matched $unpublish_paths_matched punkcheck_records $punkcheck_records]
+            return [list files_copied $files_copied files_skipped $files_skipped sources_unchanged $sources_unchanged unpublish_paths_matched $unpublish_paths_matched punkcheck_records $punkcheck_records punkcheck_folder $punkcheck_folder srcdir $srcdir tgtdir $tgtdir]
+        }
+        proc summarize_install_resultdict {resultdict} {
+            set msg ""
+            if {[dict size $resultdict]} {
+                set copied [dict get $resultdict files_copied]
+                append msg "--------------------------" \n
+                append msg  "[dict keys $resultdict]" \n
+                set tgtdir [dict get $resultdict tgtdir]
+                set checkfolder [dict get $resultdict punkcheck_folder]
+                append msg "Copied [llength $copied] files from [dict get $resultdict srcdir] to [dict get $resultdict tgtdir]" \n
+                foreach f $copied {
+                    append msg "COPIED [punkcheck::lib::path_relative $checkfolder $f]" \n
+                    append msg "    TO $tgtdir" \n
+                }
+                append msg "[llength [dict get $resultdict sources_unchanged]] unchanged source files" \n
+                append msg "[llength [dict get $resultdict files_skipped]] skipped files" \n
+                append msg "--------------------------" \n
+            }
+            return $msg
         }
 
         namespace eval recordlist {
+            set pkg punkcheck
             namespace path ::punkcheck
 
             proc records_as_target_dict {record_list} {
@@ -1590,8 +1723,8 @@ namespace eval punkcheck {
             }
             proc file_install_record_source_changes {install_record} {
                 #reject INSTALLFAILED items ?
-                if {[dict get $install_record tag] ni [list "INSTALL-RECORD" "SKIPPED" "INSTALL-INPROGRESS" "MODIFY-INPROGRESS" "MODIFY-RECORD" "VIRTUAL-INPROGRESS" "VIRTUAL-RECORD" "DELETE-RECORD" "DELETE-INPROGRESS"]} {
+                if {[dict get $install_record tag] ni [list "QUERY-INPROGRESS" "INSTALL-RECORD" "INSTALL-SKIPPED" "INSTALL-INPROGRESS" "MODIFY-INPROGRESS" "MODIFY-RECORD" "MODIFY-SKIPPED" "VIRTUAL-INPROGRESS" "VIRTUAL-RECORD" "VIRTUAL-SKIPPED" "DELETE-RECORD" "DELETE-INPROGRESS" "DELETE-SKIPPED"]} {
-                    error "file_install_record_source_changes bad file->install record: tag not INSTALL-RECORD|SKIPPED|INSTALL-INPROGRESS|MODIFY-RECORD|MODIFY-INPROGRESS|VIRTUAL-RECORD|VIRTUAL-INPROGRESS|DELETE-RECORD|DELETE-INPROGRESS"
+                    error "file_install_record_source_changes bad install record: tag '[dict get $install_record tag]' not INSTALL-RECORD|SKIPPED|INSTALL-INPROGRESS|MODIFY-RECORD|MODIFY-INPROGRESS|VIRTUAL-RECORD|VIRTUAL-INPROGRESS|DELETE-RECORD|DELETE-INPROGRESS"
                 }
                 set source_list [dict_getwithdefault $install_record body [list]]
                 set changed [list]
@@ -1742,15 +1875,19 @@ namespace eval punkcheck {
                 return $installer_record
             }
             proc file_record_latest_installrecord {file_record} {
+                tailcall file_record_latest_operationrecord INSTALL $file_record
+            }
+            proc file_record_latest_operationrecord {operation file_record} {
+                set operation [string toupper $operation]
                 if {[dict get $file_record tag] ne "FILEINFO"} {
-                    error "file_record_latest_installrecord bad file_record: tag not FILEINFO"
+                    error "file_record_latest_operationrecord bad file_record: tag not FILEINFO"
                 }
                 if {![dict exists $file_record body]} {
                     return [list]
                 }
                 set body_items [dict get $file_record body]
                 foreach item [lreverse $body_items] {
-                    if {[dict get $item tag] eq "INSTALL-RECORD"} {
+                    if {[dict get $item tag] eq "$operation-RECORD"} {
                         return $item
                     }
                 }
@@ -1758,47 +1895,6 @@ namespace eval punkcheck {
             }
 
 
-            #dead code?
-            proc file_record_add_installrecordXXX {file_record install_record} {
-                if {[dict get $file_record tag] ne "FILEINFO"} {
-                    error "file_record_add_installrecord bad file_record: tag not FILEINFO"
-                }
-                #disallow '-INPROGRESS' as it's not a final tag
-                if {[dict get $install_record tag] ni [list "INSTALL-RECORD" "SKIPPED"]} {
-                    error "file_record_add_installrecord bad install_record: tag not INSTALL-RECORD"
-                }
-                set keep 3
-                if {[dict exists $file_record -keep_installrecords]} {
-                    set keep [dict get $file_record -keep_installrecords]
-                }
-
-                if {[dict exists $file_record body]} {
-                    set body_items [dict get $file_record body]
-                } else {
-                    set body_items [list]
-                }
-                lappend body_items $install_record
-                set kept_body_items [list]
-                set kcount 0
-                foreach item [lreverse $body_items] {
-                    if {[dict get $item tag] eq "INSTALL-RECORD"} {
-                        incr kcount
-                        if {$keep < 0 || $kcount <= $keep} {
-                            lappend kept_body_items $item
-                        }
-                    } else {
-                        lappend kept_body_items $item
-                    }
-                }
-                set kept_body_items [lreverse $kept_body_items]
-
-                dict set file_record body $kept_body_items
-                return $file_record
-
-
-            }
-
-
             proc file_record_set_defaults {file_record} {
                 if {[dict get $file_record tag] ne "FILEINFO"} {
                     error "file_record_set_defaults bad file_record: tag not FILEINFO"
@@ -1881,6 +1977,7 @@ namespace eval punkcheck {
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 ##  Ready   
 package provide punkcheck [namespace eval punkcheck {
+    set pkg punkcheck
     variable version
     set version 0.1.0 
 }]

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/struct/set-2.2.3.tm

@@ -0,0 +1,189 @@
+#----------------------------------------------------------------------
+#
+# sets.tcl --
+#
+#	Definitions for the processing of sets.
+#
+# Copyright (c) 2004-2008 by Andreas Kupries.
+#
+# See the file "license.terms" for information on usage and redistribution
+# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
+#
+# RCS: @(#) $Id: sets.tcl,v 1.17 2008/03/09 04:24:37 andreas_kupries Exp $
+#
+#----------------------------------------------------------------------
+
+# @mdgen EXCLUDE: sets_c.tcl
+
+package require Tcl 8.5-
+
+namespace eval ::struct::set {}
+
+# ### ### ### ######### ######### #########
+## Management of set implementations.
+
+# ::struct::set::LoadAccelerator --
+#
+#	Loads a named implementation, if possible.
+#
+# Arguments:
+#	key	Name of the implementation to load.
+#
+# Results:
+#	A boolean flag. True if the implementation
+#	was successfully loaded; and False otherwise.
+
+proc ::struct::set::LoadAccelerator {key} {
+    variable accel
+    set r 0
+    switch -exact -- $key {
+	critcl {
+	    # Critcl implementation of set requires Tcl 8.4.
+	    if {![package vsatisfies [package provide Tcl] 8.4]} {return 0}
+	    if {[catch {package require tcllibc}]} {return 0}
+	    set r [llength [info commands ::struct::set_critcl]]
+	}
+	tcl {
+	    variable selfdir
+	    source [file join $selfdir sets_tcl.tcl]
+	    set r 1
+	}
+        default {
+            return -code error "invalid accelerator/impl. package $key:\
+                must be one of [join [KnownImplementations] {, }]"
+        }
+    }
+    set accel($key) $r
+    return $r
+}
+
+# ::struct::set::SwitchTo --
+#
+#	Activates a loaded named implementation.
+#
+# Arguments:
+#	key	Name of the implementation to activate.
+#
+# Results:
+#	None.
+
+proc ::struct::set::SwitchTo {key} {
+    variable accel
+    variable loaded
+
+    if {[string equal $key $loaded]} {
+	# No change, nothing to do.
+	return
+    } elseif {![string equal $key ""]} {
+	# Validate the target implementation of the switch.
+
+	if {![info exists accel($key)]} {
+	    return -code error "Unable to activate unknown implementation \"$key\""
+	} elseif {![info exists accel($key)] || !$accel($key)} {
+	    return -code error "Unable to activate missing implementation \"$key\""
+	}
+    }
+
+    # Deactivate the previous implementation, if there was any.
+
+    if {![string equal $loaded ""]} {
+	rename ::struct::set ::struct::set_$loaded
+    }
+
+    # Activate the new implementation, if there is any.
+
+    if {![string equal $key ""]} {
+	rename ::struct::set_$key ::struct::set
+    }
+
+    # Remember the active implementation, for deactivation by future
+    # switches.
+
+    set loaded $key
+    return
+}
+
+proc ::struct::set::Loaded {} {
+    variable loaded
+    return  $loaded
+}
+
+# ::struct::set::Implementations --
+#
+#	Determines which implementations are
+#	present, i.e. loaded.
+#
+# Arguments:
+#	None.
+#
+# Results:
+#	A list of implementation keys.
+
+proc ::struct::set::Implementations {} {
+    variable accel
+    set res {}
+    foreach n [array names accel] {
+	if {!$accel($n)} continue
+	lappend res $n
+    }
+    return $res
+}
+
+# ::struct::set::KnownImplementations --
+#
+#	Determines which implementations are known
+#	as possible implementations.
+#
+# Arguments:
+#	None.
+#
+# Results:
+#	A list of implementation keys. In the order
+#	of preference, most prefered first.
+
+proc ::struct::set::KnownImplementations {} {
+    return {critcl tcl}
+}
+
+proc ::struct::set::Names {} {
+    return {
+	critcl {tcllibc based}
+	tcl    {pure Tcl}
+    }
+}
+
+# ### ### ### ######### ######### #########
+## Initialization: Data structures.
+
+namespace eval ::struct::set {
+    variable  selfdir [file dirname [info script]]
+    variable  accel
+    array set accel   {tcl 0 critcl 0}
+    variable  loaded  {}
+}
+
+# ### ### ### ######### ######### #########
+## Initialization: Choose an implementation,
+## most prefered first. Loads only one of the
+## possible implementations. And activates it.
+
+namespace eval ::struct::set {
+    variable e
+    foreach e [KnownImplementations] {
+	if {[LoadAccelerator $e]} {
+	    SwitchTo $e
+	    break
+	}
+    }
+    unset e
+}
+
+# ### ### ### ######### ######### #########
+## Ready
+
+namespace eval ::struct {
+    # Export the constructor command.
+    namespace export set
+}
+
+package provide struct::set 2.2.3

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/struct/sets.tcl

@@ -0,0 +1,189 @@
+#----------------------------------------------------------------------
+#
+# sets.tcl --
+#
+#	Definitions for the processing of sets.
+#
+# Copyright (c) 2004-2008 by Andreas Kupries.
+#
+# See the file "license.terms" for information on usage and redistribution
+# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
+#
+# RCS: @(#) $Id: sets.tcl,v 1.17 2008/03/09 04:24:37 andreas_kupries Exp $
+#
+#----------------------------------------------------------------------
+
+# @mdgen EXCLUDE: sets_c.tcl
+
+package require Tcl 8.5-
+
+namespace eval ::struct::set {}
+
+# ### ### ### ######### ######### #########
+## Management of set implementations.
+
+# ::struct::set::LoadAccelerator --
+#
+#	Loads a named implementation, if possible.
+#
+# Arguments:
+#	key	Name of the implementation to load.
+#
+# Results:
+#	A boolean flag. True if the implementation
+#	was successfully loaded; and False otherwise.
+
+proc ::struct::set::LoadAccelerator {key} {
+    variable accel
+    set r 0
+    switch -exact -- $key {
+	critcl {
+	    # Critcl implementation of set requires Tcl 8.4.
+	    if {![package vsatisfies [package provide Tcl] 8.4]} {return 0}
+	    if {[catch {package require tcllibc}]} {return 0}
+	    set r [llength [info commands ::struct::set_critcl]]
+	}
+	tcl {
+	    variable selfdir
+	    source [file join $selfdir sets_tcl.tcl]
+	    set r 1
+	}
+        default {
+            return -code error "invalid accelerator/impl. package $key:\
+                must be one of [join [KnownImplementations] {, }]"
+        }
+    }
+    set accel($key) $r
+    return $r
+}
+
+# ::struct::set::SwitchTo --
+#
+#	Activates a loaded named implementation.
+#
+# Arguments:
+#	key	Name of the implementation to activate.
+#
+# Results:
+#	None.
+
+proc ::struct::set::SwitchTo {key} {
+    variable accel
+    variable loaded
+
+    if {[string equal $key $loaded]} {
+	# No change, nothing to do.
+	return
+    } elseif {![string equal $key ""]} {
+	# Validate the target implementation of the switch.
+
+	if {![info exists accel($key)]} {
+	    return -code error "Unable to activate unknown implementation \"$key\""
+	} elseif {![info exists accel($key)] || !$accel($key)} {
+	    return -code error "Unable to activate missing implementation \"$key\""
+	}
+    }
+
+    # Deactivate the previous implementation, if there was any.
+
+    if {![string equal $loaded ""]} {
+	rename ::struct::set ::struct::set_$loaded
+    }
+
+    # Activate the new implementation, if there is any.
+
+    if {![string equal $key ""]} {
+	rename ::struct::set_$key ::struct::set
+    }
+
+    # Remember the active implementation, for deactivation by future
+    # switches.
+
+    set loaded $key
+    return
+}
+
+proc ::struct::set::Loaded {} {
+    variable loaded
+    return  $loaded
+}
+
+# ::struct::set::Implementations --
+#
+#	Determines which implementations are
+#	present, i.e. loaded.
+#
+# Arguments:
+#	None.
+#
+# Results:
+#	A list of implementation keys.
+
+proc ::struct::set::Implementations {} {
+    variable accel
+    set res {}
+    foreach n [array names accel] {
+	if {!$accel($n)} continue
+	lappend res $n
+    }
+    return $res
+}
+
+# ::struct::set::KnownImplementations --
+#
+#	Determines which implementations are known
+#	as possible implementations.
+#
+# Arguments:
+#	None.
+#
+# Results:
+#	A list of implementation keys. In the order
+#	of preference, most prefered first.
+
+proc ::struct::set::KnownImplementations {} {
+    return {critcl tcl}
+}
+
+proc ::struct::set::Names {} {
+    return {
+	critcl {tcllibc based}
+	tcl    {pure Tcl}
+    }
+}
+
+# ### ### ### ######### ######### #########
+## Initialization: Data structures.
+
+namespace eval ::struct::set {
+    variable  selfdir [file dirname [info script]]
+    variable  accel
+    array set accel   {tcl 0 critcl 0}
+    variable  loaded  {}
+}
+
+# ### ### ### ######### ######### #########
+## Initialization: Choose an implementation,
+## most prefered first. Loads only one of the
+## possible implementations. And activates it.
+
+namespace eval ::struct::set {
+    variable e
+    foreach e [KnownImplementations] {
+	if {[LoadAccelerator $e]} {
+	    SwitchTo $e
+	    break
+	}
+    }
+    unset e
+}
+
+# ### ### ### ######### ######### #########
+## Ready
+
+namespace eval ::struct {
+    # Export the constructor command.
+    namespace export set
+}
+
+package provide struct::set 2.2.3

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/struct/sets_c.tcl

@@ -0,0 +1,93 @@
+#----------------------------------------------------------------------
+#
+# sets_tcl.tcl --
+#
+#       Definitions for the processing of sets. C implementation.
+#
+# Copyright (c) 2007 by Andreas Kupries.
+#
+# See the file "license.terms" for information on usage and redistribution
+# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
+#
+# RCS: @(#) $Id: sets_c.tcl,v 1.3 2008/03/25 07:15:34 andreas_kupries Exp $
+#
+#----------------------------------------------------------------------
+
+package require critcl
+# @sak notprovided struct_setc
+package provide struct_setc 2.1.1
+package require Tcl 8.5-
+
+namespace eval ::struct {
+    # Supporting code for the main command.
+
+    catch {
+        #critcl::cheaders -g
+        #critcl::debug memory symbols
+    }
+
+    critcl::cheaders sets/*.h
+    critcl::csources sets/*.c
+
+    critcl::ccode {
+        /* -*- c -*- */
+
+        #include <m.h>
+    }
+
+    # Main command, set creation.
+
+    critcl::ccommand set_critcl {dummy interp objc objv} {
+	/* Syntax - dispatcher to the sub commands.
+	 */
+
+        static CONST char* methods [] = {
+            "add",      "contains",     "difference",   "empty",
+            "equal","exclude",  "include",      "intersect",
+            "intersect3",       "size", "subsetof",     "subtract",
+            "symdiff",  "union",
+            NULL
+        };
+        enum methods {
+            S_add,      S_contains,     S_difference,   S_empty,
+            S_equal,S_exclude,  S_include,      S_intersect,
+            S_intersect3,       S_size, S_subsetof,     S_subtract,
+            S_symdiff,  S_union
+        };
+
+	int m;
+
+        if (objc < 2) {
+            Tcl_WrongNumArgs (interp, objc, objv, "cmd ?arg ...?");
+            return TCL_ERROR;
+        } else if (Tcl_GetIndexFromObj (interp, objv [1], methods, "option",
+            0, &m) != TCL_OK) {
+            return TCL_ERROR;
+        }
+
+        /* Dispatch to methods. They check the #args in detail before performing
+         * the requested functionality
+         */
+
+        switch (m) {
+            case S_add:        return sm_ADD        (NULL, interp, objc, objv);
+            case S_contains:   return sm_CONTAINS   (NULL, interp, objc, objv);
+            case S_difference: return sm_DIFFERENCE (NULL, interp, objc, objv);
+            case S_empty:      return sm_EMPTY      (NULL, interp, objc, objv);
+            case S_equal:      return sm_EQUAL      (NULL, interp, objc, objv);
+            case S_exclude:    return sm_EXCLUDE    (NULL, interp, objc, objv);
+            case S_include:    return sm_INCLUDE    (NULL, interp, objc, objv);
+            case S_intersect:  return sm_INTERSECT  (NULL, interp, objc, objv);
+            case S_intersect3: return sm_INTERSECT3 (NULL, interp, objc, objv);
+            case S_size:       return sm_SIZE       (NULL, interp, objc, objv);
+            case S_subsetof:   return sm_SUBSETOF   (NULL, interp, objc, objv);
+            case S_subtract:   return sm_SUBTRACT   (NULL, interp, objc, objv);
+            case S_symdiff:    return sm_SYMDIFF    (NULL, interp, objc, objv);
+            case S_union:      return sm_UNION      (NULL, interp, objc, objv);
+        }
+        /* Not coming to this place */
+    }
+}
+
+# ### ### ### ######### ######### #########
+## Ready

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/struct/sets_tcl.tcl

@@ -0,0 +1,452 @@
+#----------------------------------------------------------------------
+#
+# sets_tcl.tcl --
+#
+#	Definitions for the processing of sets.
+#
+# Copyright (c) 2004-2008 by Andreas Kupries.
+#
+# See the file "license.terms" for information on usage and redistribution
+# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
+#
+# RCS: @(#) $Id: sets_tcl.tcl,v 1.4 2008/03/09 04:38:47 andreas_kupries Exp $
+#
+#----------------------------------------------------------------------
+
+package require Tcl 8.5-
+
+namespace eval ::struct::set {
+    # Only export one command, the one used to instantiate a new tree
+    namespace export set_tcl
+}
+
+##########################
+# Public functions
+
+# ::struct::set::set --
+#
+#	Command that access all set commands.
+#
+# Arguments:
+#	cmd	Name of the subcommand to dispatch to.
+#	args	Arguments for the subcommand.
+#
+# Results:
+#	Whatever the result of the subcommand is.
+
+proc ::struct::set::set_tcl {cmd args} {
+    # Do minimal args checks here
+    if { [llength [info level 0]] == 1 } {
+	return -code error "wrong # args: should be \"$cmd ?arg arg ...?\""
+    }
+    ::set sub S_$cmd
+    if { [llength [info commands ::struct::set::$sub]] == 0 } {
+	::set optlist [info commands ::struct::set::S_*]
+	::set xlist {}
+	foreach p $optlist {
+	    lappend xlist [string range $p 17 end]
+	}
+	return -code error \
+		"bad option \"$cmd\": must be [linsert [join [lsort $xlist] ", "] "end-1" "or"]"
+    }
+    return [uplevel 1 [linsert $args 0 ::struct::set::$sub]]
+}
+
+##########################
+# Implementations of the functionality.
+#
+
+# ::struct::set::S_empty --
+#
+#       Determines emptiness of the set
+#
+# Parameters:
+#       set	-- The set to check for emptiness.
+#
+# Results:
+#       A boolean value. True indicates that the set is empty.
+#
+# Side effects:
+#       None.
+#
+# Notes:
+
+proc ::struct::set::S_empty {set} {
+    return [expr {[llength $set] == 0}]
+}
+
+# ::struct::set::S_size --
+#
+#	Computes the cardinality of the set.
+#
+# Parameters:
+#	set	-- The set to inspect.
+#
+# Results:
+#       An integer greater than or equal to zero.
+#
+# Side effects:
+#       None.
+
+proc ::struct::set::S_size {set} {
+    return [llength [Cleanup $set]]
+}
+
+# ::struct::set::S_contains --
+#
+#	Determines if the item is in the set.
+#
+# Parameters:
+#	set	-- The set to inspect.
+#	item	-- The element to look for.
+#
+# Results:
+#	A boolean value. True indicates that the element is present.
+#
+# Side effects:
+#       None.
+
+proc ::struct::set::S_contains {set item} {
+    return [expr {[lsearch -exact $set $item] >= 0}]
+}
+
+# ::struct::set::S_union --
+#
+#	Computes the union of the arguments.
+#
+# Parameters:
+#	args	-- List of sets to unify.
+#
+# Results:
+#	The union of the arguments.
+#
+# Side effects:
+#       None.
+
+proc ::struct::set::S_union {args} {
+    switch -exact -- [llength $args] {
+	0 {return {}}
+	1 {return [lindex $args 0]}
+    }
+    foreach setX $args {
+	foreach x $setX {::set ($x) {}}
+    }
+    return [array names {}]
+}
+
+
+# ::struct::set::S_intersect --
+#
+#	Computes the intersection of the arguments.
+#
+# Parameters:
+#	args	-- List of sets to intersect.
+#
+# Results:
+#	The intersection of the arguments
+#
+# Side effects:
+#       None.
+
+proc ::struct::set::S_intersect {args} {
+    switch -exact -- [llength $args] {
+	0 {return {}}
+	1 {return [lindex $args 0]}
+    }
+    ::set res [lindex $args 0]
+    foreach set [lrange $args 1 end] {
+	if {[llength $res] && [llength $set]} {
+	    ::set res [Intersect $res $set]
+	} else {
+	    # Squash 'res'. Otherwise we get the wrong result if res
+	    # is not empty, but 'set' is.
+	    ::set res {}
+	    break
+	}
+    }
+    return $res
+}
+
+proc ::struct::set::Intersect {A B} {
+    if {[llength $A] == 0} {return {}}
+    if {[llength $B] == 0} {return {}}
+
+    # This is slower than local vars, but more robust
+    if {[llength $B] > [llength $A]} {
+	::set res $A
+	::set A $B
+	::set B $res
+    }
+    ::set res {}
+    foreach x $A {::set ($x) {}}
+    foreach x $B {
+	if {[info exists ($x)]} {
+	    lappend res $x
+	}
+    }
+    return $res
+}
+
+# ::struct::set::S_difference --
+#
+#	Compute difference of two sets.
+#
+# Parameters:
+#	A, B	-- Sets to compute the difference for.
+#
+# Results:
+#	A - B
+#
+# Side effects:
+#       None.
+
+proc ::struct::set::S_difference {A B} {
+    if {[llength $A] == 0} {return {}}
+    if {[llength $B] == 0} {return $A}
+
+    array set tmp {}
+    foreach x $A {::set tmp($x) .}
+    foreach x $B {catch {unset tmp($x)}}
+    return [array names tmp]
+}
+
+if {0} {
+    # Tcllib SF Bug 1002143. We cannot use the implementation below.
+    # It will treat set elements containing '(' and ')' as array
+    # elements, and this screws up the storage of elements as the name
+    # of local vars something fierce. No way around this. Disabling
+    # this code and always using the other implementation (s.a.) is
+    # the only possible fix.
+
+    if {[package vcompare [package provide Tcl] 8.4] < 0} {
+	# Tcl 8.[23]. Use explicit array to perform the operation.
+    } else {
+	# Tcl 8.4+, has 'unset -nocomplain'
+
+	proc ::struct::set::S_difference {A B} {
+	    if {[llength $A] == 0} {return {}}
+	    if {[llength $B] == 0} {return $A}
+
+	    # Get the variable B out of the way, avoid collisions
+	    # prepare for "pure list optimization"
+	    ::set ::struct::set::tmp [lreplace $B -1 -1 unset -nocomplain]
+	    unset B
+
+	    # unset A early: no local variables left
+	    foreach [lindex [list $A [unset A]] 0] {.} {break}
+
+	    eval $::struct::set::tmp
+	    return [info locals]
+	}
+    }
+}
+
+# ::struct::set::S_symdiff --
+#
+#	Compute symmetric difference of two sets.
+#
+# Parameters:
+#	A, B	-- The sets to compute the s.difference for.
+#
+# Results:
+#	The symmetric difference of the two input sets.
+#
+# Side effects:
+#       None.
+
+proc ::struct::set::S_symdiff {A B} {
+    # symdiff == (A-B) + (B-A) == (A+B)-(A*B)
+    if {[llength $A] == 0} {return $B}
+    if {[llength $B] == 0} {return $A}
+    return [S_union \
+	    [S_difference $A $B] \
+	    [S_difference $B $A]]
+}
+
+# ::struct::set::S_intersect3 --
+#
+#	Return intersection and differences for two sets.
+#
+# Parameters:
+#	A, B	-- The sets to inspect.
+#
+# Results:
+#	List containing A*B, A-B, and B-A
+#
+# Side effects:
+#       None.
+
+proc ::struct::set::S_intersect3 {A B} {
+    return [list \
+	    [S_intersect $A $B] \
+	    [S_difference $A $B] \
+	    [S_difference $B $A]]
+}
+
+# ::struct::set::S_equal --
+#
+#	Compares two sets for equality.
+#
+# Parameters:
+#	a	First set to compare.
+#	b	Second set to compare.
+#
+# Results:
+#	A boolean. True if the lists are equal.
+#
+# Side effects:
+#       None.
+
+proc ::struct::set::S_equal {A B} {
+    ::set A [Cleanup $A]
+    ::set B [Cleanup $B]
+
+    # Equal if of same cardinality and difference is empty.
+
+    if {[::llength $A] != [::llength $B]} {return 0}
+    return [expr {[llength [S_difference $A $B]] == 0}]
+}
+
+
+proc ::struct::set::Cleanup {A} {
+    # unset A to avoid collisions
+    if {[llength $A] < 2} {return $A}
+    # We cannot use variables to avoid an explicit array. The set
+    # elements may look like namespace vars (i.e. contain ::), and
+    # such elements break that, cannot be proc-local variables.
+    array set S {}
+    foreach item $A {set S($item) .}
+    return [array names S]
+}
+
+# ::struct::set::S_include --
+#
+#	Add an element to a set.
+#
+# Parameters:
+#	Avar	-- Reference to the set variable to extend.
+#	element	-- The item to add to the set.
+#
+# Results:
+#	None.
+#
+# Side effects:
+#       The set in the variable referenced by Avar is extended
+#	by the element (if the element was not already present).
+
+proc ::struct::set::S_include {Avar element} {
+    # Avar = Avar + {element}
+    upvar 1 $Avar A
+    if {![info exists A] || ![S_contains $A $element]} {
+	lappend A $element
+    }
+    return
+}
+
+# ::struct::set::S_exclude --
+#
+#	Remove an element from a set.
+#
+# Parameters:
+#	Avar	-- Reference to the set variable to shrink.
+#	element	-- The item to remove from the set.
+#
+# Results:
+#	None.
+#
+# Side effects:
+#       The set in the variable referenced by Avar is shrunk,
+#	the element remove (if the element was actually present).
+
+proc ::struct::set::S_exclude {Avar element} {
+    # Avar = Avar - {element}
+    upvar 1 $Avar A
+    if {![info exists A]} {return -code error "can't read \"$Avar\": no such variable"}
+    while {[::set pos [lsearch -exact $A $element]] >= 0} {
+	::set A [lreplace [K $A [::set A {}]] $pos $pos]
+    }
+    return
+}
+
+# ::struct::set::S_add --
+#
+#	Add a set to a set. Similar to 'union', but the first argument
+#	is a variable.
+#
+# Parameters:
+#	Avar	-- Reference to the set variable to extend.
+#	B	-- The set to add to the set in Avar.
+#
+# Results:
+#	None.
+#
+# Side effects:
+#       The set in the variable referenced by Avar is extended
+#	by all the elements in B.
+
+proc ::struct::set::S_add {Avar B} {
+    # Avar = Avar + B
+    upvar 1 $Avar A
+    if {![info exists A]} {set A {}}
+    ::set A [S_union [K $A [::set A {}]] $B]
+    return
+}
+
+# ::struct::set::S_subtract --
+#
+#	Remove a set from a set. Similar to 'difference', but the first argument
+#	is a variable.
+#
+# Parameters:
+#	Avar	-- Reference to the set variable to shrink.
+#	B	-- The set to remove from the set in Avar.
+#
+# Results:
+#	None.
+#
+# Side effects:
+#       The set in the variable referenced by Avar is shrunk,
+#	all elements of B are removed.
+
+proc ::struct::set::S_subtract {Avar B} {
+    # Avar = Avar - B
+    upvar 1 $Avar A
+    if {![info exists A]} {return -code error "can't read \"$Avar\": no such variable"}
+    ::set A [S_difference [K $A [::set A {}]] $B]
+    return
+}
+
+# ::struct::set::S_subsetof --
+#
+#	A predicate checking if the first set is a subset
+#	or equal to the second set.
+#
+# Parameters:
+#	A	-- The possible subset.
+#	B	-- The set to compare to.
+#
+# Results:
+#	A boolean value, true if A is subset of or equal to B
+#
+# Side effects:
+#       None.
+
+proc ::struct::set::S_subsetof {A B} {
+    # A subset|== B <=> (A == A*B)
+    return [S_equal $A [S_intersect $A $B]]
+}
+
+# ::struct::set::K --
+# Performance helper command.
+
+proc ::struct::set::K {x y} {::set x}
+
+# ### ### ### ######### ######### #########
+## Ready
+
+namespace eval ::struct {
+    # Put 'set::set' into the general structure namespace
+    # for pickup by the main management.
+
+    namespace import -force set::set_tcl
+}

src/modules/punk/mix/templates/layouts/project/src/doc/include/changes_0.1.inc

@@ -1,25 +1,28 @@
 [subsection {Changes for version 0.1}]
-[vset punkshell_project https://www.gitea1.intx.com.au/jn/punkshell]
 
 This release 0.1 of project %project%
 
-[para] In detail:
+[para]	Summary
+[list_begin enumerated]
+    [enum] feature 1
+    [enum] feature 2
+[list_end]
+
 
+[para] In detail:
 [list_begin enumerated]
     [comment {- - -- --- ----- -------- ------------- ---------------------}]
+
     [enum] 	%project% requires Tcl 8.6 or higher. Tcl 8.5 or less is not
         supported.
-[comment {- - -- --- ----- -------- ------------- ---------------------}]
+
-[comment {Please consider retaining a link to PunkShell to support the project}]
-[enum]   This project uses [uri [vset punkshell_project] {PunkShell}] as a deployment management and documentation tool.
     [comment {- - -- --- ----- -------- ------------- ---------------------}]
 
+    [enum]   
 
-[para]	Summary
+    [comment {- - -- --- ----- -------- ------------- ---------------------}]
-[list_begin enumerated]
-[enum] feature 1
-[enum] feature 2
 [list_end]
 
+
+
 [comment {- - -- --- ----- -------- ------------- ---------------------}]
-[list_end]

src/modules/punk/mix/templates/layouts/project/src/make.tcl

@@ -2,8 +2,6 @@
 #
 #make any tclkits and modules in src and place them and associated data files/scripts in the parent folder of src.
 #e.g in 'bin' and 'modules' folders at same level as 'src' folder.
-#It is assumed the src folder has been placed somewhere where appropriate 
-#(e.g not in /usr  or c:/  - unless you intend it to directly make and place folders and files in those locations)
 
 set hashline "# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###"
 puts $hashline
@@ -25,13 +23,18 @@ if {"::try" ni [info commands ::try]} {
 #------------------------------------------------------------------------------
 #Module loading from /src/bootsupport or src/*.vfs if script is within a .vfs folder
 #------------------------------------------------------------------------------
-#If the there is a folder directly under the current directory /src/bootsupport/modules which contains .tm files when the starts
+#If the there is a folder  under the current directory /src/bootsupport/modules which contains .tm files 
 # - then it will attempt to preference these modules
-# This allows a source update via 'fossil update' 'git pull' etc to pull in support modules for the make script
+# This allows a source update via 'fossil update' 'git pull' etc to pull in a minimal set of support modules for the make script
-# and load these in preference to ones that may have been in the interps tcl::tm::list or auto_path due to environment variables
+# and load these in preference to ones that may have been in the interp's tcl::tm::list or auto_path due to environment variables
 set startdir [pwd]
+if {[file exists [file join $startdir src bootsupport]]} {
     set bootsupport_mod [file join $startdir src bootsupport modules]
     set bootsupport_lib [file join $startdir src bootsupport lib]
+} else {
+    set bootsupport_mod [file join $startdir bootsupport modules]
+    set bootsupport_lib [file join $startdir bootsupport lib]
+}
 if {[file exists $bootsupport_mod] || [file exists $bootsupport_lib]} {
 
     set original_tm_list [tcl::tm::list]
@@ -60,31 +63,16 @@ if {[file exists $bootsupport_mod] || [file exists $bootsupport_lib]} {
     }
 
 
-    #todo - review usecase
-    if {[string match "*.vfs/*" [info script]]} {
-        #src/xxx.vfs/lib/app-punk/repl.tcl
-        #we assume if calling directly into .vfs that the user would prefer to use src/modules - so go up 4 levels
-        set modulefolder [file dirname [file dirname [file dirname [file dirname [info script]]]]]/modules
-
-    } else {
-        #  .../bin/punkXX.exe  look for ../modules  (i.e modules folder at same level as bin folder)
-        set modulefolder [file dirname [file dirname [info nameofexecutable]]]/modules
-    }
-
-    if {[file exists $modulefolder]} {
-        tcl::tm::add $modulefolder
-    } else {
-        puts stderr "Warning unable to find module folder at: $modulefolder"
-    }
     if {[file exists [pwd]/modules]} {
         tcl::tm::add [pwd]/modules
     }
 
     #package require Thread
-    #These are strong dependencies 
+    # - the full repl requires Threading and punk,shellfilter,shellrun to call and display properly.
-    # - the repl requires Threading and punk,shellfilter,shellrun to call and display properly.
-    # tm list already indexed - need 'package forget' to find modules based on current tcl::tm::list
     
+    
+    # tm list already indexed - need 'package forget' to find modules based on current tcl::tm::list
+    #These are strong dependencies 
     package forget  punk::mix
     package require punk::mix
     package forget  punk::repo 
@@ -144,6 +132,8 @@ proc punkmake_gethelp {args} {
     append h "  $scriptname project ?-k?" \n
     append h "  - this is the literal word project - and confirms you want to run the project build" \n
     append h "  - the optional -k flag will terminate processes running as the executable being built (if applicable)" \n \n
+    append h "  $scriptname bootsupport"  \n
+    append h "  - update the src/bootsupport modules as well as the mixtemplates/layouts/<layoutname>/src/bootsupport modules if the folder exists" \n \n
     append h "  $scriptname get-project-info" \n
     append h "  - show the name and base folder of the project to be built" \n
     append h "" \n
@@ -189,7 +179,7 @@ if {[llength $commands_found] != 1 } {
 }
 if {$do_help} {
     puts stderr [punkmake_gethelp] 
-    exit 1
+    exit 0
 }
 
 set ::punkmake::command [lindex $commands_found 0]
@@ -224,6 +214,8 @@ if {![string length [set projectroot [punk::repo::find_project $scriptfolder]]]}
     
 }
 
+set sourcefolder $projectroot/src
+
 if {$::punkmake::command eq "get-project-info"} {
     puts stdout "- -- --- --- --- --- --- --- --- --- ---"
     puts stdout "- -- get-project-info -- -"
@@ -260,9 +252,128 @@ if {$::punkmake::command eq "shell"} {
 }
 
 if {$::punkmake::command eq "bootsupport"} {
+    puts "projectroot: $projectroot"
+    puts "script: [info script]"
+    #puts "-- [tcl::tm::list] --"
+    puts stdout "Updating bootsupport from local files"
+
+    proc bootsupport_localupdate {projectroot} {
+        set bootsupport_modules [list]
+        set bootsupport_config  $projectroot/src/bootsupport/include_modules.config ;#
+        if {[file exists $bootsupport_config]} {
+            source $bootsupport_config ;#populate $bootsupport_modules with project-specific list
+            if {![llength $bootsupport_modules]} {
+                puts stderr "No local bootsupport modules configured for updating"
+                return
+            }
+            set targetroot $projectroot/src/bootsupport/modules
+
+            if {[catch {
+                #----------
+                set boot_installer [punkcheck::installtrack new make.tcl $projectroot/src/bootsupport/.punkcheck]
+                $boot_installer set_source_target $projectroot $projectroot/src/bootsupport
+                set boot_event [$boot_installer start_event {-make_step bootsupport}]
+                #----------
+            } errM]} {
+                puts stderr "Unable to use punkcheck for bootsupport error: $errM"
+                set boot_event ""
+            }
+
+            foreach {relpath module} $bootsupport_modules {
+                set module [string trim $module :]
+                set module_subpath [string map [list :: /] [namespace qualifiers $module]]
+                set srclocation [file join $projectroot $relpath $module_subpath]
+                #puts stdout "$relpath  $module $module_subpath $srclocation"
+                set pkgmatches [glob -nocomplain -dir $srclocation -tail [namespace tail $module]-*]
+                #lsort won't sort version numbers properly e.g with -dictionary 0.1.1 comes before 0.1
+                if {![llength $pkgmatches]} {
+                    puts stderr "Missing source for bootsupport module $module - not found in $srclocation"
+                    continue
+                }
+                set latestfile [lindex $pkgmatches 0]
+                set latestver [lindex [split [file rootname $latestfile] -] 1]
+                foreach m $pkgmatches {
+                    lassign [split [file rootname $m] -] _pkg ver
+                    #puts "comparing $ver vs $latestver"
+                    if {[package vcompare $ver $latestver] == 1} {
+                        set latestver $ver
+                        set latestfile $m
+                    } 
+                }
+                set srcfile [file join $srclocation $latestfile]
+                set tgtfile [file join $targetroot $module_subpath $latestfile]
+                if {$boot_event ne ""} {
+                    #----------
+                    $boot_event targetset_init INSTALL   $tgtfile
+                    $boot_event targetset_addsource      $srcfile
+                    #----------
+                    if {\
+                            [llength [dict get [$boot_event targetset_source_changes] changed]]\
+                        || [llength [$boot_event get_targets_exist]] < [llength [$boot_event get_targets]]\
+                        } {
+                        $boot_event targetset_started
+                        # -- --- --- --- --- ---
+                        puts "BOOTSUPPORT update: $srcfile -> $tgtfile"
+                        if {[catch {
+                            file copy -force $srcfile  $tgtfile
+                        } errM]} {
+                            $boot_event targetset_end FAILED
+                        } else {
+                            $boot_event targetset_end OK
+                        }
+                        # -- --- --- --- --- ---
+                    } else {
+                        puts -nonewline stderr "."
+                        $boot_event targetset_end SKIPPED
+                    }
+                    $boot_event end
+                } else {
+                    file copy -force $srcfile  $tgtfile
+                }
+            }
+            if {$boot_event ne ""} {
+                puts \n
+                $boot_event destroy
+                $boot_installer destroy
+            }
+        }
+    }
 
+    bootsupport_localupdate $projectroot
 
-    exit 1
+    #/modules/punk/mix/templates/layouts only applies if the project has it's own copy of the punk/mix modules. Generally this should only apply to the punkshell project itself.
+    set layout_bases [list\
+        $sourcefolder/mixtemplates/layouts\
+        $sourcefolder/modules/punk/mix/templates/layouts\
+    ]
+    foreach project_layout_base $layout_bases {
+        if {[file exists $project_layout_base]} {
+            set project_layouts [glob -nocomplain -dir $project_layout_base -type d -tail *]
+            foreach layoutname $project_layouts {
+                if {[file exists  [file join $project_layout_base $layoutname/src/bootsupport]]} {
+                    set unpublish [list\
+                        README.md\
+                    ]
+                    set sourcemodules $projectroot/src/bootsupport/modules
+                    set targetroot [file join $project_layout_base $layoutname/src/bootsupport/modules]
+                    file mkdir $targetroot
+
+                    puts stdout "BOOTSUPPORT layouts/$layoutname: copying from $sourcemodules to $targetroot (if source file changed)"
+                    set resultdict [punkcheck::install $sourcemodules $targetroot -overwrite installedsourcechanged-targets -unpublish_paths $unpublish]
+                    puts stdout [punkcheck::summarize_install_resultdict $resultdict]
+                    flush stdout
+                }
+            }
+        } else {
+            puts stderr "No layout base at $project_layout_base"
+        }
+    }
+    puts stdout " bootsupport done "
+    flush stderr
+    flush stdout
+    #punk86 can hang if calling make.tcl via 'run' without this 'after' delay. punk87 unaffected. cause unknown.
+    #after 500
+    ::exit 0
 }
 
 
@@ -273,7 +384,6 @@ if {$::punkmake::command ne "project"} {
 }
 
 
-set sourcefolder $projectroot/src
 
 #only a single consolidated /modules folder used for target
 set target_modules_base $projectroot/modules
@@ -749,8 +859,14 @@ foreach vfs $vfs_folders {
                                 } errMsg]} {
                             puts stderr "$killcmd returned an error:"
                             puts stderr $errMsg
+                            if {!$forcekill} {
                                 puts stderr "(try '[info script] -k' option to force kill)"
+                            }
+                            #avoid exiting if the kill failure was because the task has already exited
+                            #review - *no running instance* works with windows taskkill - "*No such process*" works with kill -9 on FreeBSD and linux - other platforms?
+                            if {![string match "*no running instance*" $errMsg] && ![string match "*No such process*" $errMsg]} {
                                 exit 4
+                            }
                         } else {
                             puts stderr "$killcmd ran without error"
                             incr count_killed
@@ -790,6 +906,9 @@ foreach vfs $vfs_folders {
             $bin_installer set_source_target $buildfolder $deployment_folder
             set bin_event [$bin_installer start_event {-make-step final_kit_install}]
             $bin_event targetset_init INSTALL $deployment_folder/$targetkit
+            #todo - move final deployment step outside of the build vfs loop? (final deployment can fail and then isn't rerun even though _build and deployed versions differ, unless .vfs modified again)
+            #set last_completion [$bin_event targetset_last_complete]
+
             $bin_event targetset_addsource $buildfolder/$targetkit
             $bin_event targetset_started
             # -- ----------
@@ -802,7 +921,6 @@ foreach vfs $vfs_folders {
                     file delete $deployment_folder/$targetkit
                     } errMsg]} {
                     puts stderr "deletion of deployed version at $deployment_folder/$targetkit failed: $errMsg"
-                    #exit 5
                     set delete_failed 1
                 }
             }
@@ -818,6 +936,7 @@ foreach vfs $vfs_folders {
                 # -- ----------
             } else {
                 $bin_event targetset_end FAILED -note "could not delete"
+                exit 5
             }
             $bin_event destroy
             $bin_installer destroy

src/modules/punk/mix/templates/layouts/project/src/mixtemplates/layouts/basic/src/bootsupport/modules/README.md

@@ -0,0 +1,24 @@
+This is primarily for tcl .tm modules required for your bootstrapping/make/build process.
+It could include other files necessary for this process.
+
+The .tm modules here may be required for your build script if it intended the installation operator uses an existing tclsh or other shell as opposed to a tclkit you may have for distribution which is more likely to include necessary libraries.
+
+The modules here are loaded by your initialisation scripts and so can be a snapshot of different versions than those in your project src.
+The modules can be your own, or 3rd party such as individual items from tcllib.
+
+You can copy modules from a running punk shell to this location using the pmix command.
+
+e.g
+>pmix visible_lib_copy_to_modulefolder some::module::lib bootsupport
+
+The pmix command will help you pick the latest version, and will create any necessary file structure matching the namespace of the package.
+
+e.g the result might be a file such as
+<projectname>/src/bootsupport/some/module/lib-0.1.tm 
+
+The originating library may not yet be in .tm form.
+You can copy a pkgIndex.tcl based library that is composed of a single .tcl file the same way using the above process and it will automatically name and file it appropriately but you need to check that the library doesn't require/load additional files - and that it is Tcl script only. 
+
+Always verify that the library is copyable in this manner and test in a shell with tcl::tm::path pointed to ./bootsupport  that it works.
+
+

src/modules/punk/mix/templates/layouts/project/src/sample.vfs/modules/shellthread-1.6.tm

@@ -604,6 +604,10 @@ namespace eval shellthread::manager {
         return $taginfo_list
     }
 
+    #TODO - important.
+    #REVIEW!
+    #since moving to the unsubscribe mechansm -  close_worker $source isn't being called
+    # - we need to set a limit to the number of free threads and shut down excess when detected during unsubscription 
     #instruction to shut-down the thread that has this source. 
     proc close_worker {source {timeout 2500}} {
         variable workers

src/modules/punk/overlay-0.1.tm

@@ -89,7 +89,11 @@ namespace eval ::punk::overlay {
     #This won't stop the commandset being used with other cli/bases unless the import is done by looking up the callers namespace.
     #The basic principle is that the commandset is loaded into the caller(s) with a prefix 
     #- but commandsets should explicitly package require if they have any backwards dependencies on cli/base (which they may or may not be loaded into)
-    proc import_commandset {prefix cmdnamespace} {
+    proc import_commandset {prefix separator cmdnamespace} {
+        set bad_seps [list "::"]
+        if {$separator in $bad_seps} {
+            error "import_commandset invalid separator '$separator'"
+        }
         #namespace may or may not be a package
         # allow with or without leading ::
         if {[string range $cmdnamespace 0 1] eq "::"} {
@@ -110,7 +114,7 @@ namespace eval ::punk::overlay {
                 } else {
                     set provinfo "(package $cmdpackage not present)"
                 }
-                error "punk::mix::base::lib::import_commandset supplied namespace '$cmdnamespace' doesn't exist. $provinfo   Usage: import_commandset prefix namespace"
+                error "punk::overlay::import_commandset supplied namespace '$cmdnamespace' doesn't exist. $provinfo   Usage: import_commandset prefix separator namespace"
             }
         }
      
@@ -130,12 +134,17 @@ namespace eval ::punk::overlay {
         if {[catch {
             namespace eval ${nscaller}::temp_import [list namespace import ${cmdnamespace}::*]
             foreach cmd [info commands ${nscaller}::temp_import::*] {
-                set import_as ${nscaller}::$prefix[namespace tail $cmd]
+                set cmdtail [namespace tail $cmd]
+                if {$cmdtail eq "_default"} {
+                    set import_as ${nscaller}::${prefix}
+                } else {
+                    set import_as ${nscaller}::${prefix}${separator}${cmdtail}
+                }
                 rename $cmd  $import_as
                 lappend imported_commands $import_as
             }
         } errM]} {
-            puts stderr "Error loading commandset $prefix $cmdnamespace"
+            puts stderr "Error loading commandset $prefix $separator $cmdnamespace"
             puts stderr "err: $errM"
         }
         return $imported_commands

src/modules/punk/repl-0.1.tm

@@ -133,7 +133,8 @@ package require term::ansi::code::ctrl
 if {$::tcl_platform(platform) eq "windows"} {
     package require zzzload
     zzzload::pkg_require twapi
-    after idle [list after 2000 {  
+    after idle [list after 1000 {  
+        #puts stdout "===============repl loading twapi==========="
         zzzload::pkg_wait twapi
 
         if {![catch {package require twapi}]} {

src/modules/punkcheck-0.1.0.tm

@@ -234,10 +234,12 @@ namespace eval punkcheck {
                 #related - installfile_begin
                 #call init before we know if we are going to run the operation vs skip
                 method targetset_init {operation targetset} {
-                    set known_ops [list INSTALL MODIFY DELETE VIRTUAL]
+                    set known_ops [list QUERY INSTALL MODIFY DELETE VIRTUAL]
                     if {[string toupper $operation] ni $known_ops} {
                         error "[self] add_target unknown operation '$operation'. Known operations $known_ops"
                     }
+                    set o_operation [string toupper $operation]
+
                     if {$o_operation_start_ts ne ""} {
                         error "[self] targetset_tart $o_operation operation already in progress. Use targetset_finished or targetset_complete to finish."
                     }
@@ -245,18 +247,24 @@ namespace eval punkcheck {
                     set seconds [expr {$o_operation_start_ts / 1000000}]
                     set tsiso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"]
                     set punkcheck_file [$o_installer get_checkfile]
+                    set punkcheck_folder [file dirname $punkcheck_file]
 
                     set relativepath_targetset [list]
+                    if {$o_operation eq "VIRTUAL"} {
+                        foreach p $targetset {
+                            lappend relativepath_targetset $p
+                        }
+                    } else {
                         foreach p $targetset {
                             if {[file pathtype $p] eq "absolute"} {
-                            lappend relativepath_targetset [punkcheck::lib::path_relative [file dirname $punkcheck_file] $p]
+                                lappend relativepath_targetset [punkcheck::lib::path_relative $punkcheck_folder $p]
                             } else {
                                 lappend relativepath_targetset $p
                             }
                         }
+                    }
 
 
-                    set o_operation $operation
                     set fields [list\
                         -tsiso $tsiso\
                         -ts $o_operation_start_ts\
@@ -280,7 +288,7 @@ namespace eval punkcheck {
                     #-installer and -eventid keys are added here
                     set new_inprogress_record [dict create tag [string toupper $operation]-INPROGRESS {*}$fields -tempcontext [my as_record] body {}]
                     #set existing_body [dict_getwithdefault $o_fileset_record body [list]]
-                    #todo - look for existing "-INPROGRESS" records - mark as failed?
+                    #todo - look for existing "-INPROGRESS" records - mark as failed or incomplete?
                     dict lappend o_fileset_record body $new_inprogress_record
 
                     if {$isnew} {
@@ -288,15 +296,36 @@ namespace eval punkcheck {
                     } else {
                         set record_list [linsert $record_list[unset record_list] $oldposition $o_fileset_record]
                     }
-
+                    if {$o_operation ne "QUERY"} {
                         punkcheck::save_records_to_file $record_list $punkcheck_file
+                    }
                     return $o_fileset_record
 
                 }
                 #operation has been started
+                #todo - upgrade .punkcheck format to hold more than just list of SOURCE entries in each record.
+                # - allow arbitrary targetset_startphase <name> targetset_endphase <name> calls to store timestamps and calculate elapsed time
                 method targetset_started {} {
                     set punkcheck_folder [file dirname [$o_installer get_checkfile]]
-                    set o_fileset_record [punkcheck::installfile_started_install $punkcheck_folder $o_fileset_record]
+                    if {$o_operation eq "QUERY"} {
+                        set fileinfo_body [dict get $o_fileset_record body] ;#body of FILEINFO record
+                        set installing_record [lindex $fileinfo_body end]
+
+                        set ts_start [dict get $installing_record -ts]
+                        set ts_now [clock microseconds] 
+                        set metadata_us [expr {$ts_now - $ts_start}]
+
+                        dict set installing_record -metadata_us $metadata_us
+                        dict set installing_record -ts_start_transfer $ts_now
+
+                        lset fileinfo_body end $installing_record
+                        
+                        return [dict set o_fileset_record body $fileinfo_body]
+                    } else {
+                        #legacy call
+                        #saves to .punkcheck file
+                        return [set o_fileset_record [punkcheck::installfile_started_install $punkcheck_folder $o_fileset_record]]
+                    }
                 }
                 method targetset_end {status args} {
                     set defaults [dict create\
@@ -311,7 +340,6 @@ namespace eval punkcheck {
                         dict unset opts -note
                     }
 
-
                     set status [string toupper $status]
                     set statusdict [dict create OK RECORD SKIPPED SKIPPED FAILED FAILED] 
                     if {$o_operation_start_ts eq ""} {
@@ -332,6 +360,7 @@ namespace eval punkcheck {
                     set file_record_body [dict get $o_fileset_record body]
                     set installing_record [lindex $file_record_body end]
                     set punkcheck_file [$o_installer get_checkfile]
+                    set punkcheck_folder [file dirname $punkcheck_file]
                     set record_list [punkcheck::load_records_from_file $punkcheck_file]
                     if {[dict exists $installing_record -ts_start_transfer]} {
                         set ts_start_transfer [dict get $installing_record -ts_start_transfer]
@@ -345,6 +374,23 @@ namespace eval punkcheck {
                     dict set installing_record -elapsed_us $elapsed_us
                     dict unset installing_record -tempcontext
                     dict set installing_record tag "${o_operation}-[dict get $statusdict $status]" ;# e.g INSTALL-RECORD, INSTALL-SKIPPED
+                    if {$o_operation in [list INSTALL MODIFY] && [dict get $statusdict $status] eq "RECORD"} {
+                        #only calculate and store post operation target cksums on successful INSTALL or MODIFY, doesn't make sense for DELETE or VIRTUAL operations
+                        set new_targets_cksums [list] ;#ordered list of cksums matching targetset order
+                        set cksum_all_opts "" ;#same cksum opts for each target so we store it once
+                        set ts_begin_cksum [clock microseconds]
+                        foreach p $o_targets {
+                            set tgt_cksum_info [punk::mix::base::lib::cksum_path [file join $punkcheck_folder $p]]
+                            lappend new_targets_cksums [dict get $tgt_cksum_info cksum]
+                            if {$cksum_all_opts eq ""} {
+                                set cksum_all_opts [dict get $tgt_cksum_info opts]
+                            }
+                        }
+                        set cksum_us [expr {[clock microseconds] - $ts_begin_cksum}]
+                        dict set installing_record -targets_cksums $new_targets_cksums 
+                        dict set installing_record -cksum_all_opts $cksum_all_opts
+                        dict set installing_record -cksum_us $cksum_us
+                    }
                     lset file_record_body end $installing_record
                     dict set o_fileset_record body $file_record_body 
                     set o_fileset_record [punkcheck::recordlist::file_record_prune $o_fileset_record]
@@ -356,7 +402,9 @@ namespace eval punkcheck {
                     } else {
                         lset record_list $old_posn $o_fileset_record
                     }
+                    if {$o_operation ne "QUERY"} {
                         punkcheck::save_records_to_file $record_list $punkcheck_file
+                    }
                     set o_operation_start_ts ""
                     set o_operation ""
                     return $o_fileset_record
@@ -372,6 +420,20 @@ namespace eval punkcheck {
 
                     set o_fileset_record [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $rel_source_path $o_fileset_record]
 
+                }
+                method targetset_last_complete {} {
+                    #retrieve last completed record for the fileset ie exclude SKIPPED,INSTALL-INPROGRESS,DELETE-INPROGRESS,MODIFY-INPROGRESS
+                    set body [punkcheck::dict_getwithdefault $o_fileset_record body [list]]
+                    set previous_records [lrange $body 0 end]
+                    #get last that is tagged INSTALL-RECORD,MODIFY-RECORD,DELETE-RECORD
+                    set revlist [lreverse $previous_records]
+                    foreach rec $revlist {
+                        if {[dict get $rec tag] in [list "INSTALL-RECORD" "MODIFY-RECORD" "DELETE-RECORD" "VIRTUAL-RECORD"]} {
+                            return $rec
+                        }
+                    }
+                    return [list]
+
                 }
                 method targetset_source_changes {} {
                     punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $o_fileset_record body] end]
@@ -551,16 +613,6 @@ namespace eval punkcheck {
                 method get_event {} {
                     return $o_active_event
                 }
-                if 0 {
-                method unknown {args} {
-                    puts "[self] unknown called with args:$args"
-                    if {[llength $args]} {
-
-                    } else {
-
-                    }
-                }
-                }
             }
         }
         proc start_installer_event {punkcheck_file installername from_fullpath to_fullpath config} {
@@ -722,6 +774,10 @@ namespace eval punkcheck {
             } else {
                 set cksum_opts ""
             }
+
+            #todo - accept argument of cached source cksum info (for client calling multiple targets with same source in quick succession e.g when building .vfs kits with multiple runtimes)
+            #if same cksum_opts - then use cached data instead of checksumming here.
+
             #allow nonexistant as a source 
             set fpath [file join $punkcheck_folder $source_relpath]
             if {![file exists $fpath]} {
@@ -857,7 +913,7 @@ namespace eval punkcheck {
             set tsnow [clock microseconds]
             set elapsed_us [expr {$tsnow - $ts_start}]
             dict set installing_record -elapsed_us $elapsed_us
-            dict set installing_record tag "SKIPPED" 
+            dict set installing_record tag "INSTALL-SKIPPED" 
              
             lset file_record_body end $installing_record
             dict set file_record body $file_record_body
@@ -879,13 +935,14 @@ namespace eval punkcheck {
         #then: file_record_add_installrecord
 
         namespace eval lib {
+            set pkg punkcheck
             namespace path ::punkcheck
             proc is_file_record_inprogress {file_record} {
                 if {[dict get $file_record tag] ne "FILEINFO"} {
                     return 0
                 }
                 set installing_record [lindex [dict_getwithdefault $file_record body [list]] end]
-                if {[dict_getwithdefault $installing_record tag [list]] ni [list INSTALL-INPROGRESS MODIFY-INPROGRESS DELETE-INPROGRESS VIRTUAL-INPROGRESS]} {
+                if {[dict_getwithdefault $installing_record tag [list]] ni [list QUERY-INPROGRESS INSTALL-INPROGRESS MODIFY-INPROGRESS DELETE-INPROGRESS VIRTUAL-INPROGRESS]} {
                     return 0
                 }
                 return 1
@@ -1021,9 +1078,16 @@ namespace eval punkcheck {
         }
         proc install_non_tm_files {srcdir basedir args} {
             #set keys [dict keys $args]
+            #adjust the default anti_glob_dir_core entries so that .fossil-custom, .fossil-settings are copied
+            set antiglob_dir_core [punkcheck::default_antiglob_dir_core]
+            set posn [lsearch $antiglob_dir_core ".fossil*"]
+            if {$posn >=0} {
+                set antiglob_dir_core [lreplace $antiglob_dir_core $posn $posn]
+            }
             set defaults [list\
                 -glob *\
                 -antiglob_file [list "*.tm" "*-buildversion.txt" "*.exe"]\
+                -antiglob_dir_core $antiglob_dir_core\
                 -installer punkcheck::install_non_tm_files\
                 ]
             set opts [dict merge $defaults $args]
@@ -1070,7 +1134,8 @@ namespace eval punkcheck {
         #           -overwrite newer-targets will copy files with older source timestamp over newer target timestamp and those missing at the target (a form of 'restore' operation)
         #           -overwrite older-targets will copy files with newer source timestamp over older target timestamp and those missing at the target
         #           -overwrite all-targets will copy regardless of timestamp at target
-        #           -overwrite installedsourcechanged-targets 
+        #           -overwrite installedsourcechanged-targets will copy if the target doesn't exist or the source changed 
+        #           -overwrite synced-targets will copy if the target doesn't exist or the source changed and the target cksum is the same as the last INSTALL-RECORD targets_cksums entry
         #           review - timestamps unreliable
         #            - what about slightly mismatched system clocks and mounted filesystems? caller responsibility to verify first? 
         #           if timestamp exactly equal -  should we check content-hash? This is presumably only likely to occur deliberately(maliciously?) 
@@ -1091,11 +1156,13 @@ namespace eval punkcheck {
         #  -source_checksum  compare|store|comparestore|false|true  where true == comparestore
         #  -punkcheck_folder target|source|project|<absolutepath>  target is default and is generally recommended
         #  -punkcheck_records empty string | parsed TDL records ie {tag xxx k v}  structure
+        # install creates FILEINFO records with a single entry in the -targets field (it is legitimate to have a list of targets for an installation operation - the oo interface supports this)
         proc install {srcdir tgtdir args} {
             set defaults [list\
                 -call-depth-internal 0\
                 -max_depth 1000\
                 -subdirlist {}\
+                -createdir 0\
                 -glob *\
                 -antiglob_file_core "\uFFFF"\
                 -antiglob_file "" \
@@ -1130,11 +1197,15 @@ namespace eval punkcheck {
             set max_depth                               [dict get $opts -max_depth]
             set subdirlist                              [dict get $opts -subdirlist] ;# generally should be same length as CALLDEPTH - but user could prefill
             set fileglob                                [dict get $opts -glob]
+            set createdir                               [dict get $opts -createdir] ;#defaults to zero to help avoid mistakes with initial target dir - required target subdirs are created regardless of this setting
 
             if {$CALLDEPTH == 0} {
                 #expensive to normalize  but we need to do it at least once
                 set srcdir [file normalize $srcdir]
                 set tgtdir [file normalize $tgtdir]
+                if {$createdir} {
+                    file mkdir $tgtdir
+                }
                 #now the values we build from these will be properly cased
             }
             # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
@@ -1157,7 +1228,7 @@ namespace eval punkcheck {
             set opt_unpublish_paths                     [dict get $opts -unpublish_paths] ;#todo - combine with config file in source tree .punkcheckpublish (?)
             set unpublish_paths_matched [list]
             # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
-            set known_whats [list no-targets newer-targets older-targets all-targets installedsourcechanged-targets]
+            set known_whats [list no-targets newer-targets older-targets all-targets installedsourcechanged-targets synced-targets]
             set overwrite_what                          [string tolower [dict get $opts -overwrite]]; #accept any case for value to allow emphasis by caller e.g -overwrite NEWER-TARGETS
             if {$overwrite_what ni $known_whats} {
                 error "punkcheck::install received unrecognised value for -overwrite. Received value '$overwrite_what' vs known values '$known_whats'"
@@ -1246,9 +1317,9 @@ namespace eval punkcheck {
             }
 
             if {[string match *store* $opt_source_checksum]} {
-                set store_cksums 1
+                set store_source_cksums 1
             } else {
-                set store_cksums 0
+                set store_source_cksums 0
             }
             
 
@@ -1280,7 +1351,7 @@ namespace eval punkcheck {
                 #puts "testing folder - globmatchpath $unpub $relative_source_dir"
                 if {[globmatchpath $unpub $relative_source_dir]} {
                     lappend unpublish_paths_matched $current_source_dir
-                    return [list files_copied {} files_skipped {} sources_unchanged {} punkcheck_records $punkcheck_records unpublish_paths_matched $unpublish_paths_matched]
+                    return [list files_copied {} files_skipped {} sources_unchanged {} punkcheck_records $punkcheck_records unpublish_paths_matched $unpublish_paths_matched srcdir $srcdir tgtdir $tgtdir punkcheck_folder $punkcheck_folder]
                 }
             }
 
@@ -1343,6 +1414,7 @@ namespace eval punkcheck {
 
             #puts stdout     "Current target dir: $current_target_dir"
             foreach m $match_list {
+                set new_tgt_cksum_info [list]
                 set relative_target_path [file join $relative_target_dir $m]
                 set relative_source_path [file join $relative_source_dir $m]
                 set punkcheck_target_relpath [file join $target_relative_to_punkcheck_dir $m]
@@ -1401,9 +1473,13 @@ namespace eval punkcheck {
                 #It doesn't save to .punkcheck (the only punkcheck::installfile_ method which doesn't)
                 set filerec [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $relative_source_path $filerec]
 
+
+
+                #changeinfo comes from last record in body - which is the record we are working on and so will always exist
                 set changeinfo [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $filerec body] end]]
                 set changed [dict get $changeinfo changed]
                 set unchanged [dict get $changeinfo unchanged]
+
                 if {[llength $unchanged]} {
                     lappend sources_unchanged $current_source_dir/$m
                 }
@@ -1415,6 +1491,7 @@ namespace eval punkcheck {
                 } else {
                     if {![file exists $current_target_dir/$m]} {
                         file copy $current_source_dir/$m $current_target_dir
+                        set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
                         lappend files_copied $current_source_dir/$m
                         incr filecount_new
                     } else {
@@ -1422,7 +1499,40 @@ namespace eval punkcheck {
                             if {[llength $changed]} {
                                 #An unrecorded installation is considered a source change (from unknown/unrecorded source to recorded)
                                 file copy -force $current_source_dir/$m $current_target_dir
+                                set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
+                                lappend files_copied $current_source_dir/$m
+                            } else {
+                                set is_skip 1
+                                lappend files_skipped $current_source_dir/$m
+                            }
+                        } elseif {$overwrite_what eq "synced-targets"} {
+                            if {[llength $changed]} {
+                                #only overwrite if the target checksum equals the last installed checksum (ie target is in sync with source and so hasn't been customized)
+                                set existing_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
+                                set is_target_unmodified_since_install 0
+                                set target_cksum_compare "unknown"
+                                set latest_install_record [punkcheck::recordlist::file_record_latest_installrecord $filerec] ;#may be no such record - in which case we get an empty list
+                                if {[dict exists $latest_install_record -targets_cksums]} {
+                                set last_install_cksum [dict get $latest_install_record -targets_cksums] ;#in this case we know there is only one as 'install' always uses targetset size of 1. (FILEINFO record per file in source folder)
+                                    if {[dict get $existing_tgt_cksum_info cksum] eq $last_install_cksum} {
+                                        set is_target_unmodified_since_install 1
+                                        set target_cksum_compare "match"
+                                    } else {
+                                        set target_cksum_compare "nomatch"
+                                    }
+                                } else {
+                                    set target_cksum_compare "norecord"
+                                }
+                                if {$is_target_unmodified_since_install} {
+                                    file copy -force $current_source_dir/$m $current_target_dir
+                                    set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
                                     lappend files_copied $current_source_dir/$m
+                                } else {
+                                    #either cksum is different or we were unable to verify the record. Either way we can't know if the target is in sync so we must skip it
+                                    set is_skip 1
+                                    puts stderr "Skipping file copy $m target $current_target_dir/$m - require synced_target to overwrite - current target cksum compared to previous install: $target_cksum_compare" 
+                                    lappend files_skipped $current_source_dir/$m
+                                }
                             } else {
                                 set is_skip 1
                                 lappend files_skipped $current_source_dir/$m
@@ -1430,7 +1540,7 @@ namespace eval punkcheck {
                         } else {
                             set is_skip 1
                             puts stderr "Skipping file copy $m target $current_target_dir/$m already exists (use -overwrite all-targets  to overwrite)"
-                            #TODO! implement newer-targets older-targets
+                            #TODO! implement newer-targets older-targets? (note ctimes/mtimes are unreliable - may not be worth implementing)
                             lappend files_skipped $current_source_dir/$m
                         }
                     }
@@ -1440,7 +1550,8 @@ namespace eval punkcheck {
                 set ts_now    [clock microseconds]
                 set elapsed_us [expr {$ts_now - $ts_start}]
 
-                if {$store_cksums} {
+                #if {$store_source_cksums} {
+                #}
 
                 set install_records [dict get $filerec body] 
                 set current_install_record [lindex $install_records end]
@@ -1452,6 +1563,10 @@ namespace eval punkcheck {
                 }
                 dict set current_install_record tag $tag
                 dict set current_install_record -elapsed_us $elapsed_us
+                if {[llength $new_tgt_cksum_info]} {
+                    dict set current_install_record -targets_cksums [list [dict get $new_tgt_cksum_info cksum]]
+                    dict set current_install_record -cksum_all_opts [dict get $new_tgt_cksum_info opts]
+                }
                 lset install_records end $current_install_record
                 dict set filerec body $install_records
                 set filerec [punkcheck::recordlist::file_record_prune $filerec] ;#prune now that tag is finalized
@@ -1461,7 +1576,6 @@ namespace eval punkcheck {
                 } else {
                     lset punkcheck_records $existing_filerec_posn $filerec
                 }
-                }
 
             }
 
@@ -1536,11 +1650,10 @@ namespace eval punkcheck {
                 #puts "subdirlist: $subdirlist"
                 if {$CALLDEPTH == 0} {
                     if {[llength $files_copied] || [llength $files_skipped]} {
-                        puts stdout ">>>>>>>>>>>>>>>>>>>"
+                        #puts stdout ">>>>>>>>>>>>>>>>>>>"
                         set saveresult [punkcheck::save_records_to_file $punkcheck_records $punkcheck_file]
-                        puts stdout "[dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file"
+                        puts stdout "punkcheck::install [dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file copied: [llength $files_copied] skipped: [llength $files_skipped]"
-                        puts stdout "copied: [llength $files_copied] skipped: [llength $files_skipped]"
+                        #puts stdout ">>>>>>>>>>>>>>>>>>>"
-                        puts stdout ">>>>>>>>>>>>>>>>>>>"
                     } else {
                         #todo - write db INSTALLER record if -debug true
                         
@@ -1551,10 +1664,30 @@ namespace eval punkcheck {
                 }
             }
 
-            return [list files_copied $files_copied files_skipped $files_skipped sources_unchanged $sources_unchanged unpublish_paths_matched $unpublish_paths_matched punkcheck_records $punkcheck_records]
+            return [list files_copied $files_copied files_skipped $files_skipped sources_unchanged $sources_unchanged unpublish_paths_matched $unpublish_paths_matched punkcheck_records $punkcheck_records punkcheck_folder $punkcheck_folder srcdir $srcdir tgtdir $tgtdir]
+        }
+        proc summarize_install_resultdict {resultdict} {
+            set msg ""
+            if {[dict size $resultdict]} {
+                set copied [dict get $resultdict files_copied]
+                append msg "--------------------------" \n
+                append msg  "[dict keys $resultdict]" \n
+                set tgtdir [dict get $resultdict tgtdir]
+                set checkfolder [dict get $resultdict punkcheck_folder]
+                append msg "Copied [llength $copied] files from [dict get $resultdict srcdir] to [dict get $resultdict tgtdir]" \n
+                foreach f $copied {
+                    append msg "COPIED [punkcheck::lib::path_relative $checkfolder $f]" \n
+                    append msg "    TO $tgtdir" \n
+                }
+                append msg "[llength [dict get $resultdict sources_unchanged]] unchanged source files" \n
+                append msg "[llength [dict get $resultdict files_skipped]] skipped files" \n
+                append msg "--------------------------" \n
+            }
+            return $msg
         }
 
         namespace eval recordlist {
+            set pkg punkcheck
             namespace path ::punkcheck
 
             proc records_as_target_dict {record_list} {
@@ -1590,8 +1723,8 @@ namespace eval punkcheck {
             }
             proc file_install_record_source_changes {install_record} {
                 #reject INSTALLFAILED items ?
-                if {[dict get $install_record tag] ni [list "INSTALL-RECORD" "SKIPPED" "INSTALL-INPROGRESS" "MODIFY-INPROGRESS" "MODIFY-RECORD" "VIRTUAL-INPROGRESS" "VIRTUAL-RECORD" "DELETE-RECORD" "DELETE-INPROGRESS"]} {
+                if {[dict get $install_record tag] ni [list "QUERY-INPROGRESS" "INSTALL-RECORD" "INSTALL-SKIPPED" "INSTALL-INPROGRESS" "MODIFY-INPROGRESS" "MODIFY-RECORD" "MODIFY-SKIPPED" "VIRTUAL-INPROGRESS" "VIRTUAL-RECORD" "VIRTUAL-SKIPPED" "DELETE-RECORD" "DELETE-INPROGRESS" "DELETE-SKIPPED"]} {
-                    error "file_install_record_source_changes bad file->install record: tag not INSTALL-RECORD|SKIPPED|INSTALL-INPROGRESS|MODIFY-RECORD|MODIFY-INPROGRESS|VIRTUAL-RECORD|VIRTUAL-INPROGRESS|DELETE-RECORD|DELETE-INPROGRESS"
+                    error "file_install_record_source_changes bad install record: tag '[dict get $install_record tag]' not INSTALL-RECORD|SKIPPED|INSTALL-INPROGRESS|MODIFY-RECORD|MODIFY-INPROGRESS|VIRTUAL-RECORD|VIRTUAL-INPROGRESS|DELETE-RECORD|DELETE-INPROGRESS"
                 }
                 set source_list [dict_getwithdefault $install_record body [list]]
                 set changed [list]
@@ -1742,15 +1875,19 @@ namespace eval punkcheck {
                 return $installer_record
             }
             proc file_record_latest_installrecord {file_record} {
+                tailcall file_record_latest_operationrecord INSTALL $file_record
+            }
+            proc file_record_latest_operationrecord {operation file_record} {
+                set operation [string toupper $operation]
                 if {[dict get $file_record tag] ne "FILEINFO"} {
-                    error "file_record_latest_installrecord bad file_record: tag not FILEINFO"
+                    error "file_record_latest_operationrecord bad file_record: tag not FILEINFO"
                 }
                 if {![dict exists $file_record body]} {
                     return [list]
                 }
                 set body_items [dict get $file_record body]
                 foreach item [lreverse $body_items] {
-                    if {[dict get $item tag] eq "INSTALL-RECORD"} {
+                    if {[dict get $item tag] eq "$operation-RECORD"} {
                         return $item
                     }
                 }
@@ -1758,47 +1895,6 @@ namespace eval punkcheck {
             }
 
 
-            #dead code?
-            proc file_record_add_installrecordXXX {file_record install_record} {
-                if {[dict get $file_record tag] ne "FILEINFO"} {
-                    error "file_record_add_installrecord bad file_record: tag not FILEINFO"
-                }
-                #disallow '-INPROGRESS' as it's not a final tag
-                if {[dict get $install_record tag] ni [list "INSTALL-RECORD" "SKIPPED"]} {
-                    error "file_record_add_installrecord bad install_record: tag not INSTALL-RECORD"
-                }
-                set keep 3
-                if {[dict exists $file_record -keep_installrecords]} {
-                    set keep [dict get $file_record -keep_installrecords]
-                }
-
-                if {[dict exists $file_record body]} {
-                    set body_items [dict get $file_record body]
-                } else {
-                    set body_items [list]
-                }
-                lappend body_items $install_record
-                set kept_body_items [list]
-                set kcount 0
-                foreach item [lreverse $body_items] {
-                    if {[dict get $item tag] eq "INSTALL-RECORD"} {
-                        incr kcount
-                        if {$keep < 0 || $kcount <= $keep} {
-                            lappend kept_body_items $item
-                        }
-                    } else {
-                        lappend kept_body_items $item
-                    }
-                }
-                set kept_body_items [lreverse $kept_body_items]
-
-                dict set file_record body $kept_body_items
-                return $file_record
-
-
-            }
-
-
             proc file_record_set_defaults {file_record} {
                 if {[dict get $file_record tag] ne "FILEINFO"} {
                     error "file_record_set_defaults bad file_record: tag not FILEINFO"
@@ -1881,6 +1977,7 @@ namespace eval punkcheck {
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 ##  Ready   
 package provide punkcheck [namespace eval punkcheck {
+    set pkg punkcheck
     variable version
     set version 0.1.0 
 }]

src/modules/shellfilter-0.1.8.tm

@@ -11,7 +11,7 @@
 
 
 namespace eval shellfilter::log {
-    variable allow_adhoc_tags 0
+    variable allow_adhoc_tags 1
     variable open_logs [dict create]
 
     #'tag' is an identifier for the log source. 
@@ -22,6 +22,7 @@ namespace eval shellfilter::log {
         if {![dict exists $settingsdict -tag]} {
             dict set settingsdict -tag $tag
         } else {
+            #review
             if {$tag ne [dict get $settingsdict -tag]} {
                 error "shellfilter::log::open first argument tag: '$tag' does not match -tag '[dict get $settingsdict -tag]' omit -tag, or supply same value"
             }
@@ -36,6 +37,13 @@ namespace eval shellfilter::log {
         return $worker_tid
     }
     proc write {tag msg} {
+        upvar ::shellfilter::sources sourcelist
+        variable allow_adhoc_tags
+        if {!$allow_adhoc_tags} {
+            if {$tag ni $sourcelist} {
+                error "shellfilter::log::write tag '$tag' hasn't been initialised with a call to shellfilter::log::open $tag <settings>, and allow_adhoc_tags has been set false. use shellfilter::log::require_open false to allow adhoc tags"
+            }
+        }
         shellthread::manager::write_log $tag $msg
     }
     #write_sync - synchronous processing with logging thread, slower but potentially useful for debugging/testing or forcing delay til log written
@@ -47,7 +55,8 @@ namespace eval shellfilter::log {
         shellthread::manager::unsubscribe [list $tag]; #workertid will be added back to free list if no tags remain subscribed
     }
 
-    #todo -implement
+    #review
+    #configure whether we can call shellfilter::log::write without having called open first
     proc require_open {{is_open_required {}}} {
         variable allow_adhoc_tags
         if {![string length $is_open_required]} {
@@ -131,7 +140,7 @@ namespace eval shellfilter::ansi2 {
         overline    53  nooverline      55  frame           51  framecircle 52  noframe     54
     }
     variable SGR_colour_map {
-        black       30  red             31  green           32  yellow      33  blue        4       purple 35   cyan        36      white 37
+        black       30  red             31  green           32  yellow      33  blue        34      purple 35   cyan        36      white 37
         Black       40  Red             41  Green           42  Yellow      43  Blue        44      Purple 45   Cyan        46      White 47
         BLACK       100 RED             101 GREEN           102 YELLOW      103 BLUE        104     PURPLE 105  CYAN        106     WHITE 107
     }
@@ -1010,12 +1019,15 @@ namespace eval shellfilter::stack {
 
         set pipecount [dict size $pipelines]
         set tableprefix "$pipecount pipelines active\n"
+        foreach p [dict keys $pipelines] {
+            append tableprefix "    " $p \n
+        }
         package require overtype
         #todo -verbose
         set table ""
         set ac1 [string repeat " " 15]
-        set ac2 [string repeat " " 32]
+        set ac2 [string repeat " " 42]
-        set ac3 [string repeat " " 80]
+        set ac3 [string repeat " " 70]
         append table "[overtype::left $ac1 channel-ident] "
         append table "[overtype::left $ac2 device-info] "
         append table "[overtype::left $ac3 stack-info]"
@@ -1028,10 +1040,11 @@ namespace eval shellfilter::stack {
 
         foreach k [dict keys $pipelines $pipename] {
             set lc [dict get $pipelines $k device localchan]
+            set tid [dict get $pipelines $k device workertid]
 
 
             set col1 [overtype::left $ac1 $k]
-            set col2 [overtype::left $ac2 "localchan: $lc"]
+            set col2 [overtype::left $ac2 "localchan: $lc tid:$tid"]
 
             set stack [dict get $pipelines $k stack]
             if {![llength $stack]} {
@@ -1148,15 +1161,24 @@ namespace eval shellfilter::stack {
         dict set pipelines $pipename [list]
     }
     #todo
-    proc delete {pipename} {
+    proc delete {pipename {wait 0}} {
-        set pipeinfo [dict get $pipename]
+        variable pipelines
+        set pipeinfo [dict get $pipelines $pipename]
         set deviceinfo [dict get $pipeinfo device] 
         set localchan [dict get $deviceinfo localchan]
         unwind $pipename
 
+        #release associated thread
+        set tid       [dict get $deviceinfo workertid]
+        if {$wait} {
+            thread::release -wait $tid
+        } else {
+            thread::release $tid
+        }
 
         chan close $localchan
     }
+    #review - proc name clarity is questionable. remove_stackitem? 
     proc remove {pipename remove_id} {
         variable pipelines
         if {![dict exists $pipelines $pipename]} {
@@ -2087,8 +2109,14 @@ namespace eval shellfilter {
         set worker_errorlist [list]
         set tidied_sources [list]
         set tidytag "logtidy"
-        set tid [::shellfilter::log::open $tidytag {-syslog 127.0.0.1:514}]
+
-        ::shellfilter::log::write $tidytag " logtidyuptags '$tags'"
+
+        # opening a thread or writing to a log/syslog close to possible process exit is probably not a great idea.
+        # we should ensure the thread already exists early on if we really need logging here.
+        # 
+        #set tid [::shellfilter::log::open $tidytag {-syslog 127.0.0.1:514}]
+        #::shellfilter::log::write $tidytag " logtidyuptags '$tags'"
+
         foreach s $sources {
             if {$s eq $tidytag} {
                 continue
@@ -2112,7 +2140,10 @@ namespace eval shellfilter {
                 lappend remaining_sources $s
             }
         } 
-        set sources [concat $remaining_sources $tidytag]
+
+        #set sources [concat $remaining_sources $tidytag]
+        set sources $remaining_sources
+
         #shellfilter::stack::unwind stdout
         #shellfilter::stack::unwind stderr
         return [list tidied $tidied_sources errors $worker_errorlist]
@@ -2705,7 +2736,7 @@ namespace eval shellfilter {
                         #} else {
                         #    #false alarm, or ? we've reached eof with data but didn't get an empty blocking read just prior
                         #    #Not known if this occurs
-                        #    #debugging output inline with data - don't leave enabled
+                        #    #debugging output inline with data - don't leave enabled
                         #    puts $outchan "!!!prev read didn't block: instate  b:[chan blocked $chan] eof:[chan eof $chan] pend:[chan pending output $chan] count:$countchunk"
                         #}
                     }
@@ -2747,15 +2778,17 @@ namespace eval shellfilter {
                         }
                     } trap CHILDSTATUS {result options} {
                         set code [lindex [dict get $options -errorcode] 2]
+                        set ::shellfilter::shellcommandvars($call_id,exitcode) $code
                         if {$debug} {
                             ::shellfilter::log::write $debugname " CHILD PROCESS EXITED with code: $code"
                         }
-                        set ::shellfilter::shellcommandvars($call_id,exitcode) $code
                     } trap CHILDKILLED {result options} {
                         #set code [lindex [dict get $options -errorcode] 2]
                         #set ::shellfilter::shellcommandvars(%id%,exitcode) $code
                         set ::shellfilter::shellcommandvars($call_id,exitcode) "childkilled"
+                        if {$debug} {
                             ::shellfilter::log::write $debugname " CHILD PROCESS EXITED with result:'$result' options:'$options'"
+                        }
 
                     } finally {
                         #puts stdout "HERE"
@@ -2774,7 +2807,7 @@ namespace eval shellfilter {
       #todo - add ability to detect activity/data-flow and change timeout to only apply for period with zero data
       #e.g x hrs with no data(?)
       #reset timeout when data detected. 
-      after $timeout [string map [list %w $waitvar %id% $call_id %wrerr% $wrerr %rdout% $rdout %rderr% $rderr %debug% $debug %debugname% $debugname] {
+      after $timeout [string map [list %w% $waitvar %id% $call_id %wrerr% $wrerr %rdout% $rdout %rderr% $rderr %debug% $debug %debugname% $debugname] {
           if {[info exists ::shellfilter::shellcommandvars(%id%,exitcode)]} {
               if {[set  ::shellfilter::shellcommandvars(%id%,exitcode)] ne ""} {
                   catch { chan close %wrerr% }
@@ -2798,7 +2831,8 @@ namespace eval shellfilter {
                       set code [lindex [dict get $options -errorcode] 2]
                       #set code [dict get $options -code]
                       #set ::shellfilter::shellcommandvars(%id%,exitcode) $code
-                      set ::shellfilter::shellcommandvars($call_id,exitcode) "childkilled-timeout"
+                      #set ::shellfilter::shellcommandvars($call_id,exitcode) "childkilled-timeout"
+                      set ::shellfilter::shellcommandvars(%id%,exitcode) "childkilled-timeout"
                       if {%debug%} {
                           ::shellfilter::log::write %debugname% "(timeout) CHILDKILLED with code: $code"
                           ::shellfilter::log::write %debugname% "(timeout) result:$result options:$options"
@@ -2807,9 +2841,8 @@ namespace eval shellfilter {
                   }
                   catch { chan close %wrerr% }
                   catch { chan close %rderr%}
-
               }
-              set %w "timeout"
+              set %w% "timeout"
           }
       }]
      

src/modules/shellrun-0.1.tm

@@ -70,6 +70,7 @@ namespace eval shellrun {
 
     #maintenance: similar used in punk::ns & punk::winrun
     #todo - take runopts + aliases as args
+    #longopts must be passed as a single item ie  --timeout=100  not --timeout 100
     proc get_run_opts {arglist} {
         if {[catch {
             set callerinfo [info level -1]
@@ -81,29 +82,49 @@ namespace eval shellrun {
 
         #we provide -nonewline even for 'run' even though run doesn't deliver stderr or stdout to the tcl return value
         #This is for compatibility with other runX commands, and the difference is also visible when calling from repl.
-        set known_runopts [list "-echo" "-e" "-nonewline" "-n" "-tcl"]
+        set known_runopts [list "-echo" "-e" "-nonewline" "-n" "-tcl" "-debug"]
-        set aliases [list "-e" "-echo" "-echo" "-echo" "-n" "-nonewline" "-nonewline" "-nonewline" "-tcl" "-tcl"] ;#include map to self
+        set known_longopts [list "--timeout"]
+        set known_longopts_msg ""
+        foreach lng $known_longopts {
+            append known_longopts_msg "${lng}=val "
+        }
+        set aliases [list "-e" "-echo" "-echo" "-echo" "-n" "-nonewline" "-nonewline" "-nonewline" "-tcl" "-tcl" "-debug" "-debug"] ;#include map to self
         set runopts [list]
+        set runoptslong [list]
         set cmdargs [list]
+
         set idx_first_cmdarg [lsearch -not $arglist "-*"]
-        set runopts [lrange $arglist 0 $idx_first_cmdarg-1]
+
+        set allopts [lrange $arglist 0 $idx_first_cmdarg-1]
         set cmdargs [lrange $arglist $idx_first_cmdarg end]
-        foreach o $runopts {
+        foreach o $allopts {
+            if {[string match --* $o]} {
+                lassign [split $o =] flagpart valpart
+                if {$valpart eq ""} {
+                    error "$caller: longopt $o seems to be missing a value - must be of form --option=value"
+                }
+                if {$flagpart ni $known_longopts} {
+                    error "$caller: Unknown runoption $o - known options $known_runopts $known_longopts_msg"
+                }
+                lappend runoptslong $flagpart $valpart
+            } else {
                 if {$o ni $known_runopts} {
-                error "$caller: Unknown runoption $o - known options $known_runopts"
+                    error "$caller: Unknown runoption $o - known options $known_runopts $known_longopts_msg"
                 }
+                lappend runopts [dict get $aliases $o]
             }
-        set runopts [lmap o $runopts {dict get $aliases $o}]
+        }
-        return [list runopts $runopts cmdargs $cmdargs]
+        return [list runopts $runopts runoptslong $runoptslong cmdargs $cmdargs]
     }
 
 
-
+    #todo - investigate cause of punk86 run hanging sometimes. An 'after 500' before exit in the called script fixes the issue. punk87 doesn't seem to be affected.
     proc run {args} {
         set_last_run_display [list]
 
         set splitargs [get_run_opts $args]
         set runopts [dict get $splitargs runopts]
+        set runoptslong [dict get $splitargs runoptslong]
         set cmdargs [dict get $splitargs cmdargs]
 
         if {"-nonewline" in $runopts} {
@@ -115,7 +136,7 @@ namespace eval shellrun {
         #we leave stdout without imposed ansi colouring - because the source may be colourised
         #stderr might have source colouring - but it usually doesn't seem to, and the visual distiction of red stderr is very handy for the run command.
         #A further enhancement could be to detect well-known options such as --color and/or use a configuration for specific commands that have useful colourised stderr,
-        #but defaulting stderr to red is a pretty reasonable compromise.
+        #but defaulting stderr to red is a compromise.
         #Note that the other run commands, runout,runerr, runx don't emit in real-time - so for those commands there may be options to detect and/or post-process stdout and stderr.
         #TODO - fix. This has no effect because the repl adds an ansiwrap transform 
         # what we probably want to do is 'aside' that transform for runxxx commands only.
@@ -125,6 +146,12 @@ namespace eval shellrun {
         if {"-tcl" in $runopts} {
             dict set callopts  -tclscript 1
         }
+        if {"-debug" in $runopts} {
+            dict set callopts -debug 1
+        }
+        if {[dict exists $runoptslong --timeout]} {
+            dict set callopts -timeout [dict get $runoptslong --timeout] ;#convert to single dash 
+        }
         #---------------------------------------------------------------------------------------------
         set exitinfo [shellfilter::run $cmdargs {*}$callopts -teehandle punk -inbuffering none -outbuffering none ]
         #---------------------------------------------------------------------------------------------

src/modules/shellthread-1.6.tm

@@ -352,13 +352,22 @@ namespace eval shellthread::worker {
     #allow any client to terminate
     proc terminate {tidclient} {
         variable sock
+        variable fd
         variable client_ids
         if {$tidclient in $client_ids} {
             catch {close $sock}
+            catch {close $fd}
             set client_ids [list]
-            return 1
+            #review use of thread::release -wait
+            #docs indicate deprecated for regular use, and that we should use thread::join
+            #however.. how can we set a timeout on a thread::join ?
+            #by telling the thread to release itself - we can wait on the thread::send  variable
+            # This needs review - because it's unclear that -wait even works on self
+            # (what does it mean to wait for the target thread to exit if the target is self??) 
+            thread::release -wait
+            return [thread::id] 
         } else {
-            return 0
+            return "" 
         }
     }
 
@@ -369,6 +378,7 @@ namespace eval shellthread::worker {
 namespace eval shellthread::manager {
     variable workers [dict create]
     variable worker_errors [list]
+    variable timeouts  
 
     variable free_threads [list]
     #variable log_threads
@@ -423,7 +433,7 @@ namespace eval shellthread::manager {
             set free_threads [lassign $free_threads tidworker]
             #todo - keep track of real ts_start of free threads... kill when too old
             set winfo [dict create tid $tidworker list_client_tids [list $tidclient] ts_start $ts_start ts_end_list [list]]
-            puts stderr "shellfilter::new_worker Re-using free worker thread: $tidworker with tag $sourcetag"
+            #puts stderr "shellfilter::new_worker Re-using free worker thread: $tidworker with tag $sourcetag"
             dict set workers $sourcetag $winfo
             return $tidworker
         }
@@ -604,11 +614,58 @@ namespace eval shellthread::manager {
         return $taginfo_list
     }
 
+    #finalisation
+    proc shutdown_free_threads {{timeout 2500}} {
+        variable free_threads
+        if {![llength $free_threads]} {
+            return
+        }
+        upvar ::shellthread::manager::timeouts timeoutarr
+        if {[info exists timeoutarr(shutdown_free_threads)]} {
+            #already called
+            return false
+        } 
+        #set timeoutarr(shutdown_free_threads) waiting 
+        #after $timeout  [list set timeoutarr(shutdown_free_threads) timed-out]
+        set ::shellthread::waitfor waiting
+        after $timeout [list set ::shellthread::waitfor]
+
+        set waiting_for [list]
+        set ended [list]
+        set timedout 0
+        foreach tid $free_threads {
+            if {[thread::exists $tid]} {
+                lappend waiting_for $tid
+                #thread::send -async $tid [list shellthread::worker::terminate [thread::id]] timeoutarr(shutdown_free_threads)
+                thread::send -async $tid [list shellthread::worker::terminate [thread::id]] ::shellthread::waitfor
+            }
+        }
+        if {[llength $waiting_for]} {
+            for {set i 0} {$i < [llength $waiting_for]} {incr i} {
+                vwait ::shellthread::waitfor 
+                if {$::shellthread::waitfor eq "timed-out"} {
+                    set timedout 1
+                    break
+                } else {
+                    lappend ended $::shellthread::waitfor
+                }
+            }
+        }
+        set free_threads [list]
+        return [dict create existed $waiting_for ended $ended timedout $timedout]
+    }
+
+    #TODO - important.
+    #REVIEW!
+    #since moving to the unsubscribe mechansm -  close_worker $source isn't being called
+    # - we need to set a limit to the number of free threads and shut down excess when detected during unsubscription 
+    #instruction to shut-down the thread that has this source. 
     #instruction to shut-down the thread that has this source. 
     proc close_worker {source {timeout 2500}} {
         variable workers
         variable worker_errors
         variable free_threads
+        upvar ::shellthread::manager::timeouts timeoutarr
         set ts_now [clock micros]
         #puts stderr "close_worker $source"
         if {[dict exists $workers $source]} {
@@ -639,6 +696,8 @@ namespace eval shellthread::manager {
 
             if {[thread::exists $tidworker]} {
                 #puts stderr "shellthread::manager::close_worker: thread $tidworker for source $source still running.. terminating"
+
+                #review - timeoutarr is local var  (?)
                 set timeoutarr($source) 0
                 after $timeout  [list set timeoutarr($source) 2]
 
@@ -676,7 +735,6 @@ namespace eval shellthread::manager {
 
     #worker errors only available for a source after close_worker called on that source
     #It is possible for there to be multiple entries for a source because new_worker can be called multiple times with same sourcetag,
-    # e.g if a thread 
     proc get_and_clear_errors {source} {
       variable worker_errors
       set source_errors [lsearch -all -inline -index 0        $worker_errors $source]

src/modules/zzzload-999999.0a1.0.tm

@@ -23,12 +23,40 @@ package require Thread
 
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 namespace eval zzzload {
-    variable loader_tid  ;#thread id 
+    variable loader_tid "" ;#thread id 
-    set loader_tid [thread::create -preserved]
+    proc stacktrace {} {
+        set stack "Stack trace:\n"
+        for {set i 1} {$i < [info level]} {incr i} {
+            set lvl [info level -$i]
+            set pname [lindex $lvl 0]
+            append stack [string repeat " " $i]$pname
+
+            if {![catch {info args $pname} pargs]} {
+                foreach value [lrange $lvl 1 end] arg $pargs {
+
+                    if {$value eq ""} {
+                        if {$arg != 0} {
+                            info default $pname $arg value
+                        }
+                    }
+                    append stack " $arg='$value'"
+                }
+            } else {
+                append stack " !unknown vars for $pname"
+            }
 
+            append stack \n
+        }
+        return $stack
+    }
     proc pkg_require {pkgname args} {
         variable loader_tid
+        if {$loader_tid eq ""} {
+            set loader_tid [thread::create -joinable -preserved]
+        }
         if {![tsv::exists zzzload_pkg $pkgname]} {
+            #puts stderr "zzzload pkg_require $pkgname"
+            #puts [stacktrace]
             tsv::set zzzload_pkg $pkgname "loading"
             tsv::set zzzload_pkg_mutex $pkgname [thread::mutex create]
             set cond [thread::cond create]
@@ -62,6 +90,14 @@ namespace eval zzzload {
             return $pkgstate
         }
     }
+    proc shutdown {} {
+        variable loader_tid
+        if {[thread::exists $loader_tid]} {
+            thread::release $loader_tid
+            thread::join $loader_tid
+            set loader_tid ""
+        }
+    }
 
 }
 

src/punk86.vfs/lib/app-punk/repl.tcl

@@ -72,6 +72,7 @@ if {[llength $currentdir_modules]} {
 
 #puts stdout "$::auto_path"
 package require Thread
+catch {package require tcllibc}
 #These are strong dependencies 
 # - the repl requires Threading and punk,shellfilter,shellrun to call and display properly.
 # tm list already indexed - need 'package forget' to find modules based on current tcl::tm::list

src/punk86.vfs/lib/app-shellspy/shellspy.tcl

@@ -26,6 +26,21 @@
 #
 package provide app-shellspy 1.0
 
+if 0 {
+    rename ::package ::package_orig
+    proc package {args} {
+        if {[lindex $args 0] eq "require"} {
+            if {[lindex $args 1] eq "twapi"} {
+                puts stderr "-------------------- loading twapi -------------"
+            } else {
+                puts stderr "-- loading [lindex $args 1] --"
+            }
+
+        }
+        tailcall ::package_orig {*}$args
+    }
+}
+
 
 #a test for windows
 #fconfigure stdin -encoding utf-16le
@@ -84,6 +99,7 @@ if {[file extension $arg1] in [list .tCl]} {
 
 #lappend auto_path c:/tcl/lib/tcllib1.20
 
+catch {package require tcllibc}
 package require Thread
 
 #NOTE: tm package index will probably already have been created so we must use 'package forget' to restrict to current tcl::tm::list path
@@ -127,7 +143,9 @@ namespace eval shellspy {
     #set id_ansistrip [shellfilter::stack::add stderr ansistrip    -settings {}]
     #set id_ansistrip [shellfilter::stack::add stdout ansistrip    -settings {}]
 
-    lassign [shellfilter::redir_output_to_log "SPY"] id_stdout_redir id_stderr_redir
+    #redir on the shellfilter stack with no log or syslog specified acts to suppress output of stdout & stderr.
+    #todo - fix shellfilter code to make this noop more efficient (avoid creating corresponding logging thread and filter?)
+    lassign [shellfilter::redir_output_to_log "SUPPRESS"] id_stdout_redir id_stderr_redir
     
     
     ###
@@ -326,6 +344,8 @@ namespace eval shellspy {
         set params [dict merge $params [get_channel_config $::testconfig]] ;#working: 5 unbuffered, 6 linebuffered
 
         
+        dict set params -debug 1
+        dict set params -timeout 1000
 
         #set exitinfo [shellfilter::run [list pwsh -nologo -noprofile -c {*}$args] -debug 1]
 
@@ -392,11 +412,12 @@ namespace eval shellspy {
 
         #shellfilter::stack::remove stdout $id_out
         
+        shellfilter::log::write $shellspy_status_log "do_in_cmdshell raw exitinfo: $exitinfo"
         
         
         if {[lindex $exitinfo 0] eq "exitcode"} {
             #exit [lindex $exitinfo 1]
-            shellfilter::log::write $shellspy_status_log "do_in_cmdshell returning $exitinfo"
+            #shellfilter::log::write $shellspy_status_log "do_in_cmdshell returning $exitinfo"
             #puts stderr "do_in_cmdshell returning $exitinfo"
         }
         return $exitinfo
@@ -514,21 +535,22 @@ namespace eval shellspy {
 
         
 
-        set id_err [shellfilter::stack::add stderr ansiwrap  -action sink-locked -settings {-colour {red bold}}]
+        #set id_err [shellfilter::stack::add stderr ansiwrap  -action sink-locked -settings {-colour {red bold}}]
 
 
 
         #todo - use glob to check capitalisation of file tail (.TCL vs .tcl .Tcl etc)
         set exitinfo [shellfilter::run [concat [auto_execok $scriptbin] $scriptpath $args] {*}$params]
+        shellfilter::log::write $shellspy_status_log "do_script_process exitinfo: $exitinfo"
 
-        shellfilter::stack::remove stderr $id_err
+        #shellfilter::stack::remove stderr $id_err
 
-        if {[lindex $exitinfo 0] eq "exitcode"} {
+        #if {[lindex $exitinfo 0] eq "exitcode"} {
-            shellfilter::log::write $shellspy_status_log "do_script_process returning $exitinfo"
+        #    shellfilter::log::write $shellspy_status_log "do_script_process returning $exitinfo"
-        }
+        #}
-        if {[dict exists $exitinfo errorCode]} {
+        #if {[dict exists $exitinfo errorCode]} {
-            exit [dict get $exitinfo $errorCode]
+        #    exit [dict get $exitinfo $errorCode]
-        }
+        #}
         return $exitinfo
     }
     proc do_script {scriptname replwhen args} {
@@ -604,7 +626,22 @@ source [file normalize $scriptname]
         #  shellfilter::log::write $shellspy_status_log "do_script returning $exitinfo"
         #}
 
-        shellfilter::log::write $shellspy_status_log "do_script returning $exitinfo"
+        shellfilter::log::write $shellspy_status_log "do_script raw exitinfo: $exitinfo"
+        if {[dict exists $exitinfo errorInfo]} {
+            #strip out the irrelevant info from the errorInfo - we don't want info beyond 'invoked from within' as this is just plumbing related to the script sourcing
+            set stacktrace [string map [list \r\n \n] [dict get $exitinfo errorInfo]]
+            set output "" 
+            set tracelines [split $stacktrace \n]
+            foreach ln $tracelines {
+                if {[string match "*invoked from within*" $ln]} {
+                    break
+                }
+                append output $ln \n 
+            }
+            set output [string trimright $output \n]
+            dict set exitinfo errorInfo $output
+            shellfilter::log::write $shellspy_status_log "do_script simplified exitinfo: $exitinfo"
+        }
         return $exitinfo
     }
 
@@ -650,13 +687,10 @@ source [file normalize $scriptname]
       
       dict set params -debug 0
       
-      
       set params [dict merge $params [get_channel_config $::testconfig]]
       
-      
       set id_out [shellfilter::stack::add stdout towindows -action sink-aside-locked -junction 1 -settings {}]
       
-      
       dict set params -teehandle shellspy  ;#shellspyout shellspyerr must exist
       set exitinfo [shellfilter::run [concat wsl -d $dist -e [shellescape $args]] {*}$params]
 
@@ -792,15 +826,17 @@ source [file normalize $scriptname]
     -values $::argv ]
 
 
-
+    set is_call_error 0
+    set arglist [list] ;#processed args result - contains dispatch info etc.
     if {[catch {
         set arglist [check_flags {*}$argdefinitions]
-    } errMsg]} {
+    } callError]} {
         puts -nonewline stderr "|shellspy-stderr> ERROR during command dispatch\n"
-        puts -nonewline stderr "|shellspy-stderr> $errMsg\n"
+        puts -nonewline stderr "|shellspy-stderr> $callError\n"
         puts -nonewline stderr "|shellspy-stderr> [set ::errorInfo]\n"
 
-        shellfilter::log::write $shellspy_status_log "check_flags error: $errMsg"
+        shellfilter::log::write $shellspy_status_log "check_flags error: $callError"
+        set is_call_error 1
     } else {
         shellfilter::log::write $shellspy_status_log "check_flags result: $arglist"
     }
@@ -820,28 +856,119 @@ source [file normalize $scriptname]
     #don't open more logs.. 
     #puts stdout ">$tidyinfo"
 
+    #lassign [shellfilter::redir_output_to_log "SUPPRESS"] id_stdout_redir id_stderr_redir
+    shellfilter::stack::remove stderr $id_stderr_redir
+    shellfilter::stack::remove stdout $id_stdout_redir
 
     #shellfilter::log::write $shellspy_status_log "logtidyup -done- $tidyinfo"
 
+    catch {
         set errorlist [dict get $tidyinfo errors]
         if {[llength $errorlist]} {
             foreach err $errorlist {
             puts -nonewline stderr "|shellspy-final> worker-error-set $err\n"
             }
         }
-    puts stdout "shellspy -done-"
+    }
+
+    #puts stdout "shellspy -done1-"
+    #flush stdout
+    
     #shellfilter::log::write $shellspy_status_log "shellspy -done-"
-    flush stdout
 
     if {[catch {
         shellfilter::logtidyup $shellspy_status_log
+        #puts stdout "shellspy logtidyup done"
+        #flush stdout
     } errMsg]} {
+        puts stdout "shellspy logtidyup error $errMsg"
+        flush stdout
         shellfilter::log::open shellspy-final {-tag shellspy-final -syslog 127.0.0.1:514}
         shellfilter::log::write shellspy-final "FINAL logtidyup error $errMsg\n [set ::errorInfo]"
-        after 200
+        after 100
+    }
+    #puts [shellfilter::stack::status shellspyout]
+    #puts [shellfilter::stack::status shellspyerr]
+
+    #sample dispatch member of $arglist
+    #dispatch {
+    #    tclscript {
+    #        command {shellspy::do_script %matched% no_repl}
+    #        matched stdout.tcl arguments {} raw {} dispatchtype raw 
+    #        asdispatched {shellspy::do_script stdout.tcl no_repl}
+    #        result {result {}}
+    #        error {}
+    #    }
+    #}
+    # or
+    #dispatch {
+    #    tclscript {
+    #        command xxx
+    #        matched error.tcl arguments {} raw {} dispatchtype raw
+    #        asdispatched {shellspy::do_script error.tcl no_repl}
+    #        result {
+    #            error {This is the error}
+    #            errorCode NONE
+    #            errorInfo This\ is\ the\ error\n\ etc
+    #        }
+    #        error {}
+    #    }
+    #}
+
+
+    shellfilter::stack::delete shellspyout
+    shellfilter::stack::delete shellspyerr
+    set free_info [shellthread::manager::shutdown_free_threads]
+    #puts stdout $free_info
+    #flush stdout
+    if {[package provide zzzload] ne ""} {
+        #puts "zzzload::loader_tid [set ::zzzload::loader_tid]"
+        zzzload::shutdown
+    }
+    #puts stdout "threads: [thread::names]"
+    #flush stdout
+    #puts stdout "calling release on remaining threads"
+    foreach tid [thread::names] {
+        thread::release $tid
+    }
+    #puts stdout "threads: [thread::names]"
+    #flush stdout
+
+
+    set colour ""; set reset ""
+    if {$is_call_error} {
+        catch {
+            set colour [punk::ansi::a+ yellow bold underline]; set reset [punk::ansi::a]
+        }
+        puts stderr $colour$callError$reset
+        flush stderr
+        exit 1
+    } else {
+        if {[dict exists $arglist dispatch tclscript result errorInfo]} {
+            catch {
+                set colour [punk::ansi::a+ yellow bold]; set reset [punk::ansi::a]
+            }
+            set err [dict get $arglist dispatch tclscript result errorInfo]
+            if {$err ne ""} {
+                puts stderr $colour$err$reset
+                flush stderr
+                exit 1
+            }
+        }
+    }
+
+    # -- --- --- 
+    # -- a deadlock hack. after 500 is not enough, after 1000 seems to work.
+    #after 1000
+    #after 500
+    # -- --- --- 
 
+    if {[dict exists $arglist errorCode]} {
+        exit [dict get $arglist errorCode]
     }
+
     #if we call exit - package require Tk scripts will exit prematurely 
+    #review
     #exit 0
 }
 

src/runtime/mapvfs.config

@@ -1,6 +1,10 @@
 #single line per runtime executable. Name of runtime followed by list of .vfs folders with path relative to src folder.
 #if runtime has no entry - it will only match a .vfs folder with a matching filename e.g runtime1.exe runtime1.vfs
+#Use a runtime with a name of dash (-) to build a .kit file from the .vfs folder using no runtime
+#e.g
+#- myproject.vfs
+#- punk86.vfs
 tclkit86bi.exe punk86.vfs
-tclkit87a5bawt.exe punk86.vfs
+#tclkit87a5bawt.exe punk86.vfs
 #tclkit86bi.exe vfs_windows/punk86win.vfs