fixes for 'pmix doc.build' doctools system, punk::path refactor, more documentation

src/bootsupport/modules/punk/cap-0.1.0.tm

@@ -17,11 +17,27 @@
 #*** !doctools
 #[manpage_begin punk::cap 0 0.1.0]
 #[copyright "2023 JMNoble - BSD licensed"]
-#[titledesc {Module API}]
+#[titledesc {capability provider and handler plugin system}]
 #[moddesc {punk capabilities plugin system}]
 #[require punk::cap]
 #[description]
-#[list_begin definitions]
+#[section Overview]
+#[para]punk::cap provides management of named capabilities and the provider packages and handler packages that implement a pluggable capability.
+#[subsection Concepts]
+#[para]A [term capability] may be something like providing a folder of files, or just a data dictionary, and/or an API 
+#
+#[para][term {capability handler}] - a package/namespace which may provide validation and standardised ways of looking up provider data
+# registered (or not) using register_capabilityname <capname> <capnamespace>
+#
+#[para][term {capability provider}] - a package which registers as providing one or more capablities.
+#[para]registered using register_package <pkg> <capabilitylist>
+#the capabilitylist is a list of 2-element lists where the first element is the capabilityname and the second element is a (possibly empty) dict of data relevant to that capability
+#A capabilityname may appear multiple times. ie a package may register that it provides the capability with multiple datasets.
+
+
+#*** !doctools
+#[section API]
+
 
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 ##  Requirements
@@ -29,63 +45,115 @@
 package require oolib
 
 
-
-# mkdoc markdown 
-#' ---
-#' author: JMNoble 
-#' ---
-#' ## Concepts:
-#' > A **capability** may be something like providing a folder of files, or just a data dictionary, and/or an API 
-#'
-#' > **capability handler** - a package/namespace which may provide validation and standardised ways of looking up provider data
-#'   registered (or not) using register_capabilityname <capname> <capnamespace>
-#'
-#' > **capability provider** - a package which registers as providing one or more capablities.
-#'  registered using register_package <pkg> <capabilitylist>
-#'  the capabilitylist is a list of 2-element lists where the first element is the capabilityname and the second element is a (possibly empty) dict of data relevant to that capability
-#'  A capabilityname may appear multiple times. ie a package may register that it provides the capability with multiple datasets.
-
-
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 namespace eval punk::cap {
     variable pkgcapsdeclared [dict create] 
     variable pkgcapsaccepted [dict create]
     variable caps [dict create]
-
     namespace eval class {
         if {[info commands [namespace current]::interface_caphandler.registry] eq ""} {
-            #Handler classes
+            #*** !doctools
+            #[subsection {Namespace punk::cap::class}]
+            #[para] class definitions
+            #[list_begin itemized] [comment {- punk::cap::class groupings -}]
+            #  [item]
+            #  [para] [emph {handler_classes}]
+            #  [list_begin enumerated]
+
             oo::class create [namespace current]::interface_caphandler.registry {
+                #*** !doctools
+                #[enum] CLASS [class interface_caphandler.registry]
+                #[list_begin definitions]
+                #    [para] [emph METHODS]
                 method pkg_register {pkg capname capdict fullcapabilitylist} {
-                    #***
-                    #[call [class interface_caphandler.registry] [method pkg_register] [arg pkg] [arg capname] [arg capdict] [arg fullcapabilitylist]]
+                    #*** !doctools
+                    #[call class::interface_caphandler.registry [method pkg_register] [arg pkg] [arg capname] [arg capdict] [arg fullcapabilitylist]]
                     #handler may override and return 0 (indicating don't register)e.g if pkg capdict data wasn't valid
                     #overridden handler must be able to handle multiple calls for same pkg - but it may return 1 or 0 as it wishes. 
                     return 1 ;#default to permit 
                 }
                 method pkg_unregister {pkg} {
-                    #***
-                    #[call [class interface_caphandler.registry] [method pkg_unregister] [arg pkg]]
+                    #*** !doctools
+                    #[call class::interface_caphandler.registry [method pkg_unregister] [arg pkg]]
                     return ;#unregistration return is ignored - review
                 }
+                #*** !doctools
+                #[list_end]
             }
+
             oo::class create [namespace current]::interface_caphandler.sysapi {
+                #*** !doctools
+                #[enum] CLASS [class interface_caphandler.sysapi]
+                #[list_begin definitions]
+                #    [para] [emph METHODS]
 
+
+                #*** !doctools
+                #[list_end]
             }
 
+            #*** !doctools
+            #  [list_end] [comment {- end enumeration handler classes -}]
+
+            #*** !doctools
+            #  [item]
+            #  [para] [emph {provider_classes}]
+            #  [list_begin enumerated]
 
             #Provider classes
             oo::class create [namespace current]::interface_capprovider.registration {
+                #*** !doctools
+                #    [enum] CLASS [class interface_cappprovider.registration]
+                #    [para]Your provider package will need to instantiate this object under a sub-namespace called [namespace capsystem] within your package namespace. 
+                #    [para]If your package namespace is mypackages::providerpkg then the object command would be at mypackages::providerpkg::capsystem::capprovider.registration
+                #    [para]Example code for your provider package to evaluate within its namespace: 
+                #    [example {
+                #namespace eval capsystem {
+                #    if {[info commands capprovider.registration] eq ""} {
+                #        punk::cap::class::interface_capprovider.registration create capprovider.registration
+                #        oo::objdefine capprovider.registration {
+                #            method get_declarations {} {
+                #                set decls [list]
+                #                lappend decls [list punk.templates {relpath ../templates}]
+                #                lappend decls [list another_capability_name {somekey blah key2 etc}]
+                #                return $decls
+                #            }
+                #        }
+                #    }
+                #}
+                #}]
+                #[para] The above example declares that your package can be registered as a provider for the capabilities named 'punk.templates' and 'another_capability_name'
+                #    [list_begin definitions]
+                #    [para] [emph METHODS]
                 method get_declarations {} {
                     #***
-                    #[call [class interface_capprovider.registration] [method pkg_unregister] [arg pkg]]
+                    #[call class::interface_capprovider.registration [method get_declarations]]
+                    #[para] This method must be overridden by your provider using oo::objdefine cappprovider.registration as in the example above.
+                    # There must be at least one 2-element list in the result for the provider to be registerable.
+                    #[para]The first element of the list is the capabilityname - which can be custom to your provider/handler packages - or a well-known name that other authors may use/implement.
+                    #[para]The second element is a dictionary of keys specific to the capability being implemented. It may be empty if the any potential capability handlers for the named capability don't require registration data.  
                     error "interface_capprovider.registration not implemented by provider"
                 }
+                #*** !doctools
+                #    [list_end]
             }
+
             oo::class create [namespace current]::interface_capprovider.provider {
+                #*** !doctools
+                #    [enum] CLASS [class interface_capprovider.provider] 
+                #    [para] Your provider package will need to instantiate this directly under it's own namespace with the command name of [emph {provider}] 
+                #    [example {
+                #       namespace eval mypackages::providerpkg { 
+                #           punk::cap::class::interface_capprovider.provider create provider mypackages::providerpkg 
+                #       }
+                #    }]
+                #    [list_begin definitions]
+                #    [para] [emph METHODS]
                 variable provider_pkg
                 variable registrationobj
                 constructor {providerpkg} {
+                    #*** !doctools
+                    #[call class::interface_capprovider.provider [method constructor] [arg providerpkg]]
                     variable provider_pkg
                     if {$providerpkg in [list "" "::"]} {
                         error "interface_capprovider.provider constructor error. Invalid provider '$providerpkg'"
@@ -103,16 +171,33 @@ namespace eval punk::cap {
 
                 }
                 method register {{capabilityname_glob *}} {
-                    #***
-                    #[call [class interface_capprovider.provider] [method register] [opt capabilityname_glob]]
+                    #*** !doctools
+                    #[comment {- -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---}]
+                    #[call class::interface_capprovider.provider [method register] [opt capabilityname_glob]]
+                    #
+                    #[para]This is the mechanism by which a user of your provider package will register your package as a provider of the capability named.
+                    #
+                    #[para]A user of your provider may elect to register all your declared capabilities:
+                    #[example {
+                    #  package require mypackages::providerpkg
+                    #  mypackages::providerpkg::provider register *
+                    #}]
+                    #[para] Or a specific capability may be registered:
+                    #[example {
+                    #  package require mypackages::providerpkg
+                    #  mypackages::providerpkg::provider register another_capability_name
+                    #}]
+                    #
                     variable provider_pkg
                     set all_decls [$registrationobj get_declarations]
                     set register_decls [lsearch -all -inline -index 0 $all_decls $capabilityname_glob]
                     punk::cap::register_package $provider_pkg $register_decls
                 }
                 method capabilities {} {
-                    #***
-                    #[call [class interface_capprovider.provider] [method capabilities]]
+                    #*** !doctools
+                    #[comment {- -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---}]
+                    #[call class::interface_capprovider.provider [method capabilities]]
+                    #[para] return a list of capabilities supported by this provider package
                     variable provider_pkg
                     variable registrationobj
 
@@ -124,28 +209,24 @@ namespace eval punk::cap {
                             lappend capabilities $capname
                         }
                     }
-                    return $capname
+                    return $capabilities
                 }
+                #*** !doctools
+                #    [list_end] [comment {- end class definitions -}]
             }
+            #*** !doctools
+            #  [list_end] [comment {- end enumeration provider_classes }]
+            #[list_end] [comment {- end itemized list punk::cap::class groupings -}]
         }
     } ;# end namespace class
-    namespace eval capsystem {
-        proc get_caphandler_registry {capname} {
-            set ns [::punk::cap::get_handler $capname]::capsystem
-            if {[namespace exists ${ns}]} {
-                if {[info command ${ns}::caphandler.registry] ne ""} {
-                    if {[info object isa object ${ns}::caphandler.registry]} {
-                        return ${ns}::caphandler.registry
-                    }
-                }
-            }
-            return ""
-        }
-    }
 
+    #*** !doctools
+    #[subsection {Namespace punk::cap}]
+    #[para] Main punk::cap API for client programs interested in using capability handler packages and associated (registered) provider packages
+    #[list_begin definitions]
 
-    #Not all capabilities have to be registered.
-    #A package registering as a provider using register_package can include capabilitynames in it's capabilitylist which have no associated capnamespace (handler).
+    #Not all capability names have to be registered.
+    #A package registering as a provider using register_package can include capabilitynames in it's capabilitylist which have no associated handler.
     #such unregistered capabilitynames may be used just to flag something, or have datamembers significant to callers cooperatively interested in that capname. 
     #we allow registering a capability with an empty handler (capnamespace) - but this means another handler could be registered later.
     proc register_capabilityname {capname capnamespace} {
@@ -161,7 +242,7 @@ namespace eval punk::cap {
         #allow register of existing capname iff there is no current handler
         #as handlers can be used to validate during provider registration - ideally handlers should be registered before any pkgs call register_package 
         #we allow loading a handler later though - but will need to validate existing data from pkgs that have already registered as providers 
-        if {[set hdlr [get_handler $capname]] ne ""} {
+        if {[set hdlr [capability_get_handler $capname]] ne ""} {
             error "register_capabilityname cannot register capability:$capname with handler:$capnamespace. There is already a registered handler:$hdlr" 
         }
         #assert: capnamespace may or may not be empty string, capname may or may not already exist in caps dict, caps $capname providers may have existing entries.
@@ -213,52 +294,33 @@ namespace eval punk::cap {
             
         }
     }
-    proc exists {capname} {
+    proc capability_exists {capname} {
         #*** !doctools
-        # [call [fun exists] [arg capname]]
+        # [call [fun capability_exists] [arg capname]]
         # Return a boolean indicating if the named capability exists  (0|1)
-
-        # mkdoc markdown
-        #'
-        #' ## **exists(capname)**
-        #'
-        #' > return a boolean indicating the existence of a capability
-        #'
-        #' > Arguments:
-        #'
-        #' > - *capname* - string indicating the name of the capability
-        #'
-        #' > Returns: 0|1
-        #'
         variable caps
         return [dict exists $caps $capname]
     }
-    proc has_handler {capname} {
+    proc capability_has_handler {capname} {
         #*** !doctools
-        # [call [fun has_handler] [arg capname]]
-        # Return a boolean indicating if the named capability has a handler package installed (0|1)
-
-
+        # [call [fun capability_has_handler] [arg capname]]
+        #Return a boolean indicating if the named capability has a handler package installed (0|1)
         variable caps
         return [expr {[dict exists $caps $capname handler] && [dict get $caps $capname handler] ne ""}]
     }
-    proc get_handler {capname} {
+    proc capability_get_handler {capname} {
+        #*** !doctools
+        # [call [fun capability_get_handler] [arg capname]]
+        #Return the base namespace of the active handler package for the named capability.
+        #[para] The base namespace for a handler will always be the package name, but prefixed with ::
         variable caps
         if {[dict exists $caps $capname]} {
             return [dict get $caps $capname handler]
         }
         return "" 
     }
-
-    #dispatch
-    #proc call_handler {capname args} {
-    #    if {[set handler [get_handler $capname]] eq ""} {
-    #        error "punk::cap::call_handler $capname $args - no handler registered for capability $capname"
-    #    }
-    #    ${handler}::[lindex $args 0] {*}[lrange $args 1 end]
-    #}
     proc call_handler {capname args} {
-        if {[set handler [get_handler $capname]] eq ""} {
+        if {[set handler [capability_get_handler $capname]] eq ""} {
             error "punk::cap::call_handler $capname $args - no handler registered for capability $capname"
         }
         set obj ${handler}::api_$capname
@@ -274,10 +336,21 @@ namespace eval punk::cap {
 
     #register package with arbitrary capnames from capabilitylist
     #The registered pkg is a module that provides some service to that capname. Possibly just data members, that the capability will use.
-    proc register_package {pkg capabilitylist} {
+    proc register_package {pkg capabilitylist args} {
         variable pkgcapsdeclared
         variable pkgcapsaccepted
         variable caps
+        set defaults [dict create\
+            -nowarnings false
+        ]
+        dict for {k v} $args {
+            if {$k ni $defaults} {
+                error "Unrecognized option $k. Known options [dict keys $defaults]"
+            }
+        }
+        set opts [dict merge $defaults $args]
+        set warnings [expr {! [dict get $opts -nowarnings]}]
+
         if {[string match ::* $pkg]} {
             set pkg [string range $pkg 2 end]
         }
@@ -286,11 +359,23 @@ namespace eval punk::cap {
         } else {
             set pkg_already_accepted [list]
         }
+        package require $pkg
+        set providerapi ::${pkg}::provider
+        if {[info commands $providerapi] eq ""} {
+            error "register_package error. pkg '$pkg' doesn't seem to be a punk::cap capability provider (no object found at $providerapi)"
+        }
+        set defined_caps [$providerapi capabilities]
         #for each capability
         # - ensure 1st element is a single word
         # - ensure that if 2nd element (capdict) is present - it is dict shaped
         foreach capspec $capabilitylist {
             lassign $capspec capname capdict
+
+            if {$warnings} {
+                if {$capname ni $defined_caps} {
+                    puts stderr "WARNING: pkg '$pkg' doesn't declare support for capability '$capname'."
+                }
+            }
             if {[llength $capname] !=1} {
                 error "register_package error. pkg: '$pkg' An entry in the capability list doesn't appear to have a single-word name. Problematic entry:'$capspec'"
             }
@@ -299,7 +384,9 @@ namespace eval punk::cap {
             }
             if {$capspec in $pkg_already_accepted} {
                 #review - multiple handlers? if so - will need to record which handler(s) accepted the capspec
-                puts stderr "register_package pkg $pkg already has capspec marked as accepted: $capspec"
+                if {$warnings} {
+                    puts stderr "WARNING: register_package pkg $pkg already has capspec marked as accepted: $capspec"
+                }
                 continue
             }
             if {[dict exists $caps $capname]} {
@@ -371,73 +458,6 @@ namespace eval punk::cap {
         }
     }
 
-    #review promote/demote doesn't always make a lot of sense ..  should possibly be per cap for multicap pkgs
-    #The idea is to provide a crude way to preference/depreference packages independently of order the packages were loaded
-    #e.g a caller or cap-handler can ascribe some meaning to the order of the 'providers' key returned from punk::cap::capabilities <capname>
-    #The order of providers will be the order the packages were loaded & registered
-    #the naming: "promote vs demote" operates on a latest-package-in-list has higher preference assumption (matching last pkg loaded)
-    #Each capability handler could implement specific preferencing methods if finer control needed.
-    #In some cases the preference/loading order may be inapplicable/irrelevant to a particular capability anyway.
-    #As this is just a basic mechanism, which can't support independent per-cap preferencing for multi-cap packages -
-    # it only allows putting the pkgs to the head or tail of the lists.
-    #Whether particular caps or users of caps do anything with this ordering is dependent on the cap-handler and/or calling code.
-    proc promote_package {pkg} {
-        variable pkgcapsdeclared
-        variable caps
-        if {[string match ::* $pkg]} {
-            set pkg [string range $pkg 2 end]
-        }
-        if {![dict exists $pkgcapsdeclared $pkg]} {
-            error "punk::cap::promote_package error pkg'$pkg' not registered. Use register_package \$pkg first"
-        }
-        if {[dict size $pkgcapsdeclared] > 1} {
-            set pkginfo [dict get $pkgcapsdeclared $pkg]
-            #remove and re-add at end of dict
-            dict unset pkgcapsdeclared $pkg
-            dict set pkgcapsdeclared $pkg $pkginfo
-            dict for {cap cap_info} $caps {
-                set cap_pkgs [dict get $cap_info providers]
-                if {$pkg in $cap_pkgs} {
-                    set posn [lsearch $cap_pkgs $pkg]
-                    if {$posn >=0} {
-                        #rewrite package list with pkg at tail of list for this capability
-                        set cap_pkgs [lreplace $cap_pkgs $posn $posn]
-                        lappend cap_pkgs $pkg
-                        dict set caps $cap providers $cap_pkgs
-                    }
-                }
-            }
-        }
-    }
-    proc demote_package {pkg} {
-        variable pkgcapsdeclared
-        variable caps
-        if {[string match ::* $pkg]} {
-            set pkg [string range $pkg 2 end]
-        }
-        if {![dict exists $pkgcapsdeclared $pkg]} {
-            error "punk::cap::promote_package error pkg'$pkg' not registered. Use register_package \$pkg first"
-        }
-        if {[dict size $pkgcapsdeclared] > 1} {
-            set pkginfo [dict get $pkgcapsdeclared $pkg]
-            #remove and re-add at start of dict
-            dict unset pkgcapsdeclared $pkg
-            dict set pkgcapsdeclared $pkg $pkginfo
-            set pkgcapsdeclared [dict merge [dict create $pkg $pkginfo] $pkgcapsdeclared]
-            dict for {cap cap_info} $caps {
-                set cap_pkgs [dict get $cap_info providers]
-                if {$pkg in $cap_pkgs} {
-                    set posn [lsearch $cap_pkgs $pkg]
-                    if {$posn >=0} {
-                        #rewrite package list with pkg at head of list for this capability
-                        set cap_pkgs [lreplace $cap_pkgs $posn $posn]
-                        set cap_pkgs [list $pkg {*}$cap_pkgs]
-                        dict set caps $cap providers $cap_pkgs
-                    }
-                }
-            }
-        }
-    }
     proc pkgcap {pkg} {
         variable pkgcapsdeclared
         variable pkgcapsaccepted
@@ -502,7 +522,121 @@ namespace eval punk::cap {
         }
         return $cap_list
     }
+    #*** !doctools
+    #[list_end] [comment {- end definitions for namespace punk::cap -}]
+
+    namespace eval advanced {
+    #*** !doctools
+    #[subsection {Namespace punk::cap::advanced}]
+    #[para] punk::cap::advanced API. Functions here are generally not the preferred way to interact with punk::cap.
+    #[para] In some cases they may allow interaction in less safe ways or may allow use of features that are unavailable in the base namespace.
+    #[para] Some functions are here because they are only marginally or rarely useful, and they are here to keep the base API simple.  
+    #[list_begin definitions]
+
+    proc promote_provider {pkg} {
+        #*** !doctools
+        # [call advanced::[fun promote_provider] [arg pkg]]
+        #[para]Move the named provider package to the preferred end of the list (tail).
+        #[para]The active handler may or may not utilise this for preferencing. See documentation for the specific handler package to confirm. 
+        #[para]
+        #[para] promote/demote doesn't always make a lot of sense ..  should preferably be configurable per capapbility for multicap provider pkgs
+        #[para]The idea is to provide a crude way to preference/depreference packages independently of order the packages were loaded
+        #e.g a caller or cap-handler can ascribe some meaning to the order of the 'providers' key returned from punk::cap::capabilities <capname>
+        #[para]The order of providers will be the order the packages were loaded & registered
+        #[para]the naming: "promote vs demote" operates on a latest-package-in-list has higher preference assumption (matching last pkg loaded)
+        #[para]Each capability handler could and should implement specific preferencing methods within its own API if finer control needed.
+        #In some cases the preference/loading order may be inapplicable/irrelevant to a particular capability anyway.
+        #[para]As this is just a basic mechanism, which can't support independent per-cap preferencing for multi-cap packages -
+        # it only allows putting the pkgs to the head or tail of the lists.
+        #[para]Whether particular caps or users of caps do anything with this ordering is dependent on the cap-handler and/or calling code.
+        variable pkgcapsdeclared
+        variable caps
+        if {[string match ::* $pkg]} {
+            set pkg [string range $pkg 2 end]
+        }
+        if {![dict exists $pkgcapsdeclared $pkg]} {
+            error "punk::cap::promote_package error pkg'$pkg' not registered. Use register_package \$pkg first"
+        }
+        if {[dict size $pkgcapsdeclared] > 1} {
+            set pkginfo [dict get $pkgcapsdeclared $pkg]
+            #remove and re-add at end of dict
+            dict unset pkgcapsdeclared $pkg
+            dict set pkgcapsdeclared $pkg $pkginfo
+            dict for {cap cap_info} $caps {
+                set cap_pkgs [dict get $cap_info providers]
+                if {$pkg in $cap_pkgs} {
+                    set posn [lsearch $cap_pkgs $pkg]
+                    if {$posn >=0} {
+                        #rewrite package list with pkg at tail of list for this capability
+                        set cap_pkgs [lreplace $cap_pkgs $posn $posn]
+                        lappend cap_pkgs $pkg
+                        dict set caps $cap providers $cap_pkgs
+                    }
+                }
+            }
+        }
+    }
+    proc demote_provider {pkg} {
+        #*** !doctools
+        # [call advanced::[fun demote_provider] [arg pkg]]
+        #[para]Move the named provider package to the preferred end of the list (tail).
+        #[para]The active handler may or may not utilise this for preferencing. See documentation for the specific handler package to confirm. 
+        variable pkgcapsdeclared
+        variable caps
+        if {[string match ::* $pkg]} {
+            set pkg [string range $pkg 2 end]
+        }
+        if {![dict exists $pkgcapsdeclared $pkg]} {
+            error "punk::cap::promote_package error pkg'$pkg' not registered. Use register_package \$pkg first"
+        }
+        if {[dict size $pkgcapsdeclared] > 1} {
+            set pkginfo [dict get $pkgcapsdeclared $pkg]
+            #remove and re-add at start of dict
+            dict unset pkgcapsdeclared $pkg
+            dict set pkgcapsdeclared $pkg $pkginfo
+            set pkgcapsdeclared [dict merge [dict create $pkg $pkginfo] $pkgcapsdeclared]
+            dict for {cap cap_info} $caps {
+                set cap_pkgs [dict get $cap_info providers]
+                if {$pkg in $cap_pkgs} {
+                    set posn [lsearch $cap_pkgs $pkg]
+                    if {$posn >=0} {
+                        #rewrite package list with pkg at head of list for this capability
+                        set cap_pkgs [lreplace $cap_pkgs $posn $posn]
+                        set cap_pkgs [list $pkg {*}$cap_pkgs]
+                        dict set caps $cap providers $cap_pkgs
+                    }
+                }
+            }
+        }
+    }
+
+    #*** !doctools
+    #[list_end]
+    }
+
 
+#*** !doctools
+#[section Internal]
+
+    namespace eval capsystem {
+        #*** !doctools
+        #[subsection {Namespace punk::cap::capsystem}]
+        #[para] Internal functions used to communicate between punk::cap and capability handlers
+        #[list_begin definitions]
+        proc get_caphandler_registry {capname} {
+            set ns [::punk::cap::capability_get_handler $capname]::capsystem
+            if {[namespace exists ${ns}]} {
+                if {[info command ${ns}::caphandler.registry] ne ""} {
+                    if {[info object isa object ${ns}::caphandler.registry]} {
+                        return ${ns}::caphandler.registry
+                    }
+                }
+            }
+            return ""
+        }
+        #*** !doctools
+        #[list_end]
+    }
 }
 
 
@@ -524,5 +658,4 @@ package provide punk::cap [namespace eval punk::cap {
 return
 
 #*** !doctools
-#[list_end]
 #[manpage_end]

src/bootsupport/modules/punk/du-0.1.0.tm

@@ -24,15 +24,14 @@ namespace eval punk::du {
     variable has_twapi 0
 }
 if {"windows" eq $::tcl_platform(platform)} {
-    #jmn disable twapi
-    #package require zzzload
-    #zzzload::pkg_require twapi
-
-    #if {[catch {package require twapi}]} {
-    #    puts stderr "Warning: punk::du - unable to load twapi. Disk operations may be much slower on windows without the twapi package"
-    #} else {
-    #    set punk::du::has_twapi 1
-    #}
+    package require zzzload
+    zzzload::pkg_require twapi
+
+    if {[catch {package require twapi}]} {
+        puts stderr "Warning: punk::du - unable to load twapi. Disk operations may be much slower on windows without the twapi package"
+    } else {
+        set punk::du::has_twapi 1
+    }
     #package require punk::winpath
 }
 
@@ -1248,10 +1247,11 @@ namespace eval punk::du {
         proc du_dirlisting_undecided {folderpath args} {
             if {"windows" eq $::tcl_platform(platform)} {
                 #jmn disable twapi
-                tailcall du_dirlisting_generic $folderpath {*}$args                        
+                #tailcall du_dirlisting_generic $folderpath {*}$args                        
 
                 set loadstate [zzzload::pkg_require twapi]
                 if {$loadstate ni [list loading failed]} {
+                    #either already loaded by zzload or ordinary package require
                     package require twapi ;#should be fast once twapi dll loaded in zzzload thread
                     set ::punk::du::has_twapi 1
                     punk::du::active::set_active_function du_dirlisting du_dirlisting_twapi

src/bootsupport/modules/punk/mix/base-0.1.tm

@@ -3,6 +3,7 @@ package provide punk::mix::base [namespace eval punk::mix::base {
 	set version 0.1
 }]
 
+package require punk::path
 
 #base internal plumbing functions
 namespace eval punk::mix::base {
@@ -364,7 +365,7 @@ namespace eval punk::mix::base {
             set folderdict [dict create]
 
             package require punk::cap
-            if {[punk::cap::has_handler punk.templates]} {
+            if {[punk::cap::capability_has_handler punk.templates]} {
                 set template_folder_dict [punk::cap::call_handler punk.templates folders]
                 dict for {dir folderinfo} $template_folder_dict {
                     dict set folderdict $dir $folderinfo
@@ -632,7 +633,8 @@ namespace eval punk::mix::base {
                 cd $base ;#cd is process-wide.. keep cd in effect for as small a scope as possible. (review for thread issues)
             
                 #temp emission to stdout.. todo - repl telemetry channel 
-                puts stdout "cksum_path: creating temporary tar archive at: $archivename .."
+                puts stdout "cksum_path: creating temporary tar archive for $path"
+                puts stdout "  at: $archivename .."
                 tar::create $archivename $target
                 if {$ftype eq "file"} {
                     set sizeinfo "(size [file size $target])"
@@ -757,7 +759,7 @@ namespace eval punk::mix::base {
                         set normbase [file normalize $base]
                         set normtarg [file normalize [file join $normbase $specifiedpath]]
                         set targetpath $normtarg
-                        set storedpath [punk::mix::util::path_relative $normbase $normtarg]
+                        set storedpath [punk::path::relative $normbase $normtarg]
                     } else {
                         set targetpath [file join $base $specifiedpath]
                         set storedpath $specifiedpath
@@ -776,7 +778,7 @@ namespace eval punk::mix::base {
                         error "get_relativecksum_from_base error: base '$base' and specifiedpath '$specifiedpath' don't share a common root. Use empty-string for base if independent absolute path is required"
                     }
                     set targetpath $specifiedpath 
-                    set storedpath [punk::mix::util::path_relative $base $specifiedpath]
+                    set storedpath [punk::path::relative $base $specifiedpath]
                     
                 }
             } else {

src/bootsupport/modules/punk/mix/cli-0.3.tm

@@ -477,7 +477,13 @@ namespace eval punk::mix::cli {
 
             set did_skip 0 ;#flag for stdout/stderr formatting only
             foreach m $src_modules {
-                #puts "build_modules_from_source_to_base >>> module $m"
+                set is_interesting 0
+                if {[string match "foobar" $current_source_dir]} {
+                    set  is_interesting 1
+                }
+                if {$is_interesting} {
+                    puts "build_modules_from_source_to_base >>> module $current_source_dir/$m"
+                }
                 set fileparts [split [file rootname $m] -]
                 set tmfile_versionsegment [lindex $fileparts end]
                 if {$tmfile_versionsegment eq $magicversion} {
@@ -582,7 +588,9 @@ namespace eval punk::mix::cli {
                             #set file_record [punkcheck::installfile_finished_install $basedir $file_record]
                             $event targetset_end OK
                         } else {
-                            #puts stdout "skipping module $current_source_dir/$m - no change in sources detected"
+                            if {$is_interesting} {
+                                puts stdout "skipping module $current_source_dir/$m - no change in sources detected"
+                            }
                             puts -nonewline stderr "."
                             set did_skip 1
                             #set file_record [punkcheck::installfile_skipped_install $basedir $file_record]
@@ -627,15 +635,18 @@ namespace eval punk::mix::cli {
                     $event targetset_started
                     # -- --- --- --- --- ---
                     if {$did_skip} {set did_skip 0; puts -nonewline stdout \n}
-                    puts stderr "Copied already versioned module $current_source_dir/$m to $target_module_dir"
                     lappend module_list $current_source_dir/$m
                     file copy -force $current_source_dir/$m $target_module_dir
+                    puts stderr "Copied already versioned module $current_source_dir/$m to $target_module_dir"
                     # -- --- --- --- --- ---
                     #set file_record [punkcheck::installfile_finished_install $basedir $file_record]
                     $event targetset_end OK -note "already versioned module"
                 } else {
                     puts -nonewline stderr "."
                     set did_skip 1
+                    if {$is_interesting} {
+                        puts stderr "$current_source_dir/$m [$event targetset_source_changes]"
+                    }
                     #set file_record [punkcheck::installfile_skipped_install $basedir $file_record]
                     $event targetset_end SKIPPED
                 }

src/bootsupport/modules/punk/mix/commandset/doc-0.1.0.tm

@@ -18,11 +18,11 @@
 ##  Requirements
 ##e.g package require frobz
 
-package require punk ;# for treefilenames
+package require punk::path ;# for treefilenames, relative
 package require punk::repo
 package require punk::docgen ;#inline doctools - generate doctools .man files at src/docgen prior to using kettle to producing .html .md etc
 package require punk::mix::cli  ;#punk::mix::cli::lib used for kettle_call
-package require punk::mix::util ;#for path_relative
+#package require punk::mix::util ;#for path_relative
 
 
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
@@ -44,7 +44,7 @@ namespace eval punk::mix::commandset::doc {
         }
         #user may delete the comment containing "--- punk::docgen::overwrites" and then manually edit, and we won't overwrite
         #we still generate output in src/docgen so user can diff and manually update if thats what they prefer
-        set oldfiles [glob -nocomplain -dir $projectdir/src/doc -type f _module_*]
+        set oldfiles [punk::path::treefilenames $projectdir/src/doc _module_*.man]
         foreach maybedoomed $oldfiles {
             set fd [open $maybedoomed r]
             set data [read $fd]
@@ -57,9 +57,18 @@ namespace eval punk::mix::commandset::doc {
         if {[dict get $generated count] > 0} {
             #review
             set doclist [dict get $generated docs]
+            set source_base [dict get $generated base]
+            set target_base $projectdir/src/doc
             foreach dinfo $doclist {
                 lassign $dinfo module fpath
-                set target $projectdir/src/doc/_module_[file tail $fpath]
+                set relpath [punk::path::relative $source_base $fpath]
+                set relfolder [file dirname $relpath]
+                if {$relfolder eq "."} {
+                    set relfolder ""
+                }
+                file mkdir [file join $target_base $relfolder]
+                set target [file join $target_base $relfolder _module_[file tail $fpath]]
+                puts stderr "target --> $target"
                 if {![file exists $target]} {
                     file copy $fpath $target
                 }
@@ -184,23 +193,24 @@ namespace eval punk::mix::commandset::doc {
         variable pkg
         set pkg punk::mix::commandset::doc
         proc  do_docgen {{project_subpath modules}} {
+            #Extract doctools comments from source code
             set projectdir [punk::repo::find_project]
-            set outdir [file join $projectdir src docgen]
-            set subpath [file join $projectdir $project_subpath]
-            if {![file isdirectory $subpath]} {
-                puts stderr "WARNING punk::mix::commandset::doc unable to find subpath $subpath during do_docgen - skipping inline doctools generation"
+            set output_base [file join $projectdir src docgen]
+            set codesource_path [file join $projectdir $project_subpath]
+            if {![file isdirectory $codesource_path]} {
+                puts stderr "WARNING punk::mix::commandset::doc unable to find codesource_path $codesource_path during do_docgen - skipping inline doctools generation"
                 return
             }
-            if {[file isdirectory $outdir]} {
+            if {[file isdirectory $output_base]} {
                 if {[catch {
-                    file delete -force $outdir
+                    file delete -force $output_base
                 }]}  {
-                    error "do_docgen failed to delete existing $outdir"
+                    error "do_docgen failed to delete existing output base folder: $output_base"
                 }
             }
-            file mkdir $outdir
+            file mkdir $output_base
 
-            set matched_paths [punk::treefilenames $subpath *.tm]
+            set matched_paths [punk::path::treefilenames $codesource_path *.tm -antiglob_paths {**/mix/templates/** **/mixtemplates/**}]
             set count 0
             set newdocs [list] 
             set docgen_header_comments ""
@@ -208,14 +218,34 @@ namespace eval punk::mix::commandset::doc {
             append docgen_header_comments {[comment {--- punk::docgen DO NOT EDIT DOCS HERE UNLESS YOU REMOVE THESE COMMENT LINES ---}]} \n
             append docgen_header_comments {[comment {--- punk::docgen overwrites this file ---}]} \n
             foreach fullpath $matched_paths {
-                set relpath [punk::mix::util::path_relative $subpath $fullpath]
-                set tailsegs [file split $relpath]
-                set module_fullname [join $tailsegs ::]
-                set docname [string map [list :: _] $module_fullname].man ;#todo - something better - need to ensure unique
                 set doctools [punk::docgen::get_doctools_comments $fullpath] 
                 if {$doctools ne ""} {
-                    puts stdout "generating doctools output from file $relpath"
-                    set outfile [file join $outdir $docname]
+                    set fname [file tail $fullpath]
+                    set mod_tail [file rootname $fname] 
+                    set relpath [punk::path::relative $codesource_path [file dirname $fullpath]]
+                    if {$relpath eq "."} {
+                        set relpath ""
+                    }
+                    set tailsegs [file split $relpath]
+                    set module_fullname [join $tailsegs ::]::$mod_tail
+                    set target_docname $fname.man
+                    set this_outdir [file join $output_base $relpath]
+
+                    if {[string length $fname] > 99} {
+                        #output needs to be tarballed to do checksum change tests in a reasonably straightforward and not-too-terribly slow way.
+                        #hack - review. Determine exact limit - test if tcllib tar fixed or if it's a limit of the particular tar format
+                        #work around tcllib tar filename length limit ( somewhere around 100?) This seems to be a limit on the length of a particular segment in the path.. not whole path length?
+                        #this case only came up because docgen used to path munge to long filenames - but left because we know there is a limit and renaming fixes it - even if it's ugly - but still allows doc generation.
+                        #review - if we're checking fname - should also test length of whole path and determine limits for tar
+                        package require md5
+                        set target_docname [md5::md5 -hex $fullpath]_overlongfilename.man
+                        puts stderr "WARNING - overlong file name - renaming $fullpath"
+                        puts stderr "          to                            [file dirname $fullpath]/$target_docname"
+                    }
+
+                    file mkdir $this_outdir
+                    puts stdout "saving [string length $doctools] bytes of doctools output from file $relpath/$fname"
+                    set outfile [file join $this_outdir $target_docname]
                     set fd [open $outfile w]
                     fconfigure $fd -translation binary
                     puts -nonewline $fd $docgen_header_comments$doctools
@@ -224,7 +254,7 @@ namespace eval punk::mix::commandset::doc {
                     lappend newdocs [list $module_fullname $outfile]
                 }
             }
-            return [list count $count docs $newdocs]
+            return [list count $count docs $newdocs base $output_base]
         }
 
     }

src/bootsupport/modules/punk/mix/commandset/layout-0.1.0.tm

@@ -141,7 +141,10 @@ namespace eval punk::mix::commandset::layout {
             #use last matching layout found.  review silent if multiple?
             if {![llength $tags]} {
                 #todo - get standard tags from somewhere
-                set tags [list %project%]
+                set tagnames [list project]
+                foreach tn $tagnames {
+                    lappend tags [string cat % $tn %]
+                }
             }
             set file_list [list] 
             util::foreach-file $layoutfolder path {

src/bootsupport/modules/punk/mix/commandset/project-0.1.0.tm

@@ -13,10 +13,72 @@
 # @@ Meta End
 
 
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+# doctools header
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+#*** !doctools
+#[manpage_begin punk::mix::commandset::project 0 0.1.0]
+#[copyright "2023"]
+#[titledesc {pmix commandset - project}]  [comment {-- Name section and table of contents description --}]
+#[moddesc {pmix CLI commandset - project}]             [comment {-- Description at end of page heading --}] 
+#[require punk::mix::commandset::project]
+#[description]
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[section Overview]
+#[para] overview of punk::mix::commandset::project
+#[para]Import into an ensemble namespace similarly to the way it is done with punk::mix::cli e.g 
+#[example {
+#    namespace eval myproject::cli {
+#        namespace export *
+#        namespace ensemble create
+#        package require punk::overlay
+#    
+#        package require punk::mix::commandset::project
+#        punk::overlay::import_commandset project    .       ::punk::mix::commandset::project
+#        punk::overlay::import_commandset projects   .       ::punk::mix::commandset::project::collection 
+#    }
+#}]
+#[para] Where the . in the above example is the prefix/command separator
+#[para]The prefix ('project' in the above example) can be any string desired to disambiguate commands imported from other commandsets.
+#[para]The above results in the availability of the ensemble command: ::myproject::cli project.new, which is implemented in ::punk::mix::commandset::project::new  
+#[para]Similarly, procs under ::punk::mix::commandset::project::collection will be available as subcommands of the ensemble as projects.<procname>
+#[para]
+#[subsection Concepts]
+#[para] see punk::overlay 
+
 
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 ##  Requirements
-##e.g package require frobz
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[subsection dependencies]
+#[para] packages used by punk::mix::commandset::project 
+#[list_begin itemized]
+
+package require Tcl 8.6
+#*** !doctools
+#[item] [package {Tcl 8.6}]
+#[item] [package punk::ns]
+#[item] [package sqlite3] (binary)
+#[item] [package overtype]
+#[item] [package textutil] (tcllib)
+
+
+# #package require frobz
+# #*** !doctools
+# #[item] [package {frobz}]
+
+#*** !doctools
+#[list_end]
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[section API]
 
 
 
@@ -24,10 +86,36 @@
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 namespace eval punk::mix::commandset::project {
     namespace export *
+    #*** !doctools
+    #[subsection {Namespace punk::mix::commandset::project}]
+    #[para] core commandset functions for punk::mix::commandset::project 
+    #[list_begin definitions]
+
+    proc _default {} {
+        package require punk::ns
+        set dispatched_to [lindex [info level 2] 0] ;#e.g ::punk::mix::cli::project
+        set dispatch_tail [punk::ns::nstail $dispatched_to]
+        set dispatch_ensemble [punk::ns::nsprefix $dispatched_to] ;#e.g ::punk::mix::cli
+        set sibling_commands [namespace eval $dispatch_ensemble {namespace export}]
+        #todo - get separator?
+        set sep "."
+        set result [list]
+        foreach sib $sibling_commands {
+            if {[string match ${dispatch_tail}${sep}* $sib]} {
+                lappend result $sib
+            }
+        }
+        return [lsort $result]
+    }
+
+
+
 
-    #new project structure - may be dedicated to one module, or contain many. 
-    #create minimal folder structure only by specifying  -modules {}
     proc new {newprojectpath_or_name args} {
+        #*** !doctools
+        # [call [fun new] [arg newprojectpath_or_name] [opt args]]
+        #new project structure - may be dedicated to one module, or contain many. 
+        #create minimal folder structure only by specifying in args:  -modules {}
         if {[file pathtype $newprojectpath_or_name] eq "absolute"} {
             set projectfullpath [file normalize $newprojectpath_or_name]
             set projectname [file tail $projectfullpath]
@@ -242,7 +330,7 @@ namespace eval punk::mix::commandset::project {
         set layout_dir $templatebase/layouts/$opt_layout
         puts stdout ">>> about to call punkcheck::install $layout_dir $projectdir" 
         set resultdict [dict create]
-        set unpublish [list\
+        set antipaths [list\
             src/doc/*\
             src/doc/include/*\
         ]
@@ -250,11 +338,11 @@ namespace eval punk::mix::commandset::project {
         #default antiglob_dir_core will stop .fossil* from being updated - which is generally desirable as these are likely to be customized
         if {$opt_force} {
             puts stdout "copying layout files - with force applied - overwrite all-targets" 
-            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new  -overwrite ALL-TARGETS -unpublish_paths $unpublish]
+            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new  -overwrite ALL-TARGETS -antiglob_paths $antipaths]
             #file copy -force $layout_dir $projectdir
         } else {
             puts stdout "copying layout files - (if source file changed)" 
-            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -overwrite installedsourcechanged-targets -unpublish_paths $unpublish]
+            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -overwrite installedsourcechanged-targets -antiglob_paths $antipaths]
         }
         puts stdout [punkcheck::summarize_install_resultdict $resultdict]
 
@@ -293,7 +381,7 @@ namespace eval punk::mix::commandset::project {
             set fpath [file join $projectdir $templatetail]
             if {[file exists $fpath]} {
                 set fd [open $fpath r]; fconfigure $fd -translation binary; set data [read $fd]; close $fd
-                set data2 [string map [list %project% $projectname] $data]
+                set data2 [string map [list [lib::template_tag project] $projectname] $data]
                 if {$data2 ne $data} {
                     puts stdout "updated template file: $fpath"
                     set fdout [open $fpath w]; fconfigure $fdout -translation binary; puts -nonewline $fdout $data2; close $fdout 
@@ -406,8 +494,10 @@ namespace eval punk::mix::commandset::project {
         puts stdout "-done- project:$projectname projectdir: $projectdir"
     }
 
+    #*** !doctools
+    #[list_end] [comment {--- end definitions namespace punk::mix::commandset::project ---}]
 
-     namespace eval collection {
+    namespace eval collection {
         namespace export *
         namespace path [namespace parent]
 
@@ -764,6 +854,12 @@ namespace eval punk::mix::commandset::project {
     }
     
     namespace eval lib {
+        proc template_tag {tagname} {
+            #todo - support different tagwrappers - it shouldn't be so likely to collide with common code idioms etc.
+            #we need to detect presence of tags intended for punk::mix system
+            #consider using punk::cap to enable multiple template-substitution providers with their own set of tagnames and/or tag wrappers, where substitution providers are all run 
+            return [string cat % $tagname %]
+        }
         #get project info only by opening the central confg-db
         #(will not have proper project-name etc)
         proc get_projects {{globlist {}} args} {
@@ -828,9 +924,8 @@ namespace eval punk::mix::commandset::project {
 }
 
 
-
-
-
+#*** !doctools
+#[manpage_end]
 
 
 

src/bootsupport/modules/punk/mix/util-0.1.0.tm

@@ -131,83 +131,6 @@ namespace eval punk::mix::util {
     }
     #----------------------------------------
 
-    #maint warning - also in punkcheck
-    proc path_relative {base dst} {
-        #see also kettle
-        # Modified copy of ::fileutil::relative (tcllib)
-        # Adapted to 8.5 ({*}).
-        #
-        #    Taking two _directory_ paths, a base and a destination, computes the path
-        #    of the destination relative to the base.
-        #
-        # Arguments:
-        #    base    The path to make the destination relative to.
-        #    dst     The destination path
-        #
-        # Results:
-        #    The path of the destination, relative to the base.
-
-        # Ensure that the link to directory 'dst' is properly done relative to
-        # the directory 'base'.
-
-        #review - check volume info on windows.. UNC paths?
-        if {[file pathtype $base] ne [file pathtype $dst]} {
-            return -code error "Unable to compute relation for paths of different pathtypes: [file pathtype $base] vs. [file pathtype $dst], ($base vs. $dst)"
-        }
-
-        #avoid normalizing if possible (file normalize *very* expensive on windows)
-        set do_normalize 0
-        if {[file pathtype $base] eq "relative"} {
-            #if base is relative so is dst
-            if {[regexp {[.]{2}} [list $base $dst]]} {
-                set do_normalize 1
-            }
-            if {[regexp {[.]/} [list $base $dst]]} {
-                set do_normalize 1
-            }
-        } else {
-            set do_normalize 1
-        }
-        if {$do_normalize} {
-            set base [file normalize $base]
-            set dst  [file normalize $dst]
-        }
-
-        set save $dst
-        set base [file split $base]
-        set dst  [file split $dst]
-
-        while {[lindex $dst 0] eq [lindex $base 0]} {
-            set dst  [lrange $dst  1 end]
-            set base [lrange $base 1 end]
-            if {![llength $dst]} {break}
-        }
-
-        set dstlen  [llength $dst]
-        set baselen [llength $base]
-
-        if {($dstlen == 0) && ($baselen == 0)} {
-            # Cases:
-            # (a) base == dst
-
-            set dst .
-            } else {
-            # Cases:
-            # (b) base is: base/sub = sub
-            #     dst  is: base     = {}
-
-            # (c) base is: base     = {}
-            #     dst  is: base/sub = sub
-
-            while {$baselen > 0} {
-                set dst [linsert $dst 0 ..]
-                incr baselen -1
-            }
-            set dst [file join {*}$dst]
-        }
-
-        return $dst
-    }
     #namespace import ::punk::ns::nsimport_noclobber 
 
     proc namespace_import_pattern_to_namespace_noclobber {pattern ns} {

src/bootsupport/modules/punk/overlay-0.1.tm

@@ -132,6 +132,7 @@ namespace eval ::punk::overlay {
         set imported_commands [list]
         set nscaller [uplevel 1 [list namespace current]]
         if {[catch {
+            #review - noclobber?
             namespace eval ${nscaller}::temp_import [list namespace import ${cmdnamespace}::*]
             foreach cmd [info commands ${nscaller}::temp_import::*] {
                 set cmdtail [namespace tail $cmd]

src/bootsupport/modules/punk/repo-0.1.1.tm

@@ -33,8 +33,9 @@ if {$::tcl_platform(platform) eq "windows"} {
     catch {package require punk::winpath}
 }
 package require fileutil; #tcllib
+package require punk::path
 package require punk::mix::base ;#uses core functions from punk::mix::base::lib namespace e.g cksum_path 
-package require punk::mix::util
+package require punk::mix::util ;#do_in_path
 
 
 # -- --- --- --- --- --- --- --- --- --- ---
@@ -357,7 +358,7 @@ namespace eval punk::repo {
         if {$repodir eq ""} {
             error "workingdir_state error: No repository found at or above path '$abspath'" 
         }
-        set subpath [punk::mix::util::path_relative $repodir $abspath]
+        set subpath [punk::path::relative $repodir $abspath]
         if {$subpath eq "."} {
             set subpath ""
         }
@@ -708,10 +709,10 @@ namespace eval punk::repo {
                 append msg "** starting folder : $start_dir" \n
                 append msg "** untracked       : $candroot" \n
                 if {$closest_fossil_len} {
-                    append msg "** fossil root     : $closest_fossil    ([punk::mix::util::path_relative $start_dir $closest_fossil])" \n
+                    append msg "** fossil root     : $closest_fossil    ([punk::path::relative $start_dir $closest_fossil])" \n
                 }
                 if {$closest_git_len} {
-                    append msg "** git root        : $closest_git    ([punk::mix::util::path_relative $start_dir $closest_git])" \n
+                    append msg "** git root        : $closest_git    ([punk::path::relative $start_dir $closest_git])" \n
                 }
                 append msg "**" \n
         }

src/bootsupport/modules/punkcheck-0.1.0.tm

@@ -19,6 +19,7 @@
 ##e.g package require frobz
 
 package require punk::tdl
+package require punk::path
 package require punk::repo
 package require punk::mix::util
 
@@ -1106,29 +1107,6 @@ namespace eval punkcheck {
                 return [lindex $args end] 
             }
         }
-        proc pathglob_as_re {glob} {
-            #any segment that is not just * must match exactly one segment in the path
-            set pats [list]
-            foreach seg [file split $glob] {
-                if {$seg eq "*"} {
-                    lappend pats {[^/]*}
-                } elseif {$seg eq "**"} {
-                    lappend pats {.*}
-                } else {
-                    set seg [string map [list  . {[.]}] $seg]
-                    if {[regexp {[*?]} $seg]} {
-                        set pat [string map [list * {[^/]*} ? {[^/]}] $seg]
-                        lappend pats "$pat"
-                    } else {
-                        lappend pats "$seg"
-                    }
-                }
-            }
-            return  "^[join $pats /]\$"
-        }
-        proc globmatchpath {glob path} {
-            return [regexp [pathglob_as_re $glob] $path]
-        }
         ## unidirectional file transfer to possibly non empty folder
         #default of -overwrite no-targets will only copy files that are missing at the target
         #           -overwrite newer-targets will copy files with older source timestamp over newer target timestamp and those missing at the target (a form of 'restore' operation)
@@ -1349,7 +1327,7 @@ namespace eval punkcheck {
             }
             foreach unpub $opt_antiglob_paths {
                 #puts "testing folder - globmatchpath $unpub $relative_source_dir"
-                if {[globmatchpath $unpub $relative_source_dir]} {
+                if {[punk::path::globmatchpath $unpub $relative_source_dir]} {
                     lappend antiglob_paths_matched $current_source_dir
                     return [list files_copied {} files_skipped {} sources_unchanged {} punkcheck_records $punkcheck_records antiglob_paths_matched $antiglob_paths_matched srcdir $srcdir tgtdir $tgtdir punkcheck_folder $punkcheck_folder]
                 }
@@ -1421,7 +1399,7 @@ namespace eval punkcheck {
                 set is_antipath 0
                 foreach antipath $opt_antiglob_paths {
                     #puts "testing file - globmatchpath $antipath  vs $relative_source_path" 
-                    if {[globmatchpath $antipath $relative_source_path]} {
+                    if {[punk::path::globmatchpath $antipath $relative_source_path]} {
                         lappend antiglob_paths_matched $current_source_dir
                         set is_antipath 1
                         break

src/doc/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.man

@@ -1,23 +0,0 @@
-[comment {--- punk::docgen generated from inline doctools comments ---}]
-[comment {--- punk::docgen DO NOT EDIT DOCS HERE UNLESS YOU REMOVE THESE COMMENT LINES ---}]
-[comment {--- punk::docgen overwrites this file ---}]
-[manpage_begin punk::cap 0 0.1.0]
-[copyright "2023 JMNoble - BSD licensed"]
-[titledesc {Module API}]
-[moddesc {punk capabilities plugin system}]
-[require punk::cap]
-[description]
-[list_begin definitions]
-[call [class interface_caphandler.registry] [method pkg_register] [arg pkg] [arg capname] [arg capdict] [arg fullcapabilitylist]]
-handler may override and return 0 (indicating don't register)e.g if pkg capdict data wasn't valid
-overridden handler must be able to handle multiple calls for same pkg - but it may return 1 or 0 as it wishes.
-[call [class interface_caphandler.registry] [method pkg_unregister] [arg pkg]]
-[call [class interface_capprovider.registration] [method pkg_unregister] [arg pkg]]
-[call [class interface_capprovider.provider] [method register] [opt capabilityname_glob]]
-[call [class interface_capprovider.provider] [method capabilities]]
- [call [fun exists] [arg capname]]
- Return a boolean indicating if the named capability exists  (0|1)
- [call [fun has_handler] [arg capname]]
- Return a boolean indicating if the named capability has a handler package installed (0|1)
-[list_end]
-[manpage_end]

src/doc/_module_punk_mix_templates_modules_template_module-0.0.1.tm.man

@@ -1,11 +0,0 @@
-[comment {--- punk::docgen generated from inline doctools comments ---}]
-[comment {--- punk::docgen DO NOT EDIT DOCS HERE UNLESS YOU REMOVE THESE COMMENT LINES ---}]
-[manpage_begin %pkg% 0 999999.0a1.0]
-[copyright "%year%"]
-[titledesc {Module API}]
-[moddesc {-}]
-[require %pkg%]
-[description]
-[list_begin definitions]
-[list_end]
-[manpage_end]

src/doc/punk/_module_cap-0.1.0.tm.man

@@ -26,22 +26,24 @@ A capabilityname may appear multiple times. ie a package may register that it pr
   [item]
   [para] [emph {handler_classes}]
   [list_begin enumerated]
-[enum] [emph {CLASS interface_caphandler.registry}]
+[enum] CLASS [class interface_caphandler.registry]
 [list_begin definitions]
-[call class::[class interface_caphandler.registry] [method pkg_register] [arg pkg] [arg capname] [arg capdict] [arg fullcapabilitylist]]
+    [para] [emph METHODS]
+[call class::interface_caphandler.registry [method pkg_register] [arg pkg] [arg capname] [arg capdict] [arg fullcapabilitylist]]
 handler may override and return 0 (indicating don't register)e.g if pkg capdict data wasn't valid
 overridden handler must be able to handle multiple calls for same pkg - but it may return 1 or 0 as it wishes.
-[call class::[class interface_caphandler.registry] [method pkg_unregister] [arg pkg]]
+[call class::interface_caphandler.registry [method pkg_unregister] [arg pkg]]
 [list_end]
-[enum] [emph {CLASS interface_caphandler.sysapi}]
+[enum] CLASS [class interface_caphandler.sysapi]
 [list_begin definitions]
+    [para] [emph METHODS]
 [list_end]
   [list_end] [comment {- end enumeration handler classes -}]
   [item]
   [para] [emph {provider_classes}]
   [list_begin enumerated]
-    [enum] [emph {CLASS interface_cappprovider.registration}]
-    Your provider package will need to instantiate this object under a sub-namespace called [namespace capsystem] within your package namespace.
+    [enum] CLASS [class interface_cappprovider.registration]
+    [para]Your provider package will need to instantiate this object under a sub-namespace called [namespace capsystem] within your package namespace.
     [para]If your package namespace is mypackages::providerpkg then the object command would be at mypackages::providerpkg::capsystem::capprovider.registration
     [para]Example code for your provider package to evaluate within its namespace:
     [example {
@@ -61,13 +63,14 @@ namespace eval capsystem {
 }]
 [para] The above example declares that your package can be registered as a provider for the capabilities named 'punk.templates' and 'another_capability_name'
     [list_begin definitions]
-[call class::[class interface_capprovider.registration] [method get_declarations]]
+    [para] [emph METHODS]
+[call class::interface_capprovider.registration [method get_declarations]]
 [para] This method must be overridden by your provider using oo::objdefine cappprovider.registration as in the example above.
  There must be at least one 2-element list in the result for the provider to be registerable.
 [para]The first element of the list is the capabilityname - which can be custom to your provider/handler packages - or a well-known name that other authors may use/implement.
 [para]The second element is a dictionary of keys specific to the capability being implemented. It may be empty if the any potential capability handlers for the named capability don't require registration data.
     [list_end]
-    [enum] [emph {CLASS interface_capprovider.provider}]
+    [enum] CLASS [class interface_capprovider.provider]
     [para] Your provider package will need to instantiate this directly under it's own namespace with the command name of [emph {provider}]
     [example {
        namespace eval mypackages::providerpkg {
@@ -75,9 +78,10 @@ namespace eval capsystem {
        }
     }]
     [list_begin definitions]
-[call class::[class interface_capprovider.provider] [method constructor] [arg providerpkg]]
+    [para] [emph METHODS]
+[call class::interface_capprovider.provider [method constructor] [arg providerpkg]]
 [comment {- -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---}]
-[call class::[class interface_capprovider.provider] [method register] [opt capabilityname_glob]]
+[call class::interface_capprovider.provider [method register] [opt capabilityname_glob]]
 
 [para]This is the mechanism by which a user of your provider package will register your package as a provider of the capability named.
 
@@ -93,7 +97,7 @@ namespace eval capsystem {
 }]
 
 [comment {- -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---}]
-[call class::[class interface_capprovider.provider] [method capabilities]]
+[call class::interface_capprovider.provider [method capabilities]]
 [para] return a list of capabilities supported by this provider package
     [list_end] [comment {- end class definitions -}]
   [list_end] [comment {- end enumeration provider_classes }]

src/doc/punk/_module_path-0.1.0.tm.man

@@ -0,0 +1,84 @@
+[comment {--- punk::docgen generated from inline doctools comments ---}]
+[comment {--- punk::docgen DO NOT EDIT DOCS HERE UNLESS YOU REMOVE THESE COMMENT LINES ---}]
+[comment {--- punk::docgen overwrites this file ---}]
+[manpage_begin punk::path 0 0.1.0]
+[copyright "2023"]
+[titledesc {Filesystem path utilities}]  [comment {-- Name section and table of contents description --}]
+[moddesc {punk path filesystem utils}]             [comment {-- Description at end of page heading --}]
+[require punk::path]
+[description]
+[section Overview]
+[para] overview of punk::path
+[para] Filesystem path utility functions
+[subsection Concepts]
+[para] -
+[subsection dependencies]
+[para] packages used by punk::path
+[list_begin itemized]
+[item] [package {Tcl 8.6}]
+[list_end]
+[section API]
+[subsection {Namespace punk::path::class}]
+[para] class definitions
+[list_begin enumerated]
+[list_end] [comment {--- end class enumeration ---}]
+[subsection {Namespace punk::path}]
+[para] Core API functions for punk::path
+[list_begin definitions]
+[call [fun pathglob_as_re] [arg pathglob]]
+[para] Returns a regular expression for matching a path to a glob pattern which can contain glob chars *|? in any segment of the path structure
+[para] ** matches any number of subdirectories.
+[para] e.g /etc/**/*.txt will match any .txt files at any depth below /etc  (except directly within /etc itself)
+[para] e.g /etc/**.txt will match any .txt files at any depth below /etc
+[para] any segment that does not contain ** must match exactly one segment in the path
+[para] e.g the glob /etc/*/*.doc  - will match any .doc files that are exactly one tree level below /etc
+[para] The pathglob doesn't have to contain glob characters, in which case the returned regex will match the pathglob exactly as specified.
+[para] Regular expression syntax is deliberateley not supported within the pathglob string so that supplied regex characters will be treated as literals
+[call [fun globmatchpath] [arg pathglob] [arg path] [opt {option value...}]]
+[para] Return true if the pathglob matches the path
+[para] see [fun pathglob_as_re] for pathglob description
+[para] Caller must ensure that file separator is forward slash. (e.g use file normalize on windows)
+[para]
+[para] Known options:
+[para]     -nocase 0|1  (default 0 - case sensitive)
+[para] If -nocase is not supplied - default to case sensitive *except for driveletter*
+[para] ie - the driveletter alone in paths such as c:/etc will still be case insensitive. (ie c:/ETC/* will match C:/ETC/blah but not C:/etc/blah)
+[para] Explicitly specifying -nocase 0 will require the entire case to match including the driveletter.
+[call [fun treefilenames] [arg basepath] [arg tailglob] [opt {option value...}]]
+basic (glob based) list of filenames matching tailglob - recursive
+no natsorting - so order is dependent on filesystem
+[call [fun relative] [arg reference] [arg location]]
+[para]    Taking two directory paths, a reference and a location, computes the path
+    of the location relative to the reference.
+[list_begin itemized]
+[item]
+[para] Arguments:
+       [list_begin arguments]
+       [arg_def string reference]       The path from which the relative path to location is determined.
+       [arg_def string location]        The location path which may be above or below the reference path
+       [list_end]
+[item]
+[para] Results:
+[para]    The relative path of the location to the reference path.
+[para]    Will return a single dot "." if the paths are the same
+[item]
+[para] Notes:
+[para]    Both paths must be the same type - ie both absolute or both relative
+[para]    Case sensitive.  ie  relative /etc /etC
+      will return ../etC
+[para]    On windows, the drive-letter component (only) is not case sensitive
+[para]    ie relative c:/etc  C:/etc  returns .
+[para]    but relative c:/etc C:/Etc returns ../Etc
+[para]    On windows, if the paths are absolute and specifiy different volumes, only the location will be returned.
+    ie  relative c:/etc d:/etc/blah
+    returns d:/etc/blah
+[list_end]
+[list_end] [comment {--- end definitions namespace punk::path ---}]
+[subsection {Namespace punk::path::lib}]
+[para] Secondary functions that are part of the API
+[list_begin definitions]
+[list_end] [comment {--- end definitions namespace punk::path::lib ---}]
+[section Internal]
+[subsection {Namespace punk::path::system}]
+[para] Internal functions that are not part of the API
+[manpage_end]

src/doc/punk/mix/commandset/_module_project-0.1.0.tm.man

@@ -0,0 +1,48 @@
+[comment {--- punk::docgen generated from inline doctools comments ---}]
+[comment {--- punk::docgen DO NOT EDIT DOCS HERE UNLESS YOU REMOVE THESE COMMENT LINES ---}]
+[comment {--- punk::docgen overwrites this file ---}]
+[manpage_begin punk::mix::commandset::project 0 0.1.0]
+[copyright "2023"]
+[titledesc {pmix commandset - project}]  [comment {-- Name section and table of contents description --}]
+[moddesc {pmix CLI commandset - project}]             [comment {-- Description at end of page heading --}]
+[require punk::mix::commandset::project]
+[description]
+[section Overview]
+[para] overview of punk::mix::commandset::project
+[para]Import into an ensemble namespace similarly to the way it is done with punk::mix::cli e.g
+[example {
+    namespace eval myproject::cli {
+        namespace export *
+        namespace ensemble create
+        package require punk::overlay
+
+        package require punk::mix::commandset::project
+        punk::overlay::import_commandset project    .       ::punk::mix::commandset::project
+        punk::overlay::import_commandset projects   .       ::punk::mix::commandset::project::collection
+    }
+}]
+[para] Where the . in the above example is the prefix/command separator
+[para]The prefix ('project' in the above example) can be any string desired to disambiguate commands imported from other commandsets.
+[para]The above results in the availability of the ensemble command: ::myproject::cli project.new, which is implemented in ::punk::mix::commandset::project::new
+[para]Similarly, procs under ::punk::mix::commandset::project::collection will be available as subcommands of the ensemble as projects.<procname>
+[para]
+[subsection Concepts]
+[para] see punk::overlay
+[subsection dependencies]
+[para] packages used by punk::mix::commandset::project
+[list_begin itemized]
+[item] [package {Tcl 8.6}]
+[item] [package punk::ns]
+[item] [package sqlite3] (binary)
+[item] [package overtype]
+[item] [package textutil] (tcllib)
+[list_end]
+[section API]
+[subsection {Namespace punk::mix::commandset::project}]
+[para] core commandset functions for punk::mix::commandset::project
+[list_begin definitions]
+ [call [fun new] [arg newprojectpath_or_name] [opt args]]
+new project structure - may be dedicated to one module, or contain many.
+create minimal folder structure only by specifying in args:  -modules {}
+[list_end] [comment {--- end definitions namespace punk::mix::commandset::project ---}]
+[manpage_end]

src/embedded/man/files/punk/_module_cap-0.1.0.tm.n

@@ -1,5 +1,5 @@
 '\"
-'\" Generated from file '_module_punk_cap-0\&.1\&.0\&.tm\&.man' by tcllib/doctools with format 'nroff'
+'\" Generated from file '_module_cap-0\&.1\&.0\&.tm\&.man' by tcllib/doctools with format 'nroff'
 '\" Copyright (c) 2023 JMNoble - BSD licensed
 '\"
 .TH "punk::cap" 0 0\&.1\&.0 doc "punk capabilities plugin system"
@@ -276,17 +276,17 @@ punk::cap \- capability provider and handler plugin system
 .SH SYNOPSIS
 package require \fBpunk::cap \fR
 .sp
-class::\fBinterface_caphandler\&.registry\fR \fBpkg_register\fR \fIpkg\fR \fIcapname\fR \fIcapdict\fR \fIfullcapabilitylist\fR
+class::interface_caphandler\&.registry \fBpkg_register\fR \fIpkg\fR \fIcapname\fR \fIcapdict\fR \fIfullcapabilitylist\fR
 .sp
-class::\fBinterface_caphandler\&.registry\fR \fBpkg_unregister\fR \fIpkg\fR
+class::interface_caphandler\&.registry \fBpkg_unregister\fR \fIpkg\fR
 .sp
-class::\fBinterface_capprovider\&.registration\fR \fBget_declarations\fR
+class::interface_capprovider\&.registration \fBget_declarations\fR
 .sp
-class::\fBinterface_capprovider\&.provider\fR \fBconstructor\fR \fIproviderpkg\fR
+class::interface_capprovider\&.provider \fBconstructor\fR \fIproviderpkg\fR
 .sp
-class::\fBinterface_capprovider\&.provider\fR \fBregister\fR ?capabilityname_glob?
+class::interface_capprovider\&.provider \fBregister\fR ?capabilityname_glob?
 .sp
-class::\fBinterface_capprovider\&.provider\fR \fBcapabilities\fR
+class::interface_capprovider\&.provider \fBcapabilities\fR
 .sp
 \fBcapability_exists\fR \fIcapname\fR
 .sp
@@ -324,18 +324,22 @@ class definitions
 \fIhandler_classes\fR
 .RS
 .IP [1]
-\fICLASS interface_caphandler\&.registry\fR
+CLASS \fBinterface_caphandler\&.registry\fR
 .RS
+.sp
+\fIMETHODS\fR
 .TP
-class::\fBinterface_caphandler\&.registry\fR \fBpkg_register\fR \fIpkg\fR \fIcapname\fR \fIcapdict\fR \fIfullcapabilitylist\fR
+class::interface_caphandler\&.registry \fBpkg_register\fR \fIpkg\fR \fIcapname\fR \fIcapdict\fR \fIfullcapabilitylist\fR
 handler may override and return 0 (indicating don't register)e\&.g if pkg capdict data wasn't valid
 overridden handler must be able to handle multiple calls for same pkg - but it may return 1 or 0 as it wishes\&.
 .TP
-class::\fBinterface_caphandler\&.registry\fR \fBpkg_unregister\fR \fIpkg\fR
+class::interface_caphandler\&.registry \fBpkg_unregister\fR \fIpkg\fR
 .RE
 .IP [2]
-\fICLASS interface_caphandler\&.sysapi\fR
+CLASS \fBinterface_caphandler\&.sysapi\fR
 .RS
+.sp
+\fIMETHODS\fR
 .RE
 .RE
 .IP \(bu
@@ -343,7 +347,8 @@ class::\fBinterface_caphandler\&.registry\fR \fBpkg_unregister\fR \fIpkg\fR
 \fIprovider_classes\fR
 .RS
 .IP [1]
-\fICLASS interface_cappprovider\&.registration\fR
+CLASS \fBinterface_cappprovider\&.registration\fR
+.sp
 Your provider package will need to instantiate this object under a sub-namespace called \fBcapsystem\fR within your package namespace\&.
 .sp
 If your package namespace is mypackages::providerpkg then the object command would be at mypackages::providerpkg::capsystem::capprovider\&.registration
@@ -370,8 +375,10 @@ namespace eval capsystem {
 .sp
 The above example declares that your package can be registered as a provider for the capabilities named 'punk\&.templates' and 'another_capability_name'
 .RS
+.sp
+\fIMETHODS\fR
 .TP
-class::\fBinterface_capprovider\&.registration\fR \fBget_declarations\fR
+class::interface_capprovider\&.registration \fBget_declarations\fR
 .sp
 This method must be overridden by your provider using oo::objdefine cappprovider\&.registration as in the example above\&.
 There must be at least one 2-element list in the result for the provider to be registerable\&.
@@ -381,7 +388,7 @@ The first element of the list is the capabilityname - which can be custom to you
 The second element is a dictionary of keys specific to the capability being implemented\&. It may be empty if the any potential capability handlers for the named capability don't require registration data\&.
 .RE
 .IP [2]
-\fICLASS interface_capprovider\&.provider\fR
+CLASS \fBinterface_capprovider\&.provider\fR
 .sp
 Your provider package will need to instantiate this directly under it's own namespace with the command name of \fIprovider\fR
 .CS
@@ -393,10 +400,12 @@ Your provider package will need to instantiate this directly under it's own name
 
 .CE
 .RS
+.sp
+\fIMETHODS\fR
 .TP
-class::\fBinterface_capprovider\&.provider\fR \fBconstructor\fR \fIproviderpkg\fR
+class::interface_capprovider\&.provider \fBconstructor\fR \fIproviderpkg\fR
 .TP
-class::\fBinterface_capprovider\&.provider\fR \fBregister\fR ?capabilityname_glob?
+class::interface_capprovider\&.provider \fBregister\fR ?capabilityname_glob?
 .sp
 This is the mechanism by which a user of your provider package will register your package as a provider of the capability named\&.
 .sp
@@ -418,7 +427,7 @@ Or a specific capability may be registered:
 
 .CE
 .TP
-class::\fBinterface_capprovider\&.provider\fR \fBcapabilities\fR
+class::interface_capprovider\&.provider \fBcapabilities\fR
 .sp
 return a list of capabilities supported by this provider package
 .RE

src/embedded/man/files/punk/_module_path-0.1.0.tm.n

@@ -1,8 +1,8 @@
 '\"
-'\" Generated from file '_module_punk_mix_templates_modules_template_module-0\&.0\&.1\&.tm\&.man' by tcllib/doctools with format 'nroff'
-'\" Copyright (c) %year%
+'\" Generated from file '_module_path-0\&.1\&.0\&.tm\&.man' by tcllib/doctools with format 'nroff'
+'\" Copyright (c) 2023
 '\"
-.TH "%pkg%" 0 999999\&.0a1\&.0 doc "-"
+.TH "punk::path" 0 0\&.1\&.0 doc "punk path filesystem utils"
 .\" The -*- nroff -*- definitions below are for supplemental macros used
 .\" in Tcl/Tk manual entries.
 .\"
@@ -272,15 +272,137 @@ Database Class:	\\fB\\$3\\fR
 ..
 .BS
 .SH NAME
-%pkg% \- Module API
+punk::path \- Filesystem path utilities
 .SH SYNOPSIS
-package require \fB%pkg% \fR
+package require \fBpunk::path \fR
+.sp
+\fBpathglob_as_re\fR \fIpathglob\fR
+.sp
+\fBglobmatchpath\fR \fIpathglob\fR \fIpath\fR ?option value\&.\&.\&.?
+.sp
+\fBtreefilenames\fR \fIbasepath\fR \fItailglob\fR ?option value\&.\&.\&.?
+.sp
+\fBrelative\fR \fIreference\fR \fIlocation\fR
 .sp
 .BE
 .SH DESCRIPTION
+.SH OVERVIEW
+.PP
+overview of punk::path
+.PP
+Filesystem path utility functions
+.SS CONCEPTS
+.PP
+-
+.SS DEPENDENCIES
+.PP
+packages used by punk::path
+.IP \(bu
+\fBTcl 8\&.6\fR
+.PP
+.SH API
+.SS "NAMESPACE PUNK::PATH::CLASS"
+.PP
+class definitions
+.PP
+.SS "NAMESPACE PUNK::PATH"
+.PP
+Core API functions for punk::path
+.TP
+\fBpathglob_as_re\fR \fIpathglob\fR
+.sp
+Returns a regular expression for matching a path to a glob pattern which can contain glob chars *|? in any segment of the path structure
+.sp
+** matches any number of subdirectories\&.
+.sp
+e\&.g /etc/**/*\&.txt will match any \&.txt files at any depth below /etc  (except directly within /etc itself)
+.sp
+e\&.g /etc/**\&.txt will match any \&.txt files at any depth below /etc
+.sp
+any segment that does not contain ** must match exactly one segment in the path
+.sp
+e\&.g the glob /etc/*/*\&.doc  - will match any \&.doc files that are exactly one tree level below /etc
+.sp
+The pathglob doesn't have to contain glob characters, in which case the returned regex will match the pathglob exactly as specified\&.
+.sp
+Regular expression syntax is deliberateley not supported within the pathglob string so that supplied regex characters will be treated as literals
+.TP
+\fBglobmatchpath\fR \fIpathglob\fR \fIpath\fR ?option value\&.\&.\&.?
+.sp
+Return true if the pathglob matches the path
+.sp
+see \fBpathglob_as_re\fR for pathglob description
+.sp
+Caller must ensure that file separator is forward slash\&. (e\&.g use file normalize on windows)
+.sp
+.sp
+Known options:
+.sp
+-nocase 0|1  (default 0 - case sensitive)
+.sp
+If -nocase is not supplied - default to case sensitive *except for driveletter*
+.sp
+ie - the driveletter alone in paths such as c:/etc will still be case insensitive\&. (ie c:/ETC/* will match C:/ETC/blah but not C:/etc/blah)
+.sp
+Explicitly specifying -nocase 0 will require the entire case to match including the driveletter\&.
+.TP
+\fBtreefilenames\fR \fIbasepath\fR \fItailglob\fR ?option value\&.\&.\&.?
+basic (glob based) list of filenames matching tailglob - recursive
+no natsorting - so order is dependent on filesystem
+.TP
+\fBrelative\fR \fIreference\fR \fIlocation\fR
+.sp
+Taking two directory paths, a reference and a location, computes the path
+of the location relative to the reference\&.
+.RS
+.IP \(bu
+.sp
+Arguments:
+.RS
+.TP
+string \fIreference\fR
+The path from which the relative path to location is determined\&.
+.TP
+string \fIlocation\fR
+The location path which may be above or below the reference path
+.RE
+.IP \(bu
+.sp
+Results:
+.sp
+The relative path of the location to the reference path\&.
+.sp
+Will return a single dot "\&." if the paths are the same
+.IP \(bu
+.sp
+Notes:
+.sp
+Both paths must be the same type - ie both absolute or both relative
+.sp
+Case sensitive\&.  ie  relative /etc /etC
+will return \&.\&./etC
+.sp
+On windows, the drive-letter component (only) is not case sensitive
+.sp
+ie relative c:/etc  C:/etc  returns \&.
+.sp
+but relative c:/etc C:/Etc returns \&.\&./Etc
+.sp
+On windows, if the paths are absolute and specifiy different volumes, only the location will be returned\&.
+ie  relative c:/etc d:/etc/blah
+returns d:/etc/blah
+.RE
+.PP
+.SS "NAMESPACE PUNK::PATH::LIB"
+.PP
+Secondary functions that are part of the API
+.PP
+.SH INTERNAL
+.SS "NAMESPACE PUNK::PATH::SYSTEM"
 .PP
+Internal functions that are not part of the API
 .SH COPYRIGHT
 .nf
-Copyright (c) %year%
+Copyright (c) 2023
 
 .fi

src/embedded/man/files/punk/mix/commandset/_module_project-0.1.0.tm.n

@@ -1,8 +1,8 @@
 '\"
-'\" Generated from file '_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0\&.1\&.0\&.tm\&.man' by tcllib/doctools with format 'nroff'
-'\" Copyright (c) 2023 JMNoble - BSD licensed
+'\" Generated from file '_module_project-0\&.1\&.0\&.tm\&.man' by tcllib/doctools with format 'nroff'
+'\" Copyright (c) 2023
 '\"
-.TH "punk::cap" 0 0\&.1\&.0 doc "punk capabilities plugin system"
+.TH "punk::mix::commandset::project" 0 0\&.1\&.0 doc "pmix CLI commandset - project"
 .\" The -*- nroff -*- definitions below are for supplemental macros used
 .\" in Tcl/Tk manual entries.
 .\"
@@ -272,47 +272,70 @@ Database Class:	\\fB\\$3\\fR
 ..
 .BS
 .SH NAME
-punk::cap \- Module API
+punk::mix::commandset::project \- pmix commandset - project
 .SH SYNOPSIS
-package require \fBpunk::cap \fR
+package require \fBpunk::mix::commandset::project \fR
 .sp
-\fBinterface_caphandler\&.registry\fR \fBpkg_register\fR \fIpkg\fR \fIcapname\fR \fIcapdict\fR \fIfullcapabilitylist\fR
-.sp
-\fBinterface_caphandler\&.registry\fR \fBpkg_unregister\fR \fIpkg\fR
-.sp
-\fBinterface_capprovider\&.registration\fR \fBpkg_unregister\fR \fIpkg\fR
-.sp
-\fBinterface_capprovider\&.provider\fR \fBregister\fR ?capabilityname_glob?
-.sp
-\fBinterface_capprovider\&.provider\fR \fBcapabilities\fR
-.sp
-\fBexists\fR \fIcapname\fR
-.sp
-\fBhas_handler\fR \fIcapname\fR
+\fBnew\fR \fInewprojectpath_or_name\fR ?args?
 .sp
 .BE
 .SH DESCRIPTION
+.SH OVERVIEW
+.PP
+overview of punk::mix::commandset::project
+.PP
+Import into an ensemble namespace similarly to the way it is done with punk::mix::cli e\&.g
+.CS
+
+
+    namespace eval myproject::cli {
+        namespace export *
+        namespace ensemble create
+        package require punk::overlay
+
+        package require punk::mix::commandset::project
+        punk::overlay::import_commandset project    \&.       ::punk::mix::commandset::project
+        punk::overlay::import_commandset projects   \&.       ::punk::mix::commandset::project::collection
+    }
+
+.CE
+.PP
+Where the \&. in the above example is the prefix/command separator
+.PP
+The prefix ('project' in the above example) can be any string desired to disambiguate commands imported from other commandsets\&.
+.PP
+The above results in the availability of the ensemble command: ::myproject::cli project\&.new, which is implemented in ::punk::mix::commandset::project::new
+.PP
+Similarly, procs under ::punk::mix::commandset::project::collection will be available as subcommands of the ensemble as projects\&.<procname>
+.PP
+.SS CONCEPTS
+.PP
+see punk::overlay
+.SS DEPENDENCIES
+.PP
+packages used by punk::mix::commandset::project
+.IP \(bu
+\fBTcl 8\&.6\fR
+.IP \(bu
+\fBpunk::ns\fR
+.IP \(bu
+\fBsqlite3\fR (binary)
+.IP \(bu
+\fBovertype\fR
+.IP \(bu
+\fBtextutil\fR (tcllib)
+.PP
+.SH API
+.SS "NAMESPACE PUNK::MIX::COMMANDSET::PROJECT"
+.PP
+core commandset functions for punk::mix::commandset::project
 .TP
-\fBinterface_caphandler\&.registry\fR \fBpkg_register\fR \fIpkg\fR \fIcapname\fR \fIcapdict\fR \fIfullcapabilitylist\fR
-handler may override and return 0 (indicating don't register)e\&.g if pkg capdict data wasn't valid
-overridden handler must be able to handle multiple calls for same pkg - but it may return 1 or 0 as it wishes\&.
-.TP
-\fBinterface_caphandler\&.registry\fR \fBpkg_unregister\fR \fIpkg\fR
-.TP
-\fBinterface_capprovider\&.registration\fR \fBpkg_unregister\fR \fIpkg\fR
-.TP
-\fBinterface_capprovider\&.provider\fR \fBregister\fR ?capabilityname_glob?
-.TP
-\fBinterface_capprovider\&.provider\fR \fBcapabilities\fR
-.TP
-\fBexists\fR \fIcapname\fR
-Return a boolean indicating if the named capability exists  (0|1)
-.TP
-\fBhas_handler\fR \fIcapname\fR
-Return a boolean indicating if the named capability has a handler package installed (0|1)
+\fBnew\fR \fInewprojectpath_or_name\fR ?args?
+new project structure - may be dedicated to one module, or contain many\&.
+create minimal folder structure only by specifying in args:  -modules {}
 .PP
 .SH COPYRIGHT
 .nf
-Copyright (c) 2023 JMNoble - BSD licensed
+Copyright (c) 2023
 
 .fi

src/embedded/man/toc.n

@@ -273,14 +273,14 @@ Database Class:	\\fB\\$3\\fR
 doc
 .RS
 .TP
-\fB%pkg%\fR
-\fIfiles/_module_punk_mix_templates_modules_template_module-0\&.0\&.1\&.tm\&.n\fR: Module API
-.TP
 \fBpunk::cap\fR
-\fIfiles/_module_punk_cap-0\&.1\&.0\&.tm\&.n\fR: capability provider and handler plugin system
+\fIfiles/punk/_module_cap-0\&.1\&.0\&.tm\&.n\fR: capability provider and handler plugin system
 .TP
-\fBpunk::cap\fR
-\fIfiles/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0\&.1\&.0\&.tm\&.n\fR: Module API
+\fBpunk::mix::commandset::project\fR
+\fIfiles/punk/mix/commandset/_module_project-0\&.1\&.0\&.tm\&.n\fR: pmix commandset - project
+.TP
+\fBpunk::path\fR
+\fIfiles/punk/_module_path-0\&.1\&.0\&.tm\&.n\fR: Filesystem path utilities
 .TP
 \fBpunkshell\fR
 \fIfiles/main\&.n\fR: punkshell - Core

src/embedded/md/.doc/tocdoc

@@ -1,6 +1,6 @@
 [toc_begin {Table Of Contents} doc]
-[item doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.md                          %pkg%     {Module API}]
-[item doc/files/_module_punk_cap-0.1.0.tm.md                                                            punk::cap {capability provider and handler plugin system}]
-[item doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.md punk::cap {Module API}]
-[item doc/files/main.md                                                                                 punkshell {punkshell - Core}]
+[item doc/files/punk/_module_cap-0.1.0.tm.md                    punk::cap                      {capability provider and handler plugin system}]
+[item doc/files/punk/mix/commandset/_module_project-0.1.0.tm.md punk::mix::commandset::project {pmix commandset - project}]
+[item doc/files/punk/_module_path-0.1.0.tm.md                   punk::path                     {Filesystem path utilities}]
+[item doc/files/main.md                                         punkshell                      {punkshell - Core}]
 [toc_end]

src/embedded/md/.toc

@@ -1 +1 @@
-doc {doc/toc {{doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.md %pkg% {Module API}} {doc/files/_module_punk_cap-0.1.0.tm.md punk::cap {capability provider and handler plugin system}} {doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.md punk::cap {Module API}} {doc/files/main.md punkshell {punkshell - Core}}}}
+doc {doc/toc {{doc/files/punk/_module_cap-0.1.0.tm.md punk::cap {capability provider and handler plugin system}} {doc/files/punk/_module_path-0.1.0.tm.md punk::path {Filesystem path utilities}} {doc/files/punk/mix/commandset/_module_project-0.1.0.tm.md punk::mix::commandset::project {pmix commandset - project}} {doc/files/main.md punkshell {punkshell - Core}}}}

src/embedded/md/.xrf

@@ -1 +1 @@
-{capability provider and handler plugin system} doc/files/_module_punk_cap-0.1.0.tm.md repl {index.md repl} %pkg% doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.md kw,punk {index.md punk} %pkg%(0) doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.md punkshell(n) doc/files/main.md sa,punk::cap doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.md punkshell doc/files/main.md sa,punk::cap(0) doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.md {Module API} doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.md shell {index.md shell} kw,repl {index.md repl} sa,%pkg% doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.md {punkshell - Core} doc/files/main.md sa,%pkg%(0) doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.md sa,punkshell(n) doc/files/main.md punk::cap doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.md sa,punkshell doc/files/main.md kw,shell {index.md shell} punk {index.md punk} punk::cap(0) doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.md
+sa,punk::path(0) doc/files/punk/_module_path-0.1.0.tm.md {capability provider and handler plugin system} doc/files/punk/_module_cap-0.1.0.tm.md punk::mix::commandset::project doc/files/punk/mix/commandset/_module_project-0.1.0.tm.md punk::mix::commandset::project(0) doc/files/punk/mix/commandset/_module_project-0.1.0.tm.md repl {index.md repl} kw,punk {index.md punk} punkshell(n) doc/files/main.md sa,punk::cap doc/files/punk/_module_cap-0.1.0.tm.md {Filesystem path utilities} doc/files/punk/_module_path-0.1.0.tm.md punkshell doc/files/main.md sa,punk::cap(0) doc/files/punk/_module_cap-0.1.0.tm.md sa,punk::path doc/files/punk/_module_path-0.1.0.tm.md punk::path(0) doc/files/punk/_module_path-0.1.0.tm.md shell {index.md shell} kw,repl {index.md repl} sa,punk::mix::commandset::project doc/files/punk/mix/commandset/_module_project-0.1.0.tm.md sa,punk::mix::commandset::project(0) doc/files/punk/mix/commandset/_module_project-0.1.0.tm.md {punkshell - Core} doc/files/main.md {pmix commandset - project} doc/files/punk/mix/commandset/_module_project-0.1.0.tm.md sa,punkshell(n) doc/files/main.md punk::cap doc/files/punk/_module_cap-0.1.0.tm.md sa,punkshell doc/files/main.md kw,shell {index.md shell} punk {index.md punk} punk::cap(0) doc/files/punk/_module_cap-0.1.0.tm.md punk::path doc/files/punk/_module_path-0.1.0.tm.md

src/embedded/md/doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.md

@@ -1,64 +0,0 @@
-
-[//000000001]: # (punk::cap \- punk capabilities plugin system)
-[//000000002]: # (Generated from file '\_module\_punk\_mix\_templates\_layouts\_project\_src\_bootsupport\_modules\_punk\_cap\-0\.1\.0\.tm\.man' by tcllib/doctools with format 'markdown')
-[//000000003]: # (Copyright © 2023 JMNoble \- BSD licensed)
-[//000000004]: # (punk::cap\(0\) 0\.1\.0 doc "punk capabilities plugin system")
-
-<hr> [ <a href="../../toc.md">Main Table Of Contents</a> &#124; <a
-href="../toc.md">Table Of Contents</a> &#124; <a
-href="../../index.md">Keyword Index</a> ] <hr>
-
-# NAME
-
-punk::cap \- Module API
-
-# <a name='toc'></a>Table Of Contents
-
-  - [Table Of Contents](#toc)
-
-  - [Synopsis](#synopsis)
-
-  - [Description](#section1)
-
-  - [Copyright](#copyright)
-
-# <a name='synopsis'></a>SYNOPSIS
-
-package require punk::cap  
-
-[__interface\_caphandler\.registry__ __pkg\_register__ *pkg* *capname* *capdict* *fullcapabilitylist*](#1)  
-[__interface\_caphandler\.registry__ __pkg\_unregister__ *pkg*](#2)  
-[__interface\_capprovider\.registration__ __pkg\_unregister__ *pkg*](#3)  
-[__interface\_capprovider\.provider__ __register__ ?capabilityname\_glob?](#4)  
-[__interface\_capprovider\.provider__ __capabilities__](#5)  
-[__exists__ *capname*](#6)  
-[__has\_handler__ *capname*](#7)  
-
-# <a name='description'></a>DESCRIPTION
-
-  - <a name='1'></a>__interface\_caphandler\.registry__ __pkg\_register__ *pkg* *capname* *capdict* *fullcapabilitylist*
-
-    handler may override and return 0 \(indicating don't register\)e\.g if pkg
-    capdict data wasn't valid overridden handler must be able to handle multiple
-    calls for same pkg \- but it may return 1 or 0 as it wishes\.
-
-  - <a name='2'></a>__interface\_caphandler\.registry__ __pkg\_unregister__ *pkg*
-
-  - <a name='3'></a>__interface\_capprovider\.registration__ __pkg\_unregister__ *pkg*
-
-  - <a name='4'></a>__interface\_capprovider\.provider__ __register__ ?capabilityname\_glob?
-
-  - <a name='5'></a>__interface\_capprovider\.provider__ __capabilities__
-
-  - <a name='6'></a>__exists__ *capname*
-
-    Return a boolean indicating if the named capability exists \(0&#124;1\)
-
-  - <a name='7'></a>__has\_handler__ *capname*
-
-    Return a boolean indicating if the named capability has a handler package
-    installed \(0&#124;1\)
-
-# <a name='copyright'></a>COPYRIGHT
-
-Copyright &copy; 2023 JMNoble \- BSD licensed

src/embedded/md/doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.md

@@ -1,33 +0,0 @@
-
-[//000000001]: # (%pkg% \- \-)
-[//000000002]: # (Generated from file '\_module\_punk\_mix\_templates\_modules\_template\_module\-0\.0\.1\.tm\.man' by tcllib/doctools with format 'markdown')
-[//000000003]: # (Copyright © %year%)
-[//000000004]: # (%pkg%\(0\) 999999\.0a1\.0 doc "\-")
-
-<hr> [ <a href="../../toc.md">Main Table Of Contents</a> &#124; <a
-href="../toc.md">Table Of Contents</a> &#124; <a
-href="../../index.md">Keyword Index</a> ] <hr>
-
-# NAME
-
-%pkg% \- Module API
-
-# <a name='toc'></a>Table Of Contents
-
-  - [Table Of Contents](#toc)
-
-  - [Synopsis](#synopsis)
-
-  - [Description](#section1)
-
-  - [Copyright](#copyright)
-
-# <a name='synopsis'></a>SYNOPSIS
-
-package require %pkg%  
-
-# <a name='description'></a>DESCRIPTION
-
-# <a name='copyright'></a>COPYRIGHT
-
-Copyright &copy; %year%

src/embedded/md/doc/files/punk/_module_cap-0.1.0.tm.md

@@ -1,12 +1,12 @@
 
 [//000000001]: # (punk::cap \- punk capabilities plugin system)
-[//000000002]: # (Generated from file '\_module\_punk\_cap\-0\.1\.0\.tm\.man' by tcllib/doctools with format 'markdown')
+[//000000002]: # (Generated from file '\_module\_cap\-0\.1\.0\.tm\.man' by tcllib/doctools with format 'markdown')
 [//000000003]: # (Copyright © 2023 JMNoble \- BSD licensed)
 [//000000004]: # (punk::cap\(0\) 0\.1\.0 doc "punk capabilities plugin system")
 
-<hr> [ <a href="../../toc.md">Main Table Of Contents</a> &#124; <a
-href="../toc.md">Table Of Contents</a> &#124; <a
-href="../../index.md">Keyword Index</a> ] <hr>
+<hr> [ <a href="../../../toc.md">Main Table Of Contents</a> &#124; <a
+href="../../toc.md">Table Of Contents</a> &#124; <a
+href="../../../index.md">Keyword Index</a> ] <hr>
 
 # NAME
 
@@ -42,12 +42,12 @@ punk::cap \- capability provider and handler plugin system
 
 package require punk::cap  
 
-[class::__interface\_caphandler\.registry__ __pkg\_register__ *pkg* *capname* *capdict* *fullcapabilitylist*](#1)  
-[class::__interface\_caphandler\.registry__ __pkg\_unregister__ *pkg*](#2)  
-[class::__interface\_capprovider\.registration__ __get\_declarations__](#3)  
-[class::__interface\_capprovider\.provider__ __constructor__ *providerpkg*](#4)  
-[class::__interface\_capprovider\.provider__ __register__ ?capabilityname\_glob?](#5)  
-[class::__interface\_capprovider\.provider__ __capabilities__](#6)  
+[class::interface\_caphandler\.registry __pkg\_register__ *pkg* *capname* *capdict* *fullcapabilitylist*](#1)  
+[class::interface\_caphandler\.registry __pkg\_unregister__ *pkg*](#2)  
+[class::interface\_capprovider\.registration __get\_declarations__](#3)  
+[class::interface\_capprovider\.provider __constructor__ *providerpkg*](#4)  
+[class::interface\_capprovider\.provider __register__ ?capabilityname\_glob?](#5)  
+[class::interface\_capprovider\.provider __capabilities__](#6)  
 [__capability\_exists__ *capname*](#7)  
 [__capability\_has\_handler__ *capname*](#8)  
 [__capability\_get\_handler__ *capname*](#9)  
@@ -87,24 +87,27 @@ class definitions
 
   - *handler\_classes*
 
-      1. *CLASS interface\_caphandler\.registry*
+      1. CLASS __interface\_caphandler\.registry__
 
-           * <a name='1'></a>class::__interface\_caphandler\.registry__ __pkg\_register__ *pkg* *capname* *capdict* *fullcapabilitylist*
+           * <a name='1'></a>class::interface\_caphandler\.registry __pkg\_register__ *pkg* *capname* *capdict* *fullcapabilitylist*
 
-             handler may override and return 0 \(indicating don't register\)e\.g if
-             pkg capdict data wasn't valid overridden handler must be able to
-             handle multiple calls for same pkg \- but it may return 1 or 0 as it
-             wishes\.
+             *METHODS* handler may override and return 0 \(indicating don't
+             register\)e\.g if pkg capdict data wasn't valid overridden handler
+             must be able to handle multiple calls for same pkg \- but it may
+             return 1 or 0 as it wishes\.
 
-           * <a name='2'></a>class::__interface\_caphandler\.registry__ __pkg\_unregister__ *pkg*
+           * <a name='2'></a>class::interface\_caphandler\.registry __pkg\_unregister__ *pkg*
 
-      1. *CLASS interface\_caphandler\.sysapi*
+      1. CLASS __interface\_caphandler\.sysapi__
 
-  - *provider\_classes*
+  - *METHODS*
 
-      1. *CLASS interface\_cappprovider\.registration* Your provider package
-         will need to instantiate this object under a sub\-namespace called
-         __capsystem__ within your package namespace\.
+    *provider\_classes*
+
+      1. CLASS __interface\_cappprovider\.registration__
+
+         Your provider package will need to instantiate this object under a
+         sub\-namespace called __capsystem__ within your package namespace\.
 
          If your package namespace is mypackages::providerpkg then the object
          command would be at
@@ -131,7 +134,9 @@ class definitions
          provider for the capabilities named 'punk\.templates' and
          'another\_capability\_name'
 
-           * <a name='3'></a>class::__interface\_capprovider\.registration__ __get\_declarations__
+           * <a name='3'></a>class::interface\_capprovider\.registration __get\_declarations__
+
+             *METHODS*
 
              This method must be overridden by your provider using oo::objdefine
              cappprovider\.registration as in the example above\. There must be at
@@ -147,7 +152,7 @@ class definitions
              capability handlers for the named capability don't require
              registration data\.
 
-      1. *CLASS interface\_capprovider\.provider*
+      1. CLASS __interface\_capprovider\.provider__
 
          Your provider package will need to instantiate this directly under it's
          own namespace with the command name of *provider*
@@ -156,9 +161,11 @@ class definitions
                  punk::cap::class::interface_capprovider.provider create provider mypackages::providerpkg
              }
 
-           * <a name='4'></a>class::__interface\_capprovider\.provider__ __constructor__ *providerpkg*
+           * <a name='4'></a>class::interface\_capprovider\.provider __constructor__ *providerpkg*
+
+             *METHODS*
 
-           * <a name='5'></a>class::__interface\_capprovider\.provider__ __register__ ?capabilityname\_glob?
+           * <a name='5'></a>class::interface\_capprovider\.provider __register__ ?capabilityname\_glob?
 
              This is the mechanism by which a user of your provider package will
              register your package as a provider of the capability named\.
@@ -174,7 +181,7 @@ class definitions
                  package require mypackages::providerpkg
                  mypackages::providerpkg::provider register another_capability_name
 
-           * <a name='6'></a>class::__interface\_capprovider\.provider__ __capabilities__
+           * <a name='6'></a>class::interface\_capprovider\.provider __capabilities__
 
              return a list of capabilities supported by this provider package
 

src/embedded/md/doc/files/punk/_module_path-0.1.0.tm.md

@@ -0,0 +1,176 @@
+
+[//000000001]: # (punk::path \- punk path filesystem utils)
+[//000000002]: # (Generated from file '\_module\_path\-0\.1\.0\.tm\.man' by tcllib/doctools with format 'markdown')
+[//000000003]: # (Copyright © 2023)
+[//000000004]: # (punk::path\(0\) 0\.1\.0 doc "punk path filesystem utils")
+
+<hr> [ <a href="../../../toc.md">Main Table Of Contents</a> &#124; <a
+href="../../toc.md">Table Of Contents</a> &#124; <a
+href="../../../index.md">Keyword Index</a> ] <hr>
+
+# NAME
+
+punk::path \- Filesystem path utilities
+
+# <a name='toc'></a>Table Of Contents
+
+  - [Table Of Contents](#toc)
+
+  - [Synopsis](#synopsis)
+
+  - [Description](#section1)
+
+  - [Overview](#section2)
+
+      - [Concepts](#subsection1)
+
+      - [dependencies](#subsection2)
+
+  - [API](#section3)
+
+      - [Namespace punk::path::class](#subsection3)
+
+      - [Namespace punk::path](#subsection4)
+
+      - [Namespace punk::path::lib](#subsection5)
+
+  - [Internal](#section4)
+
+      - [Namespace punk::path::system](#subsection6)
+
+  - [Copyright](#copyright)
+
+# <a name='synopsis'></a>SYNOPSIS
+
+package require punk::path  
+
+[__pathglob\_as\_re__ *pathglob*](#1)  
+[__globmatchpath__ *pathglob* *path* ?option value\.\.\.?](#2)  
+[__treefilenames__ *basepath* *tailglob* ?option value\.\.\.?](#3)  
+[__relative__ *reference* *location*](#4)  
+
+# <a name='description'></a>DESCRIPTION
+
+# <a name='section2'></a>Overview
+
+overview of punk::path
+
+Filesystem path utility functions
+
+## <a name='subsection1'></a>Concepts
+
+\-
+
+## <a name='subsection2'></a>dependencies
+
+packages used by punk::path
+
+  - __Tcl 8\.6__
+
+# <a name='section3'></a>API
+
+## <a name='subsection3'></a>Namespace punk::path::class
+
+class definitions
+
+## <a name='subsection4'></a>Namespace punk::path
+
+  - <a name='1'></a>__pathglob\_as\_re__ *pathglob*
+
+    Returns a regular expression for matching a path to a glob pattern which can
+    contain glob chars \*&#124;? in any segment of the path structure
+
+    \*\* matches any number of subdirectories\.
+
+    e\.g /etc/\*\*/\*\.txt will match any \.txt files at any depth below /etc \(except
+    directly within /etc itself\)
+
+    e\.g /etc/\*\*\.txt will match any \.txt files at any depth below /etc
+
+    any segment that does not contain \*\* must match exactly one segment in the
+    path
+
+    e\.g the glob /etc/\*/\*\.doc \- will match any \.doc files that are exactly one
+    tree level below /etc
+
+    The pathglob doesn't have to contain glob characters, in which case the
+    returned regex will match the pathglob exactly as specified\.
+
+    Regular expression syntax is deliberateley not supported within the pathglob
+    string so that supplied regex characters will be treated as literals
+
+  - <a name='2'></a>__globmatchpath__ *pathglob* *path* ?option value\.\.\.?
+
+    Return true if the pathglob matches the path
+
+    see __pathglob\_as\_re__ for pathglob description
+
+    Caller must ensure that file separator is forward slash\. \(e\.g use file
+    normalize on windows\)
+
+    Known options:
+
+    \-nocase 0&#124;1 \(default 0 \- case sensitive\)
+
+    If \-nocase is not supplied \- default to case sensitive \*except for
+    driveletter\*
+
+    ie \- the driveletter alone in paths such as c:/etc will still be case
+    insensitive\. \(ie c:/ETC/\* will match C:/ETC/blah but not C:/etc/blah\)
+
+    Explicitly specifying \-nocase 0 will require the entire case to match
+    including the driveletter\.
+
+  - <a name='3'></a>__treefilenames__ *basepath* *tailglob* ?option value\.\.\.?
+
+    basic \(glob based\) list of filenames matching tailglob \- recursive no
+    natsorting \- so order is dependent on filesystem
+
+  - <a name='4'></a>__relative__ *reference* *location*
+
+    Taking two directory paths, a reference and a location, computes the path of
+    the location relative to the reference\.
+
+      * Arguments:
+
+          + string *reference*
+
+            The path from which the relative path to location is determined\.
+
+          + string *location*
+
+            The location path which may be above or below the reference path
+
+      * Results:
+
+        The relative path of the location to the reference path\.
+
+        Will return a single dot "\." if the paths are the same
+
+      * Notes:
+
+        Both paths must be the same type \- ie both absolute or both relative
+
+        Case sensitive\. ie relative /etc /etC will return \.\./etC
+
+        On windows, the drive\-letter component \(only\) is not case sensitive
+
+        ie relative c:/etc C:/etc returns \.
+
+        but relative c:/etc C:/Etc returns \.\./Etc
+
+        On windows, if the paths are absolute and specifiy different volumes,
+        only the location will be returned\. ie relative c:/etc d:/etc/blah
+        returns d:/etc/blah
+
+## <a name='subsection5'></a>Namespace punk::path::lib
+
+Secondary functions that are part of the API
+
+# <a name='section4'></a>Internal
+
+## <a name='subsection6'></a>Namespace punk::path::system
+
+# <a name='copyright'></a>COPYRIGHT
+
+Copyright &copy; 2023

src/embedded/md/doc/files/punk/mix/commandset/_module_project-0.1.0.tm.md

@@ -0,0 +1,102 @@
+
+[//000000001]: # (punk::mix::commandset::project \- pmix CLI commandset \- project)
+[//000000002]: # (Generated from file '\_module\_project\-0\.1\.0\.tm\.man' by tcllib/doctools with format 'markdown')
+[//000000003]: # (Copyright © 2023)
+[//000000004]: # (punk::mix::commandset::project\(0\) 0\.1\.0 doc "pmix CLI commandset \- project")
+
+<hr> [ <a href="../../../../../toc.md">Main Table Of Contents</a> &#124;
+<a href="../../../../toc.md">Table Of Contents</a> &#124; <a
+href="../../../../../index.md">Keyword Index</a> ] <hr>
+
+# NAME
+
+punk::mix::commandset::project \- pmix commandset \- project
+
+# <a name='toc'></a>Table Of Contents
+
+  - [Table Of Contents](#toc)
+
+  - [Synopsis](#synopsis)
+
+  - [Description](#section1)
+
+  - [Overview](#section2)
+
+      - [Concepts](#subsection1)
+
+      - [dependencies](#subsection2)
+
+  - [API](#section3)
+
+      - [Namespace punk::mix::commandset::project](#subsection3)
+
+  - [Copyright](#copyright)
+
+# <a name='synopsis'></a>SYNOPSIS
+
+package require punk::mix::commandset::project  
+
+[__new__ *newprojectpath\_or\_name* ?args?](#1)  
+
+# <a name='description'></a>DESCRIPTION
+
+# <a name='section2'></a>Overview
+
+overview of punk::mix::commandset::project
+
+Import into an ensemble namespace similarly to the way it is done with
+punk::mix::cli e\.g
+
+    namespace eval myproject::cli {
+        namespace export *
+        namespace ensemble create
+        package require punk::overlay
+
+        package require punk::mix::commandset::project
+        punk::overlay::import_commandset project    .       ::punk::mix::commandset::project
+        punk::overlay::import_commandset projects   .       ::punk::mix::commandset::project::collection
+    }
+
+Where the \. in the above example is the prefix/command separator
+
+The prefix \('project' in the above example\) can be any string desired to
+disambiguate commands imported from other commandsets\.
+
+The above results in the availability of the ensemble command: ::myproject::cli
+project\.new, which is implemented in ::punk::mix::commandset::project::new
+
+Similarly, procs under ::punk::mix::commandset::project::collection will be
+available as subcommands of the ensemble as projects\.<procname>
+
+## <a name='subsection1'></a>Concepts
+
+see punk::overlay
+
+## <a name='subsection2'></a>dependencies
+
+packages used by punk::mix::commandset::project
+
+  - __Tcl 8\.6__
+
+  - __punk::ns__
+
+  - __sqlite3__ \(binary\)
+
+  - __overtype__
+
+  - __textutil__ \(tcllib\)
+
+# <a name='section3'></a>API
+
+## <a name='subsection3'></a>Namespace punk::mix::commandset::project
+
+core commandset functions for punk::mix::commandset::project
+
+  - <a name='1'></a>__new__ *newprojectpath\_or\_name* ?args?
+
+    new project structure \- may be dedicated to one module, or contain many\.
+    create minimal folder structure only by specifying in args: \-modules \{\}
+
+# <a name='copyright'></a>COPYRIGHT
+
+Copyright &copy; 2023

src/embedded/md/doc/toc.md

@@ -3,10 +3,10 @@
 
 # Table Of Contents \-\- doc
 
-  - [%pkg%](doc/files/\_module\_punk\_mix\_templates\_modules\_template\_module\-0\.0\.1\.tm\.md) Module API
+  - [punk::cap](doc/files/punk/\_module\_cap\-0\.1\.0\.tm\.md) capability provider and handler plugin system
 
-  - [punk::cap](doc/files/\_module\_punk\_cap\-0\.1\.0\.tm\.md) capability provider and handler plugin system
+  - [punk::mix::commandset::project](doc/files/punk/mix/commandset/\_module\_project\-0\.1\.0\.tm\.md) pmix commandset \- project
 
-  - [punk::cap](doc/files/\_module\_punk\_mix\_templates\_layouts\_project\_src\_bootsupport\_modules\_punk\_cap\-0\.1\.0\.tm\.md) Module API
+  - [punk::path](doc/files/punk/\_module\_path\-0\.1\.0\.tm\.md) Filesystem path utilities
 
   - [punkshell](doc/files/main\.md) punkshell \- Core

src/embedded/md/toc.md

@@ -3,10 +3,10 @@
 
 # Table Of Contents \-\- doc
 
-  - [%pkg%](doc/files/\_module\_punk\_mix\_templates\_modules\_template\_module\-0\.0\.1\.tm\.md) Module API
+  - [punk::cap](doc/files/punk/\_module\_cap\-0\.1\.0\.tm\.md) capability provider and handler plugin system
 
-  - [punk::cap](doc/files/\_module\_punk\_cap\-0\.1\.0\.tm\.md) capability provider and handler plugin system
+  - [punk::mix::commandset::project](doc/files/punk/mix/commandset/\_module\_project\-0\.1\.0\.tm\.md) pmix commandset \- project
 
-  - [punk::cap](doc/files/\_module\_punk\_mix\_templates\_layouts\_project\_src\_bootsupport\_modules\_punk\_cap\-0\.1\.0\.tm\.md) Module API
+  - [punk::path](doc/files/punk/\_module\_path\-0\.1\.0\.tm\.md) Filesystem path utilities
 
   - [punkshell](doc/files/main\.md) punkshell \- Core

src/embedded/www/.doc/tocdoc

@@ -1,6 +1,6 @@
 [toc_begin {Table Of Contents} doc]
-[item doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.html                          %pkg%     {Module API}]
-[item doc/files/_module_punk_cap-0.1.0.tm.html                                                            punk::cap {capability provider and handler plugin system}]
-[item doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.html punk::cap {Module API}]
-[item doc/files/main.html                                                                                 punkshell {punkshell - Core}]
+[item doc/files/punk/_module_cap-0.1.0.tm.html                    punk::cap                      {capability provider and handler plugin system}]
+[item doc/files/punk/mix/commandset/_module_project-0.1.0.tm.html punk::mix::commandset::project {pmix commandset - project}]
+[item doc/files/punk/_module_path-0.1.0.tm.html                   punk::path                     {Filesystem path utilities}]
+[item doc/files/main.html                                         punkshell                      {punkshell - Core}]
 [toc_end]

src/embedded/www/.toc

@@ -1 +1 @@
-doc {doc/toc {{doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.html %pkg% {Module API}} {doc/files/_module_punk_cap-0.1.0.tm.html punk::cap {capability provider and handler plugin system}} {doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.html punk::cap {Module API}} {doc/files/main.html punkshell {punkshell - Core}}}}
+doc {doc/toc {{doc/files/punk/_module_cap-0.1.0.tm.html punk::cap {capability provider and handler plugin system}} {doc/files/punk/_module_path-0.1.0.tm.html punk::path {Filesystem path utilities}} {doc/files/punk/mix/commandset/_module_project-0.1.0.tm.html punk::mix::commandset::project {pmix commandset - project}} {doc/files/main.html punkshell {punkshell - Core}}}}

src/embedded/www/.xrf

@@ -1 +1 @@
-{capability provider and handler plugin system} doc/files/_module_punk_cap-0.1.0.tm.html repl {index.html repl} %pkg% doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.html kw,punk {index.html punk} %pkg%(0) doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.html punkshell(n) doc/files/main.html sa,punk::cap doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.html punkshell doc/files/main.html sa,punk::cap(0) doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.html {Module API} doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.html shell {index.html shell} kw,repl {index.html repl} sa,%pkg% doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.html {punkshell - Core} doc/files/main.html sa,%pkg%(0) doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.html sa,punkshell(n) doc/files/main.html punk::cap doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.html sa,punkshell doc/files/main.html kw,shell {index.html shell} punk {index.html punk} punk::cap(0) doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.html
+sa,punk::path(0) doc/files/punk/_module_path-0.1.0.tm.html {capability provider and handler plugin system} doc/files/punk/_module_cap-0.1.0.tm.html punk::mix::commandset::project doc/files/punk/mix/commandset/_module_project-0.1.0.tm.html punk::mix::commandset::project(0) doc/files/punk/mix/commandset/_module_project-0.1.0.tm.html repl {index.html repl} kw,punk {index.html punk} punkshell(n) doc/files/main.html sa,punk::cap doc/files/punk/_module_cap-0.1.0.tm.html {Filesystem path utilities} doc/files/punk/_module_path-0.1.0.tm.html punkshell doc/files/main.html sa,punk::cap(0) doc/files/punk/_module_cap-0.1.0.tm.html sa,punk::path doc/files/punk/_module_path-0.1.0.tm.html punk::path(0) doc/files/punk/_module_path-0.1.0.tm.html shell {index.html shell} kw,repl {index.html repl} sa,punk::mix::commandset::project doc/files/punk/mix/commandset/_module_project-0.1.0.tm.html sa,punk::mix::commandset::project(0) doc/files/punk/mix/commandset/_module_project-0.1.0.tm.html {punkshell - Core} doc/files/main.html {pmix commandset - project} doc/files/punk/mix/commandset/_module_project-0.1.0.tm.html sa,punkshell(n) doc/files/main.html punk::cap doc/files/punk/_module_cap-0.1.0.tm.html sa,punkshell doc/files/main.html kw,shell {index.html shell} punk {index.html punk} punk::cap(0) doc/files/punk/_module_cap-0.1.0.tm.html punk::path doc/files/punk/_module_path-0.1.0.tm.html

src/embedded/www/doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.html

@@ -1,156 +0,0 @@
-<!DOCTYPE html><html><head>
-<title>punk::cap - punk capabilities plugin system</title>
-<style type="text/css"><!--
-    HTML {
-	background: 	#FFFFFF;
-	color: 		black;
-    }
-    BODY {
-	background: 	#FFFFFF;
-	color:	 	black;
-    }
-    DIV.doctools {
-	margin-left:	10%;
-	margin-right:	10%;
-    }
-    DIV.doctools H1,DIV.doctools H2 {
-	margin-left:	-5%;
-    }
-    H1, H2, H3, H4 {
-	margin-top: 	1em;
-	font-family:	sans-serif;
-	font-size:	large;
-	color:		#005A9C;
-	background: 	transparent;
-	text-align:		left;
-    }
-    H1.doctools_title {
-	text-align: center;
-    }
-    UL,OL {
-	margin-right: 0em;
-	margin-top: 3pt;
-	margin-bottom: 3pt;
-    }
-    UL LI {
-	list-style: disc;
-    }
-    OL LI {
-	list-style: decimal;
-    }
-    DT {
-	padding-top: 	1ex;
-    }
-    UL.doctools_toc,UL.doctools_toc UL, UL.doctools_toc UL UL {
-	font:		normal 12pt/14pt sans-serif;
-	list-style:	none;
-    }
-    LI.doctools_section, LI.doctools_subsection {
-	list-style: 	none;
-	margin-left: 	0em;
-	text-indent:	0em;
-	padding: 	0em;
-    }
-    PRE {
-	display: 	block;
-	font-family:	monospace;
-	white-space:	pre;
-	margin:		0%;
-	padding-top:	0.5ex;
-	padding-bottom:	0.5ex;
-	padding-left:	1ex;
-	padding-right:	1ex;
-	width:		100%;
-    }
-    PRE.doctools_example {
-	color: 		black;
-	background: 	#f5dcb3;
-	border:		1px solid black;
-    }
-    UL.doctools_requirements LI, UL.doctools_syntax LI {
-	list-style: 	none;
-	margin-left: 	0em;
-	text-indent:	0em;
-	padding:	0em;
-    }
-    DIV.doctools_synopsis {
-	color: 		black;
-	background: 	#80ffff;
-	border:		1px solid black;
-	font-family:	serif;
-	margin-top: 	1em;
-	margin-bottom: 	1em;
-    }
-    UL.doctools_syntax {
-	margin-top: 	1em;
-	border-top:	1px solid black;
-    }
-    UL.doctools_requirements {
-	margin-bottom: 	1em;
-	border-bottom:	1px solid black;
-    }
---></style>
-</head>
-<!-- Generated from file '_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.man' by tcllib/doctools with format 'html'
-   -->
-<!-- Copyright &amp;copy; 2023 JMNoble - BSD licensed
-   -->
-<!-- punk::cap.0
-   -->
-<body><hr> [
-   <a href="../../toc.html">Main Table Of Contents</a>
-&#124; <a href="../toc.html">Table Of Contents</a>
-&#124; <a href="../../index.html">Keyword Index</a>
- ] <hr>
-<div class="doctools">
-<h1 class="doctools_title">punk::cap(0) 0.1.0 doc &quot;punk capabilities plugin system&quot;</h1>
-<div id="name" class="doctools_section"><h2><a name="name">Name</a></h2>
-<p>punk::cap - Module API</p>
-</div>
-<div id="toc" class="doctools_section"><h2><a name="toc">Table Of Contents</a></h2>
-<ul class="doctools_toc">
-<li class="doctools_section"><a href="#toc">Table Of Contents</a></li>
-<li class="doctools_section"><a href="#synopsis">Synopsis</a></li>
-<li class="doctools_section"><a href="#section1">Description</a></li>
-<li class="doctools_section"><a href="#copyright">Copyright</a></li>
-</ul>
-</div>
-<div id="synopsis" class="doctools_section"><h2><a name="synopsis">Synopsis</a></h2>
-<div class="doctools_synopsis">
-<ul class="doctools_requirements">
-<li>package require <b class="pkgname">punk::cap</b></li>
-</ul>
-<ul class="doctools_syntax">
-<li><a href="#1"><b class="class">interface_caphandler.registry</b> <b class="method">pkg_register</b> <i class="arg">pkg</i> <i class="arg">capname</i> <i class="arg">capdict</i> <i class="arg">fullcapabilitylist</i></a></li>
-<li><a href="#2"><b class="class">interface_caphandler.registry</b> <b class="method">pkg_unregister</b> <i class="arg">pkg</i></a></li>
-<li><a href="#3"><b class="class">interface_capprovider.registration</b> <b class="method">pkg_unregister</b> <i class="arg">pkg</i></a></li>
-<li><a href="#4"><b class="class">interface_capprovider.provider</b> <b class="method">register</b> <span class="opt">?capabilityname_glob?</span></a></li>
-<li><a href="#5"><b class="class">interface_capprovider.provider</b> <b class="method">capabilities</b></a></li>
-<li><a href="#6"><b class="function">exists</b> <i class="arg">capname</i></a></li>
-<li><a href="#7"><b class="function">has_handler</b> <i class="arg">capname</i></a></li>
-</ul>
-</div>
-</div>
-<div id="section1" class="doctools_section"><h2><a name="section1">Description</a></h2>
-<dl class="doctools_definitions">
-<dt><a name="1"><b class="class">interface_caphandler.registry</b> <b class="method">pkg_register</b> <i class="arg">pkg</i> <i class="arg">capname</i> <i class="arg">capdict</i> <i class="arg">fullcapabilitylist</i></a></dt>
-<dd><p>handler may override and return 0 (indicating don't register)e.g if pkg capdict data wasn't valid
-overridden handler must be able to handle multiple calls for same pkg - but it may return 1 or 0 as it wishes.</p></dd>
-<dt><a name="2"><b class="class">interface_caphandler.registry</b> <b class="method">pkg_unregister</b> <i class="arg">pkg</i></a></dt>
-<dd></dd>
-<dt><a name="3"><b class="class">interface_capprovider.registration</b> <b class="method">pkg_unregister</b> <i class="arg">pkg</i></a></dt>
-<dd></dd>
-<dt><a name="4"><b class="class">interface_capprovider.provider</b> <b class="method">register</b> <span class="opt">?capabilityname_glob?</span></a></dt>
-<dd></dd>
-<dt><a name="5"><b class="class">interface_capprovider.provider</b> <b class="method">capabilities</b></a></dt>
-<dd></dd>
-<dt><a name="6"><b class="function">exists</b> <i class="arg">capname</i></a></dt>
-<dd><p>Return a boolean indicating if the named capability exists  (0|1)</p></dd>
-<dt><a name="7"><b class="function">has_handler</b> <i class="arg">capname</i></a></dt>
-<dd><p>Return a boolean indicating if the named capability has a handler package installed (0|1)</p></dd>
-</dl>
-</div>
-<div id="copyright" class="doctools_section"><h2><a name="copyright">Copyright</a></h2>
-<p>Copyright &copy; 2023 JMNoble - BSD licensed</p>
-</div>
-</div></body></html>

src/embedded/www/doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.html

@@ -1,132 +0,0 @@
-<!DOCTYPE html><html><head>
-<title>%pkg% - -</title>
-<style type="text/css"><!--
-    HTML {
-	background: 	#FFFFFF;
-	color: 		black;
-    }
-    BODY {
-	background: 	#FFFFFF;
-	color:	 	black;
-    }
-    DIV.doctools {
-	margin-left:	10%;
-	margin-right:	10%;
-    }
-    DIV.doctools H1,DIV.doctools H2 {
-	margin-left:	-5%;
-    }
-    H1, H2, H3, H4 {
-	margin-top: 	1em;
-	font-family:	sans-serif;
-	font-size:	large;
-	color:		#005A9C;
-	background: 	transparent;
-	text-align:		left;
-    }
-    H1.doctools_title {
-	text-align: center;
-    }
-    UL,OL {
-	margin-right: 0em;
-	margin-top: 3pt;
-	margin-bottom: 3pt;
-    }
-    UL LI {
-	list-style: disc;
-    }
-    OL LI {
-	list-style: decimal;
-    }
-    DT {
-	padding-top: 	1ex;
-    }
-    UL.doctools_toc,UL.doctools_toc UL, UL.doctools_toc UL UL {
-	font:		normal 12pt/14pt sans-serif;
-	list-style:	none;
-    }
-    LI.doctools_section, LI.doctools_subsection {
-	list-style: 	none;
-	margin-left: 	0em;
-	text-indent:	0em;
-	padding: 	0em;
-    }
-    PRE {
-	display: 	block;
-	font-family:	monospace;
-	white-space:	pre;
-	margin:		0%;
-	padding-top:	0.5ex;
-	padding-bottom:	0.5ex;
-	padding-left:	1ex;
-	padding-right:	1ex;
-	width:		100%;
-    }
-    PRE.doctools_example {
-	color: 		black;
-	background: 	#f5dcb3;
-	border:		1px solid black;
-    }
-    UL.doctools_requirements LI, UL.doctools_syntax LI {
-	list-style: 	none;
-	margin-left: 	0em;
-	text-indent:	0em;
-	padding:	0em;
-    }
-    DIV.doctools_synopsis {
-	color: 		black;
-	background: 	#80ffff;
-	border:		1px solid black;
-	font-family:	serif;
-	margin-top: 	1em;
-	margin-bottom: 	1em;
-    }
-    UL.doctools_syntax {
-	margin-top: 	1em;
-	border-top:	1px solid black;
-    }
-    UL.doctools_requirements {
-	margin-bottom: 	1em;
-	border-bottom:	1px solid black;
-    }
---></style>
-</head>
-<!-- Generated from file '_module_punk_mix_templates_modules_template_module-0.0.1.tm.man' by tcllib/doctools with format 'html'
-   -->
-<!-- Copyright &amp;copy; %year%
-   -->
-<!-- %pkg%.0
-   -->
-<body><hr> [
-   <a href="../../toc.html">Main Table Of Contents</a>
-&#124; <a href="../toc.html">Table Of Contents</a>
-&#124; <a href="../../index.html">Keyword Index</a>
- ] <hr>
-<div class="doctools">
-<h1 class="doctools_title">%pkg%(0) 999999.0a1.0 doc &quot;-&quot;</h1>
-<div id="name" class="doctools_section"><h2><a name="name">Name</a></h2>
-<p>%pkg% - Module API</p>
-</div>
-<div id="toc" class="doctools_section"><h2><a name="toc">Table Of Contents</a></h2>
-<ul class="doctools_toc">
-<li class="doctools_section"><a href="#toc">Table Of Contents</a></li>
-<li class="doctools_section"><a href="#synopsis">Synopsis</a></li>
-<li class="doctools_section"><a href="#section1">Description</a></li>
-<li class="doctools_section"><a href="#copyright">Copyright</a></li>
-</ul>
-</div>
-<div id="synopsis" class="doctools_section"><h2><a name="synopsis">Synopsis</a></h2>
-<div class="doctools_synopsis">
-<ul class="doctools_requirements">
-<li>package require <b class="pkgname">%pkg%</b></li>
-</ul>
-</div>
-</div>
-<div id="section1" class="doctools_section"><h2><a name="section1">Description</a></h2>
-<dl class="doctools_definitions">
-</dl>
-</div>
-<div id="copyright" class="doctools_section"><h2><a name="copyright">Copyright</a></h2>
-<p>Copyright &copy; %year%</p>
-</div>
-</div></body></html>

src/embedded/www/doc/files/punk/_module_cap-0.1.0.tm.html

@@ -91,16 +91,16 @@
     }
 --></style>
 </head>
-<!-- Generated from file '_module_punk_cap-0.1.0.tm.man' by tcllib/doctools with format 'html'
+<!-- Generated from file '_module_cap-0.1.0.tm.man' by tcllib/doctools with format 'html'
    -->
 <!-- Copyright &amp;copy; 2023 JMNoble - BSD licensed
    -->
 <!-- punk::cap.0
    -->
 <body><hr> [
-   <a href="../../toc.html">Main Table Of Contents</a>
-&#124; <a href="../toc.html">Table Of Contents</a>
-&#124; <a href="../../index.html">Keyword Index</a>
+   <a href="../../../toc.html">Main Table Of Contents</a>
+&#124; <a href="../../toc.html">Table Of Contents</a>
+&#124; <a href="../../../index.html">Keyword Index</a>
  ] <hr>
 <div class="doctools">
 <h1 class="doctools_title">punk::cap(0) 0.1.0 doc &quot;punk capabilities plugin system&quot;</h1>
@@ -138,12 +138,12 @@
 <li>package require <b class="pkgname">punk::cap</b></li>
 </ul>
 <ul class="doctools_syntax">
-<li><a href="#1">class::<b class="class">interface_caphandler.registry</b> <b class="method">pkg_register</b> <i class="arg">pkg</i> <i class="arg">capname</i> <i class="arg">capdict</i> <i class="arg">fullcapabilitylist</i></a></li>
-<li><a href="#2">class::<b class="class">interface_caphandler.registry</b> <b class="method">pkg_unregister</b> <i class="arg">pkg</i></a></li>
-<li><a href="#3">class::<b class="class">interface_capprovider.registration</b> <b class="method">get_declarations</b></a></li>
-<li><a href="#4">class::<b class="class">interface_capprovider.provider</b> <b class="method">constructor</b> <i class="arg">providerpkg</i></a></li>
-<li><a href="#5">class::<b class="class">interface_capprovider.provider</b> <b class="method">register</b> <span class="opt">?capabilityname_glob?</span></a></li>
-<li><a href="#6">class::<b class="class">interface_capprovider.provider</b> <b class="method">capabilities</b></a></li>
+<li><a href="#1">class::interface_caphandler.registry <b class="method">pkg_register</b> <i class="arg">pkg</i> <i class="arg">capname</i> <i class="arg">capdict</i> <i class="arg">fullcapabilitylist</i></a></li>
+<li><a href="#2">class::interface_caphandler.registry <b class="method">pkg_unregister</b> <i class="arg">pkg</i></a></li>
+<li><a href="#3">class::interface_capprovider.registration <b class="method">get_declarations</b></a></li>
+<li><a href="#4">class::interface_capprovider.provider <b class="method">constructor</b> <i class="arg">providerpkg</i></a></li>
+<li><a href="#5">class::interface_capprovider.provider <b class="method">register</b> <span class="opt">?capabilityname_glob?</span></a></li>
+<li><a href="#6">class::interface_capprovider.provider <b class="method">capabilities</b></a></li>
 <li><a href="#7"><b class="function">capability_exists</b> <i class="arg">capname</i></a></li>
 <li><a href="#8"><b class="function">capability_has_handler</b> <i class="arg">capname</i></a></li>
 <li><a href="#9"><b class="function">capability_get_handler</b> <i class="arg">capname</i></a></li>
@@ -173,17 +173,19 @@ A capabilityname may appear multiple times. ie a package may register that it pr
   
 <li><p><em>handler_classes</em></p>
 <ol class="doctools_enumerated">
-<li><p><em>CLASS interface_caphandler.registry</em></p>
+<li><p>CLASS <b class="class">interface_caphandler.registry</b></p>
 <dl class="doctools_definitions">
-<dt><a name="1">class::<b class="class">interface_caphandler.registry</b> <b class="method">pkg_register</b> <i class="arg">pkg</i> <i class="arg">capname</i> <i class="arg">capdict</i> <i class="arg">fullcapabilitylist</i></a></dt>
+    <p><em>METHODS</em></p>
+<dt><a name="1">class::interface_caphandler.registry <b class="method">pkg_register</b> <i class="arg">pkg</i> <i class="arg">capname</i> <i class="arg">capdict</i> <i class="arg">fullcapabilitylist</i></a></dt>
 <dd><p>handler may override and return 0 (indicating don't register)e.g if pkg capdict data wasn't valid
 overridden handler must be able to handle multiple calls for same pkg - but it may return 1 or 0 as it wishes.</p></dd>
-<dt><a name="2">class::<b class="class">interface_caphandler.registry</b> <b class="method">pkg_unregister</b> <i class="arg">pkg</i></a></dt>
+<dt><a name="2">class::interface_caphandler.registry <b class="method">pkg_unregister</b> <i class="arg">pkg</i></a></dt>
 <dd></dd>
 </dl>
 </li>
-<li><p><em>CLASS interface_caphandler.sysapi</em></p>
+<li><p>CLASS <b class="class">interface_caphandler.sysapi</b></p>
 <dl class="doctools_definitions">
+    <p><em>METHODS</em></p>
 </dl>
 </li>
 </ol>
@@ -191,8 +193,8 @@ overridden handler must be able to handle multiple calls for same pkg - but it m
 <li><p><em>provider_classes</em></p>
 <ol class="doctools_enumerated">
     
-<li><p><em>CLASS interface_cappprovider.registration</em>
-    Your provider package will need to instantiate this object under a sub-namespace called <b class="namespace">capsystem</b> within your package namespace.</p>
+<li><p>CLASS <b class="class">interface_cappprovider.registration</b></p>
+<p>Your provider package will need to instantiate this object under a sub-namespace called <b class="namespace">capsystem</b> within your package namespace.</p>
 <p>If your package namespace is mypackages::providerpkg then the object command would be at mypackages::providerpkg::capsystem::capprovider.registration</p>
 <p>Example code for your provider package to evaluate within its namespace:</p>
 <pre class="doctools_example">
@@ -212,14 +214,15 @@ namespace eval capsystem {
 </pre>
 <p>The above example declares that your package can be registered as a provider for the capabilities named 'punk.templates' and 'another_capability_name'</p>
 <dl class="doctools_definitions">
-<dt><a name="3">class::<b class="class">interface_capprovider.registration</b> <b class="method">get_declarations</b></a></dt>
+    <p><em>METHODS</em></p>
+<dt><a name="3">class::interface_capprovider.registration <b class="method">get_declarations</b></a></dt>
 <dd><p>This method must be overridden by your provider using oo::objdefine cappprovider.registration as in the example above.
  There must be at least one 2-element list in the result for the provider to be registerable.</p>
 <p>The first element of the list is the capabilityname - which can be custom to your provider/handler packages - or a well-known name that other authors may use/implement.</p>
 <p>The second element is a dictionary of keys specific to the capability being implemented. It may be empty if the any potential capability handlers for the named capability don't require registration data.</p></dd>
 </dl>
 </li>
-<li><p><em>CLASS interface_capprovider.provider</em></p>
+<li><p>CLASS <b class="class">interface_capprovider.provider</b></p>
 <p>Your provider package will need to instantiate this directly under it's own namespace with the command name of <em>provider</em></p>
 <pre class="doctools_example">
        namespace eval mypackages::providerpkg {
@@ -227,9 +230,10 @@ namespace eval capsystem {
        }
     </pre>
 <dl class="doctools_definitions">
-<dt><a name="4">class::<b class="class">interface_capprovider.provider</b> <b class="method">constructor</b> <i class="arg">providerpkg</i></a></dt>
+    <p><em>METHODS</em></p>
+<dt><a name="4">class::interface_capprovider.provider <b class="method">constructor</b> <i class="arg">providerpkg</i></a></dt>
 <dd></dd>
-<dt><a name="5">class::<b class="class">interface_capprovider.provider</b> <b class="method">register</b> <span class="opt">?capabilityname_glob?</span></a></dt>
+<dt><a name="5">class::interface_capprovider.provider <b class="method">register</b> <span class="opt">?capabilityname_glob?</span></a></dt>
 <dd><p>This is the mechanism by which a user of your provider package will register your package as a provider of the capability named.</p>
 <p>A user of your provider may elect to register all your declared capabilities:</p>
 <pre class="doctools_example">
@@ -242,7 +246,7 @@ namespace eval capsystem {
   mypackages::providerpkg::provider register another_capability_name
 </pre>
 </dd>
-<dt><a name="6">class::<b class="class">interface_capprovider.provider</b> <b class="method">capabilities</b></a></dt>
+<dt><a name="6">class::interface_capprovider.provider <b class="method">capabilities</b></a></dt>
 <dd><p>return a list of capabilities supported by this provider package</p></dd>
 </dl>
 </li>

src/embedded/www/doc/files/punk/_module_path-0.1.0.tm.html

@@ -0,0 +1,237 @@
+<!DOCTYPE html><html><head>
+<title>punk::path - punk path filesystem utils</title>
+<style type="text/css"><!--
+    HTML {
+	background: 	#FFFFFF;
+	color: 		black;
+    }
+    BODY {
+	background: 	#FFFFFF;
+	color:	 	black;
+    }
+    DIV.doctools {
+	margin-left:	10%;
+	margin-right:	10%;
+    }
+    DIV.doctools H1,DIV.doctools H2 {
+	margin-left:	-5%;
+    }
+    H1, H2, H3, H4 {
+	margin-top: 	1em;
+	font-family:	sans-serif;
+	font-size:	large;
+	color:		#005A9C;
+	background: 	transparent;
+	text-align:		left;
+    }
+    H1.doctools_title {
+	text-align: center;
+    }
+    UL,OL {
+	margin-right: 0em;
+	margin-top: 3pt;
+	margin-bottom: 3pt;
+    }
+    UL LI {
+	list-style: disc;
+    }
+    OL LI {
+	list-style: decimal;
+    }
+    DT {
+	padding-top: 	1ex;
+    }
+    UL.doctools_toc,UL.doctools_toc UL, UL.doctools_toc UL UL {
+	font:		normal 12pt/14pt sans-serif;
+	list-style:	none;
+    }
+    LI.doctools_section, LI.doctools_subsection {
+	list-style: 	none;
+	margin-left: 	0em;
+	text-indent:	0em;
+	padding: 	0em;
+    }
+    PRE {
+	display: 	block;
+	font-family:	monospace;
+	white-space:	pre;
+	margin:		0%;
+	padding-top:	0.5ex;
+	padding-bottom:	0.5ex;
+	padding-left:	1ex;
+	padding-right:	1ex;
+	width:		100%;
+    }
+    PRE.doctools_example {
+	color: 		black;
+	background: 	#f5dcb3;
+	border:		1px solid black;
+    }
+    UL.doctools_requirements LI, UL.doctools_syntax LI {
+	list-style: 	none;
+	margin-left: 	0em;
+	text-indent:	0em;
+	padding:	0em;
+    }
+    DIV.doctools_synopsis {
+	color: 		black;
+	background: 	#80ffff;
+	border:		1px solid black;
+	font-family:	serif;
+	margin-top: 	1em;
+	margin-bottom: 	1em;
+    }
+    UL.doctools_syntax {
+	margin-top: 	1em;
+	border-top:	1px solid black;
+    }
+    UL.doctools_requirements {
+	margin-bottom: 	1em;
+	border-bottom:	1px solid black;
+    }
+--></style>
+</head>
+<!-- Generated from file '_module_path-0.1.0.tm.man' by tcllib/doctools with format 'html'
+   -->
+<!-- Copyright &amp;copy; 2023
+   -->
+<!-- punk::path.0
+   -->
+<body><hr> [
+   <a href="../../../toc.html">Main Table Of Contents</a>
+&#124; <a href="../../toc.html">Table Of Contents</a>
+&#124; <a href="../../../index.html">Keyword Index</a>
+ ] <hr>
+<div class="doctools">
+<h1 class="doctools_title">punk::path(0) 0.1.0 doc &quot;punk path filesystem utils&quot;</h1>
+<div id="name" class="doctools_section"><h2><a name="name">Name</a></h2>
+<p>punk::path - Filesystem path utilities</p>
+</div>
+<div id="toc" class="doctools_section"><h2><a name="toc">Table Of Contents</a></h2>
+<ul class="doctools_toc">
+<li class="doctools_section"><a href="#toc">Table Of Contents</a></li>
+<li class="doctools_section"><a href="#synopsis">Synopsis</a></li>
+<li class="doctools_section"><a href="#section1">Description</a></li>
+<li class="doctools_section"><a href="#section2">Overview</a>
+<ul>
+<li class="doctools_subsection"><a href="#subsection1">Concepts</a></li>
+<li class="doctools_subsection"><a href="#subsection2">dependencies</a></li>
+</ul>
+</li>
+<li class="doctools_section"><a href="#section3">API</a>
+<ul>
+<li class="doctools_subsection"><a href="#subsection3">Namespace punk::path::class</a></li>
+<li class="doctools_subsection"><a href="#subsection4">Namespace punk::path</a></li>
+<li class="doctools_subsection"><a href="#subsection5">Namespace punk::path::lib</a></li>
+</ul>
+</li>
+<li class="doctools_section"><a href="#section4">Internal</a>
+<ul>
+<li class="doctools_subsection"><a href="#subsection6">Namespace punk::path::system</a></li>
+</ul>
+</li>
+<li class="doctools_section"><a href="#copyright">Copyright</a></li>
+</ul>
+</div>
+<div id="synopsis" class="doctools_section"><h2><a name="synopsis">Synopsis</a></h2>
+<div class="doctools_synopsis">
+<ul class="doctools_requirements">
+<li>package require <b class="pkgname">punk::path</b></li>
+</ul>
+<ul class="doctools_syntax">
+<li><a href="#1"><b class="function">pathglob_as_re</b> <i class="arg">pathglob</i></a></li>
+<li><a href="#2"><b class="function">globmatchpath</b> <i class="arg">pathglob</i> <i class="arg">path</i> <span class="opt">?option value...?</span></a></li>
+<li><a href="#3"><b class="function">treefilenames</b> <i class="arg">basepath</i> <i class="arg">tailglob</i> <span class="opt">?option value...?</span></a></li>
+<li><a href="#4"><b class="function">relative</b> <i class="arg">reference</i> <i class="arg">location</i></a></li>
+</ul>
+</div>
+</div>
+<div id="section1" class="doctools_section"><h2><a name="section1">Description</a></h2>
+</div>
+<div id="section2" class="doctools_section"><h2><a name="section2">Overview</a></h2>
+<p>overview of punk::path</p>
+<p>Filesystem path utility functions</p>
+<div id="subsection1" class="doctools_subsection"><h3><a name="subsection1">Concepts</a></h3>
+<p>-</p>
+</div>
+<div id="subsection2" class="doctools_subsection"><h3><a name="subsection2">dependencies</a></h3>
+<p>packages used by punk::path</p>
+<ul class="doctools_itemized">
+<li><p><b class="package">Tcl 8.6</b></p></li>
+</ul>
+</div>
+</div>
+<div id="section3" class="doctools_section"><h2><a name="section3">API</a></h2>
+<div id="subsection3" class="doctools_subsection"><h3><a name="subsection3">Namespace punk::path::class</a></h3>
+<p>class definitions</p>
+<ol class="doctools_enumerated">
+</ol>
+</div>
+<div id="subsection4" class="doctools_subsection"><h3><a name="subsection4">Namespace punk::path</a></h3>
+<p>Core API functions for punk::path</p>
+<dl class="doctools_definitions">
+<dt><a name="1"><b class="function">pathglob_as_re</b> <i class="arg">pathglob</i></a></dt>
+<dd><p>Returns a regular expression for matching a path to a glob pattern which can contain glob chars *|? in any segment of the path structure</p>
+<p>** matches any number of subdirectories.</p>
+<p>e.g /etc/**/*.txt will match any .txt files at any depth below /etc  (except directly within /etc itself)</p>
+<p>e.g /etc/**.txt will match any .txt files at any depth below /etc</p>
+<p>any segment that does not contain ** must match exactly one segment in the path</p>
+<p>e.g the glob /etc/*/*.doc  - will match any .doc files that are exactly one tree level below /etc</p>
+<p>The pathglob doesn't have to contain glob characters, in which case the returned regex will match the pathglob exactly as specified.</p>
+<p>Regular expression syntax is deliberateley not supported within the pathglob string so that supplied regex characters will be treated as literals</p></dd>
+<dt><a name="2"><b class="function">globmatchpath</b> <i class="arg">pathglob</i> <i class="arg">path</i> <span class="opt">?option value...?</span></a></dt>
+<dd><p>Return true if the pathglob matches the path</p>
+<p>see <b class="function">pathglob_as_re</b> for pathglob description</p>
+<p>Caller must ensure that file separator is forward slash. (e.g use file normalize on windows)</p>
+<p>Known options:</p>
+<p>-nocase 0|1  (default 0 - case sensitive)</p>
+<p>If -nocase is not supplied - default to case sensitive *except for driveletter*</p>
+<p>ie - the driveletter alone in paths such as c:/etc will still be case insensitive. (ie c:/ETC/* will match C:/ETC/blah but not C:/etc/blah)</p>
+<p>Explicitly specifying -nocase 0 will require the entire case to match including the driveletter.</p></dd>
+<dt><a name="3"><b class="function">treefilenames</b> <i class="arg">basepath</i> <i class="arg">tailglob</i> <span class="opt">?option value...?</span></a></dt>
+<dd><p>basic (glob based) list of filenames matching tailglob - recursive
+no natsorting - so order is dependent on filesystem</p></dd>
+<dt><a name="4"><b class="function">relative</b> <i class="arg">reference</i> <i class="arg">location</i></a></dt>
+<dd><p>Taking two directory paths, a reference and a location, computes the path
+    of the location relative to the reference.</p>
+<ul class="doctools_itemized">
+<li><p>Arguments:</p>
+<dl class="doctools_arguments">
+       
+<dt>string <i class="arg">reference</i></dt>
+<dd><p>The path from which the relative path to location is determined.</p></dd>
+<dt>string <i class="arg">location</i></dt>
+<dd><p>The location path which may be above or below the reference path</p></dd>
+</dl>
+</li>
+<li><p>Results:</p>
+<p>The relative path of the location to the reference path.</p>
+<p>Will return a single dot &quot;.&quot; if the paths are the same</p></li>
+<li><p>Notes:</p>
+<p>Both paths must be the same type - ie both absolute or both relative</p>
+<p>Case sensitive.  ie  relative /etc /etC
+      will return ../etC</p>
+<p>On windows, the drive-letter component (only) is not case sensitive</p>
+<p>ie relative c:/etc  C:/etc  returns .</p>
+<p>but relative c:/etc C:/Etc returns ../Etc</p>
+<p>On windows, if the paths are absolute and specifiy different volumes, only the location will be returned.
+    ie  relative c:/etc d:/etc/blah
+    returns d:/etc/blah</p></li>
+</ul></dd>
+</dl>
+</div>
+<div id="subsection5" class="doctools_subsection"><h3><a name="subsection5">Namespace punk::path::lib</a></h3>
+<p>Secondary functions that are part of the API</p>
+<dl class="doctools_definitions">
+</dl>
+</div>
+</div>
+<div id="section4" class="doctools_section"><h2><a name="section4">Internal</a></h2>
+<div id="subsection6" class="doctools_subsection"><h3><a name="subsection6">Namespace punk::path::system</a></h3>
+<p>Internal functions that are not part of the API</p>
+</div>
+</div>
+<div id="copyright" class="doctools_section"><h2><a name="copyright">Copyright</a></h2>
+<p>Copyright &copy; 2023</p>
+</div>
+</div></body></html>

src/embedded/www/doc/files/punk/mix/commandset/_module_project-0.1.0.tm.html

@@ -0,0 +1,186 @@
+<!DOCTYPE html><html><head>
+<title>punk::mix::commandset::project - pmix CLI commandset - project</title>
+<style type="text/css"><!--
+    HTML {
+	background: 	#FFFFFF;
+	color: 		black;
+    }
+    BODY {
+	background: 	#FFFFFF;
+	color:	 	black;
+    }
+    DIV.doctools {
+	margin-left:	10%;
+	margin-right:	10%;
+    }
+    DIV.doctools H1,DIV.doctools H2 {
+	margin-left:	-5%;
+    }
+    H1, H2, H3, H4 {
+	margin-top: 	1em;
+	font-family:	sans-serif;
+	font-size:	large;
+	color:		#005A9C;
+	background: 	transparent;
+	text-align:		left;
+    }
+    H1.doctools_title {
+	text-align: center;
+    }
+    UL,OL {
+	margin-right: 0em;
+	margin-top: 3pt;
+	margin-bottom: 3pt;
+    }
+    UL LI {
+	list-style: disc;
+    }
+    OL LI {
+	list-style: decimal;
+    }
+    DT {
+	padding-top: 	1ex;
+    }
+    UL.doctools_toc,UL.doctools_toc UL, UL.doctools_toc UL UL {
+	font:		normal 12pt/14pt sans-serif;
+	list-style:	none;
+    }
+    LI.doctools_section, LI.doctools_subsection {
+	list-style: 	none;
+	margin-left: 	0em;
+	text-indent:	0em;
+	padding: 	0em;
+    }
+    PRE {
+	display: 	block;
+	font-family:	monospace;
+	white-space:	pre;
+	margin:		0%;
+	padding-top:	0.5ex;
+	padding-bottom:	0.5ex;
+	padding-left:	1ex;
+	padding-right:	1ex;
+	width:		100%;
+    }
+    PRE.doctools_example {
+	color: 		black;
+	background: 	#f5dcb3;
+	border:		1px solid black;
+    }
+    UL.doctools_requirements LI, UL.doctools_syntax LI {
+	list-style: 	none;
+	margin-left: 	0em;
+	text-indent:	0em;
+	padding:	0em;
+    }
+    DIV.doctools_synopsis {
+	color: 		black;
+	background: 	#80ffff;
+	border:		1px solid black;
+	font-family:	serif;
+	margin-top: 	1em;
+	margin-bottom: 	1em;
+    }
+    UL.doctools_syntax {
+	margin-top: 	1em;
+	border-top:	1px solid black;
+    }
+    UL.doctools_requirements {
+	margin-bottom: 	1em;
+	border-bottom:	1px solid black;
+    }
+--></style>
+</head>
+<!-- Generated from file '_module_project-0.1.0.tm.man' by tcllib/doctools with format 'html'
+   -->
+<!-- Copyright &amp;copy; 2023
+   -->
+<!-- punk::mix::commandset::project.0
+   -->
+<body><hr> [
+   <a href="../../../../../toc.html">Main Table Of Contents</a>
+&#124; <a href="../../../../toc.html">Table Of Contents</a>
+&#124; <a href="../../../../../index.html">Keyword Index</a>
+ ] <hr>
+<div class="doctools">
+<h1 class="doctools_title">punk::mix::commandset::project(0) 0.1.0 doc &quot;pmix CLI commandset - project&quot;</h1>
+<div id="name" class="doctools_section"><h2><a name="name">Name</a></h2>
+<p>punk::mix::commandset::project - pmix commandset - project</p>
+</div>
+<div id="toc" class="doctools_section"><h2><a name="toc">Table Of Contents</a></h2>
+<ul class="doctools_toc">
+<li class="doctools_section"><a href="#toc">Table Of Contents</a></li>
+<li class="doctools_section"><a href="#synopsis">Synopsis</a></li>
+<li class="doctools_section"><a href="#section1">Description</a></li>
+<li class="doctools_section"><a href="#section2">Overview</a>
+<ul>
+<li class="doctools_subsection"><a href="#subsection1">Concepts</a></li>
+<li class="doctools_subsection"><a href="#subsection2">dependencies</a></li>
+</ul>
+</li>
+<li class="doctools_section"><a href="#section3">API</a>
+<ul>
+<li class="doctools_subsection"><a href="#subsection3">Namespace punk::mix::commandset::project</a></li>
+</ul>
+</li>
+<li class="doctools_section"><a href="#copyright">Copyright</a></li>
+</ul>
+</div>
+<div id="synopsis" class="doctools_section"><h2><a name="synopsis">Synopsis</a></h2>
+<div class="doctools_synopsis">
+<ul class="doctools_requirements">
+<li>package require <b class="pkgname">punk::mix::commandset::project</b></li>
+</ul>
+<ul class="doctools_syntax">
+<li><a href="#1"><b class="function">new</b> <i class="arg">newprojectpath_or_name</i> <span class="opt">?args?</span></a></li>
+</ul>
+</div>
+</div>
+<div id="section1" class="doctools_section"><h2><a name="section1">Description</a></h2>
+</div>
+<div id="section2" class="doctools_section"><h2><a name="section2">Overview</a></h2>
+<p>overview of punk::mix::commandset::project</p>
+<p>Import into an ensemble namespace similarly to the way it is done with punk::mix::cli e.g</p>
+<pre class="doctools_example">
+    namespace eval myproject::cli {
+        namespace export *
+        namespace ensemble create
+        package require punk::overlay
+        package require punk::mix::commandset::project
+        punk::overlay::import_commandset project    .       ::punk::mix::commandset::project
+        punk::overlay::import_commandset projects   .       ::punk::mix::commandset::project::collection
+    }
+</pre>
+<p>Where the . in the above example is the prefix/command separator</p>
+<p>The prefix ('project' in the above example) can be any string desired to disambiguate commands imported from other commandsets.</p>
+<p>The above results in the availability of the ensemble command: ::myproject::cli project.new, which is implemented in ::punk::mix::commandset::project::new</p>
+<p>Similarly, procs under ::punk::mix::commandset::project::collection will be available as subcommands of the ensemble as projects.&lt;procname&gt;</p>
+<div id="subsection1" class="doctools_subsection"><h3><a name="subsection1">Concepts</a></h3>
+<p>see punk::overlay</p>
+</div>
+<div id="subsection2" class="doctools_subsection"><h3><a name="subsection2">dependencies</a></h3>
+<p>packages used by punk::mix::commandset::project</p>
+<ul class="doctools_itemized">
+<li><p><b class="package">Tcl 8.6</b></p></li>
+<li><p><b class="package">punk::ns</b></p></li>
+<li><p><b class="package">sqlite3</b> (binary)</p></li>
+<li><p><b class="package">overtype</b></p></li>
+<li><p><b class="package">textutil</b> (tcllib)</p></li>
+</ul>
+</div>
+</div>
+<div id="section3" class="doctools_section"><h2><a name="section3">API</a></h2>
+<div id="subsection3" class="doctools_subsection"><h3><a name="subsection3">Namespace punk::mix::commandset::project</a></h3>
+<p>core commandset functions for punk::mix::commandset::project</p>
+<dl class="doctools_definitions">
+ 
+<dt><a name="1"><b class="function">new</b> <i class="arg">newprojectpath_or_name</i> <span class="opt">?args?</span></a></dt>
+<dd><p>new project structure - may be dedicated to one module, or contain many.
+create minimal folder structure only by specifying in args:  -modules {}</p></dd>
+</dl>
+</div>
+</div>
+<div id="copyright" class="doctools_section"><h2><a name="copyright">Copyright</a></h2>
+<p>Copyright &copy; 2023</p>
+</div>
+</div></body></html>

src/embedded/www/doc/toc.html

@@ -13,16 +13,16 @@
 <hr><dl><dt><h2>doc</h2></dt><dd>
 <table class="#doctools_toc">
 <tr class="#doctools_toceven" >
-<td class="#doctools_tocleft" ><a name='_pkg_'><a href="files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.html">%pkg%</a></td>
-<td class="#doctools_tocright">Module API</td>
+<td class="#doctools_tocleft" ><a name='punk_cap'><a href="files/punk/_module_cap-0.1.0.tm.html">punk::cap</a></td>
+<td class="#doctools_tocright">capability provider and handler plugin system</td>
 </tr>
 <tr class="#doctools_tocodd"  >
-<td class="#doctools_tocleft" ><a name='punk_cap'><a href="files/_module_punk_cap-0.1.0.tm.html">punk::cap</a></td>
-<td class="#doctools_tocright">capability provider and handler plugin system</td>
+<td class="#doctools_tocleft" ><a name='punk_mix_commandset_project'><a href="files/punk/mix/commandset/_module_project-0.1.0.tm.html">punk::mix::commandset::project</a></td>
+<td class="#doctools_tocright">pmix commandset - project</td>
 </tr>
 <tr class="#doctools_toceven" >
-<td class="#doctools_tocleft" ><a name='punk_cap'><a href="files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.html">punk::cap</a></td>
-<td class="#doctools_tocright">Module API</td>
+<td class="#doctools_tocleft" ><a name='punk_path'><a href="files/punk/_module_path-0.1.0.tm.html">punk::path</a></td>
+<td class="#doctools_tocright">Filesystem path utilities</td>
 </tr>
 <tr class="#doctools_tocodd"  >
 <td class="#doctools_tocleft" ><a name='punkshell'><a href="files/main.html">punkshell</a></td>

src/embedded/www/toc.html

@@ -13,16 +13,16 @@
 <hr><dl><dt><h2>doc</h2></dt><dd>
 <table class="#doctools_toc">
 <tr class="#doctools_toceven" >
-<td class="#doctools_tocleft" ><a name='_pkg_'><a href="doc/files/_module_punk_mix_templates_modules_template_module-0.0.1.tm.html">%pkg%</a></td>
-<td class="#doctools_tocright">Module API</td>
+<td class="#doctools_tocleft" ><a name='punk_cap'><a href="doc/files/punk/_module_cap-0.1.0.tm.html">punk::cap</a></td>
+<td class="#doctools_tocright">capability provider and handler plugin system</td>
 </tr>
 <tr class="#doctools_tocodd"  >
-<td class="#doctools_tocleft" ><a name='punk_cap'><a href="doc/files/_module_punk_cap-0.1.0.tm.html">punk::cap</a></td>
-<td class="#doctools_tocright">capability provider and handler plugin system</td>
+<td class="#doctools_tocleft" ><a name='punk_mix_commandset_project'><a href="doc/files/punk/mix/commandset/_module_project-0.1.0.tm.html">punk::mix::commandset::project</a></td>
+<td class="#doctools_tocright">pmix commandset - project</td>
 </tr>
 <tr class="#doctools_toceven" >
-<td class="#doctools_tocleft" ><a name='punk_cap'><a href="doc/files/_module_punk_mix_templates_layouts_project_src_bootsupport_modules_punk_cap-0.1.0.tm.html">punk::cap</a></td>
-<td class="#doctools_tocright">Module API</td>
+<td class="#doctools_tocleft" ><a name='punk_path'><a href="doc/files/punk/_module_path-0.1.0.tm.html">punk::path</a></td>
+<td class="#doctools_tocright">Filesystem path utilities</td>
 </tr>
 <tr class="#doctools_tocodd"  >
 <td class="#doctools_tocleft" ><a name='punkshell'><a href="doc/files/main.html">punkshell</a></td>

src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/cap-0.1.0.tm

@@ -17,11 +17,27 @@
 #*** !doctools
 #[manpage_begin punk::cap 0 0.1.0]
 #[copyright "2023 JMNoble - BSD licensed"]
-#[titledesc {Module API}]
+#[titledesc {capability provider and handler plugin system}]
 #[moddesc {punk capabilities plugin system}]
 #[require punk::cap]
 #[description]
-#[list_begin definitions]
+#[section Overview]
+#[para]punk::cap provides management of named capabilities and the provider packages and handler packages that implement a pluggable capability.
+#[subsection Concepts]
+#[para]A [term capability] may be something like providing a folder of files, or just a data dictionary, and/or an API 
+#
+#[para][term {capability handler}] - a package/namespace which may provide validation and standardised ways of looking up provider data
+# registered (or not) using register_capabilityname <capname> <capnamespace>
+#
+#[para][term {capability provider}] - a package which registers as providing one or more capablities.
+#[para]registered using register_package <pkg> <capabilitylist>
+#the capabilitylist is a list of 2-element lists where the first element is the capabilityname and the second element is a (possibly empty) dict of data relevant to that capability
+#A capabilityname may appear multiple times. ie a package may register that it provides the capability with multiple datasets.
+
+
+#*** !doctools
+#[section API]
+
 
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 ##  Requirements
@@ -29,63 +45,115 @@
 package require oolib
 
 
-
-# mkdoc markdown 
-#' ---
-#' author: JMNoble 
-#' ---
-#' ## Concepts:
-#' > A **capability** may be something like providing a folder of files, or just a data dictionary, and/or an API 
-#'
-#' > **capability handler** - a package/namespace which may provide validation and standardised ways of looking up provider data
-#'   registered (or not) using register_capabilityname <capname> <capnamespace>
-#'
-#' > **capability provider** - a package which registers as providing one or more capablities.
-#'  registered using register_package <pkg> <capabilitylist>
-#'  the capabilitylist is a list of 2-element lists where the first element is the capabilityname and the second element is a (possibly empty) dict of data relevant to that capability
-#'  A capabilityname may appear multiple times. ie a package may register that it provides the capability with multiple datasets.
-
-
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 namespace eval punk::cap {
     variable pkgcapsdeclared [dict create] 
     variable pkgcapsaccepted [dict create]
     variable caps [dict create]
-
     namespace eval class {
         if {[info commands [namespace current]::interface_caphandler.registry] eq ""} {
-            #Handler classes
+            #*** !doctools
+            #[subsection {Namespace punk::cap::class}]
+            #[para] class definitions
+            #[list_begin itemized] [comment {- punk::cap::class groupings -}]
+            #  [item]
+            #  [para] [emph {handler_classes}]
+            #  [list_begin enumerated]
+
             oo::class create [namespace current]::interface_caphandler.registry {
+                #*** !doctools
+                #[enum] CLASS [class interface_caphandler.registry]
+                #[list_begin definitions]
+                #    [para] [emph METHODS]
                 method pkg_register {pkg capname capdict fullcapabilitylist} {
-                    #***
-                    #[call [class interface_caphandler.registry] [method pkg_register] [arg pkg] [arg capname] [arg capdict] [arg fullcapabilitylist]]
+                    #*** !doctools
+                    #[call class::interface_caphandler.registry [method pkg_register] [arg pkg] [arg capname] [arg capdict] [arg fullcapabilitylist]]
                     #handler may override and return 0 (indicating don't register)e.g if pkg capdict data wasn't valid
                     #overridden handler must be able to handle multiple calls for same pkg - but it may return 1 or 0 as it wishes. 
                     return 1 ;#default to permit 
                 }
                 method pkg_unregister {pkg} {
-                    #***
-                    #[call [class interface_caphandler.registry] [method pkg_unregister] [arg pkg]]
+                    #*** !doctools
+                    #[call class::interface_caphandler.registry [method pkg_unregister] [arg pkg]]
                     return ;#unregistration return is ignored - review
                 }
+                #*** !doctools
+                #[list_end]
             }
+
             oo::class create [namespace current]::interface_caphandler.sysapi {
+                #*** !doctools
+                #[enum] CLASS [class interface_caphandler.sysapi]
+                #[list_begin definitions]
+                #    [para] [emph METHODS]
 
+
+                #*** !doctools
+                #[list_end]
             }
 
+            #*** !doctools
+            #  [list_end] [comment {- end enumeration handler classes -}]
+
+            #*** !doctools
+            #  [item]
+            #  [para] [emph {provider_classes}]
+            #  [list_begin enumerated]
 
             #Provider classes
             oo::class create [namespace current]::interface_capprovider.registration {
+                #*** !doctools
+                #    [enum] CLASS [class interface_cappprovider.registration]
+                #    [para]Your provider package will need to instantiate this object under a sub-namespace called [namespace capsystem] within your package namespace. 
+                #    [para]If your package namespace is mypackages::providerpkg then the object command would be at mypackages::providerpkg::capsystem::capprovider.registration
+                #    [para]Example code for your provider package to evaluate within its namespace: 
+                #    [example {
+                #namespace eval capsystem {
+                #    if {[info commands capprovider.registration] eq ""} {
+                #        punk::cap::class::interface_capprovider.registration create capprovider.registration
+                #        oo::objdefine capprovider.registration {
+                #            method get_declarations {} {
+                #                set decls [list]
+                #                lappend decls [list punk.templates {relpath ../templates}]
+                #                lappend decls [list another_capability_name {somekey blah key2 etc}]
+                #                return $decls
+                #            }
+                #        }
+                #    }
+                #}
+                #}]
+                #[para] The above example declares that your package can be registered as a provider for the capabilities named 'punk.templates' and 'another_capability_name'
+                #    [list_begin definitions]
+                #    [para] [emph METHODS]
                 method get_declarations {} {
                     #***
-                    #[call [class interface_capprovider.registration] [method pkg_unregister] [arg pkg]]
+                    #[call class::interface_capprovider.registration [method get_declarations]]
+                    #[para] This method must be overridden by your provider using oo::objdefine cappprovider.registration as in the example above.
+                    # There must be at least one 2-element list in the result for the provider to be registerable.
+                    #[para]The first element of the list is the capabilityname - which can be custom to your provider/handler packages - or a well-known name that other authors may use/implement.
+                    #[para]The second element is a dictionary of keys specific to the capability being implemented. It may be empty if the any potential capability handlers for the named capability don't require registration data.  
                     error "interface_capprovider.registration not implemented by provider"
                 }
+                #*** !doctools
+                #    [list_end]
             }
+
             oo::class create [namespace current]::interface_capprovider.provider {
+                #*** !doctools
+                #    [enum] CLASS [class interface_capprovider.provider] 
+                #    [para] Your provider package will need to instantiate this directly under it's own namespace with the command name of [emph {provider}] 
+                #    [example {
+                #       namespace eval mypackages::providerpkg { 
+                #           punk::cap::class::interface_capprovider.provider create provider mypackages::providerpkg 
+                #       }
+                #    }]
+                #    [list_begin definitions]
+                #    [para] [emph METHODS]
                 variable provider_pkg
                 variable registrationobj
                 constructor {providerpkg} {
+                    #*** !doctools
+                    #[call class::interface_capprovider.provider [method constructor] [arg providerpkg]]
                     variable provider_pkg
                     if {$providerpkg in [list "" "::"]} {
                         error "interface_capprovider.provider constructor error. Invalid provider '$providerpkg'"
@@ -103,16 +171,33 @@ namespace eval punk::cap {
 
                 }
                 method register {{capabilityname_glob *}} {
-                    #***
-                    #[call [class interface_capprovider.provider] [method register] [opt capabilityname_glob]]
+                    #*** !doctools
+                    #[comment {- -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---}]
+                    #[call class::interface_capprovider.provider [method register] [opt capabilityname_glob]]
+                    #
+                    #[para]This is the mechanism by which a user of your provider package will register your package as a provider of the capability named.
+                    #
+                    #[para]A user of your provider may elect to register all your declared capabilities:
+                    #[example {
+                    #  package require mypackages::providerpkg
+                    #  mypackages::providerpkg::provider register *
+                    #}]
+                    #[para] Or a specific capability may be registered:
+                    #[example {
+                    #  package require mypackages::providerpkg
+                    #  mypackages::providerpkg::provider register another_capability_name
+                    #}]
+                    #
                     variable provider_pkg
                     set all_decls [$registrationobj get_declarations]
                     set register_decls [lsearch -all -inline -index 0 $all_decls $capabilityname_glob]
                     punk::cap::register_package $provider_pkg $register_decls
                 }
                 method capabilities {} {
-                    #***
-                    #[call [class interface_capprovider.provider] [method capabilities]]
+                    #*** !doctools
+                    #[comment {- -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---}]
+                    #[call class::interface_capprovider.provider [method capabilities]]
+                    #[para] return a list of capabilities supported by this provider package
                     variable provider_pkg
                     variable registrationobj
 
@@ -124,28 +209,24 @@ namespace eval punk::cap {
                             lappend capabilities $capname
                         }
                     }
-                    return $capname
+                    return $capabilities
                 }
+                #*** !doctools
+                #    [list_end] [comment {- end class definitions -}]
             }
+            #*** !doctools
+            #  [list_end] [comment {- end enumeration provider_classes }]
+            #[list_end] [comment {- end itemized list punk::cap::class groupings -}]
         }
     } ;# end namespace class
-    namespace eval capsystem {
-        proc get_caphandler_registry {capname} {
-            set ns [::punk::cap::get_handler $capname]::capsystem
-            if {[namespace exists ${ns}]} {
-                if {[info command ${ns}::caphandler.registry] ne ""} {
-                    if {[info object isa object ${ns}::caphandler.registry]} {
-                        return ${ns}::caphandler.registry
-                    }
-                }
-            }
-            return ""
-        }
-    }
 
+    #*** !doctools
+    #[subsection {Namespace punk::cap}]
+    #[para] Main punk::cap API for client programs interested in using capability handler packages and associated (registered) provider packages
+    #[list_begin definitions]
 
-    #Not all capabilities have to be registered.
-    #A package registering as a provider using register_package can include capabilitynames in it's capabilitylist which have no associated capnamespace (handler).
+    #Not all capability names have to be registered.
+    #A package registering as a provider using register_package can include capabilitynames in it's capabilitylist which have no associated handler.
     #such unregistered capabilitynames may be used just to flag something, or have datamembers significant to callers cooperatively interested in that capname. 
     #we allow registering a capability with an empty handler (capnamespace) - but this means another handler could be registered later.
     proc register_capabilityname {capname capnamespace} {
@@ -161,7 +242,7 @@ namespace eval punk::cap {
         #allow register of existing capname iff there is no current handler
         #as handlers can be used to validate during provider registration - ideally handlers should be registered before any pkgs call register_package 
         #we allow loading a handler later though - but will need to validate existing data from pkgs that have already registered as providers 
-        if {[set hdlr [get_handler $capname]] ne ""} {
+        if {[set hdlr [capability_get_handler $capname]] ne ""} {
             error "register_capabilityname cannot register capability:$capname with handler:$capnamespace. There is already a registered handler:$hdlr" 
         }
         #assert: capnamespace may or may not be empty string, capname may or may not already exist in caps dict, caps $capname providers may have existing entries.
@@ -213,52 +294,33 @@ namespace eval punk::cap {
             
         }
     }
-    proc exists {capname} {
+    proc capability_exists {capname} {
         #*** !doctools
-        # [call [fun exists] [arg capname]]
+        # [call [fun capability_exists] [arg capname]]
         # Return a boolean indicating if the named capability exists  (0|1)
-
-        # mkdoc markdown
-        #'
-        #' ## **exists(capname)**
-        #'
-        #' > return a boolean indicating the existence of a capability
-        #'
-        #' > Arguments:
-        #'
-        #' > - *capname* - string indicating the name of the capability
-        #'
-        #' > Returns: 0|1
-        #'
         variable caps
         return [dict exists $caps $capname]
     }
-    proc has_handler {capname} {
+    proc capability_has_handler {capname} {
         #*** !doctools
-        # [call [fun has_handler] [arg capname]]
-        # Return a boolean indicating if the named capability has a handler package installed (0|1)
-
-
+        # [call [fun capability_has_handler] [arg capname]]
+        #Return a boolean indicating if the named capability has a handler package installed (0|1)
         variable caps
         return [expr {[dict exists $caps $capname handler] && [dict get $caps $capname handler] ne ""}]
     }
-    proc get_handler {capname} {
+    proc capability_get_handler {capname} {
+        #*** !doctools
+        # [call [fun capability_get_handler] [arg capname]]
+        #Return the base namespace of the active handler package for the named capability.
+        #[para] The base namespace for a handler will always be the package name, but prefixed with ::
         variable caps
         if {[dict exists $caps $capname]} {
             return [dict get $caps $capname handler]
         }
         return "" 
     }
-
-    #dispatch
-    #proc call_handler {capname args} {
-    #    if {[set handler [get_handler $capname]] eq ""} {
-    #        error "punk::cap::call_handler $capname $args - no handler registered for capability $capname"
-    #    }
-    #    ${handler}::[lindex $args 0] {*}[lrange $args 1 end]
-    #}
     proc call_handler {capname args} {
-        if {[set handler [get_handler $capname]] eq ""} {
+        if {[set handler [capability_get_handler $capname]] eq ""} {
             error "punk::cap::call_handler $capname $args - no handler registered for capability $capname"
         }
         set obj ${handler}::api_$capname
@@ -274,10 +336,21 @@ namespace eval punk::cap {
 
     #register package with arbitrary capnames from capabilitylist
     #The registered pkg is a module that provides some service to that capname. Possibly just data members, that the capability will use.
-    proc register_package {pkg capabilitylist} {
+    proc register_package {pkg capabilitylist args} {
         variable pkgcapsdeclared
         variable pkgcapsaccepted
         variable caps
+        set defaults [dict create\
+            -nowarnings false
+        ]
+        dict for {k v} $args {
+            if {$k ni $defaults} {
+                error "Unrecognized option $k. Known options [dict keys $defaults]"
+            }
+        }
+        set opts [dict merge $defaults $args]
+        set warnings [expr {! [dict get $opts -nowarnings]}]
+
         if {[string match ::* $pkg]} {
             set pkg [string range $pkg 2 end]
         }
@@ -286,11 +359,23 @@ namespace eval punk::cap {
         } else {
             set pkg_already_accepted [list]
         }
+        package require $pkg
+        set providerapi ::${pkg}::provider
+        if {[info commands $providerapi] eq ""} {
+            error "register_package error. pkg '$pkg' doesn't seem to be a punk::cap capability provider (no object found at $providerapi)"
+        }
+        set defined_caps [$providerapi capabilities]
         #for each capability
         # - ensure 1st element is a single word
         # - ensure that if 2nd element (capdict) is present - it is dict shaped
         foreach capspec $capabilitylist {
             lassign $capspec capname capdict
+
+            if {$warnings} {
+                if {$capname ni $defined_caps} {
+                    puts stderr "WARNING: pkg '$pkg' doesn't declare support for capability '$capname'."
+                }
+            }
             if {[llength $capname] !=1} {
                 error "register_package error. pkg: '$pkg' An entry in the capability list doesn't appear to have a single-word name. Problematic entry:'$capspec'"
             }
@@ -299,7 +384,9 @@ namespace eval punk::cap {
             }
             if {$capspec in $pkg_already_accepted} {
                 #review - multiple handlers? if so - will need to record which handler(s) accepted the capspec
-                puts stderr "register_package pkg $pkg already has capspec marked as accepted: $capspec"
+                if {$warnings} {
+                    puts stderr "WARNING: register_package pkg $pkg already has capspec marked as accepted: $capspec"
+                }
                 continue
             }
             if {[dict exists $caps $capname]} {
@@ -371,73 +458,6 @@ namespace eval punk::cap {
         }
     }
 
-    #review promote/demote doesn't always make a lot of sense ..  should possibly be per cap for multicap pkgs
-    #The idea is to provide a crude way to preference/depreference packages independently of order the packages were loaded
-    #e.g a caller or cap-handler can ascribe some meaning to the order of the 'providers' key returned from punk::cap::capabilities <capname>
-    #The order of providers will be the order the packages were loaded & registered
-    #the naming: "promote vs demote" operates on a latest-package-in-list has higher preference assumption (matching last pkg loaded)
-    #Each capability handler could implement specific preferencing methods if finer control needed.
-    #In some cases the preference/loading order may be inapplicable/irrelevant to a particular capability anyway.
-    #As this is just a basic mechanism, which can't support independent per-cap preferencing for multi-cap packages -
-    # it only allows putting the pkgs to the head or tail of the lists.
-    #Whether particular caps or users of caps do anything with this ordering is dependent on the cap-handler and/or calling code.
-    proc promote_package {pkg} {
-        variable pkgcapsdeclared
-        variable caps
-        if {[string match ::* $pkg]} {
-            set pkg [string range $pkg 2 end]
-        }
-        if {![dict exists $pkgcapsdeclared $pkg]} {
-            error "punk::cap::promote_package error pkg'$pkg' not registered. Use register_package \$pkg first"
-        }
-        if {[dict size $pkgcapsdeclared] > 1} {
-            set pkginfo [dict get $pkgcapsdeclared $pkg]
-            #remove and re-add at end of dict
-            dict unset pkgcapsdeclared $pkg
-            dict set pkgcapsdeclared $pkg $pkginfo
-            dict for {cap cap_info} $caps {
-                set cap_pkgs [dict get $cap_info providers]
-                if {$pkg in $cap_pkgs} {
-                    set posn [lsearch $cap_pkgs $pkg]
-                    if {$posn >=0} {
-                        #rewrite package list with pkg at tail of list for this capability
-                        set cap_pkgs [lreplace $cap_pkgs $posn $posn]
-                        lappend cap_pkgs $pkg
-                        dict set caps $cap providers $cap_pkgs
-                    }
-                }
-            }
-        }
-    }
-    proc demote_package {pkg} {
-        variable pkgcapsdeclared
-        variable caps
-        if {[string match ::* $pkg]} {
-            set pkg [string range $pkg 2 end]
-        }
-        if {![dict exists $pkgcapsdeclared $pkg]} {
-            error "punk::cap::promote_package error pkg'$pkg' not registered. Use register_package \$pkg first"
-        }
-        if {[dict size $pkgcapsdeclared] > 1} {
-            set pkginfo [dict get $pkgcapsdeclared $pkg]
-            #remove and re-add at start of dict
-            dict unset pkgcapsdeclared $pkg
-            dict set pkgcapsdeclared $pkg $pkginfo
-            set pkgcapsdeclared [dict merge [dict create $pkg $pkginfo] $pkgcapsdeclared]
-            dict for {cap cap_info} $caps {
-                set cap_pkgs [dict get $cap_info providers]
-                if {$pkg in $cap_pkgs} {
-                    set posn [lsearch $cap_pkgs $pkg]
-                    if {$posn >=0} {
-                        #rewrite package list with pkg at head of list for this capability
-                        set cap_pkgs [lreplace $cap_pkgs $posn $posn]
-                        set cap_pkgs [list $pkg {*}$cap_pkgs]
-                        dict set caps $cap providers $cap_pkgs
-                    }
-                }
-            }
-        }
-    }
     proc pkgcap {pkg} {
         variable pkgcapsdeclared
         variable pkgcapsaccepted
@@ -502,7 +522,121 @@ namespace eval punk::cap {
         }
         return $cap_list
     }
+    #*** !doctools
+    #[list_end] [comment {- end definitions for namespace punk::cap -}]
+
+    namespace eval advanced {
+    #*** !doctools
+    #[subsection {Namespace punk::cap::advanced}]
+    #[para] punk::cap::advanced API. Functions here are generally not the preferred way to interact with punk::cap.
+    #[para] In some cases they may allow interaction in less safe ways or may allow use of features that are unavailable in the base namespace.
+    #[para] Some functions are here because they are only marginally or rarely useful, and they are here to keep the base API simple.  
+    #[list_begin definitions]
+
+    proc promote_provider {pkg} {
+        #*** !doctools
+        # [call advanced::[fun promote_provider] [arg pkg]]
+        #[para]Move the named provider package to the preferred end of the list (tail).
+        #[para]The active handler may or may not utilise this for preferencing. See documentation for the specific handler package to confirm. 
+        #[para]
+        #[para] promote/demote doesn't always make a lot of sense ..  should preferably be configurable per capapbility for multicap provider pkgs
+        #[para]The idea is to provide a crude way to preference/depreference packages independently of order the packages were loaded
+        #e.g a caller or cap-handler can ascribe some meaning to the order of the 'providers' key returned from punk::cap::capabilities <capname>
+        #[para]The order of providers will be the order the packages were loaded & registered
+        #[para]the naming: "promote vs demote" operates on a latest-package-in-list has higher preference assumption (matching last pkg loaded)
+        #[para]Each capability handler could and should implement specific preferencing methods within its own API if finer control needed.
+        #In some cases the preference/loading order may be inapplicable/irrelevant to a particular capability anyway.
+        #[para]As this is just a basic mechanism, which can't support independent per-cap preferencing for multi-cap packages -
+        # it only allows putting the pkgs to the head or tail of the lists.
+        #[para]Whether particular caps or users of caps do anything with this ordering is dependent on the cap-handler and/or calling code.
+        variable pkgcapsdeclared
+        variable caps
+        if {[string match ::* $pkg]} {
+            set pkg [string range $pkg 2 end]
+        }
+        if {![dict exists $pkgcapsdeclared $pkg]} {
+            error "punk::cap::promote_package error pkg'$pkg' not registered. Use register_package \$pkg first"
+        }
+        if {[dict size $pkgcapsdeclared] > 1} {
+            set pkginfo [dict get $pkgcapsdeclared $pkg]
+            #remove and re-add at end of dict
+            dict unset pkgcapsdeclared $pkg
+            dict set pkgcapsdeclared $pkg $pkginfo
+            dict for {cap cap_info} $caps {
+                set cap_pkgs [dict get $cap_info providers]
+                if {$pkg in $cap_pkgs} {
+                    set posn [lsearch $cap_pkgs $pkg]
+                    if {$posn >=0} {
+                        #rewrite package list with pkg at tail of list for this capability
+                        set cap_pkgs [lreplace $cap_pkgs $posn $posn]
+                        lappend cap_pkgs $pkg
+                        dict set caps $cap providers $cap_pkgs
+                    }
+                }
+            }
+        }
+    }
+    proc demote_provider {pkg} {
+        #*** !doctools
+        # [call advanced::[fun demote_provider] [arg pkg]]
+        #[para]Move the named provider package to the preferred end of the list (tail).
+        #[para]The active handler may or may not utilise this for preferencing. See documentation for the specific handler package to confirm. 
+        variable pkgcapsdeclared
+        variable caps
+        if {[string match ::* $pkg]} {
+            set pkg [string range $pkg 2 end]
+        }
+        if {![dict exists $pkgcapsdeclared $pkg]} {
+            error "punk::cap::promote_package error pkg'$pkg' not registered. Use register_package \$pkg first"
+        }
+        if {[dict size $pkgcapsdeclared] > 1} {
+            set pkginfo [dict get $pkgcapsdeclared $pkg]
+            #remove and re-add at start of dict
+            dict unset pkgcapsdeclared $pkg
+            dict set pkgcapsdeclared $pkg $pkginfo
+            set pkgcapsdeclared [dict merge [dict create $pkg $pkginfo] $pkgcapsdeclared]
+            dict for {cap cap_info} $caps {
+                set cap_pkgs [dict get $cap_info providers]
+                if {$pkg in $cap_pkgs} {
+                    set posn [lsearch $cap_pkgs $pkg]
+                    if {$posn >=0} {
+                        #rewrite package list with pkg at head of list for this capability
+                        set cap_pkgs [lreplace $cap_pkgs $posn $posn]
+                        set cap_pkgs [list $pkg {*}$cap_pkgs]
+                        dict set caps $cap providers $cap_pkgs
+                    }
+                }
+            }
+        }
+    }
+
+    #*** !doctools
+    #[list_end]
+    }
+
 
+#*** !doctools
+#[section Internal]
+
+    namespace eval capsystem {
+        #*** !doctools
+        #[subsection {Namespace punk::cap::capsystem}]
+        #[para] Internal functions used to communicate between punk::cap and capability handlers
+        #[list_begin definitions]
+        proc get_caphandler_registry {capname} {
+            set ns [::punk::cap::capability_get_handler $capname]::capsystem
+            if {[namespace exists ${ns}]} {
+                if {[info command ${ns}::caphandler.registry] ne ""} {
+                    if {[info object isa object ${ns}::caphandler.registry]} {
+                        return ${ns}::caphandler.registry
+                    }
+                }
+            }
+            return ""
+        }
+        #*** !doctools
+        #[list_end]
+    }
 }
 
 
@@ -524,5 +658,4 @@ package provide punk::cap [namespace eval punk::cap {
 return
 
 #*** !doctools
-#[list_end]
 #[manpage_end]

src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/du-0.1.0.tm

@@ -24,15 +24,14 @@ namespace eval punk::du {
     variable has_twapi 0
 }
 if {"windows" eq $::tcl_platform(platform)} {
-    #jmn disable twapi
-    #package require zzzload
-    #zzzload::pkg_require twapi
-
-    #if {[catch {package require twapi}]} {
-    #    puts stderr "Warning: punk::du - unable to load twapi. Disk operations may be much slower on windows without the twapi package"
-    #} else {
-    #    set punk::du::has_twapi 1
-    #}
+    package require zzzload
+    zzzload::pkg_require twapi
+
+    if {[catch {package require twapi}]} {
+        puts stderr "Warning: punk::du - unable to load twapi. Disk operations may be much slower on windows without the twapi package"
+    } else {
+        set punk::du::has_twapi 1
+    }
     #package require punk::winpath
 }
 
@@ -1248,10 +1247,11 @@ namespace eval punk::du {
         proc du_dirlisting_undecided {folderpath args} {
             if {"windows" eq $::tcl_platform(platform)} {
                 #jmn disable twapi
-                tailcall du_dirlisting_generic $folderpath {*}$args                        
+                #tailcall du_dirlisting_generic $folderpath {*}$args                        
 
                 set loadstate [zzzload::pkg_require twapi]
                 if {$loadstate ni [list loading failed]} {
+                    #either already loaded by zzload or ordinary package require
                     package require twapi ;#should be fast once twapi dll loaded in zzzload thread
                     set ::punk::du::has_twapi 1
                     punk::du::active::set_active_function du_dirlisting du_dirlisting_twapi

src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/base-0.1.tm

@@ -3,6 +3,7 @@ package provide punk::mix::base [namespace eval punk::mix::base {
 	set version 0.1
 }]
 
+package require punk::path
 
 #base internal plumbing functions
 namespace eval punk::mix::base {
@@ -364,7 +365,7 @@ namespace eval punk::mix::base {
             set folderdict [dict create]
 
             package require punk::cap
-            if {[punk::cap::has_handler punk.templates]} {
+            if {[punk::cap::capability_has_handler punk.templates]} {
                 set template_folder_dict [punk::cap::call_handler punk.templates folders]
                 dict for {dir folderinfo} $template_folder_dict {
                     dict set folderdict $dir $folderinfo
@@ -632,7 +633,8 @@ namespace eval punk::mix::base {
                 cd $base ;#cd is process-wide.. keep cd in effect for as small a scope as possible. (review for thread issues)
             
                 #temp emission to stdout.. todo - repl telemetry channel 
-                puts stdout "cksum_path: creating temporary tar archive at: $archivename .."
+                puts stdout "cksum_path: creating temporary tar archive for $path"
+                puts stdout "  at: $archivename .."
                 tar::create $archivename $target
                 if {$ftype eq "file"} {
                     set sizeinfo "(size [file size $target])"
@@ -757,7 +759,7 @@ namespace eval punk::mix::base {
                         set normbase [file normalize $base]
                         set normtarg [file normalize [file join $normbase $specifiedpath]]
                         set targetpath $normtarg
-                        set storedpath [punk::mix::util::path_relative $normbase $normtarg]
+                        set storedpath [punk::path::relative $normbase $normtarg]
                     } else {
                         set targetpath [file join $base $specifiedpath]
                         set storedpath $specifiedpath
@@ -776,7 +778,7 @@ namespace eval punk::mix::base {
                         error "get_relativecksum_from_base error: base '$base' and specifiedpath '$specifiedpath' don't share a common root. Use empty-string for base if independent absolute path is required"
                     }
                     set targetpath $specifiedpath 
-                    set storedpath [punk::mix::util::path_relative $base $specifiedpath]
+                    set storedpath [punk::path::relative $base $specifiedpath]
                     
                 }
             } else {

src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/cli-0.3.tm

@@ -477,7 +477,13 @@ namespace eval punk::mix::cli {
 
             set did_skip 0 ;#flag for stdout/stderr formatting only
             foreach m $src_modules {
-                #puts "build_modules_from_source_to_base >>> module $m"
+                set is_interesting 0
+                if {[string match "foobar" $current_source_dir]} {
+                    set  is_interesting 1
+                }
+                if {$is_interesting} {
+                    puts "build_modules_from_source_to_base >>> module $current_source_dir/$m"
+                }
                 set fileparts [split [file rootname $m] -]
                 set tmfile_versionsegment [lindex $fileparts end]
                 if {$tmfile_versionsegment eq $magicversion} {
@@ -582,7 +588,9 @@ namespace eval punk::mix::cli {
                             #set file_record [punkcheck::installfile_finished_install $basedir $file_record]
                             $event targetset_end OK
                         } else {
-                            #puts stdout "skipping module $current_source_dir/$m - no change in sources detected"
+                            if {$is_interesting} {
+                                puts stdout "skipping module $current_source_dir/$m - no change in sources detected"
+                            }
                             puts -nonewline stderr "."
                             set did_skip 1
                             #set file_record [punkcheck::installfile_skipped_install $basedir $file_record]
@@ -627,15 +635,18 @@ namespace eval punk::mix::cli {
                     $event targetset_started
                     # -- --- --- --- --- ---
                     if {$did_skip} {set did_skip 0; puts -nonewline stdout \n}
-                    puts stderr "Copied already versioned module $current_source_dir/$m to $target_module_dir"
                     lappend module_list $current_source_dir/$m
                     file copy -force $current_source_dir/$m $target_module_dir
+                    puts stderr "Copied already versioned module $current_source_dir/$m to $target_module_dir"
                     # -- --- --- --- --- ---
                     #set file_record [punkcheck::installfile_finished_install $basedir $file_record]
                     $event targetset_end OK -note "already versioned module"
                 } else {
                     puts -nonewline stderr "."
                     set did_skip 1
+                    if {$is_interesting} {
+                        puts stderr "$current_source_dir/$m [$event targetset_source_changes]"
+                    }
                     #set file_record [punkcheck::installfile_skipped_install $basedir $file_record]
                     $event targetset_end SKIPPED
                 }

src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/doc-0.1.0.tm

@@ -18,11 +18,11 @@
 ##  Requirements
 ##e.g package require frobz
 
-package require punk ;# for treefilenames
+package require punk::path ;# for treefilenames, relative
 package require punk::repo
 package require punk::docgen ;#inline doctools - generate doctools .man files at src/docgen prior to using kettle to producing .html .md etc
 package require punk::mix::cli  ;#punk::mix::cli::lib used for kettle_call
-package require punk::mix::util ;#for path_relative
+#package require punk::mix::util ;#for path_relative
 
 
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
@@ -44,7 +44,7 @@ namespace eval punk::mix::commandset::doc {
         }
         #user may delete the comment containing "--- punk::docgen::overwrites" and then manually edit, and we won't overwrite
         #we still generate output in src/docgen so user can diff and manually update if thats what they prefer
-        set oldfiles [glob -nocomplain -dir $projectdir/src/doc -type f _module_*]
+        set oldfiles [punk::path::treefilenames $projectdir/src/doc _module_*.man]
         foreach maybedoomed $oldfiles {
             set fd [open $maybedoomed r]
             set data [read $fd]
@@ -57,9 +57,18 @@ namespace eval punk::mix::commandset::doc {
         if {[dict get $generated count] > 0} {
             #review
             set doclist [dict get $generated docs]
+            set source_base [dict get $generated base]
+            set target_base $projectdir/src/doc
             foreach dinfo $doclist {
                 lassign $dinfo module fpath
-                set target $projectdir/src/doc/_module_[file tail $fpath]
+                set relpath [punk::path::relative $source_base $fpath]
+                set relfolder [file dirname $relpath]
+                if {$relfolder eq "."} {
+                    set relfolder ""
+                }
+                file mkdir [file join $target_base $relfolder]
+                set target [file join $target_base $relfolder _module_[file tail $fpath]]
+                puts stderr "target --> $target"
                 if {![file exists $target]} {
                     file copy $fpath $target
                 }
@@ -184,23 +193,24 @@ namespace eval punk::mix::commandset::doc {
         variable pkg
         set pkg punk::mix::commandset::doc
         proc  do_docgen {{project_subpath modules}} {
+            #Extract doctools comments from source code
             set projectdir [punk::repo::find_project]
-            set outdir [file join $projectdir src docgen]
-            set subpath [file join $projectdir $project_subpath]
-            if {![file isdirectory $subpath]} {
-                puts stderr "WARNING punk::mix::commandset::doc unable to find subpath $subpath during do_docgen - skipping inline doctools generation"
+            set output_base [file join $projectdir src docgen]
+            set codesource_path [file join $projectdir $project_subpath]
+            if {![file isdirectory $codesource_path]} {
+                puts stderr "WARNING punk::mix::commandset::doc unable to find codesource_path $codesource_path during do_docgen - skipping inline doctools generation"
                 return
             }
-            if {[file isdirectory $outdir]} {
+            if {[file isdirectory $output_base]} {
                 if {[catch {
-                    file delete -force $outdir
+                    file delete -force $output_base
                 }]}  {
-                    error "do_docgen failed to delete existing $outdir"
+                    error "do_docgen failed to delete existing output base folder: $output_base"
                 }
             }
-            file mkdir $outdir
+            file mkdir $output_base
 
-            set matched_paths [punk::treefilenames $subpath *.tm]
+            set matched_paths [punk::path::treefilenames $codesource_path *.tm -antiglob_paths {**/mix/templates/** **/mixtemplates/**}]
             set count 0
             set newdocs [list] 
             set docgen_header_comments ""
@@ -208,14 +218,34 @@ namespace eval punk::mix::commandset::doc {
             append docgen_header_comments {[comment {--- punk::docgen DO NOT EDIT DOCS HERE UNLESS YOU REMOVE THESE COMMENT LINES ---}]} \n
             append docgen_header_comments {[comment {--- punk::docgen overwrites this file ---}]} \n
             foreach fullpath $matched_paths {
-                set relpath [punk::mix::util::path_relative $subpath $fullpath]
-                set tailsegs [file split $relpath]
-                set module_fullname [join $tailsegs ::]
-                set docname [string map [list :: _] $module_fullname].man ;#todo - something better - need to ensure unique
                 set doctools [punk::docgen::get_doctools_comments $fullpath] 
                 if {$doctools ne ""} {
-                    puts stdout "generating doctools output from file $relpath"
-                    set outfile [file join $outdir $docname]
+                    set fname [file tail $fullpath]
+                    set mod_tail [file rootname $fname] 
+                    set relpath [punk::path::relative $codesource_path [file dirname $fullpath]]
+                    if {$relpath eq "."} {
+                        set relpath ""
+                    }
+                    set tailsegs [file split $relpath]
+                    set module_fullname [join $tailsegs ::]::$mod_tail
+                    set target_docname $fname.man
+                    set this_outdir [file join $output_base $relpath]
+
+                    if {[string length $fname] > 99} {
+                        #output needs to be tarballed to do checksum change tests in a reasonably straightforward and not-too-terribly slow way.
+                        #hack - review. Determine exact limit - test if tcllib tar fixed or if it's a limit of the particular tar format
+                        #work around tcllib tar filename length limit ( somewhere around 100?) This seems to be a limit on the length of a particular segment in the path.. not whole path length?
+                        #this case only came up because docgen used to path munge to long filenames - but left because we know there is a limit and renaming fixes it - even if it's ugly - but still allows doc generation.
+                        #review - if we're checking fname - should also test length of whole path and determine limits for tar
+                        package require md5
+                        set target_docname [md5::md5 -hex $fullpath]_overlongfilename.man
+                        puts stderr "WARNING - overlong file name - renaming $fullpath"
+                        puts stderr "          to                            [file dirname $fullpath]/$target_docname"
+                    }
+
+                    file mkdir $this_outdir
+                    puts stdout "saving [string length $doctools] bytes of doctools output from file $relpath/$fname"
+                    set outfile [file join $this_outdir $target_docname]
                     set fd [open $outfile w]
                     fconfigure $fd -translation binary
                     puts -nonewline $fd $docgen_header_comments$doctools
@@ -224,7 +254,7 @@ namespace eval punk::mix::commandset::doc {
                     lappend newdocs [list $module_fullname $outfile]
                 }
             }
-            return [list count $count docs $newdocs]
+            return [list count $count docs $newdocs base $output_base]
         }
 
     }

src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/layout-0.1.0.tm

@@ -141,7 +141,10 @@ namespace eval punk::mix::commandset::layout {
             #use last matching layout found.  review silent if multiple?
             if {![llength $tags]} {
                 #todo - get standard tags from somewhere
-                set tags [list %project%]
+                set tagnames [list project]
+                foreach tn $tagnames {
+                    lappend tags [string cat % $tn %]
+                }
             }
             set file_list [list] 
             util::foreach-file $layoutfolder path {

src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/commandset/project-0.1.0.tm

@@ -13,10 +13,72 @@
 # @@ Meta End
 
 
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+# doctools header
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+#*** !doctools
+#[manpage_begin punk::mix::commandset::project 0 0.1.0]
+#[copyright "2023"]
+#[titledesc {pmix commandset - project}]  [comment {-- Name section and table of contents description --}]
+#[moddesc {pmix CLI commandset - project}]             [comment {-- Description at end of page heading --}] 
+#[require punk::mix::commandset::project]
+#[description]
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[section Overview]
+#[para] overview of punk::mix::commandset::project
+#[para]Import into an ensemble namespace similarly to the way it is done with punk::mix::cli e.g 
+#[example {
+#    namespace eval myproject::cli {
+#        namespace export *
+#        namespace ensemble create
+#        package require punk::overlay
+#    
+#        package require punk::mix::commandset::project
+#        punk::overlay::import_commandset project    .       ::punk::mix::commandset::project
+#        punk::overlay::import_commandset projects   .       ::punk::mix::commandset::project::collection 
+#    }
+#}]
+#[para] Where the . in the above example is the prefix/command separator
+#[para]The prefix ('project' in the above example) can be any string desired to disambiguate commands imported from other commandsets.
+#[para]The above results in the availability of the ensemble command: ::myproject::cli project.new, which is implemented in ::punk::mix::commandset::project::new  
+#[para]Similarly, procs under ::punk::mix::commandset::project::collection will be available as subcommands of the ensemble as projects.<procname>
+#[para]
+#[subsection Concepts]
+#[para] see punk::overlay 
+
 
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 ##  Requirements
-##e.g package require frobz
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[subsection dependencies]
+#[para] packages used by punk::mix::commandset::project 
+#[list_begin itemized]
+
+package require Tcl 8.6
+#*** !doctools
+#[item] [package {Tcl 8.6}]
+#[item] [package punk::ns]
+#[item] [package sqlite3] (binary)
+#[item] [package overtype]
+#[item] [package textutil] (tcllib)
+
+
+# #package require frobz
+# #*** !doctools
+# #[item] [package {frobz}]
+
+#*** !doctools
+#[list_end]
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[section API]
 
 
 
@@ -24,10 +86,36 @@
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 namespace eval punk::mix::commandset::project {
     namespace export *
+    #*** !doctools
+    #[subsection {Namespace punk::mix::commandset::project}]
+    #[para] core commandset functions for punk::mix::commandset::project 
+    #[list_begin definitions]
+
+    proc _default {} {
+        package require punk::ns
+        set dispatched_to [lindex [info level 2] 0] ;#e.g ::punk::mix::cli::project
+        set dispatch_tail [punk::ns::nstail $dispatched_to]
+        set dispatch_ensemble [punk::ns::nsprefix $dispatched_to] ;#e.g ::punk::mix::cli
+        set sibling_commands [namespace eval $dispatch_ensemble {namespace export}]
+        #todo - get separator?
+        set sep "."
+        set result [list]
+        foreach sib $sibling_commands {
+            if {[string match ${dispatch_tail}${sep}* $sib]} {
+                lappend result $sib
+            }
+        }
+        return [lsort $result]
+    }
+
+
+
 
-    #new project structure - may be dedicated to one module, or contain many. 
-    #create minimal folder structure only by specifying  -modules {}
     proc new {newprojectpath_or_name args} {
+        #*** !doctools
+        # [call [fun new] [arg newprojectpath_or_name] [opt args]]
+        #new project structure - may be dedicated to one module, or contain many. 
+        #create minimal folder structure only by specifying in args:  -modules {}
         if {[file pathtype $newprojectpath_or_name] eq "absolute"} {
             set projectfullpath [file normalize $newprojectpath_or_name]
             set projectname [file tail $projectfullpath]
@@ -242,7 +330,7 @@ namespace eval punk::mix::commandset::project {
         set layout_dir $templatebase/layouts/$opt_layout
         puts stdout ">>> about to call punkcheck::install $layout_dir $projectdir" 
         set resultdict [dict create]
-        set unpublish [list\
+        set antipaths [list\
             src/doc/*\
             src/doc/include/*\
         ]
@@ -250,11 +338,11 @@ namespace eval punk::mix::commandset::project {
         #default antiglob_dir_core will stop .fossil* from being updated - which is generally desirable as these are likely to be customized
         if {$opt_force} {
             puts stdout "copying layout files - with force applied - overwrite all-targets" 
-            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new  -overwrite ALL-TARGETS -unpublish_paths $unpublish]
+            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new  -overwrite ALL-TARGETS -antiglob_paths $antipaths]
             #file copy -force $layout_dir $projectdir
         } else {
             puts stdout "copying layout files - (if source file changed)" 
-            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -overwrite installedsourcechanged-targets -unpublish_paths $unpublish]
+            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -overwrite installedsourcechanged-targets -antiglob_paths $antipaths]
         }
         puts stdout [punkcheck::summarize_install_resultdict $resultdict]
 
@@ -293,7 +381,7 @@ namespace eval punk::mix::commandset::project {
             set fpath [file join $projectdir $templatetail]
             if {[file exists $fpath]} {
                 set fd [open $fpath r]; fconfigure $fd -translation binary; set data [read $fd]; close $fd
-                set data2 [string map [list %project% $projectname] $data]
+                set data2 [string map [list [lib::template_tag project] $projectname] $data]
                 if {$data2 ne $data} {
                     puts stdout "updated template file: $fpath"
                     set fdout [open $fpath w]; fconfigure $fdout -translation binary; puts -nonewline $fdout $data2; close $fdout 
@@ -406,8 +494,10 @@ namespace eval punk::mix::commandset::project {
         puts stdout "-done- project:$projectname projectdir: $projectdir"
     }
 
+    #*** !doctools
+    #[list_end] [comment {--- end definitions namespace punk::mix::commandset::project ---}]
 
-     namespace eval collection {
+    namespace eval collection {
         namespace export *
         namespace path [namespace parent]
 
@@ -764,6 +854,12 @@ namespace eval punk::mix::commandset::project {
     }
     
     namespace eval lib {
+        proc template_tag {tagname} {
+            #todo - support different tagwrappers - it shouldn't be so likely to collide with common code idioms etc.
+            #we need to detect presence of tags intended for punk::mix system
+            #consider using punk::cap to enable multiple template-substitution providers with their own set of tagnames and/or tag wrappers, where substitution providers are all run 
+            return [string cat % $tagname %]
+        }
         #get project info only by opening the central confg-db
         #(will not have proper project-name etc)
         proc get_projects {{globlist {}} args} {
@@ -828,9 +924,8 @@ namespace eval punk::mix::commandset::project {
 }
 
 
-
-
-
+#*** !doctools
+#[manpage_end]
 
 
 

src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/mix/util-0.1.0.tm

@@ -131,83 +131,6 @@ namespace eval punk::mix::util {
     }
     #----------------------------------------
 
-    #maint warning - also in punkcheck
-    proc path_relative {base dst} {
-        #see also kettle
-        # Modified copy of ::fileutil::relative (tcllib)
-        # Adapted to 8.5 ({*}).
-        #
-        #    Taking two _directory_ paths, a base and a destination, computes the path
-        #    of the destination relative to the base.
-        #
-        # Arguments:
-        #    base    The path to make the destination relative to.
-        #    dst     The destination path
-        #
-        # Results:
-        #    The path of the destination, relative to the base.
-
-        # Ensure that the link to directory 'dst' is properly done relative to
-        # the directory 'base'.
-
-        #review - check volume info on windows.. UNC paths?
-        if {[file pathtype $base] ne [file pathtype $dst]} {
-            return -code error "Unable to compute relation for paths of different pathtypes: [file pathtype $base] vs. [file pathtype $dst], ($base vs. $dst)"
-        }
-
-        #avoid normalizing if possible (file normalize *very* expensive on windows)
-        set do_normalize 0
-        if {[file pathtype $base] eq "relative"} {
-            #if base is relative so is dst
-            if {[regexp {[.]{2}} [list $base $dst]]} {
-                set do_normalize 1
-            }
-            if {[regexp {[.]/} [list $base $dst]]} {
-                set do_normalize 1
-            }
-        } else {
-            set do_normalize 1
-        }
-        if {$do_normalize} {
-            set base [file normalize $base]
-            set dst  [file normalize $dst]
-        }
-
-        set save $dst
-        set base [file split $base]
-        set dst  [file split $dst]
-
-        while {[lindex $dst 0] eq [lindex $base 0]} {
-            set dst  [lrange $dst  1 end]
-            set base [lrange $base 1 end]
-            if {![llength $dst]} {break}
-        }
-
-        set dstlen  [llength $dst]
-        set baselen [llength $base]
-
-        if {($dstlen == 0) && ($baselen == 0)} {
-            # Cases:
-            # (a) base == dst
-
-            set dst .
-            } else {
-            # Cases:
-            # (b) base is: base/sub = sub
-            #     dst  is: base     = {}
-
-            # (c) base is: base     = {}
-            #     dst  is: base/sub = sub
-
-            while {$baselen > 0} {
-                set dst [linsert $dst 0 ..]
-                incr baselen -1
-            }
-            set dst [file join {*}$dst]
-        }
-
-        return $dst
-    }
     #namespace import ::punk::ns::nsimport_noclobber 
 
     proc namespace_import_pattern_to_namespace_noclobber {pattern ns} {

src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/overlay-0.1.tm

@@ -132,6 +132,7 @@ namespace eval ::punk::overlay {
         set imported_commands [list]
         set nscaller [uplevel 1 [list namespace current]]
         if {[catch {
+            #review - noclobber?
             namespace eval ${nscaller}::temp_import [list namespace import ${cmdnamespace}::*]
             foreach cmd [info commands ${nscaller}::temp_import::*] {
                 set cmdtail [namespace tail $cmd]

src/mixtemplates/layouts/basic/src/bootsupport/modules/punk/repo-0.1.1.tm

@@ -33,8 +33,9 @@ if {$::tcl_platform(platform) eq "windows"} {
     catch {package require punk::winpath}
 }
 package require fileutil; #tcllib
+package require punk::path
 package require punk::mix::base ;#uses core functions from punk::mix::base::lib namespace e.g cksum_path 
-package require punk::mix::util
+package require punk::mix::util ;#do_in_path
 
 
 # -- --- --- --- --- --- --- --- --- --- ---
@@ -357,7 +358,7 @@ namespace eval punk::repo {
         if {$repodir eq ""} {
             error "workingdir_state error: No repository found at or above path '$abspath'" 
         }
-        set subpath [punk::mix::util::path_relative $repodir $abspath]
+        set subpath [punk::path::relative $repodir $abspath]
         if {$subpath eq "."} {
             set subpath ""
         }
@@ -708,10 +709,10 @@ namespace eval punk::repo {
                 append msg "** starting folder : $start_dir" \n
                 append msg "** untracked       : $candroot" \n
                 if {$closest_fossil_len} {
-                    append msg "** fossil root     : $closest_fossil    ([punk::mix::util::path_relative $start_dir $closest_fossil])" \n
+                    append msg "** fossil root     : $closest_fossil    ([punk::path::relative $start_dir $closest_fossil])" \n
                 }
                 if {$closest_git_len} {
-                    append msg "** git root        : $closest_git    ([punk::mix::util::path_relative $start_dir $closest_git])" \n
+                    append msg "** git root        : $closest_git    ([punk::path::relative $start_dir $closest_git])" \n
                 }
                 append msg "**" \n
         }

src/mixtemplates/layouts/basic/src/bootsupport/modules/punkcheck-0.1.0.tm

@@ -19,6 +19,7 @@
 ##e.g package require frobz
 
 package require punk::tdl
+package require punk::path
 package require punk::repo
 package require punk::mix::util
 
@@ -1106,29 +1107,6 @@ namespace eval punkcheck {
                 return [lindex $args end] 
             }
         }
-        proc pathglob_as_re {glob} {
-            #any segment that is not just * must match exactly one segment in the path
-            set pats [list]
-            foreach seg [file split $glob] {
-                if {$seg eq "*"} {
-                    lappend pats {[^/]*}
-                } elseif {$seg eq "**"} {
-                    lappend pats {.*}
-                } else {
-                    set seg [string map [list  . {[.]}] $seg]
-                    if {[regexp {[*?]} $seg]} {
-                        set pat [string map [list * {[^/]*} ? {[^/]}] $seg]
-                        lappend pats "$pat"
-                    } else {
-                        lappend pats "$seg"
-                    }
-                }
-            }
-            return  "^[join $pats /]\$"
-        }
-        proc globmatchpath {glob path} {
-            return [regexp [pathglob_as_re $glob] $path]
-        }
         ## unidirectional file transfer to possibly non empty folder
         #default of -overwrite no-targets will only copy files that are missing at the target
         #           -overwrite newer-targets will copy files with older source timestamp over newer target timestamp and those missing at the target (a form of 'restore' operation)
@@ -1349,7 +1327,7 @@ namespace eval punkcheck {
             }
             foreach unpub $opt_antiglob_paths {
                 #puts "testing folder - globmatchpath $unpub $relative_source_dir"
-                if {[globmatchpath $unpub $relative_source_dir]} {
+                if {[punk::path::globmatchpath $unpub $relative_source_dir]} {
                     lappend antiglob_paths_matched $current_source_dir
                     return [list files_copied {} files_skipped {} sources_unchanged {} punkcheck_records $punkcheck_records antiglob_paths_matched $antiglob_paths_matched srcdir $srcdir tgtdir $tgtdir punkcheck_folder $punkcheck_folder]
                 }
@@ -1421,7 +1399,7 @@ namespace eval punkcheck {
                 set is_antipath 0
                 foreach antipath $opt_antiglob_paths {
                     #puts "testing file - globmatchpath $antipath  vs $relative_source_path" 
-                    if {[globmatchpath $antipath $relative_source_path]} {
+                    if {[punk::path::globmatchpath $antipath $relative_source_path]} {
                         lappend antiglob_paths_matched $current_source_dir
                         set is_antipath 1
                         break

src/modules/punk-0.1.tm

@@ -5172,23 +5172,6 @@ namespace eval punk {
         return [dirfiles_dict_as_lines -stripbase 1 $contents]
     }
 
-    #basic (glob based) list of filenames matching tailglob - recursive - no natsorting
-    #todo - implement treefiles which acts like dirfiles but allows path globbing in the same way as punk::ns::ns/ 
-    #then review if treefiles can replace dirfiles or if both should exist (dirfiles can have literal glob chars in path segments - but that is a rare usecase)
-    proc treefilenames {basepath {tailglob *}} {
-        set files [list]
-        if {![file isdirectory $basepath]} {
-            return $files
-        }
-        #todo - account for vfs where matched path could appear to be a directory but is mounted so could be a desired match?
-        set dirfiles [glob -nocomplain -dir $basepath -type f $tailglob] 
-        lappend files  {*}$dirfiles
-        set dirdirs [glob -nocomplain -dir $basepath -type d *]
-        foreach dir $dirdirs {
-            lappend files {*}[treefilenames $dir $tailglob]
-        }
-        return $files
-    }
 
     #dirfiles dirfiles_dict always deliberately return absolute *unnormalized* path
     #e.g when cwd is c:/repo/jn/shellspy  dirfiles ../../  will return something like:

src/modules/punk/cap-999999.0a1.0.tm

@@ -62,18 +62,19 @@ namespace eval punk::cap {
 
             oo::class create [namespace current]::interface_caphandler.registry {
                 #*** !doctools
-                #[enum] [emph {CLASS interface_caphandler.registry}]
+                #[enum] CLASS [class interface_caphandler.registry]
                 #[list_begin definitions]
+                #    [para] [emph METHODS]
                 method pkg_register {pkg capname capdict fullcapabilitylist} {
                     #*** !doctools
-                    #[call class::[class interface_caphandler.registry] [method pkg_register] [arg pkg] [arg capname] [arg capdict] [arg fullcapabilitylist]]
+                    #[call class::interface_caphandler.registry [method pkg_register] [arg pkg] [arg capname] [arg capdict] [arg fullcapabilitylist]]
                     #handler may override and return 0 (indicating don't register)e.g if pkg capdict data wasn't valid
                     #overridden handler must be able to handle multiple calls for same pkg - but it may return 1 or 0 as it wishes. 
                     return 1 ;#default to permit 
                 }
                 method pkg_unregister {pkg} {
                     #*** !doctools
-                    #[call class::[class interface_caphandler.registry] [method pkg_unregister] [arg pkg]]
+                    #[call class::interface_caphandler.registry [method pkg_unregister] [arg pkg]]
                     return ;#unregistration return is ignored - review
                 }
                 #*** !doctools
@@ -82,8 +83,9 @@ namespace eval punk::cap {
 
             oo::class create [namespace current]::interface_caphandler.sysapi {
                 #*** !doctools
-                #[enum] [emph {CLASS interface_caphandler.sysapi}]
+                #[enum] CLASS [class interface_caphandler.sysapi]
                 #[list_begin definitions]
+                #    [para] [emph METHODS]
 
 
                 #*** !doctools
@@ -101,8 +103,8 @@ namespace eval punk::cap {
             #Provider classes
             oo::class create [namespace current]::interface_capprovider.registration {
                 #*** !doctools
-                #    [enum] [emph {CLASS interface_cappprovider.registration}]
-                #    Your provider package will need to instantiate this object under a sub-namespace called [namespace capsystem] within your package namespace. 
+                #    [enum] CLASS [class interface_cappprovider.registration]
+                #    [para]Your provider package will need to instantiate this object under a sub-namespace called [namespace capsystem] within your package namespace. 
                 #    [para]If your package namespace is mypackages::providerpkg then the object command would be at mypackages::providerpkg::capsystem::capprovider.registration
                 #    [para]Example code for your provider package to evaluate within its namespace: 
                 #    [example {
@@ -122,9 +124,10 @@ namespace eval punk::cap {
                 #}]
                 #[para] The above example declares that your package can be registered as a provider for the capabilities named 'punk.templates' and 'another_capability_name'
                 #    [list_begin definitions]
+                #    [para] [emph METHODS]
                 method get_declarations {} {
                     #***
-                    #[call class::[class interface_capprovider.registration] [method get_declarations]]
+                    #[call class::interface_capprovider.registration [method get_declarations]]
                     #[para] This method must be overridden by your provider using oo::objdefine cappprovider.registration as in the example above.
                     # There must be at least one 2-element list in the result for the provider to be registerable.
                     #[para]The first element of the list is the capabilityname - which can be custom to your provider/handler packages - or a well-known name that other authors may use/implement.
@@ -137,7 +140,7 @@ namespace eval punk::cap {
 
             oo::class create [namespace current]::interface_capprovider.provider {
                 #*** !doctools
-                #    [enum] [emph {CLASS interface_capprovider.provider}] 
+                #    [enum] CLASS [class interface_capprovider.provider] 
                 #    [para] Your provider package will need to instantiate this directly under it's own namespace with the command name of [emph {provider}] 
                 #    [example {
                 #       namespace eval mypackages::providerpkg { 
@@ -145,11 +148,12 @@ namespace eval punk::cap {
                 #       }
                 #    }]
                 #    [list_begin definitions]
+                #    [para] [emph METHODS]
                 variable provider_pkg
                 variable registrationobj
                 constructor {providerpkg} {
                     #*** !doctools
-                    #[call class::[class interface_capprovider.provider] [method constructor] [arg providerpkg]]
+                    #[call class::interface_capprovider.provider [method constructor] [arg providerpkg]]
                     variable provider_pkg
                     if {$providerpkg in [list "" "::"]} {
                         error "interface_capprovider.provider constructor error. Invalid provider '$providerpkg'"
@@ -169,7 +173,7 @@ namespace eval punk::cap {
                 method register {{capabilityname_glob *}} {
                     #*** !doctools
                     #[comment {- -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---}]
-                    #[call class::[class interface_capprovider.provider] [method register] [opt capabilityname_glob]]
+                    #[call class::interface_capprovider.provider [method register] [opt capabilityname_glob]]
                     #
                     #[para]This is the mechanism by which a user of your provider package will register your package as a provider of the capability named.
                     #
@@ -192,7 +196,7 @@ namespace eval punk::cap {
                 method capabilities {} {
                     #*** !doctools
                     #[comment {- -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---}]
-                    #[call class::[class interface_capprovider.provider] [method capabilities]]
+                    #[call class::interface_capprovider.provider [method capabilities]]
                     #[para] return a list of capabilities supported by this provider package
                     variable provider_pkg
                     variable registrationobj

src/modules/punk/du-999999.0a1.0.tm

@@ -24,15 +24,14 @@ namespace eval punk::du {
     variable has_twapi 0
 }
 if {"windows" eq $::tcl_platform(platform)} {
-    #jmn disable twapi
-    #package require zzzload
-    #zzzload::pkg_require twapi
-
-    #if {[catch {package require twapi}]} {
-    #    puts stderr "Warning: punk::du - unable to load twapi. Disk operations may be much slower on windows without the twapi package"
-    #} else {
-    #    set punk::du::has_twapi 1
-    #}
+    package require zzzload
+    zzzload::pkg_require twapi
+
+    if {[catch {package require twapi}]} {
+        puts stderr "Warning: punk::du - unable to load twapi. Disk operations may be much slower on windows without the twapi package"
+    } else {
+        set punk::du::has_twapi 1
+    }
     #package require punk::winpath
 }
 
@@ -1248,10 +1247,11 @@ namespace eval punk::du {
         proc du_dirlisting_undecided {folderpath args} {
             if {"windows" eq $::tcl_platform(platform)} {
                 #jmn disable twapi
-                tailcall du_dirlisting_generic $folderpath {*}$args                        
+                #tailcall du_dirlisting_generic $folderpath {*}$args                        
 
                 set loadstate [zzzload::pkg_require twapi]
                 if {$loadstate ni [list loading failed]} {
+                    #either already loaded by zzload or ordinary package require
                     package require twapi ;#should be fast once twapi dll loaded in zzzload thread
                     set ::punk::du::has_twapi 1
                     punk::du::active::set_active_function du_dirlisting du_dirlisting_twapi

src/modules/punk/mix/base-0.1.tm

@@ -3,6 +3,7 @@ package provide punk::mix::base [namespace eval punk::mix::base {
 	set version 0.1
 }]
 
+package require punk::path
 
 #base internal plumbing functions
 namespace eval punk::mix::base {
@@ -632,7 +633,8 @@ namespace eval punk::mix::base {
                 cd $base ;#cd is process-wide.. keep cd in effect for as small a scope as possible. (review for thread issues)
             
                 #temp emission to stdout.. todo - repl telemetry channel 
-                puts stdout "cksum_path: creating temporary tar archive at: $archivename .."
+                puts stdout "cksum_path: creating temporary tar archive for $path"
+                puts stdout "  at: $archivename .."
                 tar::create $archivename $target
                 if {$ftype eq "file"} {
                     set sizeinfo "(size [file size $target])"
@@ -757,7 +759,7 @@ namespace eval punk::mix::base {
                         set normbase [file normalize $base]
                         set normtarg [file normalize [file join $normbase $specifiedpath]]
                         set targetpath $normtarg
-                        set storedpath [punk::mix::util::path_relative $normbase $normtarg]
+                        set storedpath [punk::path::relative $normbase $normtarg]
                     } else {
                         set targetpath [file join $base $specifiedpath]
                         set storedpath $specifiedpath
@@ -776,7 +778,7 @@ namespace eval punk::mix::base {
                         error "get_relativecksum_from_base error: base '$base' and specifiedpath '$specifiedpath' don't share a common root. Use empty-string for base if independent absolute path is required"
                     }
                     set targetpath $specifiedpath 
-                    set storedpath [punk::mix::util::path_relative $base $specifiedpath]
+                    set storedpath [punk::path::relative $base $specifiedpath]
                     
                 }
             } else {

src/modules/punk/mix/commandset/doc-999999.0a1.0.tm

@@ -18,11 +18,11 @@
 ##  Requirements
 ##e.g package require frobz
 
-package require punk ;# for treefilenames
+package require punk::path ;# for treefilenames, relative
 package require punk::repo
 package require punk::docgen ;#inline doctools - generate doctools .man files at src/docgen prior to using kettle to producing .html .md etc
 package require punk::mix::cli  ;#punk::mix::cli::lib used for kettle_call
-package require punk::mix::util ;#for path_relative
+#package require punk::mix::util ;#for path_relative
 
 
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
@@ -44,7 +44,7 @@ namespace eval punk::mix::commandset::doc {
         }
         #user may delete the comment containing "--- punk::docgen::overwrites" and then manually edit, and we won't overwrite
         #we still generate output in src/docgen so user can diff and manually update if thats what they prefer
-        set oldfiles [glob -nocomplain -dir $projectdir/src/doc -type f _module_*]
+        set oldfiles [punk::path::treefilenames $projectdir/src/doc _module_*.man]
         foreach maybedoomed $oldfiles {
             set fd [open $maybedoomed r]
             set data [read $fd]
@@ -57,9 +57,18 @@ namespace eval punk::mix::commandset::doc {
         if {[dict get $generated count] > 0} {
             #review
             set doclist [dict get $generated docs]
+            set source_base [dict get $generated base]
+            set target_base $projectdir/src/doc
             foreach dinfo $doclist {
                 lassign $dinfo module fpath
-                set target $projectdir/src/doc/_module_[file tail $fpath]
+                set relpath [punk::path::relative $source_base $fpath]
+                set relfolder [file dirname $relpath]
+                if {$relfolder eq "."} {
+                    set relfolder ""
+                }
+                file mkdir [file join $target_base $relfolder]
+                set target [file join $target_base $relfolder _module_[file tail $fpath]]
+                puts stderr "target --> $target"
                 if {![file exists $target]} {
                     file copy $fpath $target
                 }
@@ -184,23 +193,24 @@ namespace eval punk::mix::commandset::doc {
         variable pkg
         set pkg punk::mix::commandset::doc
         proc  do_docgen {{project_subpath modules}} {
+            #Extract doctools comments from source code
             set projectdir [punk::repo::find_project]
-            set outdir [file join $projectdir src docgen]
-            set subpath [file join $projectdir $project_subpath]
-            if {![file isdirectory $subpath]} {
-                puts stderr "WARNING punk::mix::commandset::doc unable to find subpath $subpath during do_docgen - skipping inline doctools generation"
+            set output_base [file join $projectdir src docgen]
+            set codesource_path [file join $projectdir $project_subpath]
+            if {![file isdirectory $codesource_path]} {
+                puts stderr "WARNING punk::mix::commandset::doc unable to find codesource_path $codesource_path during do_docgen - skipping inline doctools generation"
                 return
             }
-            if {[file isdirectory $outdir]} {
+            if {[file isdirectory $output_base]} {
                 if {[catch {
-                    file delete -force $outdir
+                    file delete -force $output_base
                 }]}  {
-                    error "do_docgen failed to delete existing $outdir"
+                    error "do_docgen failed to delete existing output base folder: $output_base"
                 }
             }
-            file mkdir $outdir
+            file mkdir $output_base
 
-            set matched_paths [punk::treefilenames $subpath *.tm]
+            set matched_paths [punk::path::treefilenames $codesource_path *.tm -antiglob_paths {**/mix/templates/** **/mixtemplates/**}]
             set count 0
             set newdocs [list] 
             set docgen_header_comments ""
@@ -208,14 +218,34 @@ namespace eval punk::mix::commandset::doc {
             append docgen_header_comments {[comment {--- punk::docgen DO NOT EDIT DOCS HERE UNLESS YOU REMOVE THESE COMMENT LINES ---}]} \n
             append docgen_header_comments {[comment {--- punk::docgen overwrites this file ---}]} \n
             foreach fullpath $matched_paths {
-                set relpath [punk::mix::util::path_relative $subpath $fullpath]
-                set tailsegs [file split $relpath]
-                set module_fullname [join $tailsegs ::]
-                set docname [string map [list :: _] $module_fullname].man ;#todo - something better - need to ensure unique
                 set doctools [punk::docgen::get_doctools_comments $fullpath] 
                 if {$doctools ne ""} {
-                    puts stdout "generating doctools output from file $relpath"
-                    set outfile [file join $outdir $docname]
+                    set fname [file tail $fullpath]
+                    set mod_tail [file rootname $fname] 
+                    set relpath [punk::path::relative $codesource_path [file dirname $fullpath]]
+                    if {$relpath eq "."} {
+                        set relpath ""
+                    }
+                    set tailsegs [file split $relpath]
+                    set module_fullname [join $tailsegs ::]::$mod_tail
+                    set target_docname $fname.man
+                    set this_outdir [file join $output_base $relpath]
+
+                    if {[string length $fname] > 99} {
+                        #output needs to be tarballed to do checksum change tests in a reasonably straightforward and not-too-terribly slow way.
+                        #hack - review. Determine exact limit - test if tcllib tar fixed or if it's a limit of the particular tar format
+                        #work around tcllib tar filename length limit ( somewhere around 100?) This seems to be a limit on the length of a particular segment in the path.. not whole path length?
+                        #this case only came up because docgen used to path munge to long filenames - but left because we know there is a limit and renaming fixes it - even if it's ugly - but still allows doc generation.
+                        #review - if we're checking fname - should also test length of whole path and determine limits for tar
+                        package require md5
+                        set target_docname [md5::md5 -hex $fullpath]_overlongfilename.man
+                        puts stderr "WARNING - overlong file name - renaming $fullpath"
+                        puts stderr "          to                            [file dirname $fullpath]/$target_docname"
+                    }
+
+                    file mkdir $this_outdir
+                    puts stdout "saving [string length $doctools] bytes of doctools output from file $relpath/$fname"
+                    set outfile [file join $this_outdir $target_docname]
                     set fd [open $outfile w]
                     fconfigure $fd -translation binary
                     puts -nonewline $fd $docgen_header_comments$doctools
@@ -224,7 +254,7 @@ namespace eval punk::mix::commandset::doc {
                     lappend newdocs [list $module_fullname $outfile]
                 }
             }
-            return [list count $count docs $newdocs]
+            return [list count $count docs $newdocs base $output_base]
         }
 
     }

src/modules/punk/mix/commandset/layout-999999.0a1.0.tm

@@ -141,7 +141,10 @@ namespace eval punk::mix::commandset::layout {
             #use last matching layout found.  review silent if multiple?
             if {![llength $tags]} {
                 #todo - get standard tags from somewhere
-                set tags [list %project%]
+                set tagnames [list project]
+                foreach tn $tagnames {
+                    lappend tags [string cat % $tn %]
+                }
             }
             set file_list [list] 
             util::foreach-file $layoutfolder path {

src/modules/punk/mix/commandset/project-999999.0a1.0.tm

@@ -13,10 +13,72 @@
 # @@ Meta End
 
 
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+# doctools header
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+#*** !doctools
+#[manpage_begin punk::mix::commandset::project 0 999999.0a1.0]
+#[copyright "2023"]
+#[titledesc {pmix commandset - project}]  [comment {-- Name section and table of contents description --}]
+#[moddesc {pmix CLI commandset - project}]             [comment {-- Description at end of page heading --}] 
+#[require punk::mix::commandset::project]
+#[description]
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[section Overview]
+#[para] overview of punk::mix::commandset::project
+#[para]Import into an ensemble namespace similarly to the way it is done with punk::mix::cli e.g 
+#[example {
+#    namespace eval myproject::cli {
+#        namespace export *
+#        namespace ensemble create
+#        package require punk::overlay
+#    
+#        package require punk::mix::commandset::project
+#        punk::overlay::import_commandset project    .       ::punk::mix::commandset::project
+#        punk::overlay::import_commandset projects   .       ::punk::mix::commandset::project::collection 
+#    }
+#}]
+#[para] Where the . in the above example is the prefix/command separator
+#[para]The prefix ('project' in the above example) can be any string desired to disambiguate commands imported from other commandsets.
+#[para]The above results in the availability of the ensemble command: ::myproject::cli project.new, which is implemented in ::punk::mix::commandset::project::new  
+#[para]Similarly, procs under ::punk::mix::commandset::project::collection will be available as subcommands of the ensemble as projects.<procname>
+#[para]
+#[subsection Concepts]
+#[para] see punk::overlay 
+
 
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 ##  Requirements
-##e.g package require frobz
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[subsection dependencies]
+#[para] packages used by punk::mix::commandset::project 
+#[list_begin itemized]
+
+package require Tcl 8.6
+#*** !doctools
+#[item] [package {Tcl 8.6}]
+#[item] [package punk::ns]
+#[item] [package sqlite3] (binary)
+#[item] [package overtype]
+#[item] [package textutil] (tcllib)
+
+
+# #package require frobz
+# #*** !doctools
+# #[item] [package {frobz}]
+
+#*** !doctools
+#[list_end]
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[section API]
 
 
 
@@ -24,10 +86,36 @@
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 namespace eval punk::mix::commandset::project {
     namespace export *
+    #*** !doctools
+    #[subsection {Namespace punk::mix::commandset::project}]
+    #[para] core commandset functions for punk::mix::commandset::project 
+    #[list_begin definitions]
+
+    proc _default {} {
+        package require punk::ns
+        set dispatched_to [lindex [info level 2] 0] ;#e.g ::punk::mix::cli::project
+        set dispatch_tail [punk::ns::nstail $dispatched_to]
+        set dispatch_ensemble [punk::ns::nsprefix $dispatched_to] ;#e.g ::punk::mix::cli
+        set sibling_commands [namespace eval $dispatch_ensemble {namespace export}]
+        #todo - get separator?
+        set sep "."
+        set result [list]
+        foreach sib $sibling_commands {
+            if {[string match ${dispatch_tail}${sep}* $sib]} {
+                lappend result $sib
+            }
+        }
+        return [lsort $result]
+    }
+
+
+
 
-    #new project structure - may be dedicated to one module, or contain many. 
-    #create minimal folder structure only by specifying  -modules {}
     proc new {newprojectpath_or_name args} {
+        #*** !doctools
+        # [call [fun new] [arg newprojectpath_or_name] [opt args]]
+        #new project structure - may be dedicated to one module, or contain many. 
+        #create minimal folder structure only by specifying in args:  -modules {}
         if {[file pathtype $newprojectpath_or_name] eq "absolute"} {
             set projectfullpath [file normalize $newprojectpath_or_name]
             set projectname [file tail $projectfullpath]
@@ -293,7 +381,7 @@ namespace eval punk::mix::commandset::project {
             set fpath [file join $projectdir $templatetail]
             if {[file exists $fpath]} {
                 set fd [open $fpath r]; fconfigure $fd -translation binary; set data [read $fd]; close $fd
-                set data2 [string map [list %project% $projectname] $data]
+                set data2 [string map [list [lib::template_tag project] $projectname] $data]
                 if {$data2 ne $data} {
                     puts stdout "updated template file: $fpath"
                     set fdout [open $fpath w]; fconfigure $fdout -translation binary; puts -nonewline $fdout $data2; close $fdout 
@@ -406,8 +494,10 @@ namespace eval punk::mix::commandset::project {
         puts stdout "-done- project:$projectname projectdir: $projectdir"
     }
 
+    #*** !doctools
+    #[list_end] [comment {--- end definitions namespace punk::mix::commandset::project ---}]
 
-     namespace eval collection {
+    namespace eval collection {
         namespace export *
         namespace path [namespace parent]
 
@@ -764,6 +854,12 @@ namespace eval punk::mix::commandset::project {
     }
     
     namespace eval lib {
+        proc template_tag {tagname} {
+            #todo - support different tagwrappers - it shouldn't be so likely to collide with common code idioms etc.
+            #we need to detect presence of tags intended for punk::mix system
+            #consider using punk::cap to enable multiple template-substitution providers with their own set of tagnames and/or tag wrappers, where substitution providers are all run 
+            return [string cat % $tagname %]
+        }
         #get project info only by opening the central confg-db
         #(will not have proper project-name etc)
         proc get_projects {{globlist {}} args} {
@@ -828,9 +924,8 @@ namespace eval punk::mix::commandset::project {
 }
 
 
-
-
-
+#*** !doctools
+#[manpage_end]
 
 
 

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/cap-0.1.0.tm

@@ -17,11 +17,27 @@
 #*** !doctools
 #[manpage_begin punk::cap 0 0.1.0]
 #[copyright "2023 JMNoble - BSD licensed"]
-#[titledesc {Module API}]
+#[titledesc {capability provider and handler plugin system}]
 #[moddesc {punk capabilities plugin system}]
 #[require punk::cap]
 #[description]
-#[list_begin definitions]
+#[section Overview]
+#[para]punk::cap provides management of named capabilities and the provider packages and handler packages that implement a pluggable capability.
+#[subsection Concepts]
+#[para]A [term capability] may be something like providing a folder of files, or just a data dictionary, and/or an API 
+#
+#[para][term {capability handler}] - a package/namespace which may provide validation and standardised ways of looking up provider data
+# registered (or not) using register_capabilityname <capname> <capnamespace>
+#
+#[para][term {capability provider}] - a package which registers as providing one or more capablities.
+#[para]registered using register_package <pkg> <capabilitylist>
+#the capabilitylist is a list of 2-element lists where the first element is the capabilityname and the second element is a (possibly empty) dict of data relevant to that capability
+#A capabilityname may appear multiple times. ie a package may register that it provides the capability with multiple datasets.
+
+
+#*** !doctools
+#[section API]
+
 
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 ##  Requirements
@@ -29,63 +45,115 @@
 package require oolib
 
 
-
-# mkdoc markdown 
-#' ---
-#' author: JMNoble 
-#' ---
-#' ## Concepts:
-#' > A **capability** may be something like providing a folder of files, or just a data dictionary, and/or an API 
-#'
-#' > **capability handler** - a package/namespace which may provide validation and standardised ways of looking up provider data
-#'   registered (or not) using register_capabilityname <capname> <capnamespace>
-#'
-#' > **capability provider** - a package which registers as providing one or more capablities.
-#'  registered using register_package <pkg> <capabilitylist>
-#'  the capabilitylist is a list of 2-element lists where the first element is the capabilityname and the second element is a (possibly empty) dict of data relevant to that capability
-#'  A capabilityname may appear multiple times. ie a package may register that it provides the capability with multiple datasets.
-
-
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 namespace eval punk::cap {
     variable pkgcapsdeclared [dict create] 
     variable pkgcapsaccepted [dict create]
     variable caps [dict create]
-
     namespace eval class {
         if {[info commands [namespace current]::interface_caphandler.registry] eq ""} {
-            #Handler classes
+            #*** !doctools
+            #[subsection {Namespace punk::cap::class}]
+            #[para] class definitions
+            #[list_begin itemized] [comment {- punk::cap::class groupings -}]
+            #  [item]
+            #  [para] [emph {handler_classes}]
+            #  [list_begin enumerated]
+
             oo::class create [namespace current]::interface_caphandler.registry {
+                #*** !doctools
+                #[enum] CLASS [class interface_caphandler.registry]
+                #[list_begin definitions]
+                #    [para] [emph METHODS]
                 method pkg_register {pkg capname capdict fullcapabilitylist} {
-                    #***
-                    #[call [class interface_caphandler.registry] [method pkg_register] [arg pkg] [arg capname] [arg capdict] [arg fullcapabilitylist]]
+                    #*** !doctools
+                    #[call class::interface_caphandler.registry [method pkg_register] [arg pkg] [arg capname] [arg capdict] [arg fullcapabilitylist]]
                     #handler may override and return 0 (indicating don't register)e.g if pkg capdict data wasn't valid
                     #overridden handler must be able to handle multiple calls for same pkg - but it may return 1 or 0 as it wishes. 
                     return 1 ;#default to permit 
                 }
                 method pkg_unregister {pkg} {
-                    #***
-                    #[call [class interface_caphandler.registry] [method pkg_unregister] [arg pkg]]
+                    #*** !doctools
+                    #[call class::interface_caphandler.registry [method pkg_unregister] [arg pkg]]
                     return ;#unregistration return is ignored - review
                 }
+                #*** !doctools
+                #[list_end]
             }
+
             oo::class create [namespace current]::interface_caphandler.sysapi {
+                #*** !doctools
+                #[enum] CLASS [class interface_caphandler.sysapi]
+                #[list_begin definitions]
+                #    [para] [emph METHODS]
 
+
+                #*** !doctools
+                #[list_end]
             }
 
+            #*** !doctools
+            #  [list_end] [comment {- end enumeration handler classes -}]
+
+            #*** !doctools
+            #  [item]
+            #  [para] [emph {provider_classes}]
+            #  [list_begin enumerated]
 
             #Provider classes
             oo::class create [namespace current]::interface_capprovider.registration {
+                #*** !doctools
+                #    [enum] CLASS [class interface_cappprovider.registration]
+                #    [para]Your provider package will need to instantiate this object under a sub-namespace called [namespace capsystem] within your package namespace. 
+                #    [para]If your package namespace is mypackages::providerpkg then the object command would be at mypackages::providerpkg::capsystem::capprovider.registration
+                #    [para]Example code for your provider package to evaluate within its namespace: 
+                #    [example {
+                #namespace eval capsystem {
+                #    if {[info commands capprovider.registration] eq ""} {
+                #        punk::cap::class::interface_capprovider.registration create capprovider.registration
+                #        oo::objdefine capprovider.registration {
+                #            method get_declarations {} {
+                #                set decls [list]
+                #                lappend decls [list punk.templates {relpath ../templates}]
+                #                lappend decls [list another_capability_name {somekey blah key2 etc}]
+                #                return $decls
+                #            }
+                #        }
+                #    }
+                #}
+                #}]
+                #[para] The above example declares that your package can be registered as a provider for the capabilities named 'punk.templates' and 'another_capability_name'
+                #    [list_begin definitions]
+                #    [para] [emph METHODS]
                 method get_declarations {} {
                     #***
-                    #[call [class interface_capprovider.registration] [method pkg_unregister] [arg pkg]]
+                    #[call class::interface_capprovider.registration [method get_declarations]]
+                    #[para] This method must be overridden by your provider using oo::objdefine cappprovider.registration as in the example above.
+                    # There must be at least one 2-element list in the result for the provider to be registerable.
+                    #[para]The first element of the list is the capabilityname - which can be custom to your provider/handler packages - or a well-known name that other authors may use/implement.
+                    #[para]The second element is a dictionary of keys specific to the capability being implemented. It may be empty if the any potential capability handlers for the named capability don't require registration data.  
                     error "interface_capprovider.registration not implemented by provider"
                 }
+                #*** !doctools
+                #    [list_end]
             }
+
             oo::class create [namespace current]::interface_capprovider.provider {
+                #*** !doctools
+                #    [enum] CLASS [class interface_capprovider.provider] 
+                #    [para] Your provider package will need to instantiate this directly under it's own namespace with the command name of [emph {provider}] 
+                #    [example {
+                #       namespace eval mypackages::providerpkg { 
+                #           punk::cap::class::interface_capprovider.provider create provider mypackages::providerpkg 
+                #       }
+                #    }]
+                #    [list_begin definitions]
+                #    [para] [emph METHODS]
                 variable provider_pkg
                 variable registrationobj
                 constructor {providerpkg} {
+                    #*** !doctools
+                    #[call class::interface_capprovider.provider [method constructor] [arg providerpkg]]
                     variable provider_pkg
                     if {$providerpkg in [list "" "::"]} {
                         error "interface_capprovider.provider constructor error. Invalid provider '$providerpkg'"
@@ -103,16 +171,33 @@ namespace eval punk::cap {
 
                 }
                 method register {{capabilityname_glob *}} {
-                    #***
-                    #[call [class interface_capprovider.provider] [method register] [opt capabilityname_glob]]
+                    #*** !doctools
+                    #[comment {- -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---}]
+                    #[call class::interface_capprovider.provider [method register] [opt capabilityname_glob]]
+                    #
+                    #[para]This is the mechanism by which a user of your provider package will register your package as a provider of the capability named.
+                    #
+                    #[para]A user of your provider may elect to register all your declared capabilities:
+                    #[example {
+                    #  package require mypackages::providerpkg
+                    #  mypackages::providerpkg::provider register *
+                    #}]
+                    #[para] Or a specific capability may be registered:
+                    #[example {
+                    #  package require mypackages::providerpkg
+                    #  mypackages::providerpkg::provider register another_capability_name
+                    #}]
+                    #
                     variable provider_pkg
                     set all_decls [$registrationobj get_declarations]
                     set register_decls [lsearch -all -inline -index 0 $all_decls $capabilityname_glob]
                     punk::cap::register_package $provider_pkg $register_decls
                 }
                 method capabilities {} {
-                    #***
-                    #[call [class interface_capprovider.provider] [method capabilities]]
+                    #*** !doctools
+                    #[comment {- -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---}]
+                    #[call class::interface_capprovider.provider [method capabilities]]
+                    #[para] return a list of capabilities supported by this provider package
                     variable provider_pkg
                     variable registrationobj
 
@@ -124,28 +209,24 @@ namespace eval punk::cap {
                             lappend capabilities $capname
                         }
                     }
-                    return $capname
+                    return $capabilities
                 }
+                #*** !doctools
+                #    [list_end] [comment {- end class definitions -}]
             }
+            #*** !doctools
+            #  [list_end] [comment {- end enumeration provider_classes }]
+            #[list_end] [comment {- end itemized list punk::cap::class groupings -}]
         }
     } ;# end namespace class
-    namespace eval capsystem {
-        proc get_caphandler_registry {capname} {
-            set ns [::punk::cap::get_handler $capname]::capsystem
-            if {[namespace exists ${ns}]} {
-                if {[info command ${ns}::caphandler.registry] ne ""} {
-                    if {[info object isa object ${ns}::caphandler.registry]} {
-                        return ${ns}::caphandler.registry
-                    }
-                }
-            }
-            return ""
-        }
-    }
 
+    #*** !doctools
+    #[subsection {Namespace punk::cap}]
+    #[para] Main punk::cap API for client programs interested in using capability handler packages and associated (registered) provider packages
+    #[list_begin definitions]
 
-    #Not all capabilities have to be registered.
-    #A package registering as a provider using register_package can include capabilitynames in it's capabilitylist which have no associated capnamespace (handler).
+    #Not all capability names have to be registered.
+    #A package registering as a provider using register_package can include capabilitynames in it's capabilitylist which have no associated handler.
     #such unregistered capabilitynames may be used just to flag something, or have datamembers significant to callers cooperatively interested in that capname. 
     #we allow registering a capability with an empty handler (capnamespace) - but this means another handler could be registered later.
     proc register_capabilityname {capname capnamespace} {
@@ -161,7 +242,7 @@ namespace eval punk::cap {
         #allow register of existing capname iff there is no current handler
         #as handlers can be used to validate during provider registration - ideally handlers should be registered before any pkgs call register_package 
         #we allow loading a handler later though - but will need to validate existing data from pkgs that have already registered as providers 
-        if {[set hdlr [get_handler $capname]] ne ""} {
+        if {[set hdlr [capability_get_handler $capname]] ne ""} {
             error "register_capabilityname cannot register capability:$capname with handler:$capnamespace. There is already a registered handler:$hdlr" 
         }
         #assert: capnamespace may or may not be empty string, capname may or may not already exist in caps dict, caps $capname providers may have existing entries.
@@ -213,52 +294,33 @@ namespace eval punk::cap {
             
         }
     }
-    proc exists {capname} {
+    proc capability_exists {capname} {
         #*** !doctools
-        # [call [fun exists] [arg capname]]
+        # [call [fun capability_exists] [arg capname]]
         # Return a boolean indicating if the named capability exists  (0|1)
-
-        # mkdoc markdown
-        #'
-        #' ## **exists(capname)**
-        #'
-        #' > return a boolean indicating the existence of a capability
-        #'
-        #' > Arguments:
-        #'
-        #' > - *capname* - string indicating the name of the capability
-        #'
-        #' > Returns: 0|1
-        #'
         variable caps
         return [dict exists $caps $capname]
     }
-    proc has_handler {capname} {
+    proc capability_has_handler {capname} {
         #*** !doctools
-        # [call [fun has_handler] [arg capname]]
-        # Return a boolean indicating if the named capability has a handler package installed (0|1)
-
-
+        # [call [fun capability_has_handler] [arg capname]]
+        #Return a boolean indicating if the named capability has a handler package installed (0|1)
         variable caps
         return [expr {[dict exists $caps $capname handler] && [dict get $caps $capname handler] ne ""}]
     }
-    proc get_handler {capname} {
+    proc capability_get_handler {capname} {
+        #*** !doctools
+        # [call [fun capability_get_handler] [arg capname]]
+        #Return the base namespace of the active handler package for the named capability.
+        #[para] The base namespace for a handler will always be the package name, but prefixed with ::
         variable caps
         if {[dict exists $caps $capname]} {
             return [dict get $caps $capname handler]
         }
         return "" 
     }
-
-    #dispatch
-    #proc call_handler {capname args} {
-    #    if {[set handler [get_handler $capname]] eq ""} {
-    #        error "punk::cap::call_handler $capname $args - no handler registered for capability $capname"
-    #    }
-    #    ${handler}::[lindex $args 0] {*}[lrange $args 1 end]
-    #}
     proc call_handler {capname args} {
-        if {[set handler [get_handler $capname]] eq ""} {
+        if {[set handler [capability_get_handler $capname]] eq ""} {
             error "punk::cap::call_handler $capname $args - no handler registered for capability $capname"
         }
         set obj ${handler}::api_$capname
@@ -274,10 +336,21 @@ namespace eval punk::cap {
 
     #register package with arbitrary capnames from capabilitylist
     #The registered pkg is a module that provides some service to that capname. Possibly just data members, that the capability will use.
-    proc register_package {pkg capabilitylist} {
+    proc register_package {pkg capabilitylist args} {
         variable pkgcapsdeclared
         variable pkgcapsaccepted
         variable caps
+        set defaults [dict create\
+            -nowarnings false
+        ]
+        dict for {k v} $args {
+            if {$k ni $defaults} {
+                error "Unrecognized option $k. Known options [dict keys $defaults]"
+            }
+        }
+        set opts [dict merge $defaults $args]
+        set warnings [expr {! [dict get $opts -nowarnings]}]
+
         if {[string match ::* $pkg]} {
             set pkg [string range $pkg 2 end]
         }
@@ -286,11 +359,23 @@ namespace eval punk::cap {
         } else {
             set pkg_already_accepted [list]
         }
+        package require $pkg
+        set providerapi ::${pkg}::provider
+        if {[info commands $providerapi] eq ""} {
+            error "register_package error. pkg '$pkg' doesn't seem to be a punk::cap capability provider (no object found at $providerapi)"
+        }
+        set defined_caps [$providerapi capabilities]
         #for each capability
         # - ensure 1st element is a single word
         # - ensure that if 2nd element (capdict) is present - it is dict shaped
         foreach capspec $capabilitylist {
             lassign $capspec capname capdict
+
+            if {$warnings} {
+                if {$capname ni $defined_caps} {
+                    puts stderr "WARNING: pkg '$pkg' doesn't declare support for capability '$capname'."
+                }
+            }
             if {[llength $capname] !=1} {
                 error "register_package error. pkg: '$pkg' An entry in the capability list doesn't appear to have a single-word name. Problematic entry:'$capspec'"
             }
@@ -299,7 +384,9 @@ namespace eval punk::cap {
             }
             if {$capspec in $pkg_already_accepted} {
                 #review - multiple handlers? if so - will need to record which handler(s) accepted the capspec
-                puts stderr "register_package pkg $pkg already has capspec marked as accepted: $capspec"
+                if {$warnings} {
+                    puts stderr "WARNING: register_package pkg $pkg already has capspec marked as accepted: $capspec"
+                }
                 continue
             }
             if {[dict exists $caps $capname]} {
@@ -371,73 +458,6 @@ namespace eval punk::cap {
         }
     }
 
-    #review promote/demote doesn't always make a lot of sense ..  should possibly be per cap for multicap pkgs
-    #The idea is to provide a crude way to preference/depreference packages independently of order the packages were loaded
-    #e.g a caller or cap-handler can ascribe some meaning to the order of the 'providers' key returned from punk::cap::capabilities <capname>
-    #The order of providers will be the order the packages were loaded & registered
-    #the naming: "promote vs demote" operates on a latest-package-in-list has higher preference assumption (matching last pkg loaded)
-    #Each capability handler could implement specific preferencing methods if finer control needed.
-    #In some cases the preference/loading order may be inapplicable/irrelevant to a particular capability anyway.
-    #As this is just a basic mechanism, which can't support independent per-cap preferencing for multi-cap packages -
-    # it only allows putting the pkgs to the head or tail of the lists.
-    #Whether particular caps or users of caps do anything with this ordering is dependent on the cap-handler and/or calling code.
-    proc promote_package {pkg} {
-        variable pkgcapsdeclared
-        variable caps
-        if {[string match ::* $pkg]} {
-            set pkg [string range $pkg 2 end]
-        }
-        if {![dict exists $pkgcapsdeclared $pkg]} {
-            error "punk::cap::promote_package error pkg'$pkg' not registered. Use register_package \$pkg first"
-        }
-        if {[dict size $pkgcapsdeclared] > 1} {
-            set pkginfo [dict get $pkgcapsdeclared $pkg]
-            #remove and re-add at end of dict
-            dict unset pkgcapsdeclared $pkg
-            dict set pkgcapsdeclared $pkg $pkginfo
-            dict for {cap cap_info} $caps {
-                set cap_pkgs [dict get $cap_info providers]
-                if {$pkg in $cap_pkgs} {
-                    set posn [lsearch $cap_pkgs $pkg]
-                    if {$posn >=0} {
-                        #rewrite package list with pkg at tail of list for this capability
-                        set cap_pkgs [lreplace $cap_pkgs $posn $posn]
-                        lappend cap_pkgs $pkg
-                        dict set caps $cap providers $cap_pkgs
-                    }
-                }
-            }
-        }
-    }
-    proc demote_package {pkg} {
-        variable pkgcapsdeclared
-        variable caps
-        if {[string match ::* $pkg]} {
-            set pkg [string range $pkg 2 end]
-        }
-        if {![dict exists $pkgcapsdeclared $pkg]} {
-            error "punk::cap::promote_package error pkg'$pkg' not registered. Use register_package \$pkg first"
-        }
-        if {[dict size $pkgcapsdeclared] > 1} {
-            set pkginfo [dict get $pkgcapsdeclared $pkg]
-            #remove and re-add at start of dict
-            dict unset pkgcapsdeclared $pkg
-            dict set pkgcapsdeclared $pkg $pkginfo
-            set pkgcapsdeclared [dict merge [dict create $pkg $pkginfo] $pkgcapsdeclared]
-            dict for {cap cap_info} $caps {
-                set cap_pkgs [dict get $cap_info providers]
-                if {$pkg in $cap_pkgs} {
-                    set posn [lsearch $cap_pkgs $pkg]
-                    if {$posn >=0} {
-                        #rewrite package list with pkg at head of list for this capability
-                        set cap_pkgs [lreplace $cap_pkgs $posn $posn]
-                        set cap_pkgs [list $pkg {*}$cap_pkgs]
-                        dict set caps $cap providers $cap_pkgs
-                    }
-                }
-            }
-        }
-    }
     proc pkgcap {pkg} {
         variable pkgcapsdeclared
         variable pkgcapsaccepted
@@ -502,7 +522,121 @@ namespace eval punk::cap {
         }
         return $cap_list
     }
+    #*** !doctools
+    #[list_end] [comment {- end definitions for namespace punk::cap -}]
+
+    namespace eval advanced {
+    #*** !doctools
+    #[subsection {Namespace punk::cap::advanced}]
+    #[para] punk::cap::advanced API. Functions here are generally not the preferred way to interact with punk::cap.
+    #[para] In some cases they may allow interaction in less safe ways or may allow use of features that are unavailable in the base namespace.
+    #[para] Some functions are here because they are only marginally or rarely useful, and they are here to keep the base API simple.  
+    #[list_begin definitions]
+
+    proc promote_provider {pkg} {
+        #*** !doctools
+        # [call advanced::[fun promote_provider] [arg pkg]]
+        #[para]Move the named provider package to the preferred end of the list (tail).
+        #[para]The active handler may or may not utilise this for preferencing. See documentation for the specific handler package to confirm. 
+        #[para]
+        #[para] promote/demote doesn't always make a lot of sense ..  should preferably be configurable per capapbility for multicap provider pkgs
+        #[para]The idea is to provide a crude way to preference/depreference packages independently of order the packages were loaded
+        #e.g a caller or cap-handler can ascribe some meaning to the order of the 'providers' key returned from punk::cap::capabilities <capname>
+        #[para]The order of providers will be the order the packages were loaded & registered
+        #[para]the naming: "promote vs demote" operates on a latest-package-in-list has higher preference assumption (matching last pkg loaded)
+        #[para]Each capability handler could and should implement specific preferencing methods within its own API if finer control needed.
+        #In some cases the preference/loading order may be inapplicable/irrelevant to a particular capability anyway.
+        #[para]As this is just a basic mechanism, which can't support independent per-cap preferencing for multi-cap packages -
+        # it only allows putting the pkgs to the head or tail of the lists.
+        #[para]Whether particular caps or users of caps do anything with this ordering is dependent on the cap-handler and/or calling code.
+        variable pkgcapsdeclared
+        variable caps
+        if {[string match ::* $pkg]} {
+            set pkg [string range $pkg 2 end]
+        }
+        if {![dict exists $pkgcapsdeclared $pkg]} {
+            error "punk::cap::promote_package error pkg'$pkg' not registered. Use register_package \$pkg first"
+        }
+        if {[dict size $pkgcapsdeclared] > 1} {
+            set pkginfo [dict get $pkgcapsdeclared $pkg]
+            #remove and re-add at end of dict
+            dict unset pkgcapsdeclared $pkg
+            dict set pkgcapsdeclared $pkg $pkginfo
+            dict for {cap cap_info} $caps {
+                set cap_pkgs [dict get $cap_info providers]
+                if {$pkg in $cap_pkgs} {
+                    set posn [lsearch $cap_pkgs $pkg]
+                    if {$posn >=0} {
+                        #rewrite package list with pkg at tail of list for this capability
+                        set cap_pkgs [lreplace $cap_pkgs $posn $posn]
+                        lappend cap_pkgs $pkg
+                        dict set caps $cap providers $cap_pkgs
+                    }
+                }
+            }
+        }
+    }
+    proc demote_provider {pkg} {
+        #*** !doctools
+        # [call advanced::[fun demote_provider] [arg pkg]]
+        #[para]Move the named provider package to the preferred end of the list (tail).
+        #[para]The active handler may or may not utilise this for preferencing. See documentation for the specific handler package to confirm. 
+        variable pkgcapsdeclared
+        variable caps
+        if {[string match ::* $pkg]} {
+            set pkg [string range $pkg 2 end]
+        }
+        if {![dict exists $pkgcapsdeclared $pkg]} {
+            error "punk::cap::promote_package error pkg'$pkg' not registered. Use register_package \$pkg first"
+        }
+        if {[dict size $pkgcapsdeclared] > 1} {
+            set pkginfo [dict get $pkgcapsdeclared $pkg]
+            #remove and re-add at start of dict
+            dict unset pkgcapsdeclared $pkg
+            dict set pkgcapsdeclared $pkg $pkginfo
+            set pkgcapsdeclared [dict merge [dict create $pkg $pkginfo] $pkgcapsdeclared]
+            dict for {cap cap_info} $caps {
+                set cap_pkgs [dict get $cap_info providers]
+                if {$pkg in $cap_pkgs} {
+                    set posn [lsearch $cap_pkgs $pkg]
+                    if {$posn >=0} {
+                        #rewrite package list with pkg at head of list for this capability
+                        set cap_pkgs [lreplace $cap_pkgs $posn $posn]
+                        set cap_pkgs [list $pkg {*}$cap_pkgs]
+                        dict set caps $cap providers $cap_pkgs
+                    }
+                }
+            }
+        }
+    }
+
+    #*** !doctools
+    #[list_end]
+    }
+
 
+#*** !doctools
+#[section Internal]
+
+    namespace eval capsystem {
+        #*** !doctools
+        #[subsection {Namespace punk::cap::capsystem}]
+        #[para] Internal functions used to communicate between punk::cap and capability handlers
+        #[list_begin definitions]
+        proc get_caphandler_registry {capname} {
+            set ns [::punk::cap::capability_get_handler $capname]::capsystem
+            if {[namespace exists ${ns}]} {
+                if {[info command ${ns}::caphandler.registry] ne ""} {
+                    if {[info object isa object ${ns}::caphandler.registry]} {
+                        return ${ns}::caphandler.registry
+                    }
+                }
+            }
+            return ""
+        }
+        #*** !doctools
+        #[list_end]
+    }
 }
 
 
@@ -524,5 +658,4 @@ package provide punk::cap [namespace eval punk::cap {
 return
 
 #*** !doctools
-#[list_end]
 #[manpage_end]

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/du-0.1.0.tm

@@ -24,15 +24,14 @@ namespace eval punk::du {
     variable has_twapi 0
 }
 if {"windows" eq $::tcl_platform(platform)} {
-    #jmn disable twapi
-    #package require zzzload
-    #zzzload::pkg_require twapi
-
-    #if {[catch {package require twapi}]} {
-    #    puts stderr "Warning: punk::du - unable to load twapi. Disk operations may be much slower on windows without the twapi package"
-    #} else {
-    #    set punk::du::has_twapi 1
-    #}
+    package require zzzload
+    zzzload::pkg_require twapi
+
+    if {[catch {package require twapi}]} {
+        puts stderr "Warning: punk::du - unable to load twapi. Disk operations may be much slower on windows without the twapi package"
+    } else {
+        set punk::du::has_twapi 1
+    }
     #package require punk::winpath
 }
 
@@ -1248,10 +1247,11 @@ namespace eval punk::du {
         proc du_dirlisting_undecided {folderpath args} {
             if {"windows" eq $::tcl_platform(platform)} {
                 #jmn disable twapi
-                tailcall du_dirlisting_generic $folderpath {*}$args                        
+                #tailcall du_dirlisting_generic $folderpath {*}$args                        
 
                 set loadstate [zzzload::pkg_require twapi]
                 if {$loadstate ni [list loading failed]} {
+                    #either already loaded by zzload or ordinary package require
                     package require twapi ;#should be fast once twapi dll loaded in zzzload thread
                     set ::punk::du::has_twapi 1
                     punk::du::active::set_active_function du_dirlisting du_dirlisting_twapi

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/mix/base-0.1.tm

@@ -3,6 +3,7 @@ package provide punk::mix::base [namespace eval punk::mix::base {
 	set version 0.1
 }]
 
+package require punk::path
 
 #base internal plumbing functions
 namespace eval punk::mix::base {
@@ -364,7 +365,7 @@ namespace eval punk::mix::base {
             set folderdict [dict create]
 
             package require punk::cap
-            if {[punk::cap::has_handler punk.templates]} {
+            if {[punk::cap::capability_has_handler punk.templates]} {
                 set template_folder_dict [punk::cap::call_handler punk.templates folders]
                 dict for {dir folderinfo} $template_folder_dict {
                     dict set folderdict $dir $folderinfo
@@ -632,7 +633,8 @@ namespace eval punk::mix::base {
                 cd $base ;#cd is process-wide.. keep cd in effect for as small a scope as possible. (review for thread issues)
             
                 #temp emission to stdout.. todo - repl telemetry channel 
-                puts stdout "cksum_path: creating temporary tar archive at: $archivename .."
+                puts stdout "cksum_path: creating temporary tar archive for $path"
+                puts stdout "  at: $archivename .."
                 tar::create $archivename $target
                 if {$ftype eq "file"} {
                     set sizeinfo "(size [file size $target])"
@@ -757,7 +759,7 @@ namespace eval punk::mix::base {
                         set normbase [file normalize $base]
                         set normtarg [file normalize [file join $normbase $specifiedpath]]
                         set targetpath $normtarg
-                        set storedpath [punk::mix::util::path_relative $normbase $normtarg]
+                        set storedpath [punk::path::relative $normbase $normtarg]
                     } else {
                         set targetpath [file join $base $specifiedpath]
                         set storedpath $specifiedpath
@@ -776,7 +778,7 @@ namespace eval punk::mix::base {
                         error "get_relativecksum_from_base error: base '$base' and specifiedpath '$specifiedpath' don't share a common root. Use empty-string for base if independent absolute path is required"
                     }
                     set targetpath $specifiedpath 
-                    set storedpath [punk::mix::util::path_relative $base $specifiedpath]
+                    set storedpath [punk::path::relative $base $specifiedpath]
                     
                 }
             } else {

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/mix/cli-0.3.tm

@@ -477,7 +477,13 @@ namespace eval punk::mix::cli {
 
             set did_skip 0 ;#flag for stdout/stderr formatting only
             foreach m $src_modules {
-                #puts "build_modules_from_source_to_base >>> module $m"
+                set is_interesting 0
+                if {[string match "foobar" $current_source_dir]} {
+                    set  is_interesting 1
+                }
+                if {$is_interesting} {
+                    puts "build_modules_from_source_to_base >>> module $current_source_dir/$m"
+                }
                 set fileparts [split [file rootname $m] -]
                 set tmfile_versionsegment [lindex $fileparts end]
                 if {$tmfile_versionsegment eq $magicversion} {
@@ -582,7 +588,9 @@ namespace eval punk::mix::cli {
                             #set file_record [punkcheck::installfile_finished_install $basedir $file_record]
                             $event targetset_end OK
                         } else {
-                            #puts stdout "skipping module $current_source_dir/$m - no change in sources detected"
+                            if {$is_interesting} {
+                                puts stdout "skipping module $current_source_dir/$m - no change in sources detected"
+                            }
                             puts -nonewline stderr "."
                             set did_skip 1
                             #set file_record [punkcheck::installfile_skipped_install $basedir $file_record]
@@ -627,15 +635,18 @@ namespace eval punk::mix::cli {
                     $event targetset_started
                     # -- --- --- --- --- ---
                     if {$did_skip} {set did_skip 0; puts -nonewline stdout \n}
-                    puts stderr "Copied already versioned module $current_source_dir/$m to $target_module_dir"
                     lappend module_list $current_source_dir/$m
                     file copy -force $current_source_dir/$m $target_module_dir
+                    puts stderr "Copied already versioned module $current_source_dir/$m to $target_module_dir"
                     # -- --- --- --- --- ---
                     #set file_record [punkcheck::installfile_finished_install $basedir $file_record]
                     $event targetset_end OK -note "already versioned module"
                 } else {
                     puts -nonewline stderr "."
                     set did_skip 1
+                    if {$is_interesting} {
+                        puts stderr "$current_source_dir/$m [$event targetset_source_changes]"
+                    }
                     #set file_record [punkcheck::installfile_skipped_install $basedir $file_record]
                     $event targetset_end SKIPPED
                 }

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/mix/commandset/doc-0.1.0.tm

@@ -18,11 +18,11 @@
 ##  Requirements
 ##e.g package require frobz
 
-package require punk ;# for treefilenames
+package require punk::path ;# for treefilenames, relative
 package require punk::repo
 package require punk::docgen ;#inline doctools - generate doctools .man files at src/docgen prior to using kettle to producing .html .md etc
 package require punk::mix::cli  ;#punk::mix::cli::lib used for kettle_call
-package require punk::mix::util ;#for path_relative
+#package require punk::mix::util ;#for path_relative
 
 
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
@@ -44,7 +44,7 @@ namespace eval punk::mix::commandset::doc {
         }
         #user may delete the comment containing "--- punk::docgen::overwrites" and then manually edit, and we won't overwrite
         #we still generate output in src/docgen so user can diff and manually update if thats what they prefer
-        set oldfiles [glob -nocomplain -dir $projectdir/src/doc -type f _module_*]
+        set oldfiles [punk::path::treefilenames $projectdir/src/doc _module_*.man]
         foreach maybedoomed $oldfiles {
             set fd [open $maybedoomed r]
             set data [read $fd]
@@ -57,9 +57,18 @@ namespace eval punk::mix::commandset::doc {
         if {[dict get $generated count] > 0} {
             #review
             set doclist [dict get $generated docs]
+            set source_base [dict get $generated base]
+            set target_base $projectdir/src/doc
             foreach dinfo $doclist {
                 lassign $dinfo module fpath
-                set target $projectdir/src/doc/_module_[file tail $fpath]
+                set relpath [punk::path::relative $source_base $fpath]
+                set relfolder [file dirname $relpath]
+                if {$relfolder eq "."} {
+                    set relfolder ""
+                }
+                file mkdir [file join $target_base $relfolder]
+                set target [file join $target_base $relfolder _module_[file tail $fpath]]
+                puts stderr "target --> $target"
                 if {![file exists $target]} {
                     file copy $fpath $target
                 }
@@ -184,23 +193,24 @@ namespace eval punk::mix::commandset::doc {
         variable pkg
         set pkg punk::mix::commandset::doc
         proc  do_docgen {{project_subpath modules}} {
+            #Extract doctools comments from source code
             set projectdir [punk::repo::find_project]
-            set outdir [file join $projectdir src docgen]
-            set subpath [file join $projectdir $project_subpath]
-            if {![file isdirectory $subpath]} {
-                puts stderr "WARNING punk::mix::commandset::doc unable to find subpath $subpath during do_docgen - skipping inline doctools generation"
+            set output_base [file join $projectdir src docgen]
+            set codesource_path [file join $projectdir $project_subpath]
+            if {![file isdirectory $codesource_path]} {
+                puts stderr "WARNING punk::mix::commandset::doc unable to find codesource_path $codesource_path during do_docgen - skipping inline doctools generation"
                 return
             }
-            if {[file isdirectory $outdir]} {
+            if {[file isdirectory $output_base]} {
                 if {[catch {
-                    file delete -force $outdir
+                    file delete -force $output_base
                 }]}  {
-                    error "do_docgen failed to delete existing $outdir"
+                    error "do_docgen failed to delete existing output base folder: $output_base"
                 }
             }
-            file mkdir $outdir
+            file mkdir $output_base
 
-            set matched_paths [punk::treefilenames $subpath *.tm]
+            set matched_paths [punk::path::treefilenames $codesource_path *.tm -antiglob_paths {**/mix/templates/** **/mixtemplates/**}]
             set count 0
             set newdocs [list] 
             set docgen_header_comments ""
@@ -208,14 +218,34 @@ namespace eval punk::mix::commandset::doc {
             append docgen_header_comments {[comment {--- punk::docgen DO NOT EDIT DOCS HERE UNLESS YOU REMOVE THESE COMMENT LINES ---}]} \n
             append docgen_header_comments {[comment {--- punk::docgen overwrites this file ---}]} \n
             foreach fullpath $matched_paths {
-                set relpath [punk::mix::util::path_relative $subpath $fullpath]
-                set tailsegs [file split $relpath]
-                set module_fullname [join $tailsegs ::]
-                set docname [string map [list :: _] $module_fullname].man ;#todo - something better - need to ensure unique
                 set doctools [punk::docgen::get_doctools_comments $fullpath] 
                 if {$doctools ne ""} {
-                    puts stdout "generating doctools output from file $relpath"
-                    set outfile [file join $outdir $docname]
+                    set fname [file tail $fullpath]
+                    set mod_tail [file rootname $fname] 
+                    set relpath [punk::path::relative $codesource_path [file dirname $fullpath]]
+                    if {$relpath eq "."} {
+                        set relpath ""
+                    }
+                    set tailsegs [file split $relpath]
+                    set module_fullname [join $tailsegs ::]::$mod_tail
+                    set target_docname $fname.man
+                    set this_outdir [file join $output_base $relpath]
+
+                    if {[string length $fname] > 99} {
+                        #output needs to be tarballed to do checksum change tests in a reasonably straightforward and not-too-terribly slow way.
+                        #hack - review. Determine exact limit - test if tcllib tar fixed or if it's a limit of the particular tar format
+                        #work around tcllib tar filename length limit ( somewhere around 100?) This seems to be a limit on the length of a particular segment in the path.. not whole path length?
+                        #this case only came up because docgen used to path munge to long filenames - but left because we know there is a limit and renaming fixes it - even if it's ugly - but still allows doc generation.
+                        #review - if we're checking fname - should also test length of whole path and determine limits for tar
+                        package require md5
+                        set target_docname [md5::md5 -hex $fullpath]_overlongfilename.man
+                        puts stderr "WARNING - overlong file name - renaming $fullpath"
+                        puts stderr "          to                            [file dirname $fullpath]/$target_docname"
+                    }
+
+                    file mkdir $this_outdir
+                    puts stdout "saving [string length $doctools] bytes of doctools output from file $relpath/$fname"
+                    set outfile [file join $this_outdir $target_docname]
                     set fd [open $outfile w]
                     fconfigure $fd -translation binary
                     puts -nonewline $fd $docgen_header_comments$doctools
@@ -224,7 +254,7 @@ namespace eval punk::mix::commandset::doc {
                     lappend newdocs [list $module_fullname $outfile]
                 }
             }
-            return [list count $count docs $newdocs]
+            return [list count $count docs $newdocs base $output_base]
         }
 
     }

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/mix/commandset/layout-0.1.0.tm

@@ -141,7 +141,10 @@ namespace eval punk::mix::commandset::layout {
             #use last matching layout found.  review silent if multiple?
             if {![llength $tags]} {
                 #todo - get standard tags from somewhere
-                set tags [list %project%]
+                set tagnames [list project]
+                foreach tn $tagnames {
+                    lappend tags [string cat % $tn %]
+                }
             }
             set file_list [list] 
             util::foreach-file $layoutfolder path {

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/mix/commandset/project-0.1.0.tm

@@ -13,10 +13,72 @@
 # @@ Meta End
 
 
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+# doctools header
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+#*** !doctools
+#[manpage_begin punk::mix::commandset::project 0 0.1.0]
+#[copyright "2023"]
+#[titledesc {pmix commandset - project}]  [comment {-- Name section and table of contents description --}]
+#[moddesc {pmix CLI commandset - project}]             [comment {-- Description at end of page heading --}] 
+#[require punk::mix::commandset::project]
+#[description]
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[section Overview]
+#[para] overview of punk::mix::commandset::project
+#[para]Import into an ensemble namespace similarly to the way it is done with punk::mix::cli e.g 
+#[example {
+#    namespace eval myproject::cli {
+#        namespace export *
+#        namespace ensemble create
+#        package require punk::overlay
+#    
+#        package require punk::mix::commandset::project
+#        punk::overlay::import_commandset project    .       ::punk::mix::commandset::project
+#        punk::overlay::import_commandset projects   .       ::punk::mix::commandset::project::collection 
+#    }
+#}]
+#[para] Where the . in the above example is the prefix/command separator
+#[para]The prefix ('project' in the above example) can be any string desired to disambiguate commands imported from other commandsets.
+#[para]The above results in the availability of the ensemble command: ::myproject::cli project.new, which is implemented in ::punk::mix::commandset::project::new  
+#[para]Similarly, procs under ::punk::mix::commandset::project::collection will be available as subcommands of the ensemble as projects.<procname>
+#[para]
+#[subsection Concepts]
+#[para] see punk::overlay 
+
 
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 ##  Requirements
-##e.g package require frobz
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[subsection dependencies]
+#[para] packages used by punk::mix::commandset::project 
+#[list_begin itemized]
+
+package require Tcl 8.6
+#*** !doctools
+#[item] [package {Tcl 8.6}]
+#[item] [package punk::ns]
+#[item] [package sqlite3] (binary)
+#[item] [package overtype]
+#[item] [package textutil] (tcllib)
+
+
+# #package require frobz
+# #*** !doctools
+# #[item] [package {frobz}]
+
+#*** !doctools
+#[list_end]
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[section API]
 
 
 
@@ -24,10 +86,36 @@
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 namespace eval punk::mix::commandset::project {
     namespace export *
+    #*** !doctools
+    #[subsection {Namespace punk::mix::commandset::project}]
+    #[para] core commandset functions for punk::mix::commandset::project 
+    #[list_begin definitions]
+
+    proc _default {} {
+        package require punk::ns
+        set dispatched_to [lindex [info level 2] 0] ;#e.g ::punk::mix::cli::project
+        set dispatch_tail [punk::ns::nstail $dispatched_to]
+        set dispatch_ensemble [punk::ns::nsprefix $dispatched_to] ;#e.g ::punk::mix::cli
+        set sibling_commands [namespace eval $dispatch_ensemble {namespace export}]
+        #todo - get separator?
+        set sep "."
+        set result [list]
+        foreach sib $sibling_commands {
+            if {[string match ${dispatch_tail}${sep}* $sib]} {
+                lappend result $sib
+            }
+        }
+        return [lsort $result]
+    }
+
+
+
 
-    #new project structure - may be dedicated to one module, or contain many. 
-    #create minimal folder structure only by specifying  -modules {}
     proc new {newprojectpath_or_name args} {
+        #*** !doctools
+        # [call [fun new] [arg newprojectpath_or_name] [opt args]]
+        #new project structure - may be dedicated to one module, or contain many. 
+        #create minimal folder structure only by specifying in args:  -modules {}
         if {[file pathtype $newprojectpath_or_name] eq "absolute"} {
             set projectfullpath [file normalize $newprojectpath_or_name]
             set projectname [file tail $projectfullpath]
@@ -242,7 +330,7 @@ namespace eval punk::mix::commandset::project {
         set layout_dir $templatebase/layouts/$opt_layout
         puts stdout ">>> about to call punkcheck::install $layout_dir $projectdir" 
         set resultdict [dict create]
-        set unpublish [list\
+        set antipaths [list\
             src/doc/*\
             src/doc/include/*\
         ]
@@ -250,11 +338,11 @@ namespace eval punk::mix::commandset::project {
         #default antiglob_dir_core will stop .fossil* from being updated - which is generally desirable as these are likely to be customized
         if {$opt_force} {
             puts stdout "copying layout files - with force applied - overwrite all-targets" 
-            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new  -overwrite ALL-TARGETS -unpublish_paths $unpublish]
+            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new  -overwrite ALL-TARGETS -antiglob_paths $antipaths]
             #file copy -force $layout_dir $projectdir
         } else {
             puts stdout "copying layout files - (if source file changed)" 
-            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -overwrite installedsourcechanged-targets -unpublish_paths $unpublish]
+            set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -overwrite installedsourcechanged-targets -antiglob_paths $antipaths]
         }
         puts stdout [punkcheck::summarize_install_resultdict $resultdict]
 
@@ -293,7 +381,7 @@ namespace eval punk::mix::commandset::project {
             set fpath [file join $projectdir $templatetail]
             if {[file exists $fpath]} {
                 set fd [open $fpath r]; fconfigure $fd -translation binary; set data [read $fd]; close $fd
-                set data2 [string map [list %project% $projectname] $data]
+                set data2 [string map [list [lib::template_tag project] $projectname] $data]
                 if {$data2 ne $data} {
                     puts stdout "updated template file: $fpath"
                     set fdout [open $fpath w]; fconfigure $fdout -translation binary; puts -nonewline $fdout $data2; close $fdout 
@@ -406,8 +494,10 @@ namespace eval punk::mix::commandset::project {
         puts stdout "-done- project:$projectname projectdir: $projectdir"
     }
 
+    #*** !doctools
+    #[list_end] [comment {--- end definitions namespace punk::mix::commandset::project ---}]
 
-     namespace eval collection {
+    namespace eval collection {
         namespace export *
         namespace path [namespace parent]
 
@@ -764,6 +854,12 @@ namespace eval punk::mix::commandset::project {
     }
     
     namespace eval lib {
+        proc template_tag {tagname} {
+            #todo - support different tagwrappers - it shouldn't be so likely to collide with common code idioms etc.
+            #we need to detect presence of tags intended for punk::mix system
+            #consider using punk::cap to enable multiple template-substitution providers with their own set of tagnames and/or tag wrappers, where substitution providers are all run 
+            return [string cat % $tagname %]
+        }
         #get project info only by opening the central confg-db
         #(will not have proper project-name etc)
         proc get_projects {{globlist {}} args} {
@@ -828,9 +924,8 @@ namespace eval punk::mix::commandset::project {
 }
 
 
-
-
-
+#*** !doctools
+#[manpage_end]
 
 
 

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/mix/util-0.1.0.tm

@@ -131,83 +131,6 @@ namespace eval punk::mix::util {
     }
     #----------------------------------------
 
-    #maint warning - also in punkcheck
-    proc path_relative {base dst} {
-        #see also kettle
-        # Modified copy of ::fileutil::relative (tcllib)
-        # Adapted to 8.5 ({*}).
-        #
-        #    Taking two _directory_ paths, a base and a destination, computes the path
-        #    of the destination relative to the base.
-        #
-        # Arguments:
-        #    base    The path to make the destination relative to.
-        #    dst     The destination path
-        #
-        # Results:
-        #    The path of the destination, relative to the base.
-
-        # Ensure that the link to directory 'dst' is properly done relative to
-        # the directory 'base'.
-
-        #review - check volume info on windows.. UNC paths?
-        if {[file pathtype $base] ne [file pathtype $dst]} {
-            return -code error "Unable to compute relation for paths of different pathtypes: [file pathtype $base] vs. [file pathtype $dst], ($base vs. $dst)"
-        }
-
-        #avoid normalizing if possible (file normalize *very* expensive on windows)
-        set do_normalize 0
-        if {[file pathtype $base] eq "relative"} {
-            #if base is relative so is dst
-            if {[regexp {[.]{2}} [list $base $dst]]} {
-                set do_normalize 1
-            }
-            if {[regexp {[.]/} [list $base $dst]]} {
-                set do_normalize 1
-            }
-        } else {
-            set do_normalize 1
-        }
-        if {$do_normalize} {
-            set base [file normalize $base]
-            set dst  [file normalize $dst]
-        }
-
-        set save $dst
-        set base [file split $base]
-        set dst  [file split $dst]
-
-        while {[lindex $dst 0] eq [lindex $base 0]} {
-            set dst  [lrange $dst  1 end]
-            set base [lrange $base 1 end]
-            if {![llength $dst]} {break}
-        }
-
-        set dstlen  [llength $dst]
-        set baselen [llength $base]
-
-        if {($dstlen == 0) && ($baselen == 0)} {
-            # Cases:
-            # (a) base == dst
-
-            set dst .
-            } else {
-            # Cases:
-            # (b) base is: base/sub = sub
-            #     dst  is: base     = {}
-
-            # (c) base is: base     = {}
-            #     dst  is: base/sub = sub
-
-            while {$baselen > 0} {
-                set dst [linsert $dst 0 ..]
-                incr baselen -1
-            }
-            set dst [file join {*}$dst]
-        }
-
-        return $dst
-    }
     #namespace import ::punk::ns::nsimport_noclobber 
 
     proc namespace_import_pattern_to_namespace_noclobber {pattern ns} {

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/overlay-0.1.tm

@@ -132,6 +132,7 @@ namespace eval ::punk::overlay {
         set imported_commands [list]
         set nscaller [uplevel 1 [list namespace current]]
         if {[catch {
+            #review - noclobber?
             namespace eval ${nscaller}::temp_import [list namespace import ${cmdnamespace}::*]
             foreach cmd [info commands ${nscaller}::temp_import::*] {
                 set cmdtail [namespace tail $cmd]

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/repo-0.1.1.tm

@@ -33,8 +33,9 @@ if {$::tcl_platform(platform) eq "windows"} {
     catch {package require punk::winpath}
 }
 package require fileutil; #tcllib
+package require punk::path
 package require punk::mix::base ;#uses core functions from punk::mix::base::lib namespace e.g cksum_path 
-package require punk::mix::util
+package require punk::mix::util ;#do_in_path
 
 
 # -- --- --- --- --- --- --- --- --- --- ---
@@ -357,7 +358,7 @@ namespace eval punk::repo {
         if {$repodir eq ""} {
             error "workingdir_state error: No repository found at or above path '$abspath'" 
         }
-        set subpath [punk::mix::util::path_relative $repodir $abspath]
+        set subpath [punk::path::relative $repodir $abspath]
         if {$subpath eq "."} {
             set subpath ""
         }
@@ -708,10 +709,10 @@ namespace eval punk::repo {
                 append msg "** starting folder : $start_dir" \n
                 append msg "** untracked       : $candroot" \n
                 if {$closest_fossil_len} {
-                    append msg "** fossil root     : $closest_fossil    ([punk::mix::util::path_relative $start_dir $closest_fossil])" \n
+                    append msg "** fossil root     : $closest_fossil    ([punk::path::relative $start_dir $closest_fossil])" \n
                 }
                 if {$closest_git_len} {
-                    append msg "** git root        : $closest_git    ([punk::mix::util::path_relative $start_dir $closest_git])" \n
+                    append msg "** git root        : $closest_git    ([punk::path::relative $start_dir $closest_git])" \n
                 }
                 append msg "**" \n
         }

src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punkcheck-0.1.0.tm

@@ -19,6 +19,7 @@
 ##e.g package require frobz
 
 package require punk::tdl
+package require punk::path
 package require punk::repo
 package require punk::mix::util
 
@@ -1106,29 +1107,6 @@ namespace eval punkcheck {
                 return [lindex $args end] 
             }
         }
-        proc pathglob_as_re {glob} {
-            #any segment that is not just * must match exactly one segment in the path
-            set pats [list]
-            foreach seg [file split $glob] {
-                if {$seg eq "*"} {
-                    lappend pats {[^/]*}
-                } elseif {$seg eq "**"} {
-                    lappend pats {.*}
-                } else {
-                    set seg [string map [list  . {[.]}] $seg]
-                    if {[regexp {[*?]} $seg]} {
-                        set pat [string map [list * {[^/]*} ? {[^/]}] $seg]
-                        lappend pats "$pat"
-                    } else {
-                        lappend pats "$seg"
-                    }
-                }
-            }
-            return  "^[join $pats /]\$"
-        }
-        proc globmatchpath {glob path} {
-            return [regexp [pathglob_as_re $glob] $path]
-        }
         ## unidirectional file transfer to possibly non empty folder
         #default of -overwrite no-targets will only copy files that are missing at the target
         #           -overwrite newer-targets will copy files with older source timestamp over newer target timestamp and those missing at the target (a form of 'restore' operation)
@@ -1349,7 +1327,7 @@ namespace eval punkcheck {
             }
             foreach unpub $opt_antiglob_paths {
                 #puts "testing folder - globmatchpath $unpub $relative_source_dir"
-                if {[globmatchpath $unpub $relative_source_dir]} {
+                if {[punk::path::globmatchpath $unpub $relative_source_dir]} {
                     lappend antiglob_paths_matched $current_source_dir
                     return [list files_copied {} files_skipped {} sources_unchanged {} punkcheck_records $punkcheck_records antiglob_paths_matched $antiglob_paths_matched srcdir $srcdir tgtdir $tgtdir punkcheck_folder $punkcheck_folder]
                 }
@@ -1421,7 +1399,7 @@ namespace eval punkcheck {
                 set is_antipath 0
                 foreach antipath $opt_antiglob_paths {
                     #puts "testing file - globmatchpath $antipath  vs $relative_source_path" 
-                    if {[globmatchpath $antipath $relative_source_path]} {
+                    if {[punk::path::globmatchpath $antipath $relative_source_path]} {
                         lappend antiglob_paths_matched $current_source_dir
                         set is_antipath 1
                         break

src/modules/punk/mix/templates/modules/template_module-0.0.1.tm

@@ -19,12 +19,12 @@
 #*** !doctools
 #[manpage_begin %pkg% 0 999999.0a1.0]
 #[copyright "%year%"]
-#[titledesc {Module API}]
-#[moddesc {-}]
+#[titledesc {Module API}]  [comment {-- Name section and table of contents description --}]
+#[moddesc {-}]             [comment {-- Description at end of page heading --}] 
 #[require %pkg%]
 #[description]
-# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
 
 #*** !doctools
 #[section Overview]
@@ -44,11 +44,11 @@
 
 package require Tcl 8.6
 #*** !doctools
-#[item] [package Tcl 8.6]
+#[item] [package {Tcl 8.6}]
 
-##package require frobz
-##*** !doctools
-##[item] [package frobz]
+# #package require frobz
+# #*** !doctools
+# #[item] [package {frobz}]
 
 #*** !doctools
 #[list_end]
@@ -69,21 +69,21 @@ namespace eval %pkg%::class {
         #*** !doctools
         #[list_begin enumerated]
 
-        oo::class create interface_sample1 {
-            #*** !doctools
-            #[enum] [emph {CLASS interface_sample1}]
-            #[list_begin definitions]
+        # oo::class create interface_sample1 {
+        #     #*** !doctools
+        #     #[enum] CLASS [class interface_sample1]
+        #     #[list_begin definitions]
 
-            method test {arg1} {
-                #*** !doctools
-                #[call class::[class interface_sample1] [method test] [arg arg1]]
-                #[para] test method
-                puts "test: $arg1"
-            }
+        #     method test {arg1} {
+        #         #*** !doctools
+        #         #[call class::interface_sample1    [method test] [arg arg1]]
+        #         #[para] test method
+        #         puts "test: $arg1"
+        #     }
 
-            #*** !doctools
-            #[list_end] [comment {-- end definitions interface_sample1}]
-        }
+        #     #*** !doctools
+        #     #[list_end] [comment {-- end definitions interface_sample1}]
+        # }
 
         #*** !doctools
         #[list_end] [comment {--- end class enumeration ---}]
@@ -109,7 +109,7 @@ namespace eval %pkg% {
 
 
 
-
+    #*** !doctools
     #[list_end] [comment {--- end definitions namespace %pkg% ---}]
 }
 # ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++

src/modules/punk/mix/util-999999.0a1.0.tm

@@ -131,83 +131,6 @@ namespace eval punk::mix::util {
     }
     #----------------------------------------
 
-    #maint warning - also in punkcheck
-    proc path_relative {base dst} {
-        #see also kettle
-        # Modified copy of ::fileutil::relative (tcllib)
-        # Adapted to 8.5 ({*}).
-        #
-        #    Taking two _directory_ paths, a base and a destination, computes the path
-        #    of the destination relative to the base.
-        #
-        # Arguments:
-        #    base    The path to make the destination relative to.
-        #    dst     The destination path
-        #
-        # Results:
-        #    The path of the destination, relative to the base.
-
-        # Ensure that the link to directory 'dst' is properly done relative to
-        # the directory 'base'.
-
-        #review - check volume info on windows.. UNC paths?
-        if {[file pathtype $base] ne [file pathtype $dst]} {
-            return -code error "Unable to compute relation for paths of different pathtypes: [file pathtype $base] vs. [file pathtype $dst], ($base vs. $dst)"
-        }
-
-        #avoid normalizing if possible (file normalize *very* expensive on windows)
-        set do_normalize 0
-        if {[file pathtype $base] eq "relative"} {
-            #if base is relative so is dst
-            if {[regexp {[.]{2}} [list $base $dst]]} {
-                set do_normalize 1
-            }
-            if {[regexp {[.]/} [list $base $dst]]} {
-                set do_normalize 1
-            }
-        } else {
-            set do_normalize 1
-        }
-        if {$do_normalize} {
-            set base [file normalize $base]
-            set dst  [file normalize $dst]
-        }
-
-        set save $dst
-        set base [file split $base]
-        set dst  [file split $dst]
-
-        while {[lindex $dst 0] eq [lindex $base 0]} {
-            set dst  [lrange $dst  1 end]
-            set base [lrange $base 1 end]
-            if {![llength $dst]} {break}
-        }
-
-        set dstlen  [llength $dst]
-        set baselen [llength $base]
-
-        if {($dstlen == 0) && ($baselen == 0)} {
-            # Cases:
-            # (a) base == dst
-
-            set dst .
-            } else {
-            # Cases:
-            # (b) base is: base/sub = sub
-            #     dst  is: base     = {}
-
-            # (c) base is: base     = {}
-            #     dst  is: base/sub = sub
-
-            while {$baselen > 0} {
-                set dst [linsert $dst 0 ..]
-                incr baselen -1
-            }
-            set dst [file join {*}$dst]
-        }
-
-        return $dst
-    }
     #namespace import ::punk::ns::nsimport_noclobber 
 
     proc namespace_import_pattern_to_namespace_noclobber {pattern ns} {

src/modules/punk/overlay-0.1.tm

@@ -132,6 +132,7 @@ namespace eval ::punk::overlay {
         set imported_commands [list]
         set nscaller [uplevel 1 [list namespace current]]
         if {[catch {
+            #review - noclobber?
             namespace eval ${nscaller}::temp_import [list namespace import ${cmdnamespace}::*]
             foreach cmd [info commands ${nscaller}::temp_import::*] {
                 set cmdtail [namespace tail $cmd]

src/modules/punk/path-999999.0a1.0.tm

@@ -0,0 +1,396 @@
+# -*- tcl -*-
+# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from <pkg>-buildversion.txt
+#
+# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem.
+# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository.
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+# (C) 2023
+#
+# @@ Meta Begin
+# Application punk::path 999999.0a1.0
+# Meta platform     tcl
+# Meta license      <unspecified>
+# @@ Meta End
+
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+# doctools header
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+#*** !doctools
+#[manpage_begin punk::path 0 999999.0a1.0]
+#[copyright "2023"]
+#[titledesc {Filesystem path utilities}]  [comment {-- Name section and table of contents description --}]
+#[moddesc {punk path filesystem utils}]             [comment {-- Description at end of page heading --}] 
+#[require punk::path]
+#[description]
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[section Overview]
+#[para] overview of punk::path
+#[para] Filesystem path utility functions
+#[subsection Concepts]
+#[para] -
+
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+##  Requirements
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[subsection dependencies]
+#[para] packages used by punk::path
+#[list_begin itemized]
+
+package require Tcl 8.6
+#*** !doctools
+#[item] [package {Tcl 8.6}]
+
+# #package require frobz
+# #*** !doctools
+# #[item] [package {frobz}]
+
+#*** !doctools
+#[list_end]
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+#*** !doctools
+#[section API]
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+# oo::class namespace
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+namespace eval punk::path::class {
+    #*** !doctools
+    #[subsection {Namespace punk::path::class}]
+    #[para] class definitions
+    if {[info commands [namespace current]::interface_sample1] eq ""} {
+        #*** !doctools
+        #[list_begin enumerated]
+
+        # oo::class create interface_sample1 {
+        #     #*** !doctools
+        #     #[enum] CLASS [class interface_sample1]
+        #     #[list_begin definitions]
+
+        #     method test {arg1} {
+        #         #*** !doctools
+        #         #[call class::interface_sample1    [method test] [arg arg1]]
+        #         #[para] test method
+        #         puts "test: $arg1"
+        #     }
+
+        #     #*** !doctools
+        #     #[list_end] [comment {-- end definitions interface_sample1}]
+        # }
+
+        #*** !doctools
+        #[list_end] [comment {--- end class enumeration ---}]
+    }
+}
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+# Base namespace
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+namespace eval punk::path {
+    namespace export *
+    #variable xyz
+
+    #*** !doctools
+    #[subsection {Namespace punk::path}]
+    #[para] Core API functions for punk::path 
+    #[list_begin definitions]
+
+
+    proc pathglob_as_re {pathglob} {
+        #*** !doctools
+        #[call [fun pathglob_as_re] [arg pathglob]]
+        #[para] Returns a regular expression for matching a path to a glob pattern which can contain glob chars *|? in any segment of the path structure
+        #[para] ** matches any number of subdirectories. 
+        #[para] e.g /etc/**/*.txt will match any .txt files at any depth below /etc  (except directly within /etc itself)
+        #[para] e.g /etc/**.txt will match any .txt files at any depth below /etc
+        #[para] any segment that does not contain ** must match exactly one segment in the path
+        #[para] e.g the glob /etc/*/*.doc  - will match any .doc files that are exactly one tree level below /etc 
+        #[para] The pathglob doesn't have to contain glob characters, in which case the returned regex will match the pathglob exactly as specified.
+        #[para] Regular expression syntax is deliberateley not supported within the pathglob string so that supplied regex characters will be treated as literals 
+
+
+        #todo - consider whether a way to escape the glob chars ? * is practical - to allow literals ? *
+        # - would require counting immediately-preceding backslashes
+        set pats [list]
+        foreach seg [file split $pathglob] {
+            if {[string range $seg end end] eq "/"} {
+                set seg [string range $seg 0 end-1] ;# e.g c:/ -> c:   / -> ""  so that join at end doesn't double up
+            }
+            if {$seg eq "*"} {
+                lappend pats {[^/]*}
+            } elseif {$seg eq "**"} {
+                lappend pats {.*}
+            } else {
+                set seg [string map [list {^ {\^} $ {\$} [} {\[} ( {\(} \{ \\\{ \\ {\\}] $seg] ;#treat regex characters in the input as literals 
+                set seg [string map [list  . {[.]}] $seg]
+                if {[regexp {[*?]} $seg]} {
+                    set pat [string map [list ** {.*} * {[^/]*} ? {[^/]}] $seg]
+                    lappend pats "$pat"
+                } else {
+                    lappend pats "$seg"
+                }
+            }
+        }
+        return  "^[join $pats /]\$"
+    }
+    proc globmatchpath {pathglob path args} {
+        #*** !doctools
+        #[call [fun globmatchpath] [arg pathglob] [arg path] [opt {option value...}]]
+        #[para] Return true if the pathglob matches the path
+        #[para] see [fun pathglob_as_re] for pathglob description
+        #[para] Caller must ensure that file separator is forward slash. (e.g use file normalize on windows)
+        #[para]
+        #[para] Known options:
+        #[para]     -nocase 0|1  (default 0 - case sensitive)
+        #[para] If -nocase is not supplied - default to case sensitive *except for driveletter*
+        #[para] ie - the driveletter alone in paths such as c:/etc will still be case insensitive. (ie c:/ETC/* will match C:/ETC/blah but not C:/etc/blah)
+        #[para] Explicitly specifying -nocase 0 will require the entire case to match including the driveletter.
+
+        set defaults [dict create\
+            -nocase \uFFFF\
+        ]
+        set known_opts [dict keys $defaults]
+        set opts [dict merge $defaults $args]
+        dict for {k v} $args {
+            if {$k ni $known_opts} {
+                error "Unrecognised options $k - known options: $known_opts"
+            }
+        }
+        # -- --- --- --- --- --- 
+        set opt_nocase [dict get $opts -nocase]
+        set explicit_nocase 1 ;#default to disprove 
+        if {$opt_nocase eq "\uFFFF"} {
+            set opt_nocase 0
+            set explicit_nocase 0
+        } 
+        # -- --- --- --- --- --- 
+        if {$opt_nocase} {
+            return [regexp -nocase [pathglob_as_re $pathglob] $path]
+        } else {
+            set re [pathglob_as_re $pathglob]
+            if {$explicit_nocase} {
+                set ismatch [regexp $re $path] ;#explicit -nocase 0  - require exact match of path literals including driveletter
+            } else {
+                #caller is using default for -nocase - which indicates case sensitivity - but we have an exception for the driveletter.
+                set re_segments [file split $re] ;#Note that file split c:/etc gives {c:/ etc}  but file split ^c:/etc gives {^c: etc}
+                set first_seg [lindex $re_segments 0]
+                if {[regexp {^\^(.{1}):$} $first_seg _match driveletter]} {
+                    #first part of re is like "^c:" i.e a drive letter
+                    set chars [string tolower $driveletter][string toupper $driveletter]
+                    set re [join [concat "^\[$chars\]:" [lrange $re_segments 1 end]] /] ;#rebuild re with case insensitive driveletter only - use join - not file join. file join will misinterpret leading re segment.
+                }
+                #puts stderr "-->re: $re"
+                set ismatch [regexp $re $path]
+            }
+        }
+        return $ismatch
+    }
+
+    #todo - implement treefiles which acts like dirfiles but allows path globbing in the same way as punk::ns::ns/ 
+    #then review if treefiles can replace dirfiles or if both should exist (dirfiles can have literal glob chars in path segments - but that is a rare usecase)
+    proc treefilenames {basepath tailglob args} {
+        #*** !doctools
+        #[call [fun treefilenames] [arg basepath] [arg tailglob] [opt {option value...}]]
+        #basic (glob based) list of filenames matching tailglob - recursive
+        #no natsorting - so order is dependent on filesystem
+        set defaults [dict create\
+            -call-depth-internal 0\
+            -antiglob_paths {}\
+        ]
+        set opts [dict merge $defaults $args]
+        set opt_antiglob_paths [dict get $opts -antiglob_paths]
+        set CALLDEPTH [dict get $opts -call-depth-internal]
+
+        set files [list]
+        if {$CALLDEPTH == 0} {
+            if {![file isdirectory $basepath]} {
+                return [list] 
+            }
+        }
+
+        set skip 0
+        foreach anti $opt_antiglob_paths {
+            if {[globmatchpath $anti $basepath]} {
+                set skip 1
+                break
+            }
+        }
+        if {$skip} {
+            return [list]
+        }
+
+        #todo - account for vfs where matched path could appear to be a directory but is mounted so could be a desired match?
+        set dirfiles [glob -nocomplain -dir $basepath -type f $tailglob] 
+        lappend files  {*}$dirfiles
+        set dirdirs [glob -nocomplain -dir $basepath -type d *]
+        foreach dir $dirdirs {
+            set skip 0
+            foreach anti $opt_antiglob_paths {
+                if {[globmatchpath $anti $dir]} {
+                    set skip 1
+                    break 
+                }
+            }
+            if {$skip} {
+                continue
+            }
+            set nextargs [dict merge $args [list -call-depth-internal [incr CALLDEPTH]]]
+            lappend files {*}[treefilenames $dir $tailglob {*}$nextargs]
+        }
+        return $files
+    }
+
+    #maint warning - also in punkcheck
+    proc relative {reference location} {
+        #*** !doctools
+        #[call [fun relative] [arg reference] [arg location]]
+        #[para]    Taking two directory paths, a reference and a location, computes the path
+        #    of the location relative to the reference.
+        #[list_begin itemized]
+        #[item]
+        #[para] Arguments:
+        #       [list_begin arguments]
+        #       [arg_def string reference]       The path from which the relative path to location is determined. 
+        #       [arg_def string location]        The location path which may be above or below the reference path 
+        #       [list_end]
+        #[item]
+        #[para] Results:
+        #[para]    The relative path of the location to the reference path.
+        #[para]    Will return a single dot "." if the paths are the same
+        #[item]
+        #[para] Notes:
+        #[para]    Both paths must be the same type - ie both absolute or both relative
+        #[para]    Case sensitive.  ie  relative /etc /etC 
+        #      will return ../etC
+        #[para]    On windows, the drive-letter component (only) is not case sensitive
+        #[para]    ie relative c:/etc  C:/etc  returns .  
+        #[para]    but relative c:/etc C:/Etc returns ../Etc
+        #[para]    On windows, if the paths are absolute and specifiy different volumes, only the location will be returned.
+        #    ie  relative c:/etc d:/etc/blah
+        #    returns d:/etc/blah
+        #[list_end]
+
+        #see also kettle
+        # Modified copy of ::fileutil::relative (tcllib)
+        # Adapted to 8.5 ({*}).
+
+        #review - check volume info on windows.. UNC paths?
+        if {[file pathtype $reference] ne [file pathtype $location]} {
+            return -code error "Unable to compute relation for paths of different pathtypes: [file pathtype $reference] vs. [file pathtype $location], ($reference vs. $location)"
+        }
+
+        #avoid normalizing if possible (file normalize *very* expensive on windows)
+        set do_normalize 0
+        if {[file pathtype $reference] eq "relative"} {
+            #if reference is relative so is location 
+            if {[regexp {[.]{2}} [list $reference $location]]} {
+                set do_normalize 1
+            }
+            if {[regexp {[.]/} [list $reference $location]]} {
+                set do_normalize 1
+            }
+        } else {
+            set do_normalize 1
+        }
+        if {$do_normalize} {
+            set reference [file normalize $reference]
+            set location  [file normalize $location]
+        }
+
+        set save $location
+        set reference [file split $reference]
+        set location  [file split $location]
+
+        while {[lindex $location 0] eq [lindex $reference 0]} {
+            set location  [lrange $location  1 end]
+            set reference [lrange $reference 1 end]
+            if {![llength $location]} {break}
+        }
+
+        set location_len  [llength $location]
+        set reference_len [llength $reference]
+
+        if {($location_len == 0) && ($reference_len == 0)} {
+            # Cases:
+            # (a) reference == location
+
+            set location .
+        } else {
+            # Cases:
+            # (b) ref is: ref/sub = sub
+            #     loc is: ref     = {}
+
+            # (c) ref is: ref     = {}
+            #     loc is: ref/sub = sub
+
+            while {$reference_len > 0} {
+                set location [linsert $location 0 ..]
+                incr reference_len -1
+            }
+            set location [file join {*}$location]
+        }
+        return $location
+    }
+
+
+
+    #*** !doctools
+    #[list_end] [comment {--- end definitions namespace punk::path ---}]
+}
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+# Secondary API namespace
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+namespace eval punk::path::lib {
+        namespace export *
+        namespace path [namespace parent]
+        #*** !doctools
+        #[subsection {Namespace punk::path::lib}]
+        #[para] Secondary functions that are part of the API 
+        #[list_begin definitions]
+
+
+
+
+
+        #*** !doctools
+        #[list_end] [comment {--- end definitions namespace punk::path::lib ---}]
+}
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+
+
+
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+#*** !doctools
+#[section Internal]
+namespace eval punk::path::system {
+        #*** !doctools
+        #[subsection {Namespace punk::path::system}]
+        #[para] Internal functions that are not part of the API 
+
+
+
+}
+# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
+##  Ready   
+package provide punk::path [namespace eval punk::path {
+    variable pkg punk::path
+    variable version
+    set version 999999.0a1.0 
+}]
+return
+
+#*** !doctools
+#[manpage_end]
+

src/modules/punk/path-buildversion.txt

@@ -0,0 +1,3 @@
+0.1.0
+#First line must be a semantic version number
+#all other lines are ignored.

src/modules/punk/repl-0.1.tm

@@ -138,7 +138,7 @@ if {$::tcl_platform(platform) eq "windows"} {
         #puts stdout "===============repl loading twapi==========="
         #zzzload::pkg_wait twapi
 
-        if {![catch {package require twapix}]} {
+        if {![catch {package require twapi}]} {
 
             proc ::repl::term::handler_console_control {args} {
                 puts -nonewline stdout .

src/modules/punk/repo-999999.0a1.0.tm

@@ -33,8 +33,9 @@ if {$::tcl_platform(platform) eq "windows"} {
     catch {package require punk::winpath}
 }
 package require fileutil; #tcllib
+package require punk::path
 package require punk::mix::base ;#uses core functions from punk::mix::base::lib namespace e.g cksum_path 
-package require punk::mix::util
+package require punk::mix::util ;#do_in_path
 
 
 # -- --- --- --- --- --- --- --- --- --- ---
@@ -357,7 +358,7 @@ namespace eval punk::repo {
         if {$repodir eq ""} {
             error "workingdir_state error: No repository found at or above path '$abspath'" 
         }
-        set subpath [punk::mix::util::path_relative $repodir $abspath]
+        set subpath [punk::path::relative $repodir $abspath]
         if {$subpath eq "."} {
             set subpath ""
         }
@@ -708,10 +709,10 @@ namespace eval punk::repo {
                 append msg "** starting folder : $start_dir" \n
                 append msg "** untracked       : $candroot" \n
                 if {$closest_fossil_len} {
-                    append msg "** fossil root     : $closest_fossil    ([punk::mix::util::path_relative $start_dir $closest_fossil])" \n
+                    append msg "** fossil root     : $closest_fossil    ([punk::path::relative $start_dir $closest_fossil])" \n
                 }
                 if {$closest_git_len} {
-                    append msg "** git root        : $closest_git    ([punk::mix::util::path_relative $start_dir $closest_git])" \n
+                    append msg "** git root        : $closest_git    ([punk::path::relative $start_dir $closest_git])" \n
                 }
                 append msg "**" \n
         }

src/modules/punkcheck-0.1.0.tm

@@ -19,6 +19,7 @@
 ##e.g package require frobz
 
 package require punk::tdl
+package require punk::path
 package require punk::repo
 package require punk::mix::util
 
@@ -1106,29 +1107,6 @@ namespace eval punkcheck {
                 return [lindex $args end] 
             }
         }
-        proc pathglob_as_re {glob} {
-            #any segment that is not just * must match exactly one segment in the path
-            set pats [list]
-            foreach seg [file split $glob] {
-                if {$seg eq "*"} {
-                    lappend pats {[^/]*}
-                } elseif {$seg eq "**"} {
-                    lappend pats {.*}
-                } else {
-                    set seg [string map [list  . {[.]}] $seg]
-                    if {[regexp {[*?]} $seg]} {
-                        set pat [string map [list * {[^/]*} ? {[^/]}] $seg]
-                        lappend pats "$pat"
-                    } else {
-                        lappend pats "$seg"
-                    }
-                }
-            }
-            return  "^[join $pats /]\$"
-        }
-        proc globmatchpath {glob path} {
-            return [regexp [pathglob_as_re $glob] $path]
-        }
         ## unidirectional file transfer to possibly non empty folder
         #default of -overwrite no-targets will only copy files that are missing at the target
         #           -overwrite newer-targets will copy files with older source timestamp over newer target timestamp and those missing at the target (a form of 'restore' operation)
@@ -1349,7 +1327,7 @@ namespace eval punkcheck {
             }
             foreach unpub $opt_antiglob_paths {
                 #puts "testing folder - globmatchpath $unpub $relative_source_dir"
-                if {[globmatchpath $unpub $relative_source_dir]} {
+                if {[punk::path::globmatchpath $unpub $relative_source_dir]} {
                     lappend antiglob_paths_matched $current_source_dir
                     return [list files_copied {} files_skipped {} sources_unchanged {} punkcheck_records $punkcheck_records antiglob_paths_matched $antiglob_paths_matched srcdir $srcdir tgtdir $tgtdir punkcheck_folder $punkcheck_folder]
                 }
@@ -1421,7 +1399,7 @@ namespace eval punkcheck {
                 set is_antipath 0
                 foreach antipath $opt_antiglob_paths {
                     #puts "testing file - globmatchpath $antipath  vs $relative_source_path" 
-                    if {[globmatchpath $antipath $relative_source_path]} {
+                    if {[punk::path::globmatchpath $antipath $relative_source_path]} {
                         lappend antiglob_paths_matched $current_source_dir
                         set is_antipath 1
                         break

src/modules/zzzload-999999.0a1.0.tm

@@ -51,6 +51,10 @@ namespace eval zzzload {
     }
     proc pkg_require {pkgname args} {
         variable loader_tid
+        if {[set ver [package provide twapi]] ne ""} {
+            #skip the whole shebazzle if it's already loaded
+            return $ver
+        }
         if {$loader_tid eq ""} {
             set loader_tid [thread::create -joinable -preserved]
         }
@@ -75,6 +79,10 @@ namespace eval zzzload {
         }
     }
     proc pkg_wait {pkgname} {
+        if {[set ver [package provide twapi]] ne ""} {
+            return $ver
+        }
+
         set pkgstate [tsv::get zzzload_pkg $pkgname]
         if {$pkgstate eq "loading"} {
             set mutex [tsv::get zzzload_pkg_mutex $pkgname]

src/vendorlib/tar/ChangeLog

@@ -0,0 +1,186 @@
+2013-11-22  Andreas Kupries <andreask@activestate.com>
+
+	* tar.man: Reviewed the work on the pyk-tar branch. Brought
+	* tar.tcl: new testsuite up to spec. Reviewed the skip fix,
+	* tar.test: modified it to reinstate the skip limit per round
+	* test-support.tcl: without getting the bug back. Bumped version
+	  to 0.9. Thanks to PoorYorick for the initial work on the bug,
+	  fix, and testsuite. This also fixes ticket [6b7aa0aecc].
+
+2013-08-12  Andreas Kupries  <andreask@activestate.com>
+
+	* tar.man (tar::untar, contents, stat, get): Extended the
+	* tar.tcl: procedures to detect and properly handle @LongName
+	* pkgIndex.tcl: header entries as generated by GNU tar. These
+	  entries contain the file name for the next header entry as file
+	  data, for files whose name is longer than the 100-char field of
+	  the regular header. Version bumped to 0.8. This is a new
+	  feature.
+
+2013-02-01  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	*
+	* Released and tagged Tcllib 1.15 ========================
+	* 
+
+2012-09-11  Andreas Kupries  <andreask@activestate.com>
+
+	* tar.tcl (seekorskip): Fixed seekorskip which prevented its use
+	* pkgIndex.tcl: from a non-seekable channel, like stdin. The issue
+	  was that the original attempt to seek before skipping not just
+	  failed, but apparently still moved the read pointer in some way
+	  which skipped over irreplacable input, breaking the next call of
+	  readHeader. Using [tell] to check seekability does not break in
+	  this manner. Bumped version to 0.7.1.
+
+2011-12-13  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	*
+	* Released and tagged Tcllib 1.14 ========================
+	* 
+
+2011-01-24  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	*
+	* Released and tagged Tcllib 1.13 ========================
+	* 
+
+2011-01-20  Andreas Kupries  <andreask@activestate.com>
+
+	* tar.tcl: [Bug 3162548]: Applied patch by Alexandre Ferrieux,
+	* tar.man: extending various tar commands to be able to use
+	* pkgIndex.tcl: the -chan option, and channels instead of files.
+	  Version bumped to 0.7
+
+2009-12-07  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	*
+	* Released and tagged Tcllib 1.12 ========================
+	* 
+
+2009-12-03  Andreas Kupries  <andreask@activestate.com>
+
+	* tar.man: [Patch 2840147]. Applied. New options -prefix and
+	* tar.tcl: -quick for tar::add. -prefix allows specifying a
+	* tar.pcx: prefix for filenames in the archive, and -quick 1
+	* pkgIndex.tcl: changes back to the seek-from-end algorithm for
+	  finding the place where to add the new files. The new default
+	  scans from start (robust). Bumped version to 0.6.
+
+2009-05-12  Aaron Faupell  <afaupell@users.sourceforge.net>
+
+	* tar.tcl: add support for reading pre-posix archives.
+	  if a file isnt writable when extracting, try deleting
+	  before giving up.
+
+2008-12-12  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	*
+	* Released and tagged Tcllib 1.11.1 ========================
+	* 
+
+2008-11-26  Aaron Faupell  <afaupell@users.sourceforge.net>
+
+	* tar.man: add and clarify documentation
+
+2008-10-16  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	*
+	* Released and tagged Tcllib 1.11 ========================
+	* 
+
+2008-06-14  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	* tar.pcx: New file. Syntax definitions for the public commands of
+	  the tar package.
+
+2007-09-12  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	*
+	* Released and tagged Tcllib 1.10 ========================
+	* 
+
+2007-03-21  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	* tar.man: Fixed all warnings due to use of now deprecated
+	  commands. Added a section about how to give feedback.
+
+2007-02-08  Aaron Faupell  <afaupell@users.sourceforge.net>
+
+	* tar.tcl: bug fix in recursion algorithm that missed
+	some files in deep subdirs. incremented version
+
+2007-01-08  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	* tar.tcl: Bumped version to 0.3, for the bugfix described
+	* tar.man: by the last entry.
+	* pkgIndex.tcl:
+
+2006-12-20  Aaron Faupell  <afaupell@users.sourceforge.net>
+
+	* tar.tcl: fix in parseOpts which affected -file and -glob
+	  arguments to tar::untar
+	* tar.man: clarifications to add, create, and untar
+
+2006-10-03  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	*
+	* Released and tagged Tcllib 1.9 ========================
+	* 
+
+2006-29-06  Aaron Faupell  <afaupell@users.sourceforge.net>
+
+	* tar.tcl: fixed bug in parseOpts
+
+2005-11-08  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	* pkgIndex.tcl: Corrected buggy commit, synchronized version
+	* tar.man:      numbers across all relevant files.
+
+2005-11-08  Aaron Faupell  <afaupell@users.sourceforge.net>
+
+	* tar.tcl: bumped version to 0.2 because of new feature
+	* tar.man: tar::remove
+
+2005-11-07  Andreas Kupries <andreask@activestate.com>
+
+	* tar.man: Fixed error, incorrect placement of [call] markup
+	  outside of list.
+
+2005-11-04  Aaron Faupell  <afaupell@users.sourceforge.net>
+
+	* tar.man: added tar::remove command and documentation for it
+	* tar.tcl:
+
+2005-10-06  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	*
+	* Released and tagged Tcllib 1.8 ========================
+	* 
+
+2005-09-30  Andreas Kupries <andreask@activestate.com>
+
+	* tar.tcl: qualified all [open] calls with :: to ensure usag of
+	  the builtin. Apparently mitigates conflict between this package
+	  and the vfs::tar module.
+
+2004-10-05  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	*
+	* Released and tagged Tcllib 1.7 ========================
+	* 
+
+2004-10-02  Andreas Kupries  <andreas_kupries@users.sourceforge.net>
+
+	* tar.man: Added keywords and title/module description to the
+	  documentation.
+
+2004-09-10  Aaron Faupell  <afaupell@users.sourceforge.net>
+
+	* tar.tcl: Fixed typo bug in ::tar::add
+	* tar.man: Added info for ::tar::stat
+
+2004-08-23  Andreas Kupries  <andreask@activestate.com>
+
+	* tar.man: Fixed problems in the documentation.
+

src/vendorlib/tar/pkgIndex.tcl

@@ -0,0 +1,5 @@
+if {![package vsatisfies [package provide Tcl] 8.5 9]} {
+    # PRAGMA: returnok
+    return
+}
+package ifneeded tar 0.12 [list source [file join $dir tar.tcl]]

src/vendorlib/tar/tar.man

@@ -0,0 +1,202 @@
+[comment {-*- mode: tcl ; fill-column: 80 -*- doctools manpage}]
+[vset PACKAGE_VERSION 0.12]
+[manpage_begin tar n [vset PACKAGE_VERSION]]
+[keywords archive]
+[keywords {tape archive}]
+[keywords tar]
+[moddesc   {Tar file handling}]
+[titledesc {Tar file creation, extraction & manipulation}]
+[category  {File formats}]
+[require Tcl "8.5 9"]
+[require tar [opt [vset PACKAGE_VERSION]]]
+[description]
+
+[para] [strong Note]: Starting with version 0.8 the tar reader commands
+(contents, stats, get, untar) support the GNU LongName extension (header type
+'L') for large paths.
+
+[para] 
+
+[section BEWARE]
+
+For all commands, when using [option -chan] ...
+
+[list_begin enumerated]
+
+[enum] It is assumed that the channel was opened for reading, and configured for
+       binary input.
+
+[enum] It is assumed that the channel position is at the beginning of a legal
+       tar file.
+
+[enum] The commands will [emph modify] the channel position as they perform their
+       task.
+
+[enum] The commands will [emph not] close the channel.
+
+[enum] In other words, the commands leave the channel in a state very likely
+       unsuitable for use by further [cmd tar] commands. Still doing so will
+       very likely results in errors, bad data, etc. pp.
+
+[enum] It is the responsibility of the user to seek the channel back to a
+       suitable position.
+
+[enum] When using a channel transformation which is not generally seekable, for
+       example [cmd gunzip], then it is the responsibility of the user to (a)
+       unstack the transformation before seeking the channel back to a suitable
+       position, and (b) for restacking it after.
+
+[list_end]
+
+[section COMMANDS]
+
+[list_begin definitions]
+
+[call [cmd ::tar::contents] [arg tarball] [opt [option -chan]]]
+
+Returns a list of the files contained in [arg tarball]. The order is not sorted and depends on the order
+files were stored in the archive.
+[para]
+
+If the option [option -chan] is present [arg tarball] is interpreted as an open channel.
+It is assumed that the channel was opened for reading, and configured for binary input.
+The command will [emph not] close the channel.
+
+[call [cmd ::tar::stat] [arg tarball] [opt file] [opt [option -chan]]]
+
+Returns a nested dict containing information on the named [opt file] in [arg tarball],
+or all files if none is specified. The top level are pairs of filename and info. The info is a dict with the keys
+"[const mode] [const uid] [const gid] [const size] [const mtime] [const type] [const linkname] [const uname] [const gname]
+ [const devmajor] [const devminor]"
+
+[example {
+% ::tar::stat tarball.tar
+foo.jpg {mode 0644 uid 1000 gid 0 size 7580 mtime 811903867 type file linkname {} uname user gname wheel devmajor 0 devminor 0}
+}]
+
+[para]
+If the option [option -chan] is present [arg tarball] is interpreted as an open channel.
+It is assumed that the channel was opened for reading, and configured for binary input.
+The command will [emph not] close the channel.
+
+[call [cmd ::tar::untar] [arg tarball] [arg args]]
+
+Extracts [arg tarball]. [arg -file] and [arg -glob] limit the extraction
+to files which exactly match or pattern match the given argument. No error is
+thrown if no files match. Returns a list of filenames extracted and the file
+size. The size will be null for non regular files. Leading path seperators are
+stripped so paths will always be relative.
+
+[list_begin options]
+[opt_def -dir dirName]
+Directory to extract to. Uses [cmd pwd] if none is specified
+[opt_def -file fileName]
+Only extract the file with this name. The name is matched against the complete path
+stored in the archive including directories.
+[opt_def -glob pattern]
+Only extract files patching this glob style pattern. The pattern is matched against the complete path
+stored in the archive.
+[opt_def -nooverwrite]
+Dont overwrite files that already exist
+[opt_def -nomtime]
+Leave the file modification time as the current time instead of setting it to the value in the archive.
+[opt_def -noperms]
+In Unix, leave the file permissions as the current umask instead of setting them to the values in the archive.
+
+[opt_def -chan]
+If this option is present [arg tarball] is interpreted as an open channel.
+It is assumed that the channel was opened for reading, and configured for binary input.
+The command will [emph not] close the channel.
+
+[list_end]
+[para]
+
+[example {
+% foreach {file size} [::tar::untar tarball.tar -glob *.jpg] {
+puts "Extracted $file ($size bytes)"
+}
+}]
+
+[call [cmd ::tar::get] [arg tarball] [arg fileName] [opt [option -chan]]]
+
+Returns the contents of [arg fileName] from the [arg tarball].
+
+[para][example {
+% set readme [::tar::get tarball.tar doc/README] {
+% puts $readme
+}
+}]
+
+[para] If the option [option -chan] is present [arg tarball] is
+interpreted as an open channel.  It is assumed that the channel was
+opened for reading, and configured for binary input.  The command will
+[emph not] close the channel.
+
+[para] An error is thrown when [arg fileName] is not found in the tar
+archive.
+
+[call [cmd ::tar::create] [arg tarball] [arg files] [arg args]]
+
+Creates a new tar file containing the [arg files]. [arg files] must be specified
+as a single argument which is a proper list of filenames.
+
+[list_begin options]
+[opt_def -dereference]
+Normally [cmd create] will store links as an actual link pointing at a file that may
+or may not exist in the archive. Specifying this option will cause the actual file point to
+ by the link to be stored instead.
+
+[opt_def -chan]
+If this option is present [arg tarball] is interpreted as an open channel.
+It is assumed that the channel was opened for writing, and configured for binary output.
+The command will [emph not] close the channel.
+
+[list_end]
+[para]
+
+[example {
+% ::tar::create new.tar [glob -nocomplain file*]
+% ::tar::contents new.tar
+file1 file2 file3
+}]
+
+[call [cmd ::tar::add] [arg tarball] [arg files] [arg args]]
+
+Appends [arg files] to the end of the existing [arg tarball]. [arg files] must be specified
+as a single argument which is a proper list of filenames.
+
+[list_begin options]
+[opt_def -dereference]
+Normally [cmd add] will store links as an actual link pointing at a file that may
+or may not exist in the archive. Specifying this option will cause the actual file point to
+ by the link to be stored instead.
+[opt_def -prefix string]
+Normally [cmd add] will store files under exactly the name specified as
+argument. Specifying a [opt -prefix] causes the [arg string] to be
+prepended to every name.
+[opt_def -quick]
+The only sure way to find the position in the [arg tarball] where new
+files can be added is to read it from start, but if [arg tarball] was
+written with a "blocksize" of 1 (as this package does) then one can
+alternatively find this position by seeking from the end. The
+[opt -quick] option tells [cmd add] to do the latter.
+[list_end]
+[para]
+
+[call [cmd ::tar::remove] [arg tarball] [arg files]]
+
+Removes [arg files] from the [arg tarball]. No error will result if the file does not exist in the
+tarball. Directory write permission and free disk space equivalent to at least the size of the tarball
+will be needed.
+
+[example {
+% ::tar::remove new.tar {file2 file3}
+% ::tar::contents new.tar
+file3
+}]
+
+[list_end]
+
+[vset CATEGORY tar]
+[include ../common-text/feedback.inc]
+[manpage_end]

src/vendorlib/tar/tar.pcx

@@ -0,0 +1,83 @@
+# -*- tcl -*- tar.pcx
+# Syntax of the commands provided by package tar.
+#
+# For use by TclDevKit's static syntax checker (v4.1+).
+# See http://www.activestate.com/solutions/tcl/
+# See http://aspn.activestate.com/ASPN/docs/Tcl_Dev_Kit/4.0/Checker.html#pcx_api
+# for the specification of the format of the code in this file.
+#
+
+package require pcx
+pcx::register tar
+pcx::tcldep   0.4 needs tcl 8.2
+pcx::tcldep   0.5 needs tcl 8.2
+pcx::tcldep   0.6 needs tcl 8.2
+
+namespace eval ::tar {}
+
+#pcx::message FOO {... text ...} type
+#pcx::scan <VERSION> <NAME> <RULE>
+
+pcx::check 0.4 std ::tar::add \
+    {checkSimpleArgs 2 -1 {
+	checkFileName
+	{checkListValues 1 -1 checkFileName}
+	{checkSwitches 1 {
+	    {-dereference checkBoolean}
+	} {}}
+    }}
+pcx::check 0.6 std ::tar::add \
+    {checkSimpleArgs 2 -1 {
+	checkFileName
+	{checkListValues 1 -1 checkFileName}
+	{checkSwitches 1 {
+	    {-dereference checkBoolean}
+	    {-quick checkBoolean}
+	    {-prefix checkWord}
+	} {}}
+    }}
+pcx::check 0.4 std ::tar::contents \
+    {checkSimpleArgs 1 1 {
+	checkFileName
+    }}
+pcx::check 0.4 std ::tar::create \
+    {checkSimpleArgs 2 -1 {
+	checkFileName
+	{checkListValues 1 -1 checkFileName}
+	{checkSwitches 1 {
+	    {-chan        checkChannelID}
+	    {-dereference checkBoolean}
+	} {}}
+    }}
+pcx::check 0.4 std ::tar::get \
+    {checkSimpleArgs 2 2 {
+	checkFileName
+	checkFileName
+    }}
+pcx::check 0.4 std ::tar::remove \
+    {checkSimpleArgs 2 2 {
+	checkFileName
+	{checkListValues 1 -1 checkFileName}
+    }}
+pcx::check 0.4 std ::tar::stat \
+    {checkSimpleArgs 1 2 {
+	checkFileName
+	checkFileName
+    }}
+pcx::check 0.4 std ::tar::untar \
+    {checkSimpleArgs 1 -1 {
+	checkFileName
+	{checkSwitches 1 {
+	    {-chan        checkChannelID}
+	    {-dir         checkFileName}
+	    {-file        checkFileName}
+	    {-glob        checkPattern}
+	    {-nomtime     checkBoolean}
+	    {-nooverwrite checkBoolean}
+	    {-noperms     checkBoolean}
+	} {}}
+    }}
+
+# Initialization via pcx::init.
+# Use a ::tar::init procedure for non-standard initialization.
+pcx::complete

src/vendorlib/tar/tar.tcl

@@ -0,0 +1,550 @@
+# tar.tcl --
+#
+#       Creating, extracting, and listing posix tar archives
+#
+# Copyright (c) 2004    Aaron Faupell <afaupell@users.sourceforge.net>
+# Copyright (c) 2013    Andreas Kupries <andreas_kupries@users.sourceforge.net>
+#                       (GNU tar @LongLink support).
+#
+# See the file "license.terms" for information on usage and redistribution
+# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
+# 
+# RCS: @(#) $Id: tar.tcl,v 1.17 2012/09/11 17:22:24 andreas_kupries Exp $
+
+package require Tcl 8.5 9
+package provide tar 0.12
+
+namespace eval ::tar {}
+
+proc ::tar::parseOpts {acc opts} {
+    array set flags $acc
+    foreach {x y} $acc {upvar $x $x}
+    
+    set len [llength $opts]
+    set i 0
+    while {$i < $len} {
+        set name [string trimleft [lindex $opts $i] -]
+        if {![info exists flags($name)]} {
+	    return -errorcode {TAR INVALID OPTION} \
+		-code error "unknown option \"$name\""
+	}
+        if {$flags($name) == 1} {
+            set $name [lindex $opts [expr {$i + 1}]]
+            incr i $flags($name)
+        } elseif {$flags($name) > 1} {
+            set $name [lrange $opts [expr {$i + 1}] [expr {$i + $flags($name)}]]
+            incr i $flags($name)
+        } else {
+            set $name 1
+        }
+        incr i
+    }
+}
+
+proc ::tar::pad {size} {
+    set pad [expr {512 - ($size % 512)}]
+    if {$pad == 512} {return 0}
+    return $pad
+}
+
+proc ::tar::seekorskip {ch off wh} {
+    if {[tell $ch] < 0} {
+	if {$wh!="current"} {
+	    return -code error -errorcode [list TAR INVALID WHENCE $wh] \
+		"WHENCE=$wh not supported on non-seekable channel $ch"
+	}
+	skip $ch $off
+	return
+    }
+    seek $ch $off $wh
+    return
+}
+
+proc ::tar::skip {ch skipover} {
+    while {$skipover > 0} {
+	set requested $skipover
+
+	# Limit individual skips to 64K, as a compromise between speed
+	# of skipping (Number of read requests), and memory usage
+	# (Note how skipped block is read into memory!). While the
+	# read data is immediately discarded it still generates memory
+	# allocation traffic, gets copied, etc. Trying to skip the
+	# block in one go without the limit may cause us to run out of
+	# (virtual) memory, or just induce swapping, for nothing.
+
+	if {$requested > 65536} {
+	    set requested 65536
+	}
+
+	set skipped [string length [read $ch $requested]]
+
+	# Stop in short read into the end of the file.
+	if {!$skipped && [eof $ch]} break
+
+	# Keep track of how much is (not) skipped yet.
+	incr skipover -$skipped
+    }
+    return
+}
+
+proc ::tar::readHeader {data} {
+    binary scan $data a100a8a8a8a12a12a8a1a100a6a2a32a32a8a8a155 \
+                      name mode uid gid size mtime cksum type \
+                      linkname magic version uname gname devmajor devminor prefix
+
+    foreach x {name type linkname} {
+        set $x [string trim [set $x] "\x00"]
+    }
+    foreach x {uid gid size mtime cksum} {
+        set $x [format %d 0[string trim [set $x] " \x00"]]
+    }
+    set mode [string trim $mode " \x00"]
+
+    if {$magic == "ustar "} {
+        # gnu tar
+        # not fully supported
+        foreach x {uname gname prefix} {
+            set $x [string trim [set $x] "\x00"]
+        }
+        foreach x {devmajor devminor} {
+            set $x [format %d 0[string trim [set $x] " \x00"]]
+        }
+    } elseif {$magic == "ustar\x00"} {
+        # posix tar
+        foreach x {uname gname prefix} {
+            set $x [string trim [set $x] "\x00"]
+        }
+        foreach x {devmajor devminor} {
+            set $x [format %d 0[string trim [set $x] " \x00"]]
+        }
+    } else {
+        # old style tar
+        foreach x {uname gname devmajor devminor prefix} { set $x {} }
+        if {$type == ""} {
+            if {[string match */ $name]} {
+                set type 5
+            } else {
+                set type 0
+            }
+        }
+    }
+
+    return [list name $name mode $mode uid $uid gid $gid size $size mtime $mtime \
+                 cksum $cksum type $type linkname $linkname magic $magic \
+                 version $version uname $uname gname $gname devmajor $devmajor \
+                 devminor $devminor prefix $prefix]
+}
+
+proc ::tar::contents {file args} {
+    set chan 0
+    parseOpts {chan 0} $args
+    if {$chan} {
+	set fh $file
+    } else {
+	set fh [::open $file]
+	fconfigure $fh -encoding binary -translation lf -eofchar {}
+    }
+    set ret {}
+    while {![eof $fh]} {
+        array set header [readHeader [read $fh 512]]
+	HandleLongLink $fh header
+        if {$header(name) == ""} break
+	if {$header(prefix) != ""} {append header(prefix) /}
+        lappend ret $header(prefix)$header(name)
+        seekorskip $fh [expr {$header(size) + [pad $header(size)]}] current
+    }
+    if {!$chan} {
+	close $fh
+    }
+    return $ret
+}
+
+proc ::tar::stat {tar {file {}} args} {
+    set chan 0
+    parseOpts {chan 0} $args
+    if {$chan} {
+	set fh $tar
+    } else {
+	set fh [::open $tar]
+	fconfigure $fh -encoding binary -translation lf -eofchar {}
+    }
+    set ret {}
+    while {![eof $fh]} {
+        array set header [readHeader [read $fh 512]]
+	HandleLongLink $fh header
+        if {$header(name) == ""} break
+	if {$header(prefix) != ""} {append header(prefix) /}
+        seekorskip $fh [expr {$header(size) + [pad $header(size)]}] current
+        if {$file != "" && "$header(prefix)$header(name)" != $file} {continue}
+        set header(type) [string map {0 file 5 directory 3 characterSpecial 4 blockSpecial 6 fifo 2 link} $header(type)]
+        set header(mode) [string range $header(mode) 2 end]
+        lappend ret $header(prefix)$header(name) [list mode $header(mode) uid $header(uid) gid $header(gid) \
+                    size $header(size) mtime $header(mtime) type $header(type) linkname $header(linkname) \
+                    uname $header(uname) gname $header(gname) devmajor $header(devmajor) devminor $header(devminor)]
+    }
+    if {!$chan} {
+	close $fh
+    }
+    return $ret
+}
+
+proc ::tar::get {tar file args} {
+    set chan 0
+    parseOpts {chan 0} $args
+    if {$chan} {
+	set fh $tar
+    } else {
+	set fh [::open $tar]
+	fconfigure $fh -encoding binary -translation lf -eofchar {}
+    }
+    while {![eof $fh]} {
+	set data [read $fh 512]
+        array set header [readHeader $data]
+	HandleLongLink $fh header
+        if {$header(name) eq ""} break
+	if {$header(prefix) ne ""} {append header(prefix) /}
+        set name [string trimleft $header(prefix)$header(name) /]
+        if {$name eq $file} {
+            set file [read $fh $header(size)]
+            if {!$chan} {
+		close $fh
+	    }
+            return $file
+        }
+        seekorskip $fh [expr {$header(size) + [pad $header(size)]}] current
+    }
+    if {!$chan} {
+	close $fh
+    }
+    return -code error -errorcode {TAR MISSING FILE} \
+	"Tar \"$tar\": File \"$file\" not found"
+}
+
+proc ::tar::untar {tar args} {
+    set nooverwrite 0
+    set data 0
+    set nomtime 0
+    set noperms 0
+    set chan 0
+    parseOpts {dir 1 file 1 glob 1 nooverwrite 0 nomtime 0 noperms 0 chan 0} $args
+    if {![info exists dir]} {set dir [pwd]}
+    set pattern *
+    if {[info exists file]} {
+        set pattern [string map {* \\* ? \\? \\ \\\\ \[ \\\[ \] \\\]} $file]
+    } elseif {[info exists glob]} {
+        set pattern $glob
+    }
+
+    set ret {}
+    if {$chan} {
+	set fh $tar
+    } else {
+	set fh [::open $tar]
+	fconfigure $fh -encoding binary -translation lf -eofchar {}
+    }
+    while {![eof $fh]} {
+        array set header [readHeader [read $fh 512]]
+	HandleLongLink $fh header
+        if {$header(name) == ""} break
+	if {$header(prefix) != ""} {append header(prefix) /}
+        set name [string trimleft $header(prefix)$header(name) /]
+        if {![string match $pattern $name] || ($nooverwrite && [file exists $name])} {
+            seekorskip $fh [expr {$header(size) + [pad $header(size)]}] current
+            continue
+        }
+
+        set name [file join $dir $name]
+        if {![file isdirectory [file dirname $name]]} {
+            file mkdir [file dirname $name]
+            lappend ret [file dirname $name] {}
+        }
+        if {[string match {[0346]} $header(type)]} {
+            if {[catch {::open $name w+} new]} {
+                # sometimes if we dont have write permission we can still delete
+                catch {file delete -force $name}
+                set new [::open $name w+]
+            }
+            fconfigure $new -encoding binary -translation lf -eofchar {}
+            fcopy $fh $new -size $header(size)
+            close $new
+            lappend ret $name $header(size)
+        } elseif {$header(type) == 5} {
+            file mkdir $name
+            lappend ret $name {}
+        } elseif {[string match {[12]} $header(type)] && $::tcl_platform(platform) == "unix"} {
+            catch {file delete $name}
+            if {![catch {file link [string map {1 -hard 2 -symbolic} $header(type)] $name $header(linkname)}]} {
+                lappend ret $name {}
+            }
+        }
+        seekorskip $fh [pad $header(size)] current
+        if {![file exists $name]} continue
+
+        if {$::tcl_platform(platform) == "unix"} {
+            if {!$noperms} {
+                catch {file attributes $name -permissions 0o[string range $header(mode) 2 end]}
+            }
+            catch {file attributes $name -owner $header(uid) -group $header(gid)}
+            catch {file attributes $name -owner $header(uname) -group $header(gname)}
+        }
+        if {!$nomtime} {
+            file mtime $name $header(mtime)
+        }
+    }
+    if {!$chan} {
+	close $fh
+    }
+    return $ret
+}
+
+## 
+ # ::tar::statFile
+ # 
+ # Returns stat info about a filesystem object, in the form of an info 
+ # dictionary like that returned by ::tar::readHeader.
+ # 
+ # The mode, uid, gid, mtime, and type entries are always present. 
+ # The size and linkname entries are present if relevant for this type 
+ # of object. The uname and gname entries are present if the OS supports 
+ # them. No devmajor or devminor entry is present.
+ ##
+
+proc ::tar::statFile {name followlinks} {
+    if {$followlinks} {
+        file stat $name stat
+    } else {
+        file lstat $name stat
+    }
+    
+    set ret {}
+    
+    if {$::tcl_platform(platform) == "unix"} {
+        # Tcl 9 returns the permission as 0o octal number. Since this
+        # is written to the tar file and the file format expects "00"
+        # we have to rewrite.
+        lappend ret mode 1[string map {o 0} [file attributes $name -permissions]]
+        lappend ret uname [file attributes $name -owner]
+        lappend ret gname [file attributes $name -group]
+        if {$stat(type) == "link"} {
+            lappend ret linkname [file link $name]
+        }
+    } else {
+        lappend ret mode [lindex {100644 100755} [expr {$stat(type) == "directory"}]]
+    }
+    
+    lappend ret  uid $stat(uid)  gid $stat(gid)  mtime $stat(mtime) \
+      type $stat(type)
+    
+    if {$stat(type) == "file"} {lappend ret size $stat(size)}
+    
+    return $ret
+}
+
+## 
+ # ::tar::formatHeader
+ # 
+ # Opposite operation to ::tar::readHeader; takes a file name and info 
+ # dictionary as arguments, returns a corresponding (POSIX-tar) header.
+ # 
+ # The following dictionary entries must be present:
+ #   mode
+ #   type
+ # 
+ # The following dictionary entries are used if present, otherwise 
+ # the indicated default is used:
+ #   uid       0
+ #   gid       0
+ #   size      0
+ #   mtime     [clock seconds]
+ #   linkname  {}
+ #   uname     {}
+ #   gname     {}
+ #   
+ # All other dictionary entries, including devmajor and devminor, are 
+ # presently ignored.
+ ##
+
+proc ::tar::formatHeader {name info} {
+    array set A {
+        linkname ""
+        uname ""
+        gname ""
+        size 0
+        gid  0
+        uid  0
+    }
+    set A(mtime) [clock seconds]
+    array set A $info
+    array set A {devmajor "" devminor ""}
+
+    set type [string map {file 0 directory 5 characterSpecial 3 \
+      blockSpecial 4 fifo 6 link 2 socket A} $A(type)]
+    
+    set osize  [format %o $A(size)]
+    set ogid   [format %o $A(gid)]
+    set ouid   [format %o $A(uid)]
+    set omtime [format %o $A(mtime)]
+    
+    set name [string trimleft $name /]
+    if {[string length $name] > 255} {
+        return -code error -errorcode {TAR BAD PATH LENGTH} \
+	    "path name over 255 chars"
+    } elseif {[string length $name] > 100} {
+	set common [string range $name end-99 154]
+	if {[set splitpoint [string first / $common]] == -1} {
+	    return -code error -errorcode {TAR BAD PATH UNSPLITTABLE} \
+		"path name cannot be split into prefix and name"
+	}
+	set prefix [string range $name 0 end-100][string range $common 0 $splitpoint-1]
+	set name   [string range $common $splitpoint+1 end][string range $name 155 end]
+    } else {
+        set prefix ""
+    }
+
+    set header [binary format a100A8A8A8A12A12A8a1a100A6a2a32a32a8a8a155a12 \
+                              $name $A(mode)\x00 $ouid\x00 $ogid\x00\
+                              $osize\x00 $omtime\x00 {} $type \
+                              $A(linkname) ustar\x00 00 $A(uname) $A(gname)\
+                              $A(devmajor) $A(devminor) $prefix {}]
+
+    binary scan $header c* tmp
+    set cksum 0
+    foreach x $tmp {incr cksum $x}
+
+    return [string replace $header 148 155 [binary format A8 [format %o $cksum]\x00]]
+}
+
+
+proc ::tar::recurseDirs {files followlinks} {
+    foreach x $files {
+        if {[file isdirectory $x] && ([file type $x] != "link" || $followlinks)} {
+            if {[set more [glob -dir $x -nocomplain *]] != ""} {
+                eval lappend files [recurseDirs $more $followlinks]
+            } else {
+                lappend files $x
+            }
+        }
+    }
+    return $files
+}
+
+proc ::tar::writefile {in out followlinks name} {
+     puts -nonewline $out [formatHeader $name [statFile $in $followlinks]]
+     set size 0
+     if {[file type $in] == "file" || ($followlinks && [file type $in] == "link")} {
+         set in [::open $in]
+         fconfigure $in -encoding binary -translation lf -eofchar {}
+         set size [fcopy $in $out]
+         close $in
+     }
+     puts -nonewline $out [string repeat \x00 [pad $size]]
+}
+
+proc ::tar::create {tar files args} {
+    set dereference 0
+    set chan 0
+    parseOpts {dereference 0 chan 0} $args
+
+    if {$chan} {
+	set fh $tar
+    } else {
+	set fh [::open $tar w+]
+	fconfigure $fh -encoding binary -translation lf -eofchar {}
+    }
+    foreach x [recurseDirs $files $dereference] {
+        writefile $x $fh $dereference $x
+    }
+    puts -nonewline $fh [string repeat \x00 1024]
+
+    if {!$chan} {
+	close $fh
+    }
+    return $tar
+}
+
+proc ::tar::add {tar files args} {
+    set dereference 0
+    set prefix ""
+    set quick 0
+    parseOpts {dereference 0 prefix 1 quick 0} $args
+    
+    set fh [::open $tar r+]
+    fconfigure $fh -encoding binary -translation lf -eofchar {}
+    
+    if {$quick} then {
+        seek $fh -1024 end
+    } else {
+        set data [read $fh 512]
+        while {[regexp {[^\0]} $data]} {
+            array set header [readHeader $data]
+            seek $fh [expr {$header(size) + [pad $header(size)]}] current
+            set data [read $fh 512]
+        }
+        seek $fh -512 current
+    }
+
+    foreach x [recurseDirs $files $dereference] {
+        writefile $x $fh $dereference $prefix$x
+    }
+    puts -nonewline $fh [string repeat \x00 1024]
+
+    close $fh
+    return $tar
+}
+
+proc ::tar::remove {tar files} {
+    set n 0
+    while {[file exists $tar$n.tmp]} {incr n}
+    set tfh [::open $tar$n.tmp w]
+    set fh [::open $tar r]
+
+    fconfigure $fh  -encoding binary -translation lf -eofchar {}
+    fconfigure $tfh -encoding binary -translation lf -eofchar {}
+
+    while {![eof $fh]} {
+        array set header [readHeader [read $fh 512]]
+        if {$header(name) == ""} {
+            puts -nonewline $tfh [string repeat \x00 1024]
+            break
+        }
+	if {$header(prefix) != ""} {append header(prefix) /}
+        set name $header(prefix)$header(name)
+        set len [expr {$header(size) + [pad $header(size)]}]
+        if {[lsearch $files $name] > -1} {
+            seek $fh $len current
+        } else {
+            seek $fh -512 current
+            fcopy $fh $tfh -size [expr {$len + 512}]
+        }
+    }
+
+    close $fh
+    close $tfh
+
+    file rename -force $tar$n.tmp $tar
+}
+
+proc ::tar::HandleLongLink {fh hv} {
+    upvar 1 $hv header thelongname thelongname
+
+    # @LongName Part I.
+    if {$header(type) == "L"} {
+	# Size == Length of name. Read it, and pad to full 512
+	# size.  After that is a regular header for the actual
+	# file, where we have to insert the name. This is handled
+	# by the next iteration and the part II below.
+	set thelongname [string trimright [read $fh $header(size)] \000]
+	seekorskip $fh [pad $header(size)] current
+	return -code continue
+    }
+    # Not supported yet: type 'K' for LongLink (long symbolic links).
+
+    # @LongName, part II, get data from previous entry, if defined.
+    if {[info exists thelongname]} {
+	set header(name) $thelongname
+	# Prevent leakage to further entries.
+	unset thelongname
+    }
+
+    return
+}

src/vendorlib/tar/tar.test

@@ -0,0 +1,139 @@
+# -*- tcl -*-
+# These tests are in the public domain
+# -------------------------------------------------------------------------
+
+source [file join \
+	[file dirname [file dirname [file normalize [info script]]]] \
+	devtools testutilities.tcl]
+
+testsNeedTcl     8.5 ; # Virt channel support!
+testsNeedTcltest 1.0
+
+# Check if we have TclOO available.
+tcltest::testConstraint tcloo [expr {![catch {package require TclOO}]}]
+
+support {
+    if {[tcltest::testConstraint tcloo]} {
+	use virtchannel_base/memchan.tcl tcl::chan::memchan
+    }
+    useLocalFile tests/support.tcl
+}
+testing {
+    useLocal tar.tcl tar
+}
+
+# -------------------------------------------------------------------------
+
+test tar-stream {stream} -constraints tcloo -setup {
+    setup1
+} -body {
+    string length [read $chan1]
+} -cleanup {
+    cleanup1
+} -result 128000
+
+test tar-pad {pad} -body {
+    tar::pad 230 
+} -result {282}
+
+test tar-skip {skip} -constraints tcloo -setup {
+    setup1
+} -body {
+    tar::skip $chan1 10
+    lappend res [read $chan1 10]
+    tar::skip $chan1 72313
+    lappend res [read $chan1 10]
+} -cleanup {
+    cleanup1
+} -result {{6 7 8 9 10} {07 13908 1}}
+
+test tar-seekorskip-backwards {seekorskip} -constraints tcl8.6plus -setup setup1 -body {
+    # The zlib push stuff is Tcl 8.6+. Properly restrict the test.
+    zlib push gzip $chan1
+    catch {tar::seekorskip $chan1 -10 start} cres
+    lappend res $cres
+    catch {tar::seekorskip $chan1 10 start} cres
+    lappend res $cres
+    catch {tar::seekorskip $chan1 -10 end} cres
+    lappend res $cres
+    catch {tar::seekorskip $chan1 10 end} cres
+    lappend res $cres
+    lappend res [read $chan1 10]
+} -cleanup cleanup1 -match glob \
+    -result [list \
+		 {WHENCE=start not supported*} \
+		 {WHENCE=start not supported*} \
+		 {WHENCE=end not supported*} \
+		 {WHENCE=end not supported*} \
+		 {1 2 3 4 5 } \
+		]
+
+test tar-header {header} -body {
+    set file1 [dict get $filesys Dir1 File1]
+    dict set file1 path /Dir1/File1
+    set header [header_posix $file1]
+    set parsed [string trim [tar::readHeader $header]]
+    set golden "name /Dir1/File1 mode 755 uid 13103 gid 18103 size 100 mtime 5706756101 cksum 3676 type 0 linkname {} magic ustar\0 version 00 uname {} gname {} devmajor 0 devminor 0 prefix {}"
+    set len [string length $parsed]
+    foreach {key value} $golden {
+	if {[set value1 [dict get $parsed $key]] ne $value } {
+	    lappend res [list $key $value $value1]
+	}
+    }
+} -result {}
+
+test tar-add {add} -constraints tcloo -setup {
+    setup1
+} -body {
+    tar::create $chan1 [list $tmpdir/one/a $tmpdir/one/two/a $tmpdir/one/three/a] -chan
+    seek $chan1 0
+    lappend res {*}[tar::contents $chan1 -chan]
+    seek $chan1 0
+    lappend res [string trim [tar::get $chan1 $tmpdir/one/two/a -chan]]
+} -cleanup {
+    cleanup1
+} -result {tartest/one/a tartest/one/two/a tartest/one/three/a hello2}
+
+
+test tar-bug-2840180 {Ticket 2840180} -setup {
+    setup2
+} -body {
+    tar::create $chan1 [list $tmpdir/[large-path]/a] -chan
+    seek $chan1 0
+
+    # What the package sees.
+    lappend res {*}[tar::contents $chan1 -chan]
+    close $chan1
+
+    # What a regular tar package sees.
+    lappend res [exec 2> $tmpfile.err tar tvf $tmpfile]
+    join $res \n
+} -cleanup {
+    cleanup2
+} -match glob -result [join [list \
+				 tartest/[large-path]/a \
+				 "* tartest/[large-path]/a" \
+				] \n]
+
+# -------------------------------------------------------------------------
+
+test tar-tkt-9f4c0e3e95-1.0 {Ticket 9f4c0e3e95, A} -setup {
+    set tarfile [setup-tkt-9f4c0e3e95]
+} -body {
+    string trim [tar::get $tarfile 02]
+} -cleanup {
+    cleanup-tkt-9f4c0e3e95
+    unset tarfile
+} -result {zero-two}
+
+test tar-tkt-9f4c0e3e95-1.1 {Ticket 9f4c0e3e95, B, } -setup {
+    set tarfile [setup-tkt-9f4c0e3e95]
+} -body {
+    tar::get $tarfile 0b10
+} -cleanup {
+    cleanup-tkt-9f4c0e3e95
+    unset tarfile
+} -returnCodes error -result {Tar "tartest/t.tar": File "0b10" not found}
+
+# -------------------------------------------------------------------------
+testsuiteCleanup

src/vendorlib/tar/tests/support.tcl

@@ -0,0 +1,149 @@
+
+proc stream {{size 128000}} {
+    set chan [tcl::chan::memchan]
+    set line {}
+    while 1 {
+	incr i
+	set istring $i
+	set ilen [string length $istring]
+	if {$line ne {}} {
+	    append line { }
+	    incr size -1 
+	}
+	append line $istring
+	incr size -$ilen
+	if {$size < 1} {
+	    set line [string range $line 0 end-[expr {abs(1-$size)}]]
+	    puts $chan $line
+	    break
+	}
+
+	if {$i % 10 == 0} {
+	    puts $chan $line 
+	    incr size -1 ;# for the [puts] newline
+	    set line {}
+	}
+    }
+
+    seek $chan 0
+    return $chan
+}
+
+proc header_posix {tarball} {
+    dict with tarball {} 
+    tar::formatHeader $path \
+	[dict create \
+	     mode $mode \
+	     type $type \
+	     uid  $uid \
+	     gid  $gid \
+	     size $size \
+	     mtime $mtime]
+}
+
+proc setup1 {} {
+    variable chan1
+    variable res {}
+    variable tmpdir tartest
+
+    tcltest::makeDirectory $tmpdir
+
+    foreach directory {
+	one
+	one/two
+	one/three
+    } {
+	tcltest::makeDirectory $tmpdir/$directory
+	set    chan [open $tmpdir/$directory/a w]
+	puts  $chan hello[incr i]
+	close $chan
+    }
+    set chan1 [stream]
+}
+
+proc large-path {} {
+    return aaaaa/bbbbaaaaa/bbbbaaaaa/bbbbaaaaa/bbbbaaaaa/bbbbaaaaa/bbbbaaaaa/bbbbaaaaa/bbbbaaaaa/bbbbaaaaa/bbbbtcllib/modules/tar
+}
+
+proc setup2 {} {
+    variable chan1
+    variable res {}
+    variable tmpdir tartest
+    variable tmpfile tarX
+
+    tcltest::makeDirectory $tmpdir
+    tcltest::makeFile {} $tmpfile
+
+    foreach directory [list [large-path]] {
+	tcltest::makeDirectory $tmpdir/$directory
+	set    chan [open $tmpdir/$directory/a w]
+	puts  $chan hello[incr i]
+	close $chan
+    }
+    set chan1 [open $tmpfile w+]
+}
+
+proc cleanup1 {} {
+    variable chan1
+    close $chan1
+    tcltest::removeDirectory tartest
+    return
+}
+
+proc cleanup2 {} {
+    variable chan1
+    variable tmpdir
+    variable tmpfile
+    catch { close $chan1 }
+    tcltest::removeDirectory $tmpdir
+    tcltest::removeFile      $tmpfile
+    tcltest::removeFile      $tmpfile.err
+    return
+}
+
+variable filesys {
+    Dir1 {
+	File1 {
+	    type 0
+	    mode 755
+	    uid 13103
+	    gid 18103
+	    size 100
+	    mtime 5706756101
+	}
+    }
+
+    Dir2 {
+	File1 {
+	    type 0
+	    mode 644
+	    uid 15103
+	    gid 19103
+	    size 100
+	    mtime 5706776103
+	}
+    }
+}
+
+proc setup-tkt-9f4c0e3e95 {} {
+    variable tmpdir tartest
+
+    tcltest::makeDirectory $tmpdir
+    tcltest::makeFile {zero-two}   $tmpdir/02
+    tcltest::makeFile {number two} $tmpdir/2
+
+    set here [pwd]
+    cd $tmpdir
+    tar::create t.tar {2 02}
+    cd $here
+
+    return $tmpdir/t.tar
+}
+
+proc cleanup-tkt-9f4c0e3e95 {} {
+    variable tmpdir
+    tcltest::removeFile      $tmpdir/2
+    tcltest::removeFile      $tmpdir/02
+    tcltest::removeDirectory $tmpdir
+    return
+}