Browse Source

make.tcl fixes, commandline script.tcl execution fixes, layout template updates

master
Julian Noble 1 year ago
parent
commit
d1a1762403
  1. 1
      .fossil-settings/ignore-glob
  2. 1886
      src/bootsupport/modules/natsort-0.1.1.5.tm
  3. 195
      src/bootsupport/modules/oolib-0.1.tm
  4. 1039
      src/bootsupport/modules/overtype-1.5.0.tm
  5. 629
      src/bootsupport/modules/punk/du-0.1.0.tm
  6. 1484
      src/bootsupport/modules/punk/mix-0.2.tm
  7. 26
      src/bootsupport/modules/punk/repo-0.1.1.tm
  8. 187
      src/bootsupport/modules/punk/winpath-0.1.0.tm
  9. 1984
      src/bootsupport/modules/punkcheck-0.1.0.tm
  10. 2
      src/doc/include/general.inc
  11. 441
      src/make.tcl
  12. 50
      src/mixtemplates/module/template_unversioned.tm
  13. 428
      src/modules/platform-1.0.17.tm
  14. 241
      src/modules/platform/shell-1.1.4.tm
  15. 55
      src/modules/punk-0.1.tm
  16. 2
      src/modules/punk/ansi-999999.0a1.0.tm
  17. 10
      src/modules/punk/console-999999.0a1.0.tm
  18. 41
      src/modules/punk/mix/base-0.1.tm
  19. 252
      src/modules/punk/mix/cli-0.3.tm
  20. 2
      src/modules/punk/mix/commandset/buildsuite-999999.0a1.0.tm
  21. 181
      src/modules/punk/mix/commandset/doc-999999.0a1.0.tm
  22. 3
      src/modules/punk/mix/commandset/doc-buildversion.txt
  23. 2
      src/modules/punk/mix/commandset/layout-999999.0a1.0.tm
  24. 8
      src/modules/punk/mix/commandset/module-999999.0a1.0.tm
  25. 130
      src/modules/punk/mix/commandset/project-999999.0a1.0.tm
  26. 1
      src/modules/punk/mix/templates/layouts/project/.gitignore
  27. 6
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/include_modules.config
  28. 200
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/cksum-1.1.4.tm
  29. 933
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/cmdline-1.5.2.tm
  30. 2311
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/fileutil-1.16.1.tm
  31. 1886
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/natsort-0.1.1.5.tm
  32. 195
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/oolib-0.1.tm
  33. 1039
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/overtype-1.5.0.tm
  34. 1308
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/du-0.1.0.tm
  35. 15
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/mix-0.2.tm
  36. 1232
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/repo-0.1.1.tm
  37. 266
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/winpath-0.1.0.tm
  38. 299
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punkcheck-0.1.0.tm
  39. 189
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/struct/set-2.2.3.tm
  40. 189
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/struct/sets.tcl
  41. 93
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/struct/sets_c.tcl
  42. 452
      src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/struct/sets_tcl.tcl
  43. 31
      src/modules/punk/mix/templates/layouts/project/src/doc/include/changes_0.1.inc
  44. 183
      src/modules/punk/mix/templates/layouts/project/src/make.tcl
  45. 24
      src/modules/punk/mix/templates/layouts/project/src/mixtemplates/layouts/basic/src/bootsupport/modules/README.md
  46. 4
      src/modules/punk/mix/templates/layouts/project/src/sample.vfs/modules/shellthread-1.6.tm
  47. 17
      src/modules/punk/overlay-0.1.tm
  48. 3
      src/modules/punk/repl-0.1.tm
  49. 299
      src/modules/punkcheck-0.1.0.tm
  50. 69
      src/modules/shellfilter-0.1.8.tm
  51. 47
      src/modules/shellrun-0.1.tm
  52. 66
      src/modules/shellthread-1.6.tm
  53. 40
      src/modules/zzzload-999999.0a1.0.tm
  54. 1
      src/punk86.vfs/lib/app-punk/repl.tcl
  55. 183
      src/punk86.vfs/lib/app-shellspy/shellspy.tcl
  56. 16
      src/runtime/mapvfs.config

1
.fossil-settings/ignore-glob

@ -1,4 +1,5 @@
.git
CVS
bin
lib
#The directory for compiled/built Tcl modules

1886
src/bootsupport/modules/natsort-0.1.1.5.tm

File diff suppressed because it is too large Load Diff

195
src/bootsupport/modules/oolib-0.1.tm

@ -0,0 +1,195 @@
#JMN - api should be kept in sync with package patternlib where possible
#
package provide oolib [namespace eval oolib {
variable version
set version 0.1
}]
namespace eval oolib {
oo::class create collection {
variable o_data ;#dict
variable o_alias
constructor {} {
set o_data [dict create]
}
method info {} {
return [dict info $o_data]
}
method count {} {
return [dict size $o_data]
}
method isEmpty {} {
expr {[dict size $o_data] == 0}
}
method names {{globOrIdx {}}} {
if {[llength $globOrIdx]} {
if {[string is integer -strict $globOrIdx]} {
if {$idx < 0} {
set idx "end-[expr {abs($idx + 1)}]"
}
if {[catch {lindex [dict keys $o_data] $idx} result]} {
error "[self object] no such index : '$idx'"
} else {
return $result
}
} else {
#glob
return [lsearch -glob -all -inline [dict keys $o_data] $globOrIdx]
}
} else {
return [dict keys $o_data]
}
}
#like names but without globbing
method keys {} {
dict keys $o_data
}
method key {{posn 0}} {
if {$posn < 0} {
set posn "end-[expr {abs($posn + 1)}]"
}
if {[catch {lindex [dict keys $o_data] $posn} result]} {
error "[self object] no such index : '$posn'"
} else {
return $result
}
}
method hasKey {key} {
dict exists $o_data $key
}
method get {} {
return $o_data
}
method items {} {
return [dict values $o_data]
}
method item {key} {
if {[string is integer -strict $key]} {
if {$key > 0} {
set valposn [expr {(2*$key) +1}]
return [lindex $o_data $valposn]
} else {
set key "end-[expr {abs($key + 1)}]"
return [lindex [dict keys $o_data] $key]
}
}
if {[dict exists $o_data $key]} {
return [dict get $o_data $key]
}
}
#inverse lookup
method itemKeys {value} {
set value_indices [lsearch -all [dict values $o_data] $value]
set keylist [list]
foreach i $value_indices {
set idx [expr {(($i + 1) *2) -2}]
lappend keylist [lindex $o_data $idx]
}
return $keylist
}
method search {value args} {
set matches [lsearch {*}$args [dict values $o_data] $value]
if {"-inline" in $args} {
return $matches
} else {
set keylist [list]
foreach i $matches {
set idx [expr {(($i + 1) *2) -2}]
lappend keylist [lindex $o_data $idx]
}
return $keylist
}
}
#review - see patternlib. Is the intention for aliases to be configurable independent of whether the target exists?
method alias {newAlias existingKeyOrAlias} {
if {[string is integer -strict $newAlias]} {
error "[self object] collection key alias cannot be integer"
}
if {[string length $existingKeyOrAlias]} {
set o_alias($newAlias) $existingKeyOrAlias
} else {
unset o_alias($newAlias)
}
}
method aliases {{key ""}} {
if {[string length $key]} {
set result [list]
foreach {n v} [array get o_alias] {
if {$v eq $key} {
lappend result $n $v
}
}
return $result
} else {
return [array get o_alias]
}
}
#if the supplied index is an alias, return the underlying key; else return the index supplied.
method realKey {idx} {
if {[catch {set o_alias($idx)} key]} {
return $idx
} else {
return $key
}
}
method add {value key} {
if {[string is integer -strict $key]} {
error "[self object] collection key must not be an integer. Use another structure if integer keys required"
}
if {[dict exists $o_data $key]} {
error "[self object] col_processors object error: key '$key' already exists in collection"
}
dict set o_data $key $value
return [expr {[dict size $o_data] - 1}] ;#return index of item
}
method remove {idx {endRange ""}} {
if {[string length $endRange]} {
error "[self object] collection error: ranged removal not yet implemented.. remove one item at a time"
}
if {[string is integer -strict $idx]} {
if {$idx < 0} {
set idx "end-[expr {abs($idx+1)}]"
}
set key [lindex [dict keys $o_data] $idx]
set posn $idx
} else {
set key $idx
set posn [lsearch -exact [dict keys $o_data] $key]
if {$posn < 0} {
error "[self object] no such index: '$idx' in this collection"
}
}
dict unset o_data $key
return
}
method clear {} {
set o_data [dict create]
return
}
method reverse {} {
set dictnew [dict create]
foreach k [lreverse [dict keys $o_data]] {
dict set dictnew $k [dict get $o_data $k]
}
set o_data $dictnew
return
}
#review - cmd as list vs cmd as script?
method map {cmd} {
set seed [list]
dict for {k v} $o_data {
lappend seed [uplevel #0 [list {*}$cmd $v]]
}
return $seed
}
method objectmap {cmd} {
set seed [list]
dict for {k v} $o_data {
lappend seed [uplevel #0 [list $v {*}$cmd]]
}
return $seed
}
}
}

1039
src/bootsupport/modules/overtype-1.5.0.tm

File diff suppressed because it is too large Load Diff

629
src/bootsupport/modules/punk/du-0.1.0.tm

@ -9,7 +9,7 @@
# @@ Meta Begin
# Application punk::du 0.1.0
# Meta platform tcl
# Meta license <unspecified>
# Meta license BSD
# @@ Meta End
@ -17,6 +17,8 @@
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
## Requirements
##e.g package require frobz
package require punk::mix::base
namespace eval punk::du {
variable has_twapi 0
@ -29,7 +31,7 @@ if {"windows" eq $::tcl_platform(platform)} {
#} else {
# set punk::du::has_twapi 1
#}
package require punk::winpath
#package require punk::winpath
}
@ -37,7 +39,14 @@ if {"windows" eq $::tcl_platform(platform)} {
namespace eval punk::du {
proc dirlisting {{folderpath {}}} {
proc dirlisting {folderpath args} {
set defaults [dict create\
-glob *\
]
set opts [dict merge $defaults $args]
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_glob [dict get $opts -glob]
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
if {[lib::pathcharacterlen $folderpath] == 0} {
set folderpath [pwd]
} elseif {[file pathtype $folderpath] ne "absolute"} {
@ -46,7 +55,7 @@ namespace eval punk::du {
}
#run whichever of du_dirlisting_twapi, du_dirlisting_generic, du_dirlisting_unix has been activated
set dirinfo [active::du_dirlisting $folderpath]
set dirinfo [active::du_dirlisting $folderpath {*}$opts]
}
@ -151,6 +160,8 @@ namespace eval punk::du {
set opt_extra 1
}
set opt_vfs 0
#This configures whether to enter a vfsmount point
#It will have no effect if cwd already with a vfs mount point - as then opt_vfs will be set to 1 automatically anyway.
if {"--vfs" in $lc_opts} {
set opt_vfs 1
}
@ -213,10 +224,18 @@ namespace eval punk::du {
#e.g tcl glob based dirlisting doesn't support gathering file sizes at the same time
set in_vfs 0
if {$opt_vfs} {
if {[package provide vfs] ne ""} {
foreach vfsmount [vfs::filesystem info] {
if {[punk::repo::path_a_atorbelow_b $folderpath $vfsmount]} {
if {[file pathtype $folderpath] ne "absolute"} {
set testpath [file normalize $folderpath]
} else {
set testpath $folderpath
}
if {[punk::mix::base::lib::path_a_atorbelow_b $testpath $vfsmount]} {
set in_vfs 1
#if already descended to or below a vfs mount point - set opt_vfs true
set opt_vfs 1
break
}
}
@ -229,10 +248,30 @@ namespace eval punk::du {
set du_info [active::du_dirlisting $folderpath]
}
set dirs [dict get $du_info dirs]
set files [dict get $du_info files]
set filesizes [dict get $du_info filesizes]
set vfsmounts [dict get $du_info vfsmounts]
#puts "---> vfsmounts $vfsmounts "
if {$opt_vfs} {
foreach vm $vfsmounts {
#puts stderr "vm: $vm"
#check if vfs is mounted over a file or a dir
if {$vm in $files} {
puts stderr "vfs mounted over file $vm"
set mposn [lsearch $files $vm]
set files [lreplace $files $mposn $mposn]
if {[llength $filesizes]} {
set filesizes [lreplace $filesizes $mposn $mposn]
}
}
if {$vm ni $dirs} {
puts stderr "treating $vm as dir"
lappend dirs $vm
}
}
}
incr leveldirs [llength $dirs]
@ -361,7 +400,7 @@ namespace eval punk::du {
variable functions_known [dict create]
#known functions from lib namespace
dict set functions_known du_dirlisting [list du_dirlisting_twapi du_dirlisting_generic du_dirlisting_unix]
dict set functions_known du_dirlisting [list du_dirlisting_twapi du_dirlisting_generic du_dirlisting_unix du_dirlisting_undecided]
proc show_functions {} {
variable functions
@ -443,10 +482,11 @@ namespace eval punk::du {
dict set result -shortname [dict get $iteminfo altname]
dict set result -rawflags $attrinfo
set extras [list]
foreach prop {ctime atime mtime size} {
lappend extras $prop [dict get $iteminfo $prop]
}
dict set result -extras $extras
#foreach prop {ctime atime mtime size} {
# lappend extras $prop [dict get $iteminfo $prop]
#}
#dict set result -extras $extras
dict set result -raw $iteminfo
return $result
} else {
error "could not read attributes for $path"
@ -455,29 +495,101 @@ namespace eval punk::du {
catch {twapi::find_file_close $iterator}
}
}
namespace export du_dirlisting_twapi du_dirlisting_generic du_dirlisting_unix
#todo - review 'errors' key. We have errors relating to containing folder and args vs per child-item errors - additional key needed?
namespace export du_dirlisting_twapi du_dirlisting_generic du_dirlisting_unix du_dirlisting_undecided
# get listing without using unix-tools (may not be installed on the windows system)
# this dirlisting is customised for du - so only retrieves dirs,files,filesizes (minimum work needed to perform du function)
proc du_dirlisting_twapi {folderpath} {
# This also preserves path rep for elements in the dirs/folders keys etc - which can make a big difference in performance
proc du_dirlisting_twapi {folderpath args} {
set defaults [dict create\
-glob *\
-with_sizes 1\
-with_times 1\
]
set opts [dict merge $defaults $args]
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_glob [dict get $opts -glob]
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_with_sizes [dict get $opts -with_sizes]
set ftypes [list f d l]
if {"$opt_with_sizes" in {0 1}} {
#don't use string is boolean - (f false vs f file!)
#only accept 0|1
if {$opt_with_sizes} {
set sized_types $ftypes
} else {
set sized_types [list]
}
} else {
set sized_types $opt_with_sizes
}
if {[llength $sized_types]} {
foreach st $sized_types {
if {$st ni $ftypes} {
error "du_dirlisting_twapi unrecognized element in -with_sizes '$st'"
}
}
}
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_with_times [dict get $opts -with_times]
if {"$opt_with_times" in {0 1}} {
if {$opt_with_times} {
set timed_types $ftypes
} else {
set timed_types [list]
}
} else {
set timed_types $opt_with_times
}
if {[llength $timed_types]} {
foreach item $timed_types {
if {$item ni $ftypes} {
error "du_dirlisting_twapi unrecognised element in -with-times '$item'"
}
}
}
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set errors [dict create]
set altname "" ;#possible we have to use a different name e.g short windows name or dos-device path //?/
# return it so it can be stored and tried as an alternative for problem paths
#puts stderr ">>> glob: $opt_glob"
#REVIEW! windows api pattern matchttps://www.red-gate.com/simple-talk/blogs/the-unexpected-behaviour-of-directoryinfo-getfiles-with-three-letter-extensions/hing is .. weird. partly due to 8.3 filenames
#https://www.red-gate.com/simple-talk/blogs/the-unexpected-behaviour-of-directoryinfo-getfiles-with-three-letter-extensions/
#we will certainly need to check the resulting listing with our supplied glob.. but maybe we will have to change the glob passed to find_file_open too.
# using * all the time may be inefficient - so we might be able to avoid that in some cases.
try {
set iterator [twapi::find_file_open [file join $folderpath *] -detail basic] ;# -detail full only adds data to the altname field
#glob of * will return dotfiles too on windows
set iterator [twapi::find_file_open [file join $folderpath $opt_glob] -detail basic] ;# -detail full only adds data to the altname field
} on error args {
try {
if {[string match "*denied*" $args]} {
#output similar format as unixy du
puts stderr "[file dirname $folderpath] '[file tail $folderpath]':$args"
return [list dirs {} vfsmounts {} links {} files {} filesizes {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {}]
dict lappend errors $folderpath $::errorCode
return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors]
}
if {[string match "*TWAPI_WIN32 59*" $::errorCode]} {
puts stderr "[file dirname $folderpath] '[file tail $folderpath]':$args (possibly blocked by permissions or share config e.g follow symlinks = no on samba)"
puts stderr " (errorcode: $::errorCode)\n"
return [list dirs {} vfsmounts {} links {} files {} filesizes {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {}]
dict lappend errors $folderpath $::errorCode
return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors]
}
#errorcode TWAPI_WIN32 2 {The system cannot find the file specified.}
#This can be a perfectly normal failure to match the glob.. which means we shouldn't really warn or error
#The find-all glob * won't get here because it returns . & ..
#so we should return immediately only if the glob has globchars ? or * but isn't equal to just "*" ? (review)
#Note that windows glob ? seems to return more than just single char results - it includes .. - which differs to tcl glob
#also ???? seems to returns items 4 or less - not just items exactly 4 long (review - where is this documented?)
if {$opt_glob ne "*" && [regexp {[?*]} $opt_glob]} {
if {[string match "*TWAPI_WIN32 2 *" $::errorCode]} {
#looks like an ordinary no results for chosen glob
return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors]
}
}
if {[set plen [pathcharacterlen $folderpath]] >= 250} {
@ -503,15 +615,17 @@ namespace eval punk::du {
set errmsg "error reading folder: $folderpath (len:$plen)\n"
append errmsg "error: $args" \n
append errmsg "errorcode: $::errorCode" \n
set tmp_errors [list $::errorCode]
#possibly an illegal windows filename - easily happens on a machine with WSL or with drive mapped to unix share
#we can use //?/path dos device path - but not with tcl functions
#unfortunately we can't call find_file_open directly on the problem name - we have to call the parent folder and iterate through again..
#this gets problematic as we go deeper unless we rewrite the .. but we can get at least one level further here
set fixedtail ""
set parent [file dirname $folderpath]
set badtail [file tail $folderpath]
set iterator [twapi::find_file_open [file join $parent *] -detail full] ;#retrieve with altnames
set fixedtail ""
while {[twapi::find_file_next $iterator iteminfo]} {
set nm [dict get $iteminfo name]
if {$nm eq $badtail} {
@ -521,9 +635,11 @@ namespace eval punk::du {
}
if {![string length $fixedtail]} {
dict lappend errors $folderpath {*}$tmp_errors
puts stderr "[file dirname $folderpath] '[file tail $folderpath]':$args (Unable to retrieve altname to progress further with path - returning no contents for this folder)"
return [list dirs {} vfsmounts {} links {} files {} filesizes {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {}]
return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors]
}
#twapi as at 2023-08 doesn't seem to support //?/ dos device paths..
#Tcl can test only get as far as testing existence of illegal name by prefixing with //?/ - but can't glob inside it
#we can call file attributes on it - but we get no shortname (but we could get shortname for parent that way)
@ -537,21 +653,35 @@ namespace eval punk::du {
}
set iterator [twapi::find_file_open $fixedpath/* -detail basic]
if {[catch {
set iterator [twapi::find_file_open $fixedpath/* -detail basic]
} errMsg]} {
puts stderr "[file dirname $folderpath] '[file tail $folderpath]':$args (failed to read even with fixedpath:'$fixedpath')"
puts stderr " (errorcode: $::errorCode)\n"
puts stderr "$errMsg"
dict lappend errors $folderpath $::errorCode
return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors]
}
} on error args {
set errmsg "error reading folder: $folderpath\n"
append errmsg "error: $args"
append errmsg "aborting.."
error $errmsg
append errmsg "error: $args" \n
append errmsg "errorInfo: $::errorInfo" \n
puts stderr "$errmsg"
puts stderr "FAILED to collect info for folder '$folderpath'"
#append errmsg "aborting.."
#error $errmsg
return [list dirs {} vfsmounts {} links {} files {} filesizes {} sizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors]
}
}
set dirs [list]
set files [list]
set filesizes [list]
set allsizes [dict create]
set alltimes [dict create]
set links [list]
set flaggedhidden [list]
set flaggedsystem [list]
@ -559,13 +689,19 @@ namespace eval punk::du {
while {[twapi::find_file_next $iterator iteminfo]} {
set nm [dict get $iteminfo name]
#recheck glob
#review!
if {![string match $opt_glob $nm]} {
continue
}
set tail_altname [dict get $iteminfo altname] ;#altname of tail - not whole path
set attrinfo [decode_win_attributes [dict get $iteminfo attrs]]
#puts stderr "$iteminfo"
#puts stderr "$nm -> [dict get $iteminfo attrs] -> $attrinfo"
set ftype ""
#attributes applicable to any classification
set fullname [file_join_one $folderpath $nm]
if {"hidden" in $attrinfo} {
lappend flaggedhidden $fullname
}
@ -578,6 +714,13 @@ namespace eval punk::du {
#main classification
if {"reparse_point" in $attrinfo} {
#this concept doesn't correspond 1-to-1 with unix links
#https://learn.microsoft.com/en-us/windows/win32/fileio/reparse-points
#review - and see which if any actually belong in the links key of our return
#One thing it could be, is a 'mounted folder' https://learn.microsoft.com/en-us/windows/win32/fileio/determining-whether-a-directory-is-a-volume-mount-point
#
#we will treat as zero sized for du purposes.. review - option -L for symlinks like BSD du?
#Note 'file readlink' can fail on windows - reporting 'invalid argument' - according to tcl docs, 'On systems that don't support symbolic links this option is undefined'
#The link may be viewable ok in windows explorer, and cmd.exe /c dir and unix tools such as ls
@ -593,33 +736,70 @@ namespace eval punk::du {
#Note also - a shortcut created in explorer with drag and drop to an existant folder is a different animal to a symlink (file with .lnk extension) even though it looks the same in explorer window.
#
#links are techically files too, whether they point to a file/dir or nothing.
lappend links $fullname
set ftype "l"
} elseif {"directory" in $attrinfo} {
if {$nm in {. ..}} {
continue
}
lappend dirs $fullname
set ftype "d"
} else {
#review - is anything that isn't a reparse_point or a directory, some sort of 'file' in this context? What about the 'device' attribute? Can that occur in a directory listing of some sort?
lappend files $fullname
lappend filesizes [dict get $iteminfo size]
if {"f" in $sized_types} {
lappend filesizes [dict get $iteminfo size]
}
set ftype "f"
}
if {$ftype in $sized_types} {
dict set allsizes $fullname [dict create bytes [dict get $iteminfo size]]
}
if {$ftype in $timed_types} {
#convert time from windows (100ns units since jan 1, 1601) to Tcl time (seconds since Jan 1, 1970)
#We lose some precision by not passing the boolean to the large_system_time_to_secs_since_1970 function which returns fractional seconds
#but we need to maintain compatibility with other platforms and other tcl functions so if we want to return more precise times we will need another flag and/or result dict
dict set alltimes $fullname [dict create\
c [twapi::large_system_time_to_secs_since_1970 [dict get $iteminfo ctime]]\
a [twapi::large_system_time_to_secs_since_1970 [dict get $iteminfo atime]]\
m [twapi::large_system_time_to_secs_since_1970 [dict get $iteminfo mtime]]\
]
}
}
twapi::find_file_close $iterator
set vfsmounts [get_vfsmounts_in_folder $folderpath]
set effective_opts $opts
dict set effective_opts -with_times $timed_types
dict set effective_opts -with_sizes $sized_types
#also determine whether vfs. file system x is *much* faster than file attributes
#whether or not there is a corresponding file/dir add any applicable mountpoints for the containing folder
return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes $filesizes flaggedhidden $flaggedhidden flaggedsystem $flaggedsystem flaggedreadonly $flaggedreadonly altname $altname]
return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes $filesizes sizes $allsizes times $alltimes flaggedhidden $flaggedhidden flaggedsystem $flaggedsystem flaggedreadonly $flaggedreadonly altname $altname opts $effective_opts errors $errors]
}
proc get_vfsmounts_in_folder {folderpath} {
set vfsmounts [list]
if {![llength [package provide vfs]]} {
return [list]
}
set fpath [punk::objclone $folderpath]
set is_rel 0
if {[file pathtype $fpath] ne "absolute"} {
set fpath [file normalize $fpath]
set is_rel 1
}
set known_vfs_mounts [vfs::filesystem info]
foreach mount $known_vfs_mounts {
if {[punk::repo::path_a_above_b $folderpath $mount]} {
if {([llength [file split $mount]] - [llength [file split $folderpath]]) == 1} {
if {[punk::mix::base::lib::path_a_above_b $fpath $mount]} {
if {([llength [file split $mount]] - [llength [file split $fpath]]) == 1} {
#the mount is in this folder
lappend vfsmounts $mount
if {$is_rel} {
lappend vfsmounts [file join $folderpath [file tail $mount]]
} else {
lappend vfsmounts $mount
}
}
}
}
@ -636,7 +816,66 @@ namespace eval punk::du {
#this is the cross-platform pure-tcl version - which calls glob multiple times to make sure it gets everythign it needs and can ignore everything it needs to.
#These repeated calls to glob will be a killer for performance - especially on a network share or when walking a large directory structure
proc du_dirlisting_generic {folderpath} {
proc du_dirlisting_generic {folderpath args} {
set defaults [dict create\
-glob *\
-with_sizes 0\
-with_times 0\
]
set errors [dict create]
set known_opts [dict keys $defaults]
foreach k [dict keys $args] {
if {$k ni $known_opts} {
error "du_dirlisting_generic unknown-option $k"
}
}
set opts [dict merge $defaults $args]
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_glob [dict get $opts -glob]
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_with_sizes [dict get $opts -with_sizes]
set ftypes [list f d l]
if {"$opt_with_sizes" in {0 1}} {
#dn't use string is boolean (false vs f problem)
if {$opt_with_sizes} {
set sized_types $ftypes
} else {
set sized_types [list]
}
} else {
set sized_types $opt_with_sizes
}
if {[llength $sized_types]} {
foreach st $sized_types {
if {$st ni $ftypes} {
error "du_dirlisting_generic unrecognized element in -with_sizes '$st'"
}
}
}
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_with_times [dict get $opts -with_times]
if {"$opt_with_times" in {0 1}} {
if {$opt_with_times} {
set timed_types $ftypes
} else {
set timed_types [list]
}
} else {
set timed_types $opt_with_times
}
if {[llength $timed_types]} {
foreach item $timed_types {
if {$item ni $ftypes} {
error "du_dirlisting_generic unrecognised element in -with-times '$item'"
}
}
}
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
# The repeated globs are a source of slowness for this function.
#TODO - we could minimize the number of globs if we know we need to do a file stat and/or file attributes on each entry anyway
#For the case where we don't need times,sizes or other metadata - it is faster to do multiple globs
#This all makes this function complicated to gather the required data efficiently.
#note platform differences between what is considered hidden make this tricky.
# on windows 'glob .*' will not return some hidden dot items but will return . .. and glob -types hidden .* will not return some dotted items
# glob -types hidden * on windows will not necessarily return all dot files/folders
@ -644,28 +883,87 @@ namespace eval punk::du {
# we need to process * and .* in the same glob calls and remove duplicates
# if we do * and .* in separate iterations of this loop we lose the ability to filter duplicates easily
#Note - glob operations seem to be faster than looped tests like 'file isdirectory' & 'file readlink'
#set parent [lindex $folders $folderidx]
set hdirs [glob -nocomplain -dir $folderpath -types {hidden d} * .*]
#set hdirs {}
set dirs [glob -nocomplain -dir $folderpath -types d * .*]
#note - with this design, we can't glob for all except dotfiles - this is for cross-platform consistency and efficiency (Review).
#dotfiles aren't considered hidden on all platforms
#some sort of antiglob is a possible enhancement and more consistent with the view that leading dot should be treated as any other filename character in this context.
if {$opt_glob eq "*"} {
#Note - glob operations seem to be faster than looped tests like 'file isdirectory' & 'file readlink'
#set parent [lindex $folders $folderidx]
set hdirs [glob -nocomplain -dir $folderpath -types {hidden d} * .*]
#set hdirs {}
set dirs [glob -nocomplain -dir $folderpath -types d * .*]
set hlinks [glob -nocomplain -dir $folderpath -types {hidden l} * .*]
#set hlinks {}
set links [glob -nocomplain -dir $folderpath -types l * .*] ;#links may have dupes - we don't care. struct::set difference will remove
#set links [lsort -unique [concat $hlinks $links[unset links]]]
set hfiles [glob -nocomplain -dir $folderpath -types {hidden f} * .*]
#set hfiles {}
set files [glob -nocomplain -dir $folderpath -types f * .*]
#set files {}
} else {
set hdirs [glob -nocomplain -dir $folderpath -types {hidden d} $opt_glob]
set dirs [glob -nocomplain -dir $folderpath -types d $opt_glob]
set hlinks [glob -nocomplain -dir $folderpath -types {hidden l} * .*]
#set hlinks {}
set links [glob -nocomplain -dir $folderpath -types l * .*] ;#links may have dupes - we don't care. struct::set difference will remove
#set links [lsort -unique [concat $hlinks $links[unset links]]]
set hlinks [glob -nocomplain -dir $folderpath -types {hidden l} $opt_glob]
set links [glob -nocomplain -dir $folderpath -types l $opt_glob] ;#links may have dupes - we don't care. struct::set difference will remove
set hfiles [glob -nocomplain -dir $folderpath -types {hidden f} * .*]
#set hfiles {}
set files [glob -nocomplain -dir $folderpath -types f * .*]
#set files {}
set hfiles [glob -nocomplain -dir $folderpath -types {hidden f} $opt_glob]
set files [glob -nocomplain -dir $folderpath -types f $opt_glob]
}
#note struct::set difference produces unordered result
#struct::set difference removes duplicates
#remove links and . .. from directories, remove links from files
set dirs [struct::set difference [concat $hdirs $dirs[unset dirs]] [concat $links [list [file join $folderpath .] [file join $folderpath ..] ]]]
set files [struct::set difference [concat $hfiles $files[unset files]] $links]
set dirs [struct::set difference [concat $hdirs $dirs[unset dirs]] [concat $links [list [file join $folderpath .] [file join $folderpath ..] ]]]
set links [lsort -unique [concat $links $hlinks]]
set meta_dict [dict create]
set meta_types [concat $sized_types $timed_types]
#known tcl stat keys 2023 - review
set empty_stat_dict [dict create atime {} ctime {} dev {} gid {} ino {} mode {} mtime {} nlink {} size {} type {} uid {}]
#make sure we call file stat only once per item
set statkeys [list]
if {[llength $meta_types]} {
foreach ft {f d l} lvar {files dirs links} {
if {"$ft" in $meta_types} {
foreach path [set $lvar] {
#caller may have read perm on the containing folder - but not on child item - so file stat could raise an error
if {![catch {file stat $path arrstat} errM]} {
dict set meta_dict $path [dict create shorttype $ft {*}[array get arrstat]]
} else {
dict lappend errors $path "file stat error: $errM"
dict set meta_dict $path [dict create shorttype $ft {*}$empty_stat_dict]
}
}
}
}
}
set fsizes [list]
set allsizes [dict create]
set alltimes [dict create]
#review birthtime field of stat? cross-platform differences ctime etc?
dict for {path pathinfo} $meta_dict {
set ft [dict get $pathinfo shorttype]
if {$ft in $sized_types} {
dict set allsizes $path [dict create bytes [dict get $pathinfo size]]
if {$ft eq "f"} {
lappend fsizes [dict get $pathinfo size]
}
}
if {$ft in $timed_types} {
dict set alltimes $path [dict create c [dict get $pathinfo ctime] a [dict get $pathinfo atime] m [dict get $pathinfo mtime]]
}
}
if {"f" in $sized_types} {
if {[llength $fsizes] ne [llength $files]} {
dict lappend errors $folderpath "failed to retrieve all file sizes"
}
}
if {"windows" eq $::tcl_platform(platform)} {
set flaggedhidden [concat $hdirs $hfiles $hlinks]
} else {
@ -676,32 +974,202 @@ namespace eval punk::du {
set vfsmounts [get_vfsmounts_in_folder $folderpath]
set filesizes [list]; #not available in listing-call - as opposed to twapi which can do it as it goes
return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {}]
set effective_opts $opts
dict set effective_opts -with_times $timed_types
dict set effective_opts -with_sizes $sized_types
return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes $fsizes sizes $allsizes times $alltimes flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $effective_opts errors $errors]
}
#we can halve the number of round trips on unix-like systems, where 'hidden' always corresponds to dotted files
proc du_dirlisting_unix {folderpath} {
set dirs [glob -nocomplain -dir $folderpath -types d * .*] ;# also returns links to dirs
set links [glob -nocomplain -dir $folderpath -types l * .*] ;# links may have dupes - we don't care. struct::set difference will remove
set files [glob -nocomplain -dir $folderpath -types f * .*] ;# also returns links to files
proc du_dirlisting_tclvfs {folderpath args} {
set defaults [dict create\
-glob *\
-with_sizes 0\
-with_times 0\
]
set opts [dict merge $defaults $args]
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_glob [dict get $opts -glob]
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_with_sizes [dict get $opts -with_sizes]
set ftypes [list f d l]
if {"$opt_with_sizes" in {0 1}} {
#dn't use string is boolean (false vs f problem)
if {$opt_with_sizes} {
set sized_types $ftypes
} else {
set sized_types [list]
}
} else {
set sized_types $opt_with_sizes
}
if {[llength $sized_types]} {
foreach st $sized_types {
if {$st ni $ftypes} {
error "du_dirlisting_generic unrecognized element in -with_sizes '$st'"
}
}
}
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_with_times [dict get $opts -with_times]
if {"$opt_with_times" in {0 1}} {
if {$opt_with_times} {
set timed_types $ftypes
} else {
set timed_types [list]
}
} else {
set timed_types $opt_with_times
}
if {[llength $timed_types]} {
foreach item $timed_types {
if {$item ni $ftypes} {
error "du_dirlisting_generic unrecognised element in -with-times '$item'"
}
}
}
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set errors [dict create]
if {$opt_glob eq "*"} {
set dirs [glob -nocomplain -dir $folderpath -types d * .*] ;# also returns links to dirs
#review - how are links handled in vfs? presumably if the vfs is a full implementation they should work at least within the vfs?
set links [glob -nocomplain -dir $folderpath -types l * .*] ;# links may have dupes - we don't care. struct::set difference will remove
set files [glob -nocomplain -dir $folderpath -types f * .*] ;# also returns links to files
} else {
set dirs [glob -nocomplain -dir $folderpath -types d $opt_glob]
set links [glob -nocomplain -dir $folderpath -types l $opt_glob]
set files [glob -nocomplain -dir $folderpath -types f $opt_glob]
}
#remove any links from our dirs and files collections
set dirs [struct::set difference $dirs[unset dirs] [concat $links [list [file join $folderpath .] [file join $folderpath ..] ]]]
set files [struct::set difference $files[unset files] $links]
#nested vfs mount.. REVIEW - does anything need special handling?
set vfsmounts [get_vfsmounts_in_folder $folderpath]
return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {}]
set meta_dict [dict create]
set meta_types [concat $sized_types $timed_types]
#known tcl stat keys 2023 - review
set empty_stat_dict [dict create atime {} ctime {} dev {} gid {} ino {} mode {} mtime {} nlink {} size {} type {} uid {}]
#make sure we call file stat only once per item
set statkeys [list]
if {[llength $meta_types]} {
foreach ft {f d l} lvar {files dirs links} {
if {"$ft" in $meta_types} {
foreach path [set $lvar] {
#caller may have read perm on the containing folder - but not on child item - so file stat could raise an error
if {![catch {file stat $path arrstat} errM]} {
dict set meta_dict $path [dict create shorttype $ft {*}[array get arrstat]]
} else {
dict lappend errors $path "file stat error: $errM"
dict set meta_dict $path [dict create shorttype $ft {*}$empty_stat_dict]
}
}
}
}
}
set fsizes [list]
set allsizes [dict create]
set alltimes [dict create]
#review birthtime field of stat? cross-platform differences ctime etc?
dict for {path pathinfo} $meta_dict {
set ft [dict get $pathinfo shorttype]
if {$ft in $sized_types} {
dict set allsizes $path [dict create bytes [dict get $pathinfo size]]
if {$ft eq "f"} {
lappend fsizes [dict get $pathinfo size]
}
}
if {$ft in $timed_types} {
dict set alltimes $path [dict create c [dict get $pathinfo ctime] a [dict get $pathinfo atime] m [dict get $pathinfo mtime]]
}
}
if {"f" in $sized_types} {
if {[llength $fsizes] ne [llength $files]} {
dict lappend errors $folderpath "failed to retrieve all file sizes"
}
}
set effective_opts $opts
dict set effective_opts -with_times $timed_types
dict set effective_opts -with_sizes $sized_types
return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes $fsizes sizes $allsizes times $alltimes flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $effective_opts errors $errors]
}
proc du_dirlisting_tclvfs {folderpath} {
set dirs [glob -nocomplain -dir $folderpath -types d * .*] ;# also returns links to dirs
#review - how are links handled in vfs? presumably if the vfs is a full implementation they should work at least within the vfs?
set links [glob -nocomplain -dir $folderpath -types l * .*] ;# links may have dupes - we don't care. struct::set difference will remove
set files [glob -nocomplain -dir $folderpath -types f * .*] ;# also returns links to files
#we can halve the number of round trips on unix-like systems, where 'hidden' always corresponds to dotted files
proc du_dirlisting_unix {folderpath args} {
set defaults [dict create\
-glob *\
-with_sizes 0\
-with_times 0\
]
set errors [dict create]
dict lappend errors $folderpath "metdata support incomplete - prefer du_dirlisting_generic"
set opts [dict merge $defaults $args]
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_glob [dict get $opts -glob]
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_with_sizes [dict get $opts -with_sizes]
set ftypes [list f d l]
if {"$opt_with_sizes" in {0 1}} {
#dn't use string is boolean (false vs f problem)
if {$opt_with_sizes} {
set sized_types $ftypes
} else {
set sized_types [list]
}
} else {
set sized_types $opt_with_sizes
}
if {[llength $sized_types]} {
foreach st $sized_types {
if {$st ni $ftypes} {
error "du_dirlisting_generic unrecognized element in -with_sizes '$st'"
}
}
}
# -- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_with_times [dict get $opts -with_times]
if {"$opt_with_times" in {0 1}} {
if {$opt_with_times} {
set timed_types $ftypes
} else {
set timed_types [list]
}
} else {
set timed_types $opt_with_times
}
if {[llength $timed_types]} {
foreach item $timed_types {
if {$item ni $ftypes} {
error "du_dirlisting_generic unrecognised element in -with-times '$item'"
}
}
}
#this is not consistent with standard way * works for unix folders - but it is consistent with behaviour of this facility on windows
if {$opt_glob eq "*"} {
set dirs [glob -nocomplain -dir $folderpath -types d * .*] ;# also returns links to dirs
set links [glob -nocomplain -dir $folderpath -types l * .*] ;# links may have dupes - we don't care. struct::set difference will remove
set files [glob -nocomplain -dir $folderpath -types f * .*] ;# also returns links to files
} else {
set dirs [glob -nocomplain -dir $folderpath -types d $opt_glob]
set links [glob -nocomplain -dir $folderpath -types l $opt_glob]
set files [glob -nocomplain -dir $folderpath -types f $opt_glob]
}
#remove any links from our dirs and files collections
set dirs [struct::set difference $dirs[unset dirs] [concat $links [list [file join $folderpath .] [file join $folderpath ..] ]]]
set files [struct::set difference $files[unset files] $links]
#nested vfs mount.. REVIEW - does anything need special handling?
set vfsmounts [get_vfsmounts_in_folder $folderpath]
return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {}]
set effective_opts $opts
dict set effective_opts -with_times $timed_types
dict set effective_opts -with_sizes $sized_types
return [list dirs $dirs vfsmounts $vfsmounts links $links files $files filesizes {} times {} flaggedhidden {} flaggedsystem {} flaggedreadonly {} altname {} opts $opts errors $errors]
}
@ -740,9 +1208,15 @@ namespace eval punk::du {
return $newlist
}
#just an experiment
#same implementation as punk::strlen
#get length of path which has internal rep of path - maintaining path/list rep without shimmering to string representation.
proc pathcharacterlen {pathrep} {
append str2 $pathrep {}
string length $str2
}
#just an experiment
proc pathcharacterlen1 {pathrep} {
#This works - but is unnecessarily complex
set l 0
set parts [file split $pathrep]
if {[llength $parts] < 2} {
@ -769,6 +1243,26 @@ namespace eval punk::du {
}
}
proc du_dirlisting_undecided {folderpath args} {
if {"windows" eq $::tcl_platform(platform)} {
set loadstate [zzzload::pkg_require twapi]
if {$loadstate ni [list loading failed]} {
package require twapi ;#should be fast once twapi dll loaded in zzzload thread
set ::punk::du::has_twapi 1
punk::du::active::set_active_function du_dirlisting du_dirlisting_twapi
tailcall du_dirlisting_twapi $folderpath {*}$args
} else {
if {$loadstate eq "failed"} {
puts stderr "punk::du defaulting to du_dirlisting_generic because twapi load failed"
punk::du::active::set_active_function du_dirlisting du_dirlisting_generic
}
tailcall du_dirlisting_generic $folderpath {*}$args
}
} else {
punk::du::active::set_active_function du_dirlisting du_dirlisting_unix
tailcall du_dirlisting_unix $folderpath {*}$args
}
}
}
@ -791,15 +1285,8 @@ namespace eval punk::du {
variable functions_kown
upvar ::punk::du::has_twapi has_twapi
if {"windows" eq $::tcl_platform(platform)} {
if {$has_twapi} {
set_active_function du_dirlisting du_dirlisting_twapi
} else {
set_active_function du_dirlisting du_dirlisting_generic
}
} else {
set_active_function du_dirlisting du_dirlisting_unix
}
set_active_function du_dirlisting du_dirlisting_undecided
}

1484
src/bootsupport/modules/punk/mix-0.2.tm

File diff suppressed because it is too large Load Diff

26
src/bootsupport/modules/punk/repo-0.1.1.tm

@ -767,6 +767,30 @@ namespace eval punk::repo {
return $root_dict
}
proc fossil_get_repository_file {{path {}}} {
if {$path eq {}} { set path [pwd] }
set fossilcmd [auto_execok fossil]
if {[llength $fossilcmd]} {
do_in_path $path {
set fossilinfo [::exec {*}$fossilcmd info]
}
set matching_lines [punk::repo::grep {repository:*} $fossilinfo]
if {![llength $matching_lines]} {
return ""
}
set trimmedline [string trim [lindex $matching_lines 0]]
set firstcolon [string first : $trimmedline]
set repofile_path [string trim [string range $trimmedline $firstcolon+1 end]]
if {![file exists $repofile_path]} {
puts stderr "Repository file pointed to by fossil configdb doesn't exist: $repofile_path"
return ""
}
return $repofile_path
} else {
puts stderr "fossil_get_repository_file: fossil command unavailable"
return ""
}
}
proc fossil_get_repository_folder_for_project {projectname args} {
set defaults [list -parentfolder \uFFFF -extrachoice \uFFFF]
@ -1040,7 +1064,7 @@ namespace eval punk::repo {
do_in_path $path {
set info [::exec {*}$fossilcmd remote ls]
}
return [string trim $v]
return [string trim $info]
} else {
return Unknown
}

187
src/bootsupport/modules/punk/winpath-0.1.0.tm

@ -25,111 +25,18 @@
namespace eval punk::winpath {
namespace export winpath windir cdwin cdwindir illegalname_fix illegalname_test
#review - is this intended to be useful/callable on non-windows platforms?
#it should in theory be useable from another platform that wants to create a path for use on windows.
#In this case - we shouldn't examine what volumes exist (assume A: .. Z: are valid)
#review zipfs:// other uri schemes?
proc winpath {path} {
#NOTE: tcl file exists gives different answers on windows for paths like /c depending on cwd (presumably based on file pathtype of volumerelative)
#This is add odds with attempting to navigate on a windows system which has cygwin, wsl etc... It also makes it difficult for functions intended to operate independent of CWD.
#e.g there is potential confusion when there is a c folder on c: drive (c:/c)
#I will attempt to provide a coherent operation for winpath ./ ../ etc , but it may disallow for example; change to /something or /x where these don't match a driveletter or /mnt
#whereas tcl may allow cd to /something if a something folder happens to exist on the current volume based on cwd.
#I think it's preferable to require an explicit driveletter /x or /mnt when using unix-like paths on windows - but practical considerations may prove me wrong..
#It's possible that this function should also ignore the current set of driveletters - and operate completely independent of whether a path actually exists
#This makes it hard to use things like 'file normalize' - which also looks at things like current volume.
#
#Note for example the results of 'which' grep on windows can produce a path like /c/Users/somewhere/bin/grep
#which tcl's file normalize may change to C:/c/Users or X:/c/Users - based on current volumen. Given that C:/c might exist - this can be problematic in a couple of ways.
#The mixing of unix-like and windows commands on the same machine is a large part of the problem.. but this mix is now common
#
#convert /c/etc to C:/etc
set re_slash_x_slash {^/([[:alpha:]]){1}/.*}
set re_slash_else {^/([[:alpha:]]*)(.*)}
set volumes [file volumes]
#exclude things like //zipfs:/
set driveletters [list]
foreach v $volumes {
if {[regexp {^([[:alpha:]]){1}:/$} $v _ letter]} {
lappend driveletters $letter
}
}
#puts stderr "->$driveletters"
if {[regexp $re_slash_x_slash $path _ letter]} {
#upper case appears to be windows canonical form
set path [string toupper $letter]:/[string range $path 3 end]
} elseif {[regexp {^/mnt/([[:alpha:]]){1}/.*} [string tolower $path] _ letter]} {
set path [string toupper $letter]:/[string range $path 7 end]
} elseif {[regexp {^/mnt/([[:alpha:]]){1}$} [string tolower $path] _ letter]} {
set path [string toupper $letter]:/
} elseif {[regexp $re_slash_else $path _ firstpart remainder]} {
#could be for example /c or /something/users
if {[string length $firstpart] == 1} {
set letter $firstpart
set path [string toupper $letter]:/
} else {
#attempt to use cygpath helper
if {![catch {
set cygpath [runout -n cygpath -w $path] ;#!
set ::punk::last_run_display [list] ;#hack - review shouldn't really be necessary.. but because we call winpath from ./ - the repl looks for last_run_display
} errM]} {
set path [string map [list "\\" "/"] $cygpath]
} else {
error "Path '$path' does not appear to be in a standard form. For unix-like paths on windows such as /x, x must correspond to a drive letter. Consider installing cygwin's cygpath tool to see if that helps."
}
}
}
#puts stderr "=> $path"
#things like 'which' seem to return a path minus the .exe - so we'll just test the containing folder
#
#By now file normalize shouldn't do too many shannanigans related to cwd..
#We want it to look at cwd for relative paths.. but we don't consider things like /c/Users to be relative even on windows
if {![file exists [file dirname $path]]} {
set path [file normalize $path]
#may still not exist.. that's ok.
}
#file normalize may change backslashes to forward slashes.. including things like the special \\?\ prefix which is intended to stop windows api from parsing a name
#2023 - this is ok as //?/ also seems to work.. but it is unclear if that is because Tcl is re-converting to backslashes
if {[illegalname_test $path]} {
set path [illegalname_fix $path]
}
return $path
}
proc windir {path} {
if {$path eq "~"} {
#as the tilde hasn't been normalized.. we can't assume we're running on the actual platform
return ~/..
}
return [file dirname [winpath $path]]
}
#REVIEW high-coupling
proc cdwin {path} {
set path [winpath $path]
if {$::repl::running} {
repl::term::set_console_title $path
}
cd $path
}
proc cdwindir {path} {
set path [winpath $path]
if {$::repl::running} {
repl::term::set_console_title $path
}
cd [file dirname $path]
}
#\\servername\share etc or \\?\UNC\servername\share etc.
proc is_unc_path {path} {
set path [string map [list \\ /] $path] ;#normalize to forward slashes for testing purposes (and forward slashes seem to be auto-translated by windows anyway)
if {[string first "//" $path] == 0} {
set strcopy_path [punk::objclone $path]
set strcopy_path [string map [list \\ /] $strcopy_path] ;#normalize to forward slashes for testing purposes (and forward slashes seem to be auto-translated by windows anyway)
if {[string first "//" $strcopy_path] == 0} {
#check for "Dos device path" syntax
if {[string range $path 0 3] in [list "//?/" "//./"]} {
if {[string range $strcopy_path 0 3] in [list "//?/" "//./"]} {
#Note that //./ doesn't appear to be supported in Tcl as at 2023-08 - but //?/ works (except for //?/UNC/Server/share)
if {[string range $path 4 6] eq "UNC"} {
if {[string range $strcopy_path 4 6] eq "UNC"} {
return 1
} else {
#some other Dos device path. Could be a drive which is mapped to a UNC path - but the path itself isn't a unc path
@ -146,7 +53,7 @@ namespace eval punk::winpath {
#ordinary \\Servername or \\servername\share or \\servername\share\path (or forward-slash equivalent) with no dos device syntax //?/ //./ etc.
proc is_unc_path_plain {path} {
if {[is_unc_path $path]} {
if {![is_dos_device_path]} {
if {![is_dos_device_path $path]} {
return 1
} else {
return 0
@ -156,9 +63,9 @@ namespace eval punk::winpath {
}
}
#'file attributes', and therefor this operation, is expensive (on windows at least)
#int-rep path preserved - but 'file attributes', and therefor this operation, is expensive (on windows at least)
proc pwdshortname {{path {}}} {
if {![string length $path]} {
if {$path eq ""} {
set path [pwd]
} else {
if {[file pathtype $path] eq "relative"} {
@ -170,8 +77,9 @@ namespace eval punk::winpath {
#dos device path syntax allows windows api to acces extended-length paths and filenames with illegal path characters such as trailing dots or whitespace
#(can exist on server shares and on NTFS - but standard apps can't access without dos device syntax)
proc is_dos_device_path {path} {
set path [string map [list \\ /] $path] ;#normalize to forward slashes for testing purposes (and forward slashes seem to be auto-translated by windows anyway)
if {[string range $path 0 3] in [list "//?/" "//./"]} {
set strcopy_path [punk::objclone $path]
set strcopy_path [string map [list \\ /] $strcopy_path] ;#normalize to forward slashes for testing purposes (and forward slashes seem to be auto-translated by windows anyway)
if {[string range $strcopy_path 0 3] in [list "//?/" "//./"]} {
return 1
} else {
return 0
@ -192,17 +100,35 @@ namespace eval punk::winpath {
proc strip_unc_path_prefix {path} {
if {[is_unc_path $path]} {
#//?/UNC/server/etc
return [string range $path 7 end]
set strcopy_path [punk::objclone $path]
set trimmedpath [string range $strcopy_path 7 end]
file pathtype $trimmedpath ;#shimmer it to path rep
return $trimmedpath
} elseif {is_unc_path_plain $path} {
#plain unc //server
return [string range $path 2 end]
set strcopy_path [punk::objclone $path]
set trimmedpath [string range $strcopy_path 2 end]
file pathtype $trimmedpath
return $trimmedpath
} else {
return $path
}
}
#we don't validate that path is actually illegal because we don't know the full range of such names.
#The caller can apply this to any path.
#don't test for platform here - needs to be callable from any platform for potential passing to windows
#don't test for platform here - needs to be callable from any platform for potential passing to windows (what usecase? 8.3 name is not always calculable independently)
#The utility of this is questionable. prepending a dos-device path won't make a filename with illegal characters readable by windows.
#It will need the 'shortname' at least for the illegal segment - if not the whole path
#Whilst the 8.3 name algorithm - including undocumented hash function has been reverse engineered
#- it depends on the content of the directory - as collisions cause a different name (e.g incremented number)
#- it also depends on the history of the folder
#- you can't take the current dir contents and a particular *existing* longname and determine the shortname algorithmically...
#- the shortname may have been generated during a different directory state.
#- It is then stored on disk (where?) - so access to reading the existing shortname is required.
#- An implementation of the 8.3 algorithm would only be potentially useful in determining the name that will result from adding a new file
# and would be subject to potential collisions if there are race-conditions in file creation
#- Using an 8.3 algorithm externally would be dangerous in that it could appear to work a lot of the time - but return a different file entirely sometimes.
#- Conclusion is that the 8.3 name must be retrieved rathern than calclated
proc illegalname_fix {path} {
#don't add extra dos device path syntax protection-prefix if already done
if {[is_unc_path $path]} {
@ -214,6 +140,7 @@ namespace eval punk::winpath {
}
#\\servername\share theoretically maps to: \\?\UNC\servername\share in protected form. https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats
#NOTE: 2023-08 on windows 10 at least \\?\UNC\Server\share doesn't work - ie we can't use illegalname_fix on UNC paths such as \\Server\share
#(but mapped drive to same path will work)
@ -225,6 +152,10 @@ namespace eval punk::winpath {
append err \n " - because //?/UNC/Servername/share is not supported in Tcl (and only minimally even in powershell) as at 2023. (on windows use mapped drive instead)"
error $err
}
set strcopy_path [punk::objclone $path]
#Note: path could still have leading double slash if it is a Dos device path: e.g. //?/c:/etc
if {[file pathtype $path] eq "absolute"} {
if {$path eq "~"} {
@ -239,10 +170,10 @@ namespace eval punk::winpath {
} else {
#set fullpath [file normalize $path] ;#very slow on windows
#set fullpath [pwd]/$path ;#will keep ./ in middle of path - not valid for dos-device paths
if {[string range $path 0 1] eq "./"} {
set path [string range $path 2 end]
if {[string range $strcopy_path 0 1] eq "./"} {
set strcopy_path [string range $strcopy_path 2 end]
}
set fullpath [file join [pwd] $path]
set fullpath [file join [pwd] $strcopy_path]
}
#For file I/O, the "\\?\" prefix to a path string tells the Windows APIs to disable all string parsing
# and to send the string that follows it straight to the file system.
@ -252,16 +183,21 @@ namespace eval punk::winpath {
#choose //?/ as normalized version - since likely 'file normalize' will do it anyway, and experimentall, the windows API accepts both REVIEW
return ${protect2}$fullpath
set result ${protect2}$fullpath
file pathtype $result ;#make it return a path rep
return $result
}
#don't test for platform here - needs to be callable from any platform for potential passing to windows
#we can create files with windows illegal names by using //?/ dos device path syntax - but we need to detect when that is required.
#
# path int-rep preserving
proc illegalname_test {path} {
#first test if already protected - we return false even if the file would be illegal without the protection!
if {[is_dos_device_path $path]} {
return 0
}
#https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file
#according to the above: Use any character in the current code page for a name, including Unicode characters and characters in the extended character set (128–255), except for the following:
set reserved [list < > : \" / \\ | ? *]
#we need to exclude things like path/.. path/.
foreach seg [file split $path] {
if {$seg in [list . ..]} {
@ -289,14 +225,23 @@ namespace eval punk::winpath {
return 0
}
proc test_ntfs_tunneling {f1 f2 args} {
file mkdir $f1
puts stderr "waiting 15secs..."
after 5000 {puts -nonewline stderr .}
after 5000 {puts -nonewline stderr .}
after 5000 {puts -nonewline stderr .}
after 500 {puts stderr \n}
file mkdir $f2
puts stdout "$f1 [file stat $f1]"
puts stdout "$f2 [file stat $f2]"
file delete $f1
puts stdout "renaming $f2 to $f1"
file rename $f2 $f1
puts stdout "$f1 [file stat $f1]"
}
#----------------------------------------------
#leave the winpath related aliases available on all platforms
interp alias {} cdwin {} punk::winpath::cdwin
interp alias {} cdwindir {} punk::winpath::cdwindir
interp alias {} winpath {} punk::winpath::winpath
interp alias {} windir {} punk::winpath::windir
#----------------------------------------------
}

1984
src/bootsupport/modules/punkcheck-0.1.0.tm

File diff suppressed because it is too large Load Diff

2
src/doc/include/general.inc

@ -2,5 +2,5 @@
[comment {- -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---}]
[moddesc {punkshell - a Tcl }]
[category {shell}]
[keywords {shell repl}]
[keywords shell repl punk]
[require Tcl 8.6]

441
src/make.tcl

@ -2,8 +2,6 @@
#
#make any tclkits and modules in src and place them and associated data files/scripts in the parent folder of src.
#e.g in 'bin' and 'modules' folders at same level as 'src' folder.
#It is assumed the src folder has been placed somewhere where appropriate
#(e.g not in /usr or c:/ - unless you intend it to directly make and place folders and files in those locations)
set hashline "# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###"
puts $hashline
@ -25,13 +23,18 @@ if {"::try" ni [info commands ::try]} {
#------------------------------------------------------------------------------
#Module loading from /src/bootsupport or src/*.vfs if script is within a .vfs folder
#------------------------------------------------------------------------------
#If the there is a folder directly under the current directory /src/bootsupport/modules which contains .tm files when the starts
#If the there is a folder under the current directory /src/bootsupport/modules which contains .tm files
# - then it will attempt to preference these modules
# This allows a source update via 'fossil update' 'git pull' etc to pull in support modules for the make script
# and load these in preference to ones that may have been in the interps tcl::tm::list or auto_path due to environment variables
# This allows a source update via 'fossil update' 'git pull' etc to pull in a minimal set of support modules for the make script
# and load these in preference to ones that may have been in the interp's tcl::tm::list or auto_path due to environment variables
set startdir [pwd]
set bootsupport_mod [file join $startdir src bootsupport modules]
set bootsupport_lib [file join $startdir src bootsupport lib]
if {[file exists [file join $startdir src bootsupport]]} {
set bootsupport_mod [file join $startdir src bootsupport modules]
set bootsupport_lib [file join $startdir src bootsupport lib]
} else {
set bootsupport_mod [file join $startdir bootsupport modules]
set bootsupport_lib [file join $startdir bootsupport lib]
}
if {[file exists $bootsupport_mod] || [file exists $bootsupport_lib]} {
set original_tm_list [tcl::tm::list]
@ -60,31 +63,16 @@ if {[file exists $bootsupport_mod] || [file exists $bootsupport_lib]} {
}
#todo - review usecase
if {[string match "*.vfs/*" [info script]]} {
#src/xxx.vfs/lib/app-punk/repl.tcl
#we assume if calling directly into .vfs that the user would prefer to use src/modules - so go up 4 levels
set modulefolder [file dirname [file dirname [file dirname [file dirname [info script]]]]]/modules
} else {
# .../bin/punkXX.exe look for ../modules (i.e modules folder at same level as bin folder)
set modulefolder [file dirname [file dirname [info nameofexecutable]]]/modules
}
if {[file exists $modulefolder]} {
tcl::tm::add $modulefolder
} else {
puts stderr "Warning unable to find module folder at: $modulefolder"
}
if {[file exists [pwd]/modules]} {
tcl::tm::add [pwd]/modules
}
#package require Thread
#These are strong dependencies
# - the repl requires Threading and punk,shellfilter,shellrun to call and display properly.
# - the full repl requires Threading and punk,shellfilter,shellrun to call and display properly.
# tm list already indexed - need 'package forget' to find modules based on current tcl::tm::list
#These are strong dependencies
package forget punk::mix
package require punk::mix
package forget punk::repo
@ -144,6 +132,8 @@ proc punkmake_gethelp {args} {
append h " $scriptname project ?-k?" \n
append h " - this is the literal word project - and confirms you want to run the project build" \n
append h " - the optional -k flag will terminate processes running as the executable being built (if applicable)" \n \n
append h " $scriptname bootsupport" \n
append h " - update the src/bootsupport modules as well as the mixtemplates/layouts/<layoutname>/src/bootsupport modules if the folder exists" \n \n
append h " $scriptname get-project-info" \n
append h " - show the name and base folder of the project to be built" \n
append h "" \n
@ -189,7 +179,7 @@ if {[llength $commands_found] != 1 } {
}
if {$do_help} {
puts stderr [punkmake_gethelp]
exit 1
exit 0
}
set ::punkmake::command [lindex $commands_found 0]
@ -224,6 +214,8 @@ if {![string length [set projectroot [punk::repo::find_project $scriptfolder]]]}
}
set sourcefolder $projectroot/src
if {$::punkmake::command eq "get-project-info"} {
puts stdout "- -- --- --- --- --- --- --- --- --- ---"
puts stdout "- -- get-project-info -- -"
@ -260,9 +252,128 @@ if {$::punkmake::command eq "shell"} {
}
if {$::punkmake::command eq "bootsupport"} {
puts "projectroot: $projectroot"
puts "script: [info script]"
#puts "-- [tcl::tm::list] --"
puts stdout "Updating bootsupport from local files"
proc bootsupport_localupdate {projectroot} {
set bootsupport_modules [list]
set bootsupport_config $projectroot/src/bootsupport/include_modules.config ;#
if {[file exists $bootsupport_config]} {
source $bootsupport_config ;#populate $bootsupport_modules with project-specific list
if {![llength $bootsupport_modules]} {
puts stderr "No local bootsupport modules configured for updating"
return
}
set targetroot $projectroot/src/bootsupport/modules
if {[catch {
#----------
set boot_installer [punkcheck::installtrack new make.tcl $projectroot/src/bootsupport/.punkcheck]
$boot_installer set_source_target $projectroot $projectroot/src/bootsupport
set boot_event [$boot_installer start_event {-make_step bootsupport}]
#----------
} errM]} {
puts stderr "Unable to use punkcheck for bootsupport error: $errM"
set boot_event ""
}
exit 1
foreach {relpath module} $bootsupport_modules {
set module [string trim $module :]
set module_subpath [string map [list :: /] [namespace qualifiers $module]]
set srclocation [file join $projectroot $relpath $module_subpath]
#puts stdout "$relpath $module $module_subpath $srclocation"
set pkgmatches [glob -nocomplain -dir $srclocation -tail [namespace tail $module]-*]
#lsort won't sort version numbers properly e.g with -dictionary 0.1.1 comes before 0.1
if {![llength $pkgmatches]} {
puts stderr "Missing source for bootsupport module $module - not found in $srclocation"
continue
}
set latestfile [lindex $pkgmatches 0]
set latestver [lindex [split [file rootname $latestfile] -] 1]
foreach m $pkgmatches {
lassign [split [file rootname $m] -] _pkg ver
#puts "comparing $ver vs $latestver"
if {[package vcompare $ver $latestver] == 1} {
set latestver $ver
set latestfile $m
}
}
set srcfile [file join $srclocation $latestfile]
set tgtfile [file join $targetroot $module_subpath $latestfile]
if {$boot_event ne ""} {
#----------
$boot_event targetset_init INSTALL $tgtfile
$boot_event targetset_addsource $srcfile
#----------
if {\
[llength [dict get [$boot_event targetset_source_changes] changed]]\
|| [llength [$boot_event get_targets_exist]] < [llength [$boot_event get_targets]]\
} {
$boot_event targetset_started
# -- --- --- --- --- ---
puts "BOOTSUPPORT update: $srcfile -> $tgtfile"
if {[catch {
file copy -force $srcfile $tgtfile
} errM]} {
$boot_event targetset_end FAILED
} else {
$boot_event targetset_end OK
}
# -- --- --- --- --- ---
} else {
puts -nonewline stderr "."
$boot_event targetset_end SKIPPED
}
$boot_event end
} else {
file copy -force $srcfile $tgtfile
}
}
if {$boot_event ne ""} {
puts \n
$boot_event destroy
$boot_installer destroy
}
}
}
bootsupport_localupdate $projectroot
#/modules/punk/mix/templates/layouts only applies if the project has it's own copy of the punk/mix modules. Generally this should only apply to the punkshell project itself.
set layout_bases [list\
$sourcefolder/mixtemplates/layouts\
$sourcefolder/modules/punk/mix/templates/layouts\
]
foreach project_layout_base $layout_bases {
if {[file exists $project_layout_base]} {
set project_layouts [glob -nocomplain -dir $project_layout_base -type d -tail *]
foreach layoutname $project_layouts {
if {[file exists [file join $project_layout_base $layoutname/src/bootsupport]]} {
set unpublish [list\
README.md\
]
set sourcemodules $projectroot/src/bootsupport/modules
set targetroot [file join $project_layout_base $layoutname/src/bootsupport/modules]
file mkdir $targetroot
puts stdout "BOOTSUPPORT layouts/$layoutname: copying from $sourcemodules to $targetroot (if source file changed)"
set resultdict [punkcheck::install $sourcemodules $targetroot -overwrite installedsourcechanged-targets -unpublish_paths $unpublish]
puts stdout [punkcheck::summarize_install_resultdict $resultdict]
flush stdout
}
}
} else {
puts stderr "No layout base at $project_layout_base"
}
}
puts stdout " bootsupport done "
flush stderr
flush stdout
#punk86 can hang if calling make.tcl via 'run' without this 'after' delay. punk87 unaffected. cause unknown.
#after 500
::exit 0
}
@ -273,7 +384,6 @@ if {$::punkmake::command ne "project"} {
}
set sourcefolder $projectroot/src
#only a single consolidated /modules folder used for target
set target_modules_base $projectroot/modules
@ -290,37 +400,21 @@ if {[file exists $sourcefolder/vendorlib]} {
README.md\
]
puts stdout "VENDORLIB: copying from $sourcefolder/vendorlib to $projectroot/lib (if source file changed)"
set resultdict [punkcheck::install $sourcefolder/vendorlib $projectroot/lib -overwrite installedsourcechanged-targets -unpublish_paths $unpublish]
set copied [dict get $resultdict files_copied]
set sources_unchanged [dict get $resultdict sources_unchanged]
puts stdout "--------------------------"
flush stdout
puts stderr "Copied [llength $copied] vendor lib files from src/vendorlib to $projectroot/lib"
foreach f $copied {
puts stdout "COPIED $f"
}
puts stdout "[llength $sources_unchanged] unchanged source files"
puts stdout "--------------------------"
puts stdout [punkcheck::summarize_install_resultdict $resultdict]
} else {
puts stderr "NOTE: No src/vendorlib folder found."
puts stderr "VENDORLIB: No src/vendorlib folder found."
}
if {[file exists $sourcefolder/vendormodules]} {
#install .tm *and other files*
puts stdout "VENDORMODULES: copying from $sourcefolder/vendormodules to $target_modules_base (if source file changed)"
set resultdict [punkcheck::install $sourcefolder/vendormodules $target_modules_base -installer make.tcl -overwrite installedsourcechanged-targets -unpublish_paths {README.md}]
set copied [dict get $resultdict files_copied]
set sources_unchanged [dict get $resultdict sources_unchanged]
puts stdout "--------------------------"
flush stdout
puts stderr "Copied [llength $copied] vendor module files from src/vendormodules to $target_modules_base"
foreach f $copied {
puts stdout "COPIED $f"
}
puts stdout "[llength $sources_unchanged] unchanged source files"
puts stdout "--------------------------"
puts stdout [punkcheck::summarize_install_resultdict $resultdict]
} else {
puts stderr "NOTE: No src/vendormodules folder found."
puts stderr "VENDORMODULES: No src/vendormodules folder found."
}
########################################################
@ -354,6 +448,7 @@ foreach layoutinfo $layout_update_list {
foreach filepair $pairs {
lassign $filepair srcfile tgtfile
file mkdir [file dirname $tgtfile]
#----------
$tpl_event targetset_init INSTALL $tgtfile
$tpl_event targetset_addsource $srcfile
@ -386,8 +481,6 @@ foreach layoutinfo $layout_update_list {
########################################################
#default source module folder is at projectroot/src/modules
#There may be multiple other src module folders at same level (e.g folder not being other special-purpose folder and not matching name vendor* that contains at least one .tm file in its root)
set source_module_folderlist [punk::mix::cli::lib::find_source_module_paths $projectroot]
@ -403,19 +496,54 @@ foreach src_module_dir $source_module_folderlist {
set overwrite "installedsourcechanged-targets"
#set overwrite "ALL-TARGETS"
puts stdout "MODULEFOLDER non_tm_files $src_module_dir - copying to $target_modules_base (if source file changed)"
set resultdict [punkcheck::install_non_tm_files $src_module_dir $target_modules_base -installer make.tcl -overwrite $overwrite -unpublish_paths {README.md}]
set copied [dict get $resultdict files_copied]
set sources_unchanged [dict get $resultdict sources_unchanged]
puts stdout "--------------------------"
flush stdout
puts stderr "Copied [llength $copied] non-tm source files from $src_module_dir to $target_modules_base"
puts stderr "[llength $sources_unchanged] unchanged source files"
flush stderr
puts stdout "--------------------------"
puts stdout [punkcheck::summarize_install_resultdict $resultdict]
}
set installername "make.tcl"
# ----------------------------------------
if {[punk::repo::is_fossil_root $projectroot]} {
set config [dict create\
-make-step configure_fossil\
]
#----------
set installer [punkcheck::installtrack new $installername $projectroot/.punkcheck]
$installer set_source_target $projectroot $projectroot
set event [$installer start_event $config]
$event targetset_init VIRTUAL fossil_settings_mainmenu ;#VIRTUAL - since there is no actual target file
set menufile $projectroot/.fossil-custom/mainmenu
$event targetset_addsource $menufile
#----------
if {\
[llength [dict get [$event targetset_source_changes] changed]]\
} {
$event targetset_started
# -- --- --- --- --- ---
puts stdout "Configuring fossil setting: mainmenu from: $menufile"
if {[catch {
set fd [open $menufile r]
fconfigure $fd -translation binary
set data [read $fd]
close $fd
exec fossil settings mainmenu $data
} errM]} {
$event targetset_end FAILED -note "fossil update failed: $errM"
} else {
$event targetset_end OK
}
# -- --- --- --- --- ---
} else {
puts stderr "."
$event targetset_end SKIPPED
}
$event end
$event destroy
$installer destroy
}
set buildfolder [punk::mix::cli::lib::get_build_workdir $sourcefolder]
if {$buildfolder ne "$sourcefolder/_build"} {
@ -430,7 +558,8 @@ set rtfolder $sourcefolder/runtime
set runtimes [glob -nocomplain -dir $rtfolder -types {f x} -tail *]
if {![llength $runtimes]} {
puts stderr "No executable runtimes found in $rtfolder - unable to build any .vfs folders into executables."
exit 2
puts stderr "Add runtimes to $sourcefolder/runtime if required"
exit 0
}
if {[catch {exec sdx help} errM]} {
@ -465,13 +594,15 @@ if {[file exists $mapfile]} {
#drop windows .exe suffix so same config can work cross platform - extension will be re-added if necessary later
set runtime [string range $runtime 0 end-4]
}
set runtime_test $runtime
if {"windows" eq $::tcl_platform(platform)} {
set runtime_test $runtime.exe
}
if {![file exists [file join $rtfolder $runtime_test]]} {
puts stderr "WARNING: Missing runtime file $rtfolder/$runtime_test (line in mapvfs.config: $ln)"
lappend missing $runtime
if {$runtime ne "-"} {
set runtime_test $runtime
if {"windows" eq $::tcl_platform(platform)} {
set runtime_test $runtime.exe
}
if {![file exists [file join $rtfolder $runtime_test]]} {
puts stderr "WARNING: Missing runtime file $rtfolder/$runtime_test (line in mapvfs.config: $ln)"
lappend missing $runtime
}
}
foreach vfs $vfspaths {
if {![file isdirectory [file join $sourcefolder $vfs]]} {
@ -511,7 +642,6 @@ if {![llength $vfs_folders]} {
set vfs_folder_changes [dict create] ;#cache whether each .vfs folder has changes so we don't re-run tests if building from same .vfs with multiple runtime executables
set installername "make.tcl"
# -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
#set runtimefile [lindex $runtimes 0]
foreach runtimefile $runtimes {
@ -593,7 +723,7 @@ foreach vfs $vfs_folders {
set runtimes_raw $runtimes
set runtimes [list]
foreach rt $runtimes_raw {
if {![string match *.exe $rt]} {
if {![string match *.exe $rt] && $rt ne "-"} {
set rt $rt.exe
}
lappend runtimes $rt
@ -615,27 +745,36 @@ foreach vfs $vfs_folders {
#assert $runtimes is a list of executable names suffixed with .exe if on windows - whether or not specified with .exe in the mapvfs.config
#todo - non kit based - zipkit?
# $runtimes may now include a dash entry "-" (from mapvfs.config file)
foreach rtname $runtimes {
#rtname of "-" indicates build a kit without a runtime
#first configured runtime will be the one to use the same name as .vfs folder for output executable. Additional runtimes on this .vfs will need to suffix the runtime name to disambiguate.
#review: This mechanism may not be great for multiplatform builds ? We may be better off consistently combining vfsname and rtname and letting a later platform-specific step choose ones to install in bin with simpler names.
if {$::tcl_platform(platform) eq "windows"} {
set targetexe ${vfsname}.exe
if {$rtname eq "-"} {
set targetkit $vfsname.kit
} else {
set targetexe $vfsname
}
if {$targetexe in $exe_names_seen} {
#more than one runtime for this .vfs
set targetexe ${vfsname}_$rtname
if {$::tcl_platform(platform) eq "windows"} {
set targetkit ${vfsname}.exe
} else {
set targetkit $vfsname
}
if {$targetkit in $exe_names_seen} {
#more than one runtime for this .vfs
set targetkit ${vfsname}_$rtname
}
}
lappend exe_names_seen $targetexe
lappend exe_names_seen $targetkit
# -- ----------
set vfs_installer [punkcheck::installtrack new $installername $basedir/.punkcheck]
$vfs_installer set_source_target $sourcefolder $buildfolder
set vfs_event [$vfs_installer start_event {-make-step build_vfs}]
$vfs_event targetset_init INSTALL $buildfolder/$targetexe
$vfs_event targetset_init INSTALL $buildfolder/$targetkit
$vfs_event targetset_addsource $sourcefolder/$vfs
$vfs_event targetset_addsource $buildfolder/build_$rtname
if {$rtname ne "-"} {
$vfs_event targetset_addsource $buildfolder/build_$rtname
}
# -- ----------
set changed_unchanged [$vfs_event targetset_source_changes]
@ -655,9 +794,17 @@ foreach vfs $vfs_folders {
if {[catch {
exec sdx wrap $buildfolder/$vfsname.new -vfs $sourcefolder/$vfs -runtime $buildfolder/build_$rtname -verbose
if {$rtname ne "-"} {
exec sdx wrap $buildfolder/$vfsname.new -vfs $sourcefolder/$vfs -runtime $buildfolder/build_$rtname -verbose
} else {
exec sdx wrap $buildfolder/$vfsname.new -vfs $sourcefolder/$vfs -verbose
}
} result]} {
puts stderr "sdx wrap $buildfolder/$vfsname.new -vfs $sourcefolder/$vfs -runtime $buildfolder/build_$rtname -verbose failed with msg: $result"
if {$rtname ne "-"} {
puts stderr "sdx wrap $buildfolder/$vfsname.new -vfs $sourcefolder/$vfs -runtime $buildfolder/build_$rtname -verbose failed with msg: $result"
} else {
puts stderr "sdx wrap $buildfolder/$vfsname.new -vfs $sourcefolder/$vfs -verbose failed with msg: $result"
}
} else {
puts stdout "ok - finished sdx"
set separator [string repeat = 40]
@ -678,65 +825,74 @@ foreach vfs $vfs_folders {
} else {
set pscmd "ps"
}
if {![catch {
exec $pscmd | grep $vfsname
} still_running]} {
puts stdout "found $vfsname instances still running\n"
set count_killed 0
foreach ln [split $still_running \n] {
puts stdout " $ln"
if {$::tcl_platform(platform) eq "windows"} {
set pid [lindex $ln 1]
if {$forcekill} {
set killcmd [list taskkill /F /PID $pid]
#killing process doesn't apply to .kit build
if {$rtname ne "-"} {
if {![catch {
exec $pscmd | grep $vfsname
} still_running]} {
puts stdout "found $vfsname instances still running\n"
set count_killed 0
foreach ln [split $still_running \n] {
puts stdout " $ln"
if {$::tcl_platform(platform) eq "windows"} {
set pid [lindex $ln 1]
if {$forcekill} {
set killcmd [list taskkill /F /PID $pid]
} else {
set killcmd [list taskkill /PID $pid]
}
} else {
set killcmd [list taskkill /PID $pid]
set pid [lindex $ln 0]
#review!
if {$forcekill} {
set killcmd [list kill -9 $pid]
} else {
set killcmd [list kill $pid]
}
}
} else {
set pid [lindex $ln 0]
#review!
if {$forcekill} {
set killcmd [list kill -9 $pid]
puts stdout " pid: $pid (attempting to kill now using '$killcmd')"
if {[catch {
exec {*}$killcmd
} errMsg]} {
puts stderr "$killcmd returned an error:"
puts stderr $errMsg
if {!$forcekill} {
puts stderr "(try '[info script] -k' option to force kill)"
}
#avoid exiting if the kill failure was because the task has already exited
#review - *no running instance* works with windows taskkill - "*No such process*" works with kill -9 on FreeBSD and linux - other platforms?
if {![string match "*no running instance*" $errMsg] && ![string match "*No such process*" $errMsg]} {
exit 4
}
} else {
set killcmd [list kill $pid]
puts stderr "$killcmd ran without error"
incr count_killed
}
}
puts stdout " pid: $pid (attempting to kill now using '$killcmd')"
if {[catch {
exec {*}$killcmd
} errMsg]} {
puts stderr "$killcmd returned an error:"
puts stderr $errMsg
puts stderr "(try '[info script] -k' option to force kill)"
exit 4
} else {
puts stderr "$killcmd ran without error"
incr count_killed
if {$count_killed > 0} {
puts stderr "\nKilled $count_killed processes. Waiting a short time before attempting to delete executable"
after 1000
}
} else {
puts stderr "Ok.. no running '$vfsname' processes found"
}
if {$count_killed > 0} {
puts stderr "\nKilled $count_killed processes. Waiting a short time before attempting to delete executable"
after 1000
}
} else {
puts stderr "Ok.. no running '$vfsname' processes found"
}
if {[file exists $buildfolder/$targetexe]} {
puts stderr "deleting existing $buildfolder/$targetexe"
if {[file exists $buildfolder/$targetkit]} {
puts stderr "deleting existing $buildfolder/$targetkit"
if {[catch {
file delete $buildfolder/$targetexe
file delete $buildfolder/$targetkit
} msg]} {
puts stderr "Failed to delete $buildfolder/$targetexe"
puts stderr "Failed to delete $buildfolder/$targetkit"
exit 4
}
}
#WINDOWS filesystem 'tunneling' (file replacement within 15secs) could cause targetexe to copy ctime & shortname metadata from previous file!
#WINDOWS filesystem 'tunneling' (file replacement within 15secs) could cause targetkit to copy ctime & shortname metadata from previous file!
#This is probably harmless - but worth being aware of.
file rename $buildfolder/$vfsname.new $buildfolder/$targetexe
file rename $buildfolder/$vfsname.new $buildfolder/$targetkit
# -- --- --- --- --- ---
$vfs_event targetset_end OK
@ -748,36 +904,39 @@ foreach vfs $vfs_folders {
# -- ----------
set bin_installer [punkcheck::installtrack new "make.tcl" $deployment_folder/.punkcheck]
$bin_installer set_source_target $buildfolder $deployment_folder
set bin_event [$bin_installer start_event {-make-step final_exe_install}]
$bin_event targetset_init INSTALL $deployment_folder/$targetexe
$bin_event targetset_addsource $buildfolder/$targetexe
set bin_event [$bin_installer start_event {-make-step final_kit_install}]
$bin_event targetset_init INSTALL $deployment_folder/$targetkit
#todo - move final deployment step outside of the build vfs loop? (final deployment can fail and then isn't rerun even though _build and deployed versions differ, unless .vfs modified again)
#set last_completion [$bin_event targetset_last_complete]
$bin_event targetset_addsource $buildfolder/$targetkit
$bin_event targetset_started
# -- ----------
set delete_failed 0
if {[file exists $deployment_folder/$targetexe]} {
puts stderr "deleting existing deployed at $deployment_folder/$targetexe"
if {[file exists $deployment_folder/$targetkit]} {
puts stderr "deleting existing deployed at $deployment_folder/$targetkit"
if {[catch {
file delete $deployment_folder/$targetexe
file delete $deployment_folder/$targetkit
} errMsg]} {
puts stderr "deletion of deployed version at $deployment_folder/$targetexe failed: $errMsg"
#exit 5
puts stderr "deletion of deployed version at $deployment_folder/$targetkit failed: $errMsg"
set delete_failed 1
}
}
if {!$delete_failed} {
puts stdout "copying.."
puts stdout "$buildfolder/$targetexe"
puts stdout "$buildfolder/$targetkit"
puts stdout "to:"
puts stdout "$deployment_folder/$targetexe"
puts stdout "$deployment_folder/$targetkit"
after 300
file copy $buildfolder/$targetexe $deployment_folder/$targetexe
file copy $buildfolder/$targetkit $deployment_folder/$targetkit
# -- ----------
$bin_event targetset_end OK
# -- ----------
} else {
$bin_event targetset_end FAILED -note "could not delete
$bin_event targetset_end FAILED -note "could not delete"
exit 5
}
$bin_event destroy
$bin_installer destroy
@ -785,7 +944,7 @@ foreach vfs $vfs_folders {
} else {
set skipped_vfs_build 1
puts stderr "."
puts stdout "Skipping build for vfs $vfs - no change detected"
puts stdout "Skipping build for vfs $vfs with runtime $rtname - no change detected"
$vfs_event targetset_end SKIPPED
}
$vfs_event destroy

50
src/mixtemplates/module/template_unversioned.tm

@ -1,50 +0,0 @@
# -*- tcl -*-
#
# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem.
# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository.
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
# (C) %year%
#
# @@ Meta Begin
# Application %pkg% %version%
# Meta platform tcl
# Meta license %license%
# @@ Meta End
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
## Requirements
##e.g package require frobz
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
namespace eval %pkg% {
}
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
## Ready
package provide %pkg% [namespace eval %pkg% {
variable version
set version %version%
}]
return

428
src/modules/platform-1.0.17.tm

@ -1,428 +0,0 @@
# -*- tcl -*-
# ### ### ### ######### ######### #########
## Overview
# Heuristics to assemble a platform identifier from publicly available
# information. The identifier describes the platform of the currently
# running tcl shell. This is a mixture of the runtime environment and
# of build-time properties of the executable itself.
#
# Examples:
# <1> A tcl shell executing on a x86_64 processor, but having a
# wordsize of 4 was compiled for the x86 environment, i.e. 32
# bit, and loaded packages have to match that, and not the
# actual cpu.
#
# <2> The hp/solaris 32/64 bit builds of the core cannot be
# distinguished by looking at tcl_platform. As packages have to
# match the 32/64 information we have to look in more places. In
# this case we inspect the executable itself (magic numbers,
# i.e. fileutil::magic::filetype).
#
# The basic information used comes out of the 'os' and 'machine'
# entries of the 'tcl_platform' array. A number of general and
# os/machine specific transformation are applied to get a canonical
# result.
#
# General
# Only the first element of 'os' is used - we don't care whether we
# are on "Windows NT" or "Windows XP" or whatever.
#
# Machine specific
# % amd64 -> x86_64
# % arm* -> arm
# % sun4* -> sparc
# % ia32* -> ix86
# % intel -> ix86
# % i*86* -> ix86
# % Power* -> powerpc
# % x86_64 + wordSize 4 => x86 code
#
# OS specific
# % AIX are always powerpc machines
# % HP-UX 9000/800 etc means parisc
# % linux has to take glibc version into account
# % sunos -> solaris, and keep version number
#
# NOTE: A platform like linux glibc 2.3, which can use glibc 2.2 stuff
# has to provide all possible allowed platform identifiers when
# searching search. Ditto a solaris 2.8 platform can use solaris 2.6
# packages. Etc. This is handled by the other procedure, see below.
# ### ### ### ######### ######### #########
## Requirements
namespace eval ::platform {}
# ### ### ### ######### ######### #########
## Implementation
# -- platform::generic
#
# Assembles an identifier for the generic platform. It leaves out
# details like kernel version, libc version, etc.
proc ::platform::generic {} {
global tcl_platform
set plat [string tolower [lindex $tcl_platform(os) 0]]
set cpu $tcl_platform(machine)
switch -glob -- $cpu {
sun4* {
set cpu sparc
}
intel -
ia32* -
i*86* {
set cpu ix86
}
x86_64 {
if {$tcl_platform(wordSize) == 4} {
# See Example <1> at the top of this file.
set cpu ix86
}
}
ppc -
"Power*" {
set cpu powerpc
}
"arm*" {
set cpu arm
}
ia64 {
if {$tcl_platform(wordSize) == 4} {
append cpu _32
}
}
}
switch -glob -- $plat {
windows {
if {$tcl_platform(platform) == "unix"} {
set plat cygwin
} else {
set plat win32
}
if {$cpu eq "amd64"} {
# Do not check wordSize, win32-x64 is an IL32P64 platform.
set cpu x86_64
}
}
sunos {
set plat solaris
if {[string match "ix86" $cpu]} {
if {$tcl_platform(wordSize) == 8} {
set cpu x86_64
}
} elseif {![string match "ia64*" $cpu]} {
# sparc
if {$tcl_platform(wordSize) == 8} {
append cpu 64
}
}
}
darwin {
set plat macosx
# Correctly identify the cpu when running as a 64bit
# process on a machine with a 32bit kernel
if {$cpu eq "ix86"} {
if {$tcl_platform(wordSize) == 8} {
set cpu x86_64
}
}
}
aix {
set cpu powerpc
if {$tcl_platform(wordSize) == 8} {
append cpu 64
}
}
hp-ux {
set plat hpux
if {![string match "ia64*" $cpu]} {
set cpu parisc
if {$tcl_platform(wordSize) == 8} {
append cpu 64
}
}
}
osf1 {
set plat tru64
}
default {
set plat [lindex [split $plat _-] 0]
}
}
return "${plat}-${cpu}"
}
# -- platform::identify
#
# Assembles an identifier for the exact platform, by extending the
# generic identifier. I.e. it adds in details like kernel version,
# libc version, etc., if they are relevant for the loading of
# packages on the platform.
proc ::platform::identify {} {
global tcl_platform
set id [generic]
regexp {^([^-]+)-([^-]+)$} $id -> plat cpu
switch -- $plat {
solaris {
regsub {^5} $tcl_platform(osVersion) 2 text
append plat $text
return "${plat}-${cpu}"
}
macosx {
set major [lindex [split $tcl_platform(osVersion) .] 0]
if {$major > 19} {
set minor [lindex [split $tcl_platform(osVersion) .] 1]
incr major -9
append plat $major.[expr {$minor - 1}]
} else {
incr major -4
append plat 10.$major
return "${plat}-${cpu}"
}
return "${plat}-${cpu}"
}
linux {
# Look for the libc*.so and determine its version
# (libc5/6, libc6 further glibc 2.X)
set v unknown
# Determine in which directory to look. /lib, or /lib64.
# For that we use the tcl_platform(wordSize).
#
# We could use the 'cpu' info, per the equivalence below,
# that however would be restricted to intel. And this may
# be a arm, mips, etc. system. The wordsize is more
# fundamental.
#
# ix86 <=> (wordSize == 4) <=> 32 bit ==> /lib
# x86_64 <=> (wordSize == 8) <=> 64 bit ==> /lib64
#
# Do not look into /lib64 even if present, if the cpu
# doesn't fit.
# TODO: Determine the prefixes (i386, x86_64, ...) for
# other cpus. The path after the generic one is utterly
# specific to intel right now. Ok, on Ubuntu, possibly
# other Debian systems we may apparently be able to query
# the necessary CPU code. If we can't we simply use the
# hardwired fallback.
switch -exact -- $tcl_platform(wordSize) {
4 {
lappend bases /lib
if {[catch {
exec dpkg-architecture -qDEB_HOST_MULTIARCH
} res]} {
lappend bases /lib/i386-linux-gnu
} else {
# dpkg-arch returns the full tripled, not just cpu.
lappend bases /lib/$res
}
}
8 {
lappend bases /lib64
if {[catch {
exec dpkg-architecture -qDEB_HOST_MULTIARCH
} res]} {
lappend bases /lib/x86_64-linux-gnu
} else {
# dpkg-arch returns the full tripled, not just cpu.
lappend bases /lib/$res
}
}
default {
return -code error "Bad wordSize $tcl_platform(wordSize), expected 4 or 8"
}
}
foreach base $bases {
if {[LibcVersion $base -> v]} break
}
append plat -$v
return "${plat}-${cpu}"
}
}
return $id
}
proc ::platform::LibcVersion {base _->_ vv} {
upvar 1 $vv v
set libclist [lsort [glob -nocomplain -directory $base libc*]]
if {![llength $libclist]} { return 0 }
set libc [lindex $libclist 0]
# Try executing the library first. This should suceed
# for a glibc library, and return the version
# information.
if {![catch {
set vdata [lindex [split [exec $libc] \n] 0]
}]} {
regexp {version ([0-9]+(\.[0-9]+)*)} $vdata -> v
foreach {major minor} [split $v .] break
set v glibc${major}.${minor}
return 1
} else {
# We had trouble executing the library. We are now
# inspecting its name to determine the version
# number. This code by Larry McVoy.
if {[regexp -- {libc-([0-9]+)\.([0-9]+)} $libc -> major minor]} {
set v glibc${major}.${minor}
return 1
}
}
return 0
}
# -- platform::patterns
#
# Given an exact platform identifier, i.e. _not_ the generic
# identifier it assembles a list of exact platform identifier
# describing platform which should be compatible with the
# input.
#
# I.e. packages for all platforms in the result list should be
# loadable on the specified platform.
# << Should we add the generic identifier to the list as well ? In
# general it is not compatible I believe. So better not. In many
# cases the exact identifier is identical to the generic one
# anyway.
# >>
proc ::platform::patterns {id} {
set res [list $id]
if {$id eq "tcl"} {return $res}
switch -glob -- $id {
solaris*-* {
if {[regexp {solaris([^-]*)-(.*)} $id -> v cpu]} {
if {$v eq ""} {return $id}
foreach {major minor} [split $v .] break
incr minor -1
for {set j $minor} {$j >= 6} {incr j -1} {
lappend res solaris${major}.${j}-${cpu}
}
}
}
linux*-* {
if {[regexp {linux-glibc([^-]*)-(.*)} $id -> v cpu]} {
foreach {major minor} [split $v .] break
incr minor -1
for {set j $minor} {$j >= 0} {incr j -1} {
lappend res linux-glibc${major}.${j}-${cpu}
}
}
}
macosx-powerpc {
lappend res macosx-universal
}
macosx-x86_64 {
lappend res macosx-i386-x86_64
}
macosx-ix86 {
lappend res macosx-universal macosx-i386-x86_64
}
macosx*-* {
# 10.5+,11.0+
if {[regexp {macosx([^-]*)-(.*)} $id -> v cpu]} {
switch -exact -- $cpu {
ix86 {
lappend alt i386-x86_64
lappend alt universal
}
x86_64 {
if {[lindex [split $::tcl_platform(osVersion) .] 0] < 19} {
set alt i386-x86_64
} else {
set alt {}
}
}
arm {
lappend alt x86_64
}
default { set alt {} }
}
if {$v ne ""} {
foreach {major minor} [split $v .] break
set res {}
if {$major eq 11} {
# Add 11.0 to 11.minor to patterns.
for {set j $minor} {$j >= 0} {incr j -1} {
lappend res macosx${major}.${j}-${cpu}
foreach a $alt {
lappend res macosx${major}.${j}-$a
}
}
set major 10
set minor 15
}
# Add 10.5 to 10.minor to patterns.
for {set j $minor} {$j >= 5} {incr j -1} {
if {$cpu ne "arm"} {
lappend res macosx${major}.${j}-${cpu}
}
foreach a $alt {
lappend res macosx${major}.${j}-$a
}
}
# Add unversioned patterns for 10.3/10.4 builds.
lappend res macosx-${cpu}
foreach a $alt {
lappend res macosx-$a
}
} else {
# No version, just do unversioned patterns.
foreach a $alt {
lappend res macosx-$a
}
}
} else {
# no v, no cpu ... nothing
}
}
}
lappend res tcl ; # Pure tcl packages are always compatible.
return $res
}
# ### ### ### ######### ######### #########
## Ready
package provide platform 1.0.17
# ### ### ### ######### ######### #########
## Demo application
if {[info exists argv0] && ($argv0 eq [info script])} {
puts ====================================
parray tcl_platform
puts ====================================
puts Generic\ identification:\ [::platform::generic]
puts Exact\ identification:\ \ \ [::platform::identify]
puts ====================================
puts Search\ patterns:
puts *\ [join [::platform::patterns [::platform::identify]] \n*\ ]
puts ====================================
exit 0
}

241
src/modules/platform/shell-1.1.4.tm

@ -1,241 +0,0 @@
# -*- tcl -*-
# ### ### ### ######### ######### #########
## Overview
# Higher-level commands which invoke the functionality of this package
# for an arbitrary tcl shell (tclsh, wish, ...). This is required by a
# repository as while the tcl shell executing packages uses the same
# platform in general as a repository application there can be
# differences in detail (i.e. 32/64 bit builds).
# ### ### ### ######### ######### #########
## Requirements
package require platform
namespace eval ::platform::shell {}
# ### ### ### ######### ######### #########
## Implementation
# -- platform::shell::generic
proc ::platform::shell::generic {shell} {
# Argument is the path to a tcl shell.
CHECK $shell
LOCATE base out
set code {}
# Forget any pre-existing platform package, it might be in
# conflict with this one.
lappend code {package forget platform}
# Inject our platform package
lappend code [list source $base]
# Query and print the architecture
lappend code {puts [platform::generic]}
# And done
lappend code {exit 0}
set arch [RUN $shell [join $code \n]]
if {$out} {file delete -force $base}
return $arch
}
# -- platform::shell::identify
proc ::platform::shell::identify {shell} {
# Argument is the path to a tcl shell.
CHECK $shell
LOCATE base out
set code {}
# Forget any pre-existing platform package, it might be in
# conflict with this one.
lappend code {package forget platform}
# Inject our platform package
lappend code [list source $base]
# Query and print the architecture
lappend code {puts [platform::identify]}
# And done
lappend code {exit 0}
set arch [RUN $shell [join $code \n]]
if {$out} {file delete -force $base}
return $arch
}
# -- platform::shell::platform
proc ::platform::shell::platform {shell} {
# Argument is the path to a tcl shell.
CHECK $shell
set code {}
lappend code {puts $tcl_platform(platform)}
lappend code {exit 0}
return [RUN $shell [join $code \n]]
}
# ### ### ### ######### ######### #########
## Internal helper commands.
proc ::platform::shell::CHECK {shell} {
if {![file exists $shell]} {
return -code error "Shell \"$shell\" does not exist"
}
if {![file executable $shell]} {
return -code error "Shell \"$shell\" is not executable (permissions)"
}
return
}
proc ::platform::shell::LOCATE {bv ov} {
upvar 1 $bv base $ov out
# Locate the platform package for injection into the specified
# shell. We are using package management to find it, whereever it
# is, instead of using hardwired relative paths. This allows us to
# install the two packages as TMs without breaking the code
# here. If the found package is wrapped we copy the code somewhere
# where the spawned shell will be able to read it.
# This code is brittle, it needs has to adapt to whatever changes
# are made to the TM code, i.e. the provide statement generated by
# tm.tcl
set pl [package ifneeded platform [package require platform]]
set base [lindex $pl end]
set out 0
if {[lindex [file system $base]] ne "native"} {
set temp [TEMP]
file copy -force $base $temp
set base $temp
set out 1
}
return
}
proc ::platform::shell::RUN {shell code} {
set c [TEMP]
set cc [open $c w]
puts $cc $code
close $cc
set e [TEMP]
set code [catch {
exec $shell $c 2> $e
} res]
file delete $c
if {$code} {
append res \n[read [set chan [open $e r]]][close $chan]
file delete $e
return -code error "Shell \"$shell\" is not executable ($res)"
}
file delete $e
return $res
}
proc ::platform::shell::TEMP {} {
set prefix platform
# This code is copied out of Tcllib's fileutil package.
# (TempFile/tempfile)
set tmpdir [DIR]
set chars "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
set nrand_chars 10
set maxtries 10
set access [list RDWR CREAT EXCL TRUNC]
set permission 0600
set channel ""
set checked_dir_writable 0
set mypid [pid]
for {set i 0} {$i < $maxtries} {incr i} {
set newname $prefix
for {set j 0} {$j < $nrand_chars} {incr j} {
append newname [string index $chars \
[expr {int(rand()*62)}]]
}
set newname [file join $tmpdir $newname]
if {[file exists $newname]} {
after 1
} else {
if {[catch {open $newname $access $permission} channel]} {
if {!$checked_dir_writable} {
set dirname [file dirname $newname]
if {![file writable $dirname]} {
return -code error "Directory $dirname is not writable"
}
set checked_dir_writable 1
}
} else {
# Success
close $channel
return [file normalize $newname]
}
}
}
if {$channel ne ""} {
return -code error "Failed to open a temporary file: $channel"
} else {
return -code error "Failed to find an unused temporary file name"
}
}
proc ::platform::shell::DIR {} {
# This code is copied out of Tcllib's fileutil package.
# (TempDir/tempdir)
global tcl_platform env
set attempdirs [list]
foreach tmp {TMPDIR TEMP TMP} {
if { [info exists env($tmp)] } {
lappend attempdirs $env($tmp)
}
}
switch $tcl_platform(platform) {
windows {
lappend attempdirs "C:\\TEMP" "C:\\TMP" "\\TEMP" "\\TMP"
}
macintosh {
set tmpdir $env(TRASH_FOLDER) ;# a better place?
}
default {
lappend attempdirs \
[file join / tmp] \
[file join / var tmp] \
[file join / usr tmp]
}
}
lappend attempdirs [pwd]
foreach tmp $attempdirs {
if { [file isdirectory $tmp] && [file writable $tmp] } {
return [file normalize $tmp]
}
}
# Fail if nothing worked.
return -code error "Unable to determine a proper directory for temporary files"
}
# ### ### ### ######### ######### #########
## Ready
package provide platform::shell 1.1.4

55
src/modules/punk-0.1.tm

@ -3,8 +3,13 @@
namespace eval punk {
package require zzzload
zzzload::pkg_require twapi
proc lazyload {pkg} {
package require zzzload
if {[package provide $pkg] eq ""} {
zzzload::pkg_require $pkg
}
}
#lazyload twapi
catch {package require vfs} ;#attempt load now so we can use faster 'package provide' to test existence later
}
@ -165,15 +170,47 @@ namespace eval punk {
proc ::punk::K {x y} { return $x}
proc stacktrace {} {
set stack "Stack trace:\n"
for {set i 1} {$i < [info level]} {incr i} {
set lvl [info level -$i]
set pname [lindex $lvl 0]
append stack [string repeat " " $i]$pname
if {![catch {info args $pname} pargs]} {
foreach value [lrange $lvl 1 end] arg $pargs {
if {$value eq ""} {
if {$arg != 0} {
info default $pname $arg value
}
}
append stack " $arg='$value'"
}
} else {
append stack " !unknown vars for $pname"
}
append stack \n
}
return $stack
}
proc ::punk::uuid {} {
set has_twapi 0
if {"windows" eq $::tcl_platform(platform)} {
set loader [zzzload::pkg_wait twapi]
if {$loader in [list failed loading]} {
puts stderr "Unexpected problem during thread-load of pkg twapi - zzload::pkg_wait returned $loader"
}
if {[package provide twapi] ne ""} {
set has_twapi 1
if 0 {
if {"windows" eq $::tcl_platform(platform)} {
if {![catch {
set loader [zzzload::pkg_wait twapi]
} errM]} {
if {$loader in [list failed loading]} {
puts stderr "Unexpected problem during thread-load of pkg twapi - zzload::pkg_wait returned $loader"
}
} else {
package require twapi
}
if {[package provide twapi] ne ""} {
set has_twapi 1
}
}
}
if {!$has_twapi} {

2
src/modules/punk/ansi-999999.0a1.0.tm

@ -158,7 +158,7 @@ namespace eval punk::ansi {
overline 53 nooverline 55 frame 51 framecircle 52 noframe 54
}
variable SGR_colour_map {
black 30 red 31 green 32 yellow 33 blue 4 purple 35 cyan 36 white 37
black 30 red 31 green 32 yellow 33 blue 34 purple 35 cyan 36 white 37
Black 40 Red 41 Green 42 Yellow 43 Blue 44 Purple 45 Cyan 46 White 47
BLACK 100 RED 101 GREEN 102 YELLOW 103 BLUE 104 PURPLE 105 CYAN 106 WHITE 107
}

10
src/modules/punk/console-999999.0a1.0.tm

@ -21,8 +21,8 @@ package require punk::ansi
if {"windows" eq $::tcl_platform(platform)} {
package require zzzload
zzzload::pkg_require twapi
#package require zzzload
#zzzload::pkg_require twapi
}
#see https://learn.microsoft.com/en-us/windows/console/classic-vs-vt
@ -129,8 +129,12 @@ namespace eval punk::console {
}
}
proc define_windows_procs {} {
package require zzzload
set loadstate [zzzload::pkg_require twapi]
if {$loadstate ni [list loading failed]} {
if {$loadstate ni [list failed]} {
#review zzzload usage
#puts stdout "=========== console loading twapi ============="
zzzload::pkg_wait twapi
package require twapi ;#should be fast once twapi dll loaded in zzzload thread
set ::punk::console::has_twapi 1

41
src/modules/punk/mix/base-0.1.tm

@ -284,7 +284,7 @@ namespace eval punk::mix::base {
# puts stdout "base doingstuff-with-args:'$args'-in-namespace:'[namespace current]'"
#}
namespace eval lib {
variable sha3_implementation "" ;#set by cksum_algorithms (which is called by cksum_path) It looks for fossil or sqlite3. Todo - add proper Tcl implementation.
namespace export *
#-----------------------------------------------------
@ -445,9 +445,28 @@ namespace eval punk::mix::base {
#adler32 is fastest for some larger files of a few MB but slower on small files (possibly due to Tcl-based file load?)
#sha1 as at 2023 seems a good default
proc cksum_algorithms {} {
variable sha3_implementation
#sha2 is an alias for sha256
#2023 - no sha3 available in tcllib
return [list md5 sha1 sha2 sha256 cksum adler32]
#2023 - no sha3 available in tcllib - we can exec fossil for now - which will be very slow
set algs [list md5 sha1 sha2 sha256 cksum adler32]
set sha3_algs [list sha3 sha3-224 sha3-256 sha3-384 sha3-512]
if {[auto_execok sqlite3] ne ""} {
lappend algs {*}$sha3_algs
set sha3_implementation sqlite3_sha3
} else {
if {[auto_execok fossil] ne ""} {
lappend algs {*}$sha3_algs
set sha3_implementation fossil_sha3
}
}
return $algs
}
proc sqlite3_sha3 {bits filename} {
return [exec sqlite3 :memory: "select lower(hex(sha3(readfile('$filename'),$bits)))"]
}
proc fossil_sha3 {bits filename} {
return [lindex [exec fossil sha3sum -$bits $filename] 0]
}
#adler32 via file-slurp
@ -465,6 +484,7 @@ namespace eval punk::mix::base {
#-noperms only available on extraction - so that doesn't help
#Needs to operate on non-existant paths and return empty string in cksum field
proc cksum_path {path args} {
variable sha3_implementation
if {$path eq {}} { set path [pwd] }
if {[file pathtype $path] eq "relative"} {
set path [file normalize $path]
@ -473,6 +493,13 @@ namespace eval punk::mix::base {
set startdir [pwd]
set defaults [cksum_default_opts]
set known_opts [dict keys $defaults]
foreach {k v} $args {
if {$k ni $known_opts} {
error "cksum_path unknown option '$k' known_options: $known_opts"
}
}
set opts [dict merge $defaults $args]
set opts_actual $opts ;#default - auto updated to 0 or 1 later
@ -582,6 +609,14 @@ namespace eval punk::mix::base {
set cksum_command [list crc::cksum -format 0x%X -file]
} elseif {$opt_cksum_algorithm eq "adler32"} {
set cksum_command [list cksum_adler32_file]
} elseif {$opt_cksum_algorithm in [list "sha3" "sha3-256"]} {
#todo - replace with something that doesn't call another process
#set cksum_command [list apply {{file} {lindex [exec fossil sha3sum -256 $file] 0}}]
set cksum_command [list $sha3_implementation 256]
} elseif {$opt_cksum_algorithm in [list "sha3-224" "sha3-384" "sha3-512"]} {
set bits [lindex [split $opt_cksum_algorithm -] 1]
#set cksum_command [list apply {{bits file} {lindex [exec fossil sha3sum -$bits $file] 0}} $bits]
set cksum_command [list $sha3_implementation $bits]
}
set cksum ""

252
src/modules/punk/mix/cli-0.3.tm

@ -32,34 +32,42 @@ namespace eval punk::mix::cli {
package require punk::overlay
catch {
punk::overlay::import_commandset module. ::punk::mix::commandset::module
punk::overlay::import_commandset module . ::punk::mix::commandset::module
}
punk::overlay::import_commandset debug. ::punk::mix::commandset::debug
punk::overlay::import_commandset repo. ::punk::mix::commandset::repo
punk::overlay::import_commandset lib. ::punk::mix::commandset::loadedlib
punk::overlay::import_commandset debug . ::punk::mix::commandset::debug
punk::overlay::import_commandset repo . ::punk::mix::commandset::repo
punk::overlay::import_commandset lib . ::punk::mix::commandset::loadedlib
catch {
package require punk::mix::commandset::project
punk::overlay::import_commandset project. ::punk::mix::commandset::project
punk::overlay::import_commandset "" ::punk::mix::commandset::project::collection
punk::overlay::import_commandset project . ::punk::mix::commandset::project
punk::overlay::import_commandset projects . ::punk::mix::commandset::project::collection
}
if {[catch {
package require punk::mix::commandset::layout
punk::overlay::import_commandset project.layout. ::punk::mix::commandset::layout
punk::overlay::import_commandset "project." ::punk::mix::commandset::layout::collection
punk::overlay::import_commandset project.layout . ::punk::mix::commandset::layout
punk::overlay::import_commandset project.layouts . ::punk::mix::commandset::layout::collection
} errM]} {
puts stderr "error loading punk::mix::commandset::layout"
puts stderr $errM
}
if {[catch {
package require punk::mix::commandset::buildsuite
punk::overlay::import_commandset buildsuite. ::punk::mix::commandset::buildsuite
punk::overlay::import_commandset "" ::punk::mix::commandset::buildsuite::collection
punk::overlay::import_commandset buildsuite . ::punk::mix::commandset::buildsuite
punk::overlay::import_commandset buildsuites . ::punk::mix::commandset::buildsuite::collection
} errM]} {
puts stderr "error loading punk::mix::commandset::buildsuite"
puts stderr $errM
}
punk::overlay::import_commandset scriptwrap. ::punk::mix::commandset::scriptwrap
punk::overlay::import_commandset scriptwrap . ::punk::mix::commandset::scriptwrap
if {[catch {
package require punk::mix::commandset::doc
punk::overlay::import_commandset doc . ::punk::mix::commandset::doc
punk::overlay::import_commandset "" "" ::punk::mix::commandset::doc::collection
} errM]} {
puts stderr "error loading punk::mix::commandset::doc"
puts stderr $errM
}
proc help {args} {
@ -148,21 +156,28 @@ namespace eval punk::mix::cli {
set lc_proj_bin [string tolower $project_base/bin]
set lc_build_bin [string tolower $project_base/src/_build]
set is_own_exe 0
if {[string match "${lc_proj_bin}*" $lc_this_exe] || [string match "${lc_build_bin}" $lc_this_exe]} {
set is_own_exe 1
puts stderr "WARNING - running make using executable that may be created by the project being built"
set answer [util::askuser "Do you want to proceed using this executable? (build will probably stop when it is unable to update the executable) Y|N"]
if {[string tolower $answer] ne "y"} {
puts stderr "mix new aborting due to user response '$answer' (required Y|y to proceed) use -confirm 0 to avoid prompts."
return
if {"project" in $args} {
set is_own_exe 0
if {[string match "${lc_proj_bin}*" $lc_this_exe] || [string match "${lc_build_bin}" $lc_this_exe]} {
set is_own_exe 1
puts stderr "WARNING - running make using executable that may be created by the project being built"
set answer [util::askuser "Do you want to proceed using this executable? (build will probably stop when it is unable to update the executable) Y|N"]
if {[string tolower $answer] ne "y"} {
puts stderr "mix new aborting due to user response '$answer' (required Y|y to proceed) use -confirm 0 to avoid prompts."
return
}
}
}
cd $sourcefolder
#use run so that stdout visible as it goes
set exitinfo [run [info nameofexecutable] $sourcefolder/make.tcl project]
set exitcode [dict get $exitinfo exitcode]
if {![catch {run --timeout=5000 -debug [info nameofexecutable] $sourcefolder/make.tcl {*}$args} exitinfo]} {
puts stderr "exitinfo: $exitinfo"
set exitcode [dict get $exitinfo exitcode]
} else {
puts stderr "Error unable to determine exitcode. err: $exitinfo"
cd $startdir
return false
}
cd $startdir
if {$exitcode != 0} {
@ -388,7 +403,7 @@ namespace eval punk::mix::cli {
-call-depth-internal 0\
-max_depth 1000\
-subdirlist {}\
-punkcheck_eventid "\uFFFF"\
-punkcheck_eventobj "\uFFFF"\
-glob *.tm\
]
set opts [dict merge $defaults $args]
@ -405,7 +420,7 @@ namespace eval punk::mix::cli {
error "build_modules_from_source_to_base -glob '$fileglob' doesn't seem to target tcl modules."
}
# -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_punkcheck_eventid [dict get $opts -punkcheck_eventid]
set opt_punkcheck_eventobj [dict get $opts -punkcheck_eventobj]
set magicversion [punk::mix::util::magic_tm_version] ;#deliberately large so given load-preference when testing
set module_list [list]
@ -443,10 +458,15 @@ namespace eval punk::mix::cli {
-glob $fileglob\
-max_depth 0\
]
lassign [punkcheck::start_installer_event $punkcheck_file $installername $srcdir $basedir $config] _eventid punkcheck_eventid _recordset record_list
#lassign [punkcheck::start_installer_event $punkcheck_file $installername $srcdir $basedir $config] _eventid punkcheck_eventid _recordset record_list
# -- ---
set installer [punkcheck::installtrack new $installername $punkcheck_file]
$installer set_source_target $srcdir $basedir
set event [$installer start_event $config]
# -- ---
} else {
set punkcheck_eventid $opt_punkcheck_eventid
set event $opt_punkcheck_eventobj
}
#----------------------------------------
@ -482,6 +502,7 @@ namespace eval punk::mix::cli {
if {[file exists $current_source_dir/#tarjar-$basename-$magicversion]} {
#TODO
file mkdir $buildfolder
if {[file exists $current_source_dir/#tarjar-$basename-$magicversion/DESCRIPTION.txt]} {
@ -519,24 +540,31 @@ namespace eval punk::mix::cli {
#------------------------------
#
set target_relpath [punkcheck::lib::path_relative $basedir $target_module_dir/$basename-$module_build_version.tm]
set file_record [punkcheck::installfile_begin $basedir $target_relpath $installername -eventid $punkcheck_eventid]
set changed_list [list]
# -- --- --- --- --- ---
set source_relpath [punkcheck::lib::path_relative $basedir $versionfile]
set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record]
# -- --- --- --- --- ---
set source_relpath [punkcheck::lib::path_relative $basedir $current_source_dir/$m]
set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record]
# -- --- --- --- --- ---
set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]]
set changed_list [dict get $changed_unchanged changed]
if {[llength $changed_list]} {
set file_record [punkcheck::installfile_started_install $basedir $file_record]
#set target_relpath [punkcheck::lib::path_relative $basedir $target_module_dir/$basename-$module_build_version.tm]
#set file_record [punkcheck::installfile_begin $basedir $target_relpath $installername -eventid $punkcheck_eventid]
$event targetset_init INSTALL $target_module_dir/$basename-$module_build_version.tm
$event targetset_addsource $versionfile
$event targetset_addsource $current_source_dir/$m
#set changed_list [list]
## -- --- --- --- --- ---
#set source_relpath [punkcheck::lib::path_relative $basedir $versionfile]
#set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record]
## -- --- --- --- --- ---
#set source_relpath [punkcheck::lib::path_relative $basedir $current_source_dir/$m]
#set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record]
## -- --- --- --- --- ---
#set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]]
#set changed_list [dict get $changed_unchanged changed]
if {\
[llength [dict get [$event targetset_source_changes] changed]]\
|| [llength [$event get_targets_exist]] < [llength [$event get_targets]]\
} {
#set file_record [punkcheck::installfile_started_install $basedir $file_record]
$event targetset_started
# -- --- --- --- --- ---
set target $target_module_dir/$basename-$module_build_version.tm
if {$did_skip} {set did_skip 0; puts -nonewline stdout \n}
@ -550,12 +578,14 @@ namespace eval punk::mix::cli {
#file copy -force $srcdir/$m $target
lappend module_list $target
# -- --- --- --- --- ---
set file_record [punkcheck::installfile_finished_install $basedir $file_record]
#set file_record [punkcheck::installfile_finished_install $basedir $file_record]
$event targetset_end OK
} else {
#puts stdout "skipping module $current_source_dir/$m - no change in sources detected"
puts -nonewline stderr "."
set did_skip 1
set file_record [punkcheck::installfile_skipped_install $basedir $file_record]
#set file_record [punkcheck::installfile_skipped_install $basedir $file_record]
$event targetset_end SKIPPED
}
#------------------------------
@ -572,33 +602,41 @@ namespace eval punk::mix::cli {
exit 1
}
#------------------------------
#
set target_relpath [punkcheck::lib::path_relative $basedir $target_module_dir/$m]
set file_record [punkcheck::installfile_begin $basedir $target_relpath $installername -eventid $punkcheck_eventid]
set changed_list [list]
# -- --- --- --- --- ---
set source_relpath [punkcheck::lib::path_relative $basedir $current_source_dir/$m]
set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record]
# -- --- --- --- --- ---
set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]]
set changed_list [dict get $changed_unchanged changed]
if {[llength $changed_list]} {
set file_record [punkcheck::installfile_started_install $basedir $file_record]
##------------------------------
##
#set target_relpath [punkcheck::lib::path_relative $basedir $target_module_dir/$m]
#set file_record [punkcheck::installfile_begin $basedir $target_relpath $installername -eventid $punkcheck_eventid]
#set changed_list [list]
## -- --- --- --- --- ---
#set source_relpath [punkcheck::lib::path_relative $basedir $current_source_dir/$m]
#set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record]
## -- --- --- --- --- ---
#set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]]
#set changed_list [dict get $changed_unchanged changed]
#----------
$event targetset_init INSTALL $target_module_dir/$m
$event targetset_addsource $current_source_dir/$m
if {\
[llength [dict get [$event targetset_source_changes] changed]]\
|| [llength [$event get_targets_exist]] < [llength [$event get_targets]]\
} {
#set file_record [punkcheck::installfile_started_install $basedir $file_record]
$event targetset_started
# -- --- --- --- --- ---
if {$did_skip} {set did_skip 0; puts -nonewline stdout \n}
puts stderr "Copied already versioned module $current_source_dir/$m to $target_module_dir"
lappend module_list $current_source_dir/$m
file copy -force $current_source_dir/$m $target_module_dir
# -- --- --- --- --- ---
set file_record [punkcheck::installfile_finished_install $basedir $file_record]
#set file_record [punkcheck::installfile_finished_install $basedir $file_record]
$event targetset_end OK -note "already versioned module"
} else {
puts -nonewline stderr "."
set did_skip 1
set file_record [punkcheck::installfile_skipped_install $basedir $file_record]
#set file_record [punkcheck::installfile_skipped_install $basedir $file_record]
$event targetset_end SKIPPED
}
}
@ -625,17 +663,90 @@ namespace eval punk::mix::cli {
lappend module_list {*}[build_modules_from_source_to_base $srcdir $basedir\
-call-depth-internal [expr {$CALLDEPTH +1}]\
-subdirlist [list {*}$subdirlist $d]\
-punkcheck_eventid $punkcheck_eventid\
-punkcheck_eventobj $event\
-glob $fileglob\
]
}
if {$did_skip} {
puts -nonewline stdout \n
}
if {$CALLDEPTH == 0} {
$event destroy
$installer destroy
}
return $module_list
}
variable kettle_reset_bodies [dict create]
variable kettle_reset_args [dict create]
#We are abusing kettle to run in-process.
# when we change to another project we need recipes to be reloaded.
# Kettle rewrites some of it's own procs - stopping reloading of recipes when we change folders
#kettle_init stores the original proc bodies & args
proc kettle_init {} {
variable kettle_reset_bodies ;#dict
variable kettle_reset_args
set reset_procs [list\
::kettle::benchmarks\
::kettle::doc\
::kettle::figures\
::kettle::meta::scan\
::kettle::testsuite\
]
foreach p $reset_procs {
set b [info body $p]
if {[string match "*Overwrite self*" $b]} {
dict set kettle_reset_bodies $p $b
set argnames [info args $p]
set arglist [list]
foreach a $argnames {
if {[info default $p $a dval]} {
lappend arglist [list $a $dval]
} else {
lappend arglist $a
}
}
dict set kettle_reset_args $p $arglist
}
}
}
#call kettle_reinit to ensure recipes point to current project
proc kettle_reinit {} {
variable kettle_reset_bodies
variable kettle_reset_args
foreach p [dict keys $kettle_reset_bodies] {
set b [dict get $kettle_reset_bodies $p]
set argl [dict get $kettle_reset_args $p]
uplevel 1 [list ::proc $p $argl $b]
}
#todo - determine standard recipes by examining standard.tcl instead of hard coding?
set standard_recipes [list\
null\
forever\
list-recipes\
help-recipes\
help-dump\
help-recipes\
help\
list\
list-options\
help-options\
show-configuration\
show-state\
show\
meta-status\
gui\
]
#set ::kettle::recipe::recipe [dict create]
foreach r [dict keys $::kettle::recipe::recipe] {
if {$r ni $standard_recipes} {
dict unset ::kettle::recipe::recipe $r
}
}
}
proc kettle_call {calltype args} {
variable kettle_reset_bodies
if {$calltype ni [list lib shell]} {
error "pmix kettle_call 1st argument must be one of: 'lib' for direct use of kettle module or 'shell' to call as separate process"
}
@ -659,9 +770,18 @@ namespace eval punk::mix::cli {
if {![file exists $startdir/build.tcl]} {
error "pmix kettle must be run from a folder containing build.tcl (cwd: [pwd])"
}
if {[catch {package present kettle}]} {
if {[package provide kettle] eq ""} {
puts stdout "Loading kettle package - may be delay on first load ..."
package require kettle
kettle_init ;#store original procs for those kettle procs that rewrite themselves
} else {
if {[dict size $kettle_reset_bodies] == 0} {
#presumably package require kettle was called without calling our kettle_init hack.
kettle_init
} else {
#undo proc rewrites
kettle_reinit
}
}
set first [lindex $args 0]
if {[string match @* $first]} {

2
src/modules/punk/mix/commandset/buildsuite-999999.0a1.0.tm

@ -107,7 +107,7 @@ namespace eval punk::mix::commandset::buildsuite {
namespace eval collection {
namespace export *
proc buildsuites {{glob {}}} {
proc _default {{glob {}}} {
if {![string length $glob]} {
set glob *
}

181
src/modules/punk/mix/commandset/doc-999999.0a1.0.tm

@ -0,0 +1,181 @@
# -*- tcl -*-
# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from <pkg>-buildversion.txt
#
# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem.
# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository.
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
# (C) 2023
#
# @@ Meta Begin
# Application punk::mix::commandset::doc 999999.0a1.0
# Meta platform tcl
# Meta license <unspecified>
# @@ Meta End
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
## Requirements
##e.g package require frobz
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
namespace eval punk::mix::commandset::doc {
namespace export *
proc _default {} {
puts "documentation subsystem"
puts "commands: doc.build"
puts " build documentation from src/doc to src/embedded using the kettle build tool"
}
proc build {} {
puts "build docs"
set projectdir [punk::repo::find_project]
if {$projectdir eq ""} {
puts stderr "No current project dir - unable to build docs"
return
}
if {[file exists $projectdir/src/doc]} {
set original_wd [pwd]
cd $projectdir/src
#----------
set installer [punkcheck::installtrack new project.new $projectdir/src/.punkcheck]
$installer set_source_target $projectdir/src/doc $projectdir/src/embedded
set event [$installer start_event {-install_step kettledoc}]
#use same virtual id "kettle_build_doc" as project.new - review best way to keep identifiers like this in sync.
$event targetset_init VIRTUAL kettle_build_doc ;#VIRTUAL - since there is no specific target file - and we don't know all the files that will be generated
$event targetset_addsource $projectdir/src/doc ;#whole doc tree is considered the source
#----------
if {\
[llength [dict get [$event targetset_source_changes] changed]]\
} {
$event targetset_started
# -- --- --- --- --- ---
puts stdout "BUILDING DOCS at $projectdir/src/embedded from src/doc"
if {[catch {
punk::mix::cli::lib::kettle_call lib doc
#Kettle doc
} errM]} {
$event targetset_end FAILED -note "kettle_build_doc failed: $errM"
} else {
$event targetset_end OK
}
# -- --- --- --- --- ---
} else {
puts stderr "No change detected in src/doc"
$event targetset_end SKIPPED
}
$event end
$event destroy
$installer destroy
cd $original_wd
} else {
puts stderr "No doc folder found at $projectdir/src/doc"
}
}
proc status {} {
set projectdir [punk::repo::find_project]
if {$projectdir eq ""} {
puts stderr "No current project dir - unable to check doc status"
return
}
if {![file exists $projectdir/src/doc]} {
set result "No documentation source found. Expected .man files in doctools format at $projectdir/src/doc"
return $result
}
set original_wd [pwd]
cd $projectdir/src
puts stdout "Testing status of doctools source location $projectdir/src/doc ..."
flush stdout
#----------
set installer [punkcheck::installtrack new project.new $projectdir/src/.punkcheck]
$installer set_source_target $projectdir/src/doc $projectdir/src/embedded
set event [$installer start_event {-install_step kettledoc}]
#use same virtual id "kettle_build_doc" as project.new - review best way to keep identifiers like this in sync.
$event targetset_init QUERY kettle_build_doc ;#usually VIRTUAL - since there is no specific target file - and we don't know all the files that will be generated - but here we use QUERY to ensure no writes to .punkcheck
set last_completion [$event targetset_last_complete]
if {[llength $last_completion]} {
#adding a source causes it to be checksummed
$event targetset_addsource $projectdir/src/doc ;#whole doc tree is considered the source
#----------
set changeinfo [$event targetset_source_changes]
if {\
[llength [dict get $changeinfo changed]]\
} {
puts stdout "changed"
puts stdout $changeinfo
} else {
puts stdout "No changes detected in $projectdir/src/doc tree"
}
} else {
#no previous completion-record for this target - must assume changed - no need to trigger checksumming
puts stdout "No existing record of doc build in .punkcheck. Assume it needs to be rebuilt."
}
$event destroy
$installer destroy
cd $original_wd
}
proc validate {} {
set projectdir [punk::repo::find_project]
if {$projectdir eq ""} {
puts stderr "No current project dir - unable to check doc status"
return
}
if {![file exists $projectdir/src/doc]} {
set result "No documentation source found. Expected .man files in doctools format at $projectdir/src/doc"
return $result
}
set original_wd [pwd]
cd $projectdir/src
punk::mix::cli::lib::kettle_call lib validate-doc
cd $original_wd
}
namespace eval collection {
variable pkg
set pkg punk::mix::commandset::doc
namespace export *
namespace path [namespace parent]
}
namespace eval lib {
variable pkg
set pkg punk::mix::commandset::doc
}
}
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
## Ready
package provide punk::mix::commandset::doc [namespace eval punk::mix::commandset::doc {
variable pkg punk::mix::commandset::doc
variable version
set version 999999.0a1.0
}]
return

3
src/modules/punk/mix/commandset/doc-buildversion.txt

@ -0,0 +1,3 @@
0.1.0
#First line must be a semantic version number
#all other lines are ignored.

2
src/modules/punk/mix/commandset/layout-999999.0a1.0.tm

@ -68,7 +68,7 @@ namespace eval punk::mix::commandset::layout {
#layout collection functions - to be imported with punk::overlay::import_commandset separately
namespace eval collection {
namespace export *
proc layouts {{glob {}}} {
proc _default {{glob {}}} {
if {![string length $glob]} {
set glob *
}

8
src/modules/punk/mix/commandset/module-999999.0a1.0.tm

@ -102,6 +102,7 @@ namespace eval punk::mix::commandset::module {
return $table
}
#return all module templates with repeated ones suffixed with .2 .3 etc
proc templates_dict {args} {
tailcall lib::templates_dict {*}$args
}
@ -186,8 +187,8 @@ namespace eval punk::mix::commandset::module {
set opt_license [dict get $opts -license]
# -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
set opt_template [dict get $opts -template]
set lib_tpldir [file join [punk::mix::cli::lib::mix_templates_dir] module];#fallback for modulename_buildversion.txt, modulename_description.txt
set templates_dict [templates_dict]
set templates_dict [templates_dict] ;#possibly suffixed with .2 .3 etc
#todo - allow versionless name - pick latest which isn't suffixed with .2 etc
if {![dict exists $templates_dict $opt_template]} {
error "module.new unable to find template '$opt_template'. Known templates: [dict keys $templates_dict]"
@ -283,9 +284,12 @@ namespace eval punk::mix::commandset::module {
error $errmsg
}
if {[file exists $tpldir/modulename_buildversion.txt]} {
set fd [open $tpldir/modulename_buildversion.txt r]; set buildversion_filedata [read $fd]; close $fd
} else {
#mix_templates_dir warns of deprecation - review
set lib_tpldir [file join [punk::mix::cli::lib::mix_templates_dir] modules];#fallback for modulename_buildversion.txt, modulename_description.txt
set fd [open $lib_tpldir/modulename_buildversion.txt r]; set buildversion_filedata [read $fd]; close $fd
}
set buildversionfile [file join $modulefolder ${moduletail}-buildversion.txt]

130
src/modules/punk/mix/commandset/project-999999.0a1.0.tm

@ -242,44 +242,40 @@ namespace eval punk::mix::commandset::project {
set layout_dir $templatebase/layouts/$opt_layout
puts stdout ">>> about to call punkcheck::install $layout_dir $projectdir"
set resultdict [dict create]
#In this case we need to override the default dir antiglob - as .fossil- folders need to be installed from template
## default_antiglob_dir_core [list "#*" "_aside" ".git" ".fossil*"]
set override_antiglob_dir_core [list #* _aside .git]
set unpublish [list\
src/doc/*\
src/doc/include/*\
]
#default antiglob_dir_core will stop .fossil* from being updated - which is generally desirable as these are likely to be customized
if {$opt_force} {
set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite ALL-TARGETS -unpublish_paths $unpublish]
puts stdout "copying layout files - with force applied - overwrite all-targets"
set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -overwrite ALL-TARGETS -unpublish_paths $unpublish]
#file copy -force $layout_dir $projectdir
} else {
set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite installedsourcechanged-targets -unpublish_paths $unpublish]
}
if {[dict size $resultdict]} {
set copied [dict get $resultdict files_copied]
set sources_unchanged [dict get $resultdict sources_unchanged]
puts stdout "--------------------------"
flush stdout
puts stderr "Copied [llength $copied] files from $layout_dir to $projectdir"
foreach f $copied {
puts stdout "COPIED $f"
}
puts stdout "[llength $sources_unchanged] unchanged source files"
puts stdout "--------------------------"
}
set resultdict [punkcheck::install $layout_dir/src/doc $projectdir/src/doc -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite NO-TARGETS]
if {[dict size $resultdict]} {
set copied [dict get $resultdict files_copied]
set files_skipped [dict get $resultdict files_skipped]
puts stdout "--------------------------"
flush stdout
puts stderr "Copied [llength $copied] doc files from $layout_dir/src/doc to $projectdir/src/doc"
foreach f $copied {
puts stdout "COPIED $f"
}
puts stdout "[llength $files_skipped] skipped files"
puts stdout "--------------------------"
puts stdout "copying layout files - (if source file changed)"
set resultdict [punkcheck::install $layout_dir $projectdir -installer project.new -overwrite installedsourcechanged-targets -unpublish_paths $unpublish]
}
puts stdout [punkcheck::summarize_install_resultdict $resultdict]
puts stdout "copying layout src/doc files (if target missing)"
set resultdict [punkcheck::install $layout_dir/src/doc $projectdir/src/doc -punkcheck_folder $projectdir -installer project.new -overwrite SYNCED-TARGETS]
puts stdout [punkcheck::summarize_install_resultdict $resultdict]
#target folders .fossil-custom and .fossil-settings may not exist. use -createdir 1 to ensure existence.
#In this case we need to override the default dir antiglob - as .fossil-xxx folders need to be installed from template if missing, or if target is uncustomized.
## default_antiglob_dir_core [list "#*" "_aside" ".git" ".fossil*"]
set override_antiglob_dir_core [list #* _aside .git]
puts stdout "copying layout src/.fossil-custom files (if target missing or uncustomised)"
set resultdict [punkcheck::install $layout_dir/.fossil-custom $projectdir/.fossil-custom -createdir 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS]
puts stdout [punkcheck::summarize_install_resultdict $resultdict]
puts stdout "copying layout src/.fossil-settings files (if target missing or uncustomised)"
set resultdict [punkcheck::install $layout_dir/.fossil-settings $projectdir/.fossil-settings -createdir 1 -punkcheck_folder $projectdir -installer project.new -antiglob_dir_core $override_antiglob_dir_core -overwrite SYNCED-TARGETS]
puts stdout [punkcheck::summarize_install_resultdict $resultdict]
#lappend substfiles $projectdir/README.md
#lappend substfiles $projectdir/src/README.md
@ -328,8 +324,37 @@ namespace eval punk::mix::commandset::project {
#generate www/man/md output in 'embedded' folder which should be checked into repo for online documentation
if {[file exists $projectdir/src]} {
cd $projectdir/src
punk::mix::cli::lib::kettle_call lib doc
#Kettle doc
#----------
set installer [punkcheck::installtrack new project.new $projectdir/src/.punkcheck]
$installer set_source_target $projectdir/src/doc $projectdir/src/embedded
set event [$installer start_event {-install_step kettledoc}]
$event targetset_init VIRTUAL kettle_build_doc ;#VIRTUAL - since there is no specific target file - and we don't know all the files that will be generated
$event targetset_addsource $projectdir/src/doc ;#whole doc tree is considered the source
#----------
if {\
[llength [dict get [$event targetset_source_changes] changed]]\
} {
$event targetset_started
# -- --- --- --- --- ---
puts stdout "BUILDING DOCS at src/embedded from src/doc"
if {[catch {
punk::mix::cli::lib::kettle_call lib doc
#Kettle doc
} errM]} {
$event targetset_end FAILED -note "kettle_build_doc failed: $errM"
} else {
$event targetset_end OK
}
# -- --- --- --- --- ---
} else {
puts stderr "No change detected in src/doc"
$event targetset_end SKIPPED
}
$event end
$event destroy
$installer destroy
}
cd $projectdir
@ -386,7 +411,8 @@ namespace eval punk::mix::commandset::project {
namespace export *
namespace path [namespace parent]
proc projects {{glob {}} args} {
#e.g imported as 'projects'
proc _default {{glob {}} args} {
package require overtype
set db_projects [lib::get_projects $glob]
set col1items [lsearch -all -inline -index 0 -subindices $db_projects *]
@ -414,7 +440,7 @@ namespace eval punk::mix::commandset::project {
return $msg
#return [list_as_lines [lib::get_projects $glob]]
}
proc projects.detail {{glob {}} args} {
proc detail {{glob {}} args} {
package require overtype
package require textutil
set defaults [dict create\
@ -533,11 +559,11 @@ namespace eval punk::mix::commandset::project {
return $msg
#return [list_as_lines [lib::get_projects $glob]]
}
proc projects.cd {{glob {}} args} {
proc cd {{glob {}} args} {
dict set args -cd 1
projects.work $glob {*}$args
work $glob {*}$args
}
proc projects.work {{glob {}} args} {
proc work {{glob {}} args} {
package require sqlite3
set db_projects [lib::get_projects $glob]
#list of lists of the form:
@ -586,11 +612,22 @@ namespace eval punk::mix::commandset::project {
set dupid ""
}
if {$dbcount == 1} {
sqlite3 fdb $fosdb
set pname [lindex [fdb eval {select value from config where name = 'project-name'}] 0]
set pcode [lindex [fdb eval {select value from config where name = 'project-code'}] 0]
fdb close
dict set fosdb_cache $fosdb [list name $pname code $pcode]
set pname ""
set pcode ""
if {[file exists $fosdb]} {
if {[catch {
sqlite3 fdb $fosdb
set pname [lindex [fdb eval {select value from config where name = 'project-name'}] 0]
set pcode [lindex [fdb eval {select value from config where name = 'project-code'}] 0]
fdb close
dict set fosdb_cache $fosdb [list name $pname code $pcode]
} errM]} {
puts stderr "!!! problem with fossil db: $fosdb when examining workdir $wd"
puts stderr "!!! error: $errM"
}
} else {
puts stderr "!!! missing fossil db $fosdb"
}
} else {
set info [dict get $fosdb_cache $fosdb]
lassign $info _name pname _code pcode
@ -704,8 +741,13 @@ namespace eval punk::mix::commandset::project {
if {$numrows == 1} {
set workingdir [lindex $workdirs 0]
puts stdout "1 result. Changing dir to $workingdir"
cd $workingdir
return $workingdir
if {[file exists $workingdir]} {
cd $workingdir
return $workingdir
} else {
puts stderr "path $workingdir doesn't appear to exist"
return [pwd]
}
} else {
set answer [util::askuser "Change directory to working folder - select a number from 1 to [llength $col_rowids] or any other key to cancel."]
if {[string trim $answer] in $col_rowids} {

1
src/modules/punk/mix/templates/layouts/project/.gitignore vendored

@ -35,6 +35,7 @@ _FOSSIL_
#miscellaneous editor files etc
*.swp
.punkcheck
todo.txt

6
src/modules/punk/mix/templates/layouts/project/src/bootsupport/include_modules.config

@ -0,0 +1,6 @@
## e.g
#set bootsupport_modules [list\
# src/vendormodules cksum\
# modules punkcheck\
#]

200
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/cksum-1.1.4.tm

@ -0,0 +1,200 @@
# cksum.tcl - Copyright (C) 2002 Pat Thoyts <patthoyts@users.sourceforge.net>
#
# Provides a Tcl only implementation of the unix cksum(1) command. This is
# similar to the sum(1) command but the algorithm is better defined and
# standardized across multiple platforms by POSIX 1003.2/D11.2
#
# This command has been verified against the cksum command from the GNU
# textutils package version 2.0
#
# -------------------------------------------------------------------------
# See the file "license.terms" for information on usage and redistribution
# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
# -------------------------------------------------------------------------
package require Tcl 8.5-; # tcl minimum version
namespace eval ::crc {
namespace export cksum
variable cksum_tbl [list 0x0 \
0x04C11DB7 0x09823B6E 0x0D4326D9 0x130476DC 0x17C56B6B \
0x1A864DB2 0x1E475005 0x2608EDB8 0x22C9F00F 0x2F8AD6D6 \
0x2B4BCB61 0x350C9B64 0x31CD86D3 0x3C8EA00A 0x384FBDBD \
0x4C11DB70 0x48D0C6C7 0x4593E01E 0x4152FDA9 0x5F15ADAC \
0x5BD4B01B 0x569796C2 0x52568B75 0x6A1936C8 0x6ED82B7F \
0x639B0DA6 0x675A1011 0x791D4014 0x7DDC5DA3 0x709F7B7A \
0x745E66CD 0x9823B6E0 0x9CE2AB57 0x91A18D8E 0x95609039 \
0x8B27C03C 0x8FE6DD8B 0x82A5FB52 0x8664E6E5 0xBE2B5B58 \
0xBAEA46EF 0xB7A96036 0xB3687D81 0xAD2F2D84 0xA9EE3033 \
0xA4AD16EA 0xA06C0B5D 0xD4326D90 0xD0F37027 0xDDB056FE \
0xD9714B49 0xC7361B4C 0xC3F706FB 0xCEB42022 0xCA753D95 \
0xF23A8028 0xF6FB9D9F 0xFBB8BB46 0xFF79A6F1 0xE13EF6F4 \
0xE5FFEB43 0xE8BCCD9A 0xEC7DD02D 0x34867077 0x30476DC0 \
0x3D044B19 0x39C556AE 0x278206AB 0x23431B1C 0x2E003DC5 \
0x2AC12072 0x128E9DCF 0x164F8078 0x1B0CA6A1 0x1FCDBB16 \
0x018AEB13 0x054BF6A4 0x0808D07D 0x0CC9CDCA 0x7897AB07 \
0x7C56B6B0 0x71159069 0x75D48DDE 0x6B93DDDB 0x6F52C06C \
0x6211E6B5 0x66D0FB02 0x5E9F46BF 0x5A5E5B08 0x571D7DD1 \
0x53DC6066 0x4D9B3063 0x495A2DD4 0x44190B0D 0x40D816BA \
0xACA5C697 0xA864DB20 0xA527FDF9 0xA1E6E04E 0xBFA1B04B \
0xBB60ADFC 0xB6238B25 0xB2E29692 0x8AAD2B2F 0x8E6C3698 \
0x832F1041 0x87EE0DF6 0x99A95DF3 0x9D684044 0x902B669D \
0x94EA7B2A 0xE0B41DE7 0xE4750050 0xE9362689 0xEDF73B3E \
0xF3B06B3B 0xF771768C 0xFA325055 0xFEF34DE2 0xC6BCF05F \
0xC27DEDE8 0xCF3ECB31 0xCBFFD686 0xD5B88683 0xD1799B34 \
0xDC3ABDED 0xD8FBA05A 0x690CE0EE 0x6DCDFD59 0x608EDB80 \
0x644FC637 0x7A089632 0x7EC98B85 0x738AAD5C 0x774BB0EB \
0x4F040D56 0x4BC510E1 0x46863638 0x42472B8F 0x5C007B8A \
0x58C1663D 0x558240E4 0x51435D53 0x251D3B9E 0x21DC2629 \
0x2C9F00F0 0x285E1D47 0x36194D42 0x32D850F5 0x3F9B762C \
0x3B5A6B9B 0x0315D626 0x07D4CB91 0x0A97ED48 0x0E56F0FF \
0x1011A0FA 0x14D0BD4D 0x19939B94 0x1D528623 0xF12F560E \
0xF5EE4BB9 0xF8AD6D60 0xFC6C70D7 0xE22B20D2 0xE6EA3D65 \
0xEBA91BBC 0xEF68060B 0xD727BBB6 0xD3E6A601 0xDEA580D8 \
0xDA649D6F 0xC423CD6A 0xC0E2D0DD 0xCDA1F604 0xC960EBB3 \
0xBD3E8D7E 0xB9FF90C9 0xB4BCB610 0xB07DABA7 0xAE3AFBA2 \
0xAAFBE615 0xA7B8C0CC 0xA379DD7B 0x9B3660C6 0x9FF77D71 \
0x92B45BA8 0x9675461F 0x8832161A 0x8CF30BAD 0x81B02D74 \
0x857130C3 0x5D8A9099 0x594B8D2E 0x5408ABF7 0x50C9B640 \
0x4E8EE645 0x4A4FFBF2 0x470CDD2B 0x43CDC09C 0x7B827D21 \
0x7F436096 0x7200464F 0x76C15BF8 0x68860BFD 0x6C47164A \
0x61043093 0x65C52D24 0x119B4BE9 0x155A565E 0x18197087 \
0x1CD86D30 0x029F3D35 0x065E2082 0x0B1D065B 0x0FDC1BEC \
0x3793A651 0x3352BBE6 0x3E119D3F 0x3AD08088 0x2497D08D \
0x2056CD3A 0x2D15EBE3 0x29D4F654 0xC5A92679 0xC1683BCE \
0xCC2B1D17 0xC8EA00A0 0xD6AD50A5 0xD26C4D12 0xDF2F6BCB \
0xDBEE767C 0xE3A1CBC1 0xE760D676 0xEA23F0AF 0xEEE2ED18 \
0xF0A5BD1D 0xF464A0AA 0xF9278673 0xFDE69BC4 0x89B8FD09 \
0x8D79E0BE 0x803AC667 0x84FBDBD0 0x9ABC8BD5 0x9E7D9662 \
0x933EB0BB 0x97FFAD0C 0xAFB010B1 0xAB710D06 0xA6322BDF \
0xA2F33668 0xBCB4666D 0xB8757BDA 0xB5365D03 0xB1F740B4 ]
variable uid
if {![info exists uid]} {set uid 0}
}
# crc::CksumInit --
#
# Create and initialize a cksum context. This is cleaned up when we
# call CksumFinal to obtain the result.
#
proc ::crc::CksumInit {} {
variable uid
set token [namespace current]::[incr uid]
upvar #0 $token state
array set state {t 0 l 0}
return $token
}
proc ::crc::CksumUpdate {token data} {
variable cksum_tbl
upvar #0 $token state
set t $state(t)
binary scan $data c* r
foreach {n} $r {
set index [expr { (($t >> 24) ^ ($n & 0xFF)) & 0xFF }]
# Since the introduction of built-in bigInt support with Tcl
# 8.5, bit-shifting $t to the left no longer overflows,
# keeping it 32 bits long. The value grows bigger and bigger
# instead - a severe hit on performance. For this reason we
# do a bitwise AND against 0xFFFFFFFF at each step to keep the
# value within limits.
set t [expr {0xFFFFFFFF & (($t << 8) ^ [lindex $cksum_tbl $index])}]
incr state(l)
}
set state(t) $t
return
}
proc ::crc::CksumFinal {token} {
variable cksum_tbl
upvar #0 $token state
set t $state(t)
for {set i $state(l)} {$i > 0} {set i [expr {$i>>8}]} {
set index [expr {(($t >> 24) ^ $i) & 0xFF}]
set t [expr {0xFFFFFFFF & (($t << 8) ^ [lindex $cksum_tbl $index])}]
}
unset state
return [expr {~$t & 0xFFFFFFFF}]
}
# crc::Pop --
#
# Pop the nth element off a list. Used in options processing.
#
proc ::crc::Pop {varname {nth 0}} {
upvar $varname args
set r [lindex $args $nth]
set args [lreplace $args $nth $nth]
return $r
}
# Description:
# Provide a Tcl equivalent of the unix cksum(1) command.
# Options:
# -filename name - return a checksum for the specified file.
# -format string - return the checksum using this format string.
# -chunksize size - set the chunking read size
#
proc ::crc::cksum {args} {
array set opts [list -filename {} -channel {} -chunksize 4096 \
-format %u -command {}]
while {[string match -* [set option [lindex $args 0]]]} {
switch -glob -- $option {
-file* { set opts(-filename) [Pop args 1] }
-chan* { set opts(-channel) [Pop args 1] }
-chunk* { set opts(-chunksize) [Pop args 1] }
-for* { set opts(-format) [Pop args 1] }
-command { set opts(-command) [Pop args 1] }
default {
if {[llength $args] == 1} { break }
if {[string compare $option "--"] == 0} { Pop args ; break }
set err [join [lsort [array names opts -*]] ", "]
return -code error "bad option \"option\": must be $err"
}
}
Pop args
}
if {$opts(-filename) != {}} {
set opts(-channel) [open $opts(-filename) r]
fconfigure $opts(-channel) -translation binary
}
if {$opts(-channel) == {}} {
if {[llength $args] != 1} {
return -code error "wrong # args: should be\
cksum ?-format string?\
-channel chan | -filename file | string"
}
set tok [CksumInit]
CksumUpdate $tok [lindex $args 0]
set r [CksumFinal $tok]
} else {
set tok [CksumInit]
while {![eof $opts(-channel)]} {
CksumUpdate $tok [read $opts(-channel) $opts(-chunksize)]
}
set r [CksumFinal $tok]
if {$opts(-filename) != {}} {
close $opts(-channel)
}
}
return [format $opts(-format) $r]
}
# -------------------------------------------------------------------------
package provide cksum 1.1.4
# -------------------------------------------------------------------------
# Local variables:
# mode: tcl
# indent-tabs-mode: nil
# End:

933
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/cmdline-1.5.2.tm

@ -0,0 +1,933 @@
# cmdline.tcl --
#
# This package provides a utility for parsing command line
# arguments that are processed by our various applications.
# It also includes a utility routine to determine the
# application name for use in command line errors.
#
# Copyright (c) 1998-2000 by Ajuba Solutions.
# Copyright (c) 2001-2015 by Andreas Kupries <andreas_kupries@users.sf.net>.
# Copyright (c) 2003 by David N. Welton <davidw@dedasys.com>
# See the file "license.terms" for information on usage and redistribution
# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
package require Tcl 8.5-
package provide cmdline 1.5.2
namespace eval ::cmdline {
namespace export getArgv0 getopt getKnownOpt getfiles getoptions \
getKnownOptions usage
}
# ::cmdline::getopt --
#
# The cmdline::getopt works in a fashion like the standard
# C based getopt function. Given an option string and a
# pointer to an array or args this command will process the
# first argument and return info on how to proceed.
#
# Arguments:
# argvVar Name of the argv list that you
# want to process. If options are found the
# arg list is modified and the processed arguments
# are removed from the start of the list.
# optstring A list of command options that the application
# will accept. If the option ends in ".arg" the
# getopt routine will use the next argument as
# an argument to the option. Otherwise the option
# is a boolean that is set to 1 if present.
# optVar The variable pointed to by optVar
# contains the option that was found (without the
# leading '-' and without the .arg extension).
# valVar Upon success, the variable pointed to by valVar
# contains the value for the specified option.
# This value comes from the command line for .arg
# options, otherwise the value is 1.
# If getopt fails, the valVar is filled with an
# error message.
#
# Results:
# The getopt function returns 1 if an option was found, 0 if no more
# options were found, and -1 if an error occurred.
proc ::cmdline::getopt {argvVar optstring optVar valVar} {
upvar 1 $argvVar argsList
upvar 1 $optVar option
upvar 1 $valVar value
set result [getKnownOpt argsList $optstring option value]
if {$result < 0} {
# Collapse unknown-option error into any-other-error result.
set result -1
}
return $result
}
# ::cmdline::getKnownOpt --
#
# The cmdline::getKnownOpt works in a fashion like the standard
# C based getopt function. Given an option string and a
# pointer to an array or args this command will process the
# first argument and return info on how to proceed.
#
# Arguments:
# argvVar Name of the argv list that you
# want to process. If options are found the
# arg list is modified and the processed arguments
# are removed from the start of the list. Note that
# unknown options and the args that follow them are
# left in this list.
# optstring A list of command options that the application
# will accept. If the option ends in ".arg" the
# getopt routine will use the next argument as
# an argument to the option. Otherwise the option
# is a boolean that is set to 1 if present.
# optVar The variable pointed to by optVar
# contains the option that was found (without the
# leading '-' and without the .arg extension).
# valVar Upon success, the variable pointed to by valVar
# contains the value for the specified option.
# This value comes from the command line for .arg
# options, otherwise the value is 1.
# If getopt fails, the valVar is filled with an
# error message.
#
# Results:
# The getKnownOpt function returns 1 if an option was found,
# 0 if no more options were found, -1 if an unknown option was
# encountered, and -2 if any other error occurred.
proc ::cmdline::getKnownOpt {argvVar optstring optVar valVar} {
upvar 1 $argvVar argsList
upvar 1 $optVar option
upvar 1 $valVar value
# default settings for a normal return
set value ""
set option ""
set result 0
# check if we're past the end of the args list
if {[llength $argsList] != 0} {
# if we got -- or an option that doesn't begin with -, return (skipping
# the --). otherwise process the option arg.
switch -glob -- [set arg [lindex $argsList 0]] {
"--" {
set argsList [lrange $argsList 1 end]
}
"--*" -
"-*" {
set option [string range $arg 1 end]
if {[string equal [string range $option 0 0] "-"]} {
set option [string range $arg 2 end]
}
# support for format: [-]-option=value
set idx [string first "=" $option 1]
if {$idx != -1} {
set _val [string range $option [expr {$idx+1}] end]
set option [string range $option 0 [expr {$idx-1}]]
}
if {[lsearch -exact $optstring $option] != -1} {
# Booleans are set to 1 when present
set value 1
set result 1
set argsList [lrange $argsList 1 end]
} elseif {[lsearch -exact $optstring "$option.arg"] != -1} {
set result 1
set argsList [lrange $argsList 1 end]
if {[info exists _val]} {
set value $_val
} elseif {[llength $argsList]} {
set value [lindex $argsList 0]
set argsList [lrange $argsList 1 end]
} else {
set value "Option \"$option\" requires an argument"
set result -2
}
} else {
# Unknown option.
set value "Illegal option \"-$option\""
set result -1
}
}
default {
# Skip ahead
}
}
}
return $result
}
# ::cmdline::getoptions --
#
# Process a set of command line options, filling in defaults
# for those not specified. This also generates an error message
# that lists the allowed flags if an incorrect flag is specified.
#
# Arguments:
# argvVar The name of the argument list, typically argv.
# We remove all known options and their args from it.
# In other words, after the call to this command the
# referenced variable contains only the non-options,
# and unknown options.
# optlist A list-of-lists where each element specifies an option
# in the form:
# (where flag takes no argument)
# flag comment
#
# (or where flag takes an argument)
# flag default comment
#
# If flag ends in ".arg" then the value is taken from the
# command line. Otherwise it is a boolean and appears in
# the result if present on the command line. If flag ends
# in ".secret", it will not be displayed in the usage.
# usage Text to include in the usage display. Defaults to
# "options:"
#
# Results
# Name value pairs suitable for using with array set.
# A modified `argvVar`.
proc ::cmdline::getoptions {argvVar optlist {usage options:}} {
upvar 1 $argvVar argv
set opts [GetOptionDefaults $optlist result]
set argc [llength $argv]
while {[set err [getopt argv $opts opt arg]]} {
if {$err < 0} {
set result(?) ""
break
}
set result($opt) $arg
}
if {[info exist result(?)] || [info exists result(help)]} {
Error [usage $optlist $usage] USAGE
}
return [array get result]
}
# ::cmdline::getKnownOptions --
#
# Process a set of command line options, filling in defaults
# for those not specified. This ignores unknown flags, but generates
# an error message that lists the correct usage if a known option
# is used incorrectly.
#
# Arguments:
# argvVar The name of the argument list, typically argv. This
# We remove all known options and their args from it.
# In other words, after the call to this command the
# referenced variable contains only the non-options,
# and unknown options.
# optlist A list-of-lists where each element specifies an option
# in the form:
# flag default comment
# If flag ends in ".arg" then the value is taken from the
# command line. Otherwise it is a boolean and appears in
# the result if present on the command line. If flag ends
# in ".secret", it will not be displayed in the usage.
# usage Text to include in the usage display. Defaults to
# "options:"
#
# Results
# Name value pairs suitable for using with array set.
# A modified `argvVar`.
proc ::cmdline::getKnownOptions {argvVar optlist {usage options:}} {
upvar 1 $argvVar argv
set opts [GetOptionDefaults $optlist result]
# As we encounter them, keep the unknown options and their
# arguments in this list. Before we return from this procedure,
# we'll prepend these args to the argList so that the application
# doesn't lose them.
set unknownOptions [list]
set argc [llength $argv]
while {[set err [getKnownOpt argv $opts opt arg]]} {
if {$err == -1} {
# Unknown option.
# Skip over any non-option items that follow it.
# For now, add them to the list of unknownOptions.
lappend unknownOptions [lindex $argv 0]
set argv [lrange $argv 1 end]
while {([llength $argv] != 0) \
&& ![string match "-*" [lindex $argv 0]]} {
lappend unknownOptions [lindex $argv 0]
set argv [lrange $argv 1 end]
}
} elseif {$err == -2} {
set result(?) ""
break
} else {
set result($opt) $arg
}
}
# Before returning, prepend the any unknown args back onto the
# argList so that the application doesn't lose them.
set argv [concat $unknownOptions $argv]
if {[info exist result(?)] || [info exists result(help)]} {
Error [usage $optlist $usage] USAGE
}
return [array get result]
}
# ::cmdline::GetOptionDefaults --
#
# This internal procedure processes the option list (that was passed to
# the getopt or getKnownOpt procedure). The defaultArray gets an index
# for each option in the option list, the value of which is the option's
# default value.
#
# Arguments:
# optlist A list-of-lists where each element specifies an option
# in the form:
# flag default comment
# If flag ends in ".arg" then the value is taken from the
# command line. Otherwise it is a boolean and appears in
# the result if present on the command line. If flag ends
# in ".secret", it will not be displayed in the usage.
# defaultArrayVar The name of the array in which to put argument defaults.
#
# Results
# Name value pairs suitable for using with array set.
proc ::cmdline::GetOptionDefaults {optlist defaultArrayVar} {
upvar 1 $defaultArrayVar result
set opts {? help}
foreach opt $optlist {
set name [lindex $opt 0]
if {[regsub -- {\.secret$} $name {} name] == 1} {
# Need to hide this from the usage display and getopt
}
lappend opts $name
if {[regsub -- {\.arg$} $name {} name] == 1} {
# Set defaults for those that take values.
set default [lindex $opt 1]
set result($name) $default
} else {
# The default for booleans is false
set result($name) 0
}
}
return $opts
}
# ::cmdline::usage --
#
# Generate an error message that lists the allowed flags.
#
# Arguments:
# optlist As for cmdline::getoptions
# usage Text to include in the usage display. Defaults to
# "options:"
#
# Results
# A formatted usage message
proc ::cmdline::usage {optlist {usage {options:}}} {
set str "[getArgv0] $usage\n"
set longest 20
set lines {}
foreach opt [concat $optlist \
{{- "Forcibly stop option processing"} {help "Print this message"} {? "Print this message"}}] {
set name "-[lindex $opt 0]"
if {[regsub -- {\.secret$} $name {} name] == 1} {
# Hidden option
continue
}
if {[regsub -- {\.arg$} $name {} name] == 1} {
append name " value"
set desc "[lindex $opt 2] <[lindex $opt 1]>"
} else {
set desc "[lindex $opt 1]"
}
set n [string length $name]
if {$n > $longest} { set longest $n }
# max not available before 8.5 - set longest [expr {max($longest, )}]
lappend lines $name $desc
}
foreach {name desc} $lines {
append str "[string trimright [format " %-*s %s" $longest $name $desc]]\n"
}
return $str
}
# ::cmdline::getfiles --
#
# Given a list of file arguments from the command line, compute
# the set of valid files. On windows, file globbing is performed
# on each argument. On Unix, only file existence is tested. If
# a file argument produces no valid files, a warning is optionally
# generated.
#
# This code also uses the full path for each file. If not
# given it prepends [pwd] to the filename. This ensures that
# these files will never conflict with files in our zip file.
#
# Arguments:
# patterns The file patterns specified by the user.
# quiet If this flag is set, no warnings will be generated.
#
# Results:
# Returns the list of files that match the input patterns.
proc ::cmdline::getfiles {patterns quiet} {
set result {}
if {$::tcl_platform(platform) == "windows"} {
foreach pattern $patterns {
set pat [file join $pattern]
set files [glob -nocomplain -- $pat]
if {$files == {}} {
if {! $quiet} {
puts stdout "warning: no files match \"$pattern\""
}
} else {
foreach file $files {
lappend result $file
}
}
}
} else {
set result $patterns
}
set files {}
foreach file $result {
# Make file an absolute path so that we will never conflict
# with files that might be contained in our zip file.
set fullPath [file join [pwd] $file]
if {[file isfile $fullPath]} {
lappend files $fullPath
} elseif {! $quiet} {
puts stdout "warning: no files match \"$file\""
}
}
return $files
}
# ::cmdline::getArgv0 --
#
# This command returns the "sanitized" version of argv0. It will strip
# off the leading path and remove the ".bin" extensions that our apps
# use because they must be wrapped by a shell script.
#
# Arguments:
# None.
#
# Results:
# The application name that can be used in error messages.
proc ::cmdline::getArgv0 {} {
global argv0
set name [file tail $argv0]
return [file rootname $name]
}
##
# ### ### ### ######### ######### #########
##
# Now the typed versions of the above commands.
##
# ### ### ### ######### ######### #########
##
# typedCmdline.tcl --
#
# This package provides a utility for parsing typed command
# line arguments that may be processed by various applications.
#
# Copyright (c) 2000 by Ross Palmer Mohn.
# See the file "license.terms" for information on usage and redistribution
# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
#
# RCS: @(#) $Id: cmdline.tcl,v 1.28 2011/02/23 17:41:52 andreas_kupries Exp $
namespace eval ::cmdline {
namespace export typedGetopt typedGetoptions typedUsage
# variable cmdline::charclasses --
#
# Create regexp list of allowable character classes
# from "string is" error message.
#
# Results:
# String of character class names separated by "|" characters.
variable charclasses
#checker exclude badKey
catch {string is . .} charclasses
variable dummy
regexp -- {must be (.+)$} $charclasses dummy charclasses
regsub -all -- {, (or )?} $charclasses {|} charclasses
unset dummy
}
# ::cmdline::typedGetopt --
#
# The cmdline::typedGetopt works in a fashion like the standard
# C based getopt function. Given an option string and a
# pointer to a list of args this command will process the
# first argument and return info on how to proceed. In addition,
# you may specify a type for the argument to each option.
#
# Arguments:
# argvVar Name of the argv list that you want to process.
# If options are found, the arg list is modified
# and the processed arguments are removed from the
# start of the list.
#
# optstring A list of command options that the application
# will accept. If the option ends in ".xxx", where
# xxx is any valid character class to the tcl
# command "string is", then typedGetopt routine will
# use the next argument as a typed argument to the
# option. The argument must match the specified
# character classes (e.g. integer, double, boolean,
# xdigit, etc.). Alternatively, you may specify
# ".arg" for an untyped argument.
#
# optVar Upon success, the variable pointed to by optVar
# contains the option that was found (without the
# leading '-' and without the .xxx extension). If
# typedGetopt fails the variable is set to the empty
# string. SOMETIMES! Different for each -value!
#
# argVar Upon success, the variable pointed to by argVar
# contains the argument for the specified option.
# If typedGetopt fails, the variable is filled with
# an error message.
#
# Argument type syntax:
# Option that takes no argument.
# foo
#
# Option that takes a typeless argument.
# foo.arg
#
# Option that takes a typed argument. Allowable types are all
# valid character classes to the tcl command "string is".
# Currently must be one of alnum, alpha, ascii, control,
# boolean, digit, double, false, graph, integer, lower, print,
# punct, space, true, upper, wordchar, or xdigit.
# foo.double
#
# Option that takes an argument from a list.
# foo.(bar|blat)
#
# Argument quantifier syntax:
# Option that takes an optional argument.
# foo.arg?
#
# Option that takes a list of arguments terminated by "--".
# foo.arg+
#
# Option that takes an optional list of arguments terminated by "--".
# foo.arg*
#
# Argument quantifiers work on all argument types, so, for
# example, the following is a valid option specification.
# foo.(bar|blat|blah)?
#
# Argument syntax miscellany:
# Options may be specified on the command line using a unique,
# shortened version of the option name. Given that program foo
# has an option list of {bar.alpha blah.arg blat.double},
# "foo -b fob" returns an error, but "foo -ba fob"
# successfully returns {bar fob}
#
# Results:
# The typedGetopt function returns one of the following:
# 1 a valid option was found
# 0 no more options found to process
# -1 invalid option
# -2 missing argument to a valid option
# -3 argument to a valid option does not match type
#
# Known Bugs:
# When using options which include special glob characters,
# you must use the exact option. Abbreviating it can cause
# an error in the "cmdline::prefixSearch" procedure.
proc ::cmdline::typedGetopt {argvVar optstring optVar argVar} {
variable charclasses
upvar $argvVar argsList
upvar $optVar retvar
upvar $argVar optarg
# default settings for a normal return
set optarg ""
set retvar ""
set retval 0
# check if we're past the end of the args list
if {[llength $argsList] != 0} {
# if we got -- or an option that doesn't begin with -, return (skipping
# the --). otherwise process the option arg.
switch -glob -- [set arg [lindex $argsList 0]] {
"--" {
set argsList [lrange $argsList 1 end]
}
"-*" {
# Create list of options without their argument extensions
set optstr ""
foreach str $optstring {
lappend optstr [file rootname $str]
}
set _opt [string range $arg 1 end]
set i [prefixSearch $optstr [file rootname $_opt]]
if {$i != -1} {
set opt [lindex $optstring $i]
set quantifier "none"
if {[regexp -- {\.[^.]+([?+*])$} $opt dummy quantifier]} {
set opt [string range $opt 0 end-1]
}
if {[string first . $opt] == -1} {
set retval 1
set retvar $opt
set argsList [lrange $argsList 1 end]
} elseif {[regexp -- "\\.(arg|$charclasses)\$" $opt dummy charclass]
|| [regexp -- {\.\(([^)]+)\)} $opt dummy charclass]} {
if {[string equal arg $charclass]} {
set type arg
} elseif {[regexp -- "^($charclasses)\$" $charclass]} {
set type class
} else {
set type oneof
}
set argsList [lrange $argsList 1 end]
set opt [file rootname $opt]
while {1} {
if {[llength $argsList] == 0
|| [string equal "--" [lindex $argsList 0]]} {
if {[string equal "--" [lindex $argsList 0]]} {
set argsList [lrange $argsList 1 end]
}
set oneof ""
if {$type == "arg"} {
set charclass an
} elseif {$type == "oneof"} {
set oneof ", one of $charclass"
set charclass an
}
if {$quantifier == "?"} {
set retval 1
set retvar $opt
set optarg ""
} elseif {$quantifier == "+"} {
set retvar $opt
if {[llength $optarg] < 1} {
set retval -2
set optarg "Option requires at least one $charclass argument$oneof -- $opt"
} else {
set retval 1
}
} elseif {$quantifier == "*"} {
set retval 1
set retvar $opt
} else {
set optarg "Option requires $charclass argument$oneof -- $opt"
set retvar $opt
set retval -2
}
set quantifier ""
} elseif {($type == "arg")
|| (($type == "oneof")
&& [string first "|[lindex $argsList 0]|" "|$charclass|"] != -1)
|| (($type == "class")
&& [string is $charclass [lindex $argsList 0]])} {
set retval 1
set retvar $opt
lappend optarg [lindex $argsList 0]
set argsList [lrange $argsList 1 end]
} else {
set oneof ""
if {$type == "arg"} {
set charclass an
} elseif {$type == "oneof"} {
set oneof ", one of $charclass"
set charclass an
}
set optarg "Option requires $charclass argument$oneof -- $opt"
set retvar $opt
set retval -3
if {$quantifier == "?"} {
set retval 1
set optarg ""
}
set quantifier ""
}
if {![regexp -- {[+*]} $quantifier]} {
break;
}
}
} else {
Error \
"Illegal option type specification: must be one of $charclasses" \
BAD OPTION TYPE
}
} else {
set optarg "Illegal option -- $_opt"
set retvar $_opt
set retval -1
}
}
default {
# Skip ahead
}
}
}
return $retval
}
# ::cmdline::typedGetoptions --
#
# Process a set of command line options, filling in defaults
# for those not specified. This also generates an error message
# that lists the allowed options if an incorrect option is
# specified.
#
# Arguments:
# argvVar The name of the argument list, typically argv
# optlist A list-of-lists where each element specifies an option
# in the form:
#
# option default comment
#
# Options formatting is as described for the optstring
# argument of typedGetopt. Default is for optionally
# specifying a default value. Comment is for optionally
# specifying a comment for the usage display. The
# options "--", "-help", and "-?" are automatically included
# in optlist.
#
# Argument syntax miscellany:
# Options formatting and syntax is as described in typedGetopt.
# There are two additional suffixes that may be applied when
# passing options to typedGetoptions.
#
# You may add ".multi" as a suffix to any option. For options
# that take an argument, this means that the option may be used
# more than once on the command line and that each additional
# argument will be appended to a list, which is then returned
# to the application.
# foo.double.multi
#
# If a non-argument option is specified as ".multi", it is
# toggled on and off for each time it is used on the command
# line.
# foo.multi
#
# If an option specification does not contain the ".multi"
# suffix, it is not an error to use an option more than once.
# In this case, the behavior for options with arguments is that
# the last argument is the one that will be returned. For
# options that do not take arguments, using them more than once
# has no additional effect.
#
# Options may also be hidden from the usage display by
# appending the suffix ".secret" to any option specification.
# Please note that the ".secret" suffix must be the last suffix,
# after any argument type specification and ".multi" suffix.
# foo.xdigit.multi.secret
#
# Results
# Name value pairs suitable for using with array set.
proc ::cmdline::typedGetoptions {argvVar optlist {usage options:}} {
variable charclasses
upvar 1 $argvVar argv
set opts {? help}
foreach opt $optlist {
set name [lindex $opt 0]
if {[regsub -- {\.secret$} $name {} name] == 1} {
# Remove this extension before passing to typedGetopt.
}
if {[regsub -- {\.multi$} $name {} name] == 1} {
# Remove this extension before passing to typedGetopt.
regsub -- {\..*$} $name {} temp
set multi($temp) 1
}
lappend opts $name
if {[regsub -- "\\.(arg|$charclasses|\\(.+).?\$" $name {} name] == 1} {
# Set defaults for those that take values.
# Booleans are set just by being present, or not
set dflt [lindex $opt 1]
if {$dflt != {}} {
set defaults($name) $dflt
}
}
}
set argc [llength $argv]
while {[set err [typedGetopt argv $opts opt arg]]} {
if {$err == 1} {
if {[info exists result($opt)]
&& [info exists multi($opt)]} {
# Toggle boolean options or append new arguments
if {$arg == ""} {
unset result($opt)
} else {
set result($opt) "$result($opt) $arg"
}
} else {
set result($opt) "$arg"
}
} elseif {($err == -1) || ($err == -3)} {
Error [typedUsage $optlist $usage] USAGE
} elseif {$err == -2 && ![info exists defaults($opt)]} {
Error [typedUsage $optlist $usage] USAGE
}
}
if {[info exists result(?)] || [info exists result(help)]} {
Error [typedUsage $optlist $usage] USAGE
}
foreach {opt dflt} [array get defaults] {
if {![info exists result($opt)]} {
set result($opt) $dflt
}
}
return [array get result]
}
# ::cmdline::typedUsage --
#
# Generate an error message that lists the allowed flags,
# type of argument taken (if any), default value (if any),
# and an optional description.
#
# Arguments:
# optlist As for cmdline::typedGetoptions
#
# Results
# A formatted usage message
proc ::cmdline::typedUsage {optlist {usage {options:}}} {
variable charclasses
set str "[getArgv0] $usage\n"
set longest 20
set lines {}
foreach opt [concat $optlist \
{{help "Print this message"} {? "Print this message"}}] {
set name "-[lindex $opt 0]"
if {[regsub -- {\.secret$} $name {} name] == 1} {
# Hidden option
continue
}
if {[regsub -- {\.multi$} $name {} name] == 1} {
# Display something about multiple options
}
if {[regexp -- "\\.(arg|$charclasses)\$" $name dummy charclass] ||
[regexp -- {\.\(([^)]+)\)} $opt dummy charclass]
} {
regsub -- "\\..+\$" $name {} name
append name " $charclass"
set desc [lindex $opt 2]
set default [lindex $opt 1]
if {$default != ""} {
append desc " <$default>"
}
} else {
set desc [lindex $opt 1]
}
lappend accum $name $desc
set n [string length $name]
if {$n > $longest} { set longest $n }
# max not available before 8.5 - set longest [expr {max($longest, [string length $name])}]
}
foreach {name desc} $accum {
append str "[string trimright [format " %-*s %s" $longest $name $desc]]\n"
}
return $str
}
# ::cmdline::prefixSearch --
#
# Search a Tcl list for a pattern; searches first for an exact match,
# and if that fails, for a unique prefix that matches the pattern
# (i.e, first "lsearch -exact", then "lsearch -glob $pattern*"
#
# Arguments:
# list list of words
# pattern word to search for
#
# Results:
# Index of found word is returned. If no exact match or
# unique short version is found then -1 is returned.
proc ::cmdline::prefixSearch {list pattern} {
# Check for an exact match
if {[set pos [::lsearch -exact $list $pattern]] > -1} {
return $pos
}
# Check for a unique short version
set slist [lsort $list]
if {[set pos [::lsearch -glob $slist $pattern*]] > -1} {
# What if there is nothing for the check variable?
set check [lindex $slist [expr {$pos + 1}]]
if {[string first $pattern $check] != 0} {
return [::lsearch -exact $list [lindex $slist $pos]]
}
}
return -1
}
# ::cmdline::Error --
#
# Internal helper to throw errors with a proper error-code attached.
#
# Arguments:
# message text of the error message to throw.
# args additional parts of the error code to use,
# with CMDLINE as basic prefix added by this command.
#
# Results:
# An error is thrown, always.
proc ::cmdline::Error {message args} {
return -code error -errorcode [linsert $args 0 CMDLINE] $message
}

2311
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/fileutil-1.16.1.tm

File diff suppressed because it is too large Load Diff

1886
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/natsort-0.1.1.5.tm

File diff suppressed because it is too large Load Diff

195
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/oolib-0.1.tm

@ -0,0 +1,195 @@
#JMN - api should be kept in sync with package patternlib where possible
#
package provide oolib [namespace eval oolib {
variable version
set version 0.1
}]
namespace eval oolib {
oo::class create collection {
variable o_data ;#dict
variable o_alias
constructor {} {
set o_data [dict create]
}
method info {} {
return [dict info $o_data]
}
method count {} {
return [dict size $o_data]
}
method isEmpty {} {
expr {[dict size $o_data] == 0}
}
method names {{globOrIdx {}}} {
if {[llength $globOrIdx]} {
if {[string is integer -strict $globOrIdx]} {
if {$idx < 0} {
set idx "end-[expr {abs($idx + 1)}]"
}
if {[catch {lindex [dict keys $o_data] $idx} result]} {
error "[self object] no such index : '$idx'"
} else {
return $result
}
} else {
#glob
return [lsearch -glob -all -inline [dict keys $o_data] $globOrIdx]
}
} else {
return [dict keys $o_data]
}
}
#like names but without globbing
method keys {} {
dict keys $o_data
}
method key {{posn 0}} {
if {$posn < 0} {
set posn "end-[expr {abs($posn + 1)}]"
}
if {[catch {lindex [dict keys $o_data] $posn} result]} {
error "[self object] no such index : '$posn'"
} else {
return $result
}
}
method hasKey {key} {
dict exists $o_data $key
}
method get {} {
return $o_data
}
method items {} {
return [dict values $o_data]
}
method item {key} {
if {[string is integer -strict $key]} {
if {$key > 0} {
set valposn [expr {(2*$key) +1}]
return [lindex $o_data $valposn]
} else {
set key "end-[expr {abs($key + 1)}]"
return [lindex [dict keys $o_data] $key]
}
}
if {[dict exists $o_data $key]} {
return [dict get $o_data $key]
}
}
#inverse lookup
method itemKeys {value} {
set value_indices [lsearch -all [dict values $o_data] $value]
set keylist [list]
foreach i $value_indices {
set idx [expr {(($i + 1) *2) -2}]
lappend keylist [lindex $o_data $idx]
}
return $keylist
}
method search {value args} {
set matches [lsearch {*}$args [dict values $o_data] $value]
if {"-inline" in $args} {
return $matches
} else {
set keylist [list]
foreach i $matches {
set idx [expr {(($i + 1) *2) -2}]
lappend keylist [lindex $o_data $idx]
}
return $keylist
}
}
#review - see patternlib. Is the intention for aliases to be configurable independent of whether the target exists?
method alias {newAlias existingKeyOrAlias} {
if {[string is integer -strict $newAlias]} {
error "[self object] collection key alias cannot be integer"
}
if {[string length $existingKeyOrAlias]} {
set o_alias($newAlias) $existingKeyOrAlias
} else {
unset o_alias($newAlias)
}
}
method aliases {{key ""}} {
if {[string length $key]} {
set result [list]
foreach {n v} [array get o_alias] {
if {$v eq $key} {
lappend result $n $v
}
}
return $result
} else {
return [array get o_alias]
}
}
#if the supplied index is an alias, return the underlying key; else return the index supplied.
method realKey {idx} {
if {[catch {set o_alias($idx)} key]} {
return $idx
} else {
return $key
}
}
method add {value key} {
if {[string is integer -strict $key]} {
error "[self object] collection key must not be an integer. Use another structure if integer keys required"
}
if {[dict exists $o_data $key]} {
error "[self object] col_processors object error: key '$key' already exists in collection"
}
dict set o_data $key $value
return [expr {[dict size $o_data] - 1}] ;#return index of item
}
method remove {idx {endRange ""}} {
if {[string length $endRange]} {
error "[self object] collection error: ranged removal not yet implemented.. remove one item at a time"
}
if {[string is integer -strict $idx]} {
if {$idx < 0} {
set idx "end-[expr {abs($idx+1)}]"
}
set key [lindex [dict keys $o_data] $idx]
set posn $idx
} else {
set key $idx
set posn [lsearch -exact [dict keys $o_data] $key]
if {$posn < 0} {
error "[self object] no such index: '$idx' in this collection"
}
}
dict unset o_data $key
return
}
method clear {} {
set o_data [dict create]
return
}
method reverse {} {
set dictnew [dict create]
foreach k [lreverse [dict keys $o_data]] {
dict set dictnew $k [dict get $o_data $k]
}
set o_data $dictnew
return
}
#review - cmd as list vs cmd as script?
method map {cmd} {
set seed [list]
dict for {k v} $o_data {
lappend seed [uplevel #0 [list {*}$cmd $v]]
}
return $seed
}
method objectmap {cmd} {
set seed [list]
dict for {k v} $o_data {
lappend seed [uplevel #0 [list $v {*}$cmd]]
}
return $seed
}
}
}

1039
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/overtype-1.5.0.tm

File diff suppressed because it is too large Load Diff

1308
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/du-0.1.0.tm

File diff suppressed because it is too large Load Diff

15
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/mix-0.2.tm

@ -0,0 +1,15 @@
package require punk::cap
package require punk::mix::templates ;#registers 'templates' capability with punk::cap
package require punk::mix::base
package require punk::mix::cli
namespace eval punk::mix {
}
package provide punk::mix [namespace eval punk::mix {
variable version
set version 0.2
}]

1232
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/repo-0.1.1.tm

File diff suppressed because it is too large Load Diff

266
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punk/winpath-0.1.0.tm

@ -0,0 +1,266 @@
# -*- tcl -*-
# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from <pkg>-buildversion.txt
#
# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem.
# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository.
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
# (C) 2023
#
# @@ Meta Begin
# Application punk::winpath 0.1.0
# Meta platform tcl
# Meta license BSD
# @@ Meta End
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
## Requirements
##e.g package require frobz
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
namespace eval punk::winpath {
namespace export winpath windir cdwin cdwindir illegalname_fix illegalname_test
#\\servername\share etc or \\?\UNC\servername\share etc.
proc is_unc_path {path} {
set strcopy_path [punk::objclone $path]
set strcopy_path [string map [list \\ /] $strcopy_path] ;#normalize to forward slashes for testing purposes (and forward slashes seem to be auto-translated by windows anyway)
if {[string first "//" $strcopy_path] == 0} {
#check for "Dos device path" syntax
if {[string range $strcopy_path 0 3] in [list "//?/" "//./"]} {
#Note that //./ doesn't appear to be supported in Tcl as at 2023-08 - but //?/ works (except for //?/UNC/Server/share)
if {[string range $strcopy_path 4 6] eq "UNC"} {
return 1
} else {
#some other Dos device path. Could be a drive which is mapped to a UNC path - but the path itself isn't a unc path
return 0
}
} else {
#leading double slash and not dos device path syntax
return 1
}
}
return 0
}
#ordinary \\Servername or \\servername\share or \\servername\share\path (or forward-slash equivalent) with no dos device syntax //?/ //./ etc.
proc is_unc_path_plain {path} {
if {[is_unc_path $path]} {
if {![is_dos_device_path $path]} {
return 1
} else {
return 0
}
} else {
return 0
}
}
#int-rep path preserved - but 'file attributes', and therefor this operation, is expensive (on windows at least)
proc pwdshortname {{path {}}} {
if {$path eq ""} {
set path [pwd]
} else {
if {[file pathtype $path] eq "relative"} {
set path [file normalize $path]
}
}
return [dict get [file attributes $path] -shortname]
}
#dos device path syntax allows windows api to acces extended-length paths and filenames with illegal path characters such as trailing dots or whitespace
#(can exist on server shares and on NTFS - but standard apps can't access without dos device syntax)
proc is_dos_device_path {path} {
set strcopy_path [punk::objclone $path]
set strcopy_path [string map [list \\ /] $strcopy_path] ;#normalize to forward slashes for testing purposes (and forward slashes seem to be auto-translated by windows anyway)
if {[string range $strcopy_path 0 3] in [list "//?/" "//./"]} {
return 1
} else {
return 0
}
}
proc strip_dos_device_prefix {path} {
#it's unlikely to be valid to strip only //?/ from a //?/UNC path so check for it here and diver to strip that.
#(review.. or raise error because a //?/UNC path isn't *strictly* a UNC path? )
if {[is_unc_path $path]} {
return [strip_unc_path_prefix $path]
}
if {[is_dos_device_path $path]} {
return [string range $path 4 end]
} else {
return $path
}
}
proc strip_unc_path_prefix {path} {
if {[is_unc_path $path]} {
#//?/UNC/server/etc
set strcopy_path [punk::objclone $path]
set trimmedpath [string range $strcopy_path 7 end]
file pathtype $trimmedpath ;#shimmer it to path rep
return $trimmedpath
} elseif {is_unc_path_plain $path} {
#plain unc //server
set strcopy_path [punk::objclone $path]
set trimmedpath [string range $strcopy_path 2 end]
file pathtype $trimmedpath
return $trimmedpath
} else {
return $path
}
}
#we don't validate that path is actually illegal because we don't know the full range of such names.
#The caller can apply this to any path.
#don't test for platform here - needs to be callable from any platform for potential passing to windows (what usecase? 8.3 name is not always calculable independently)
#The utility of this is questionable. prepending a dos-device path won't make a filename with illegal characters readable by windows.
#It will need the 'shortname' at least for the illegal segment - if not the whole path
#Whilst the 8.3 name algorithm - including undocumented hash function has been reverse engineered
#- it depends on the content of the directory - as collisions cause a different name (e.g incremented number)
#- it also depends on the history of the folder
#- you can't take the current dir contents and a particular *existing* longname and determine the shortname algorithmically...
#- the shortname may have been generated during a different directory state.
#- It is then stored on disk (where?) - so access to reading the existing shortname is required.
#- An implementation of the 8.3 algorithm would only be potentially useful in determining the name that will result from adding a new file
# and would be subject to potential collisions if there are race-conditions in file creation
#- Using an 8.3 algorithm externally would be dangerous in that it could appear to work a lot of the time - but return a different file entirely sometimes.
#- Conclusion is that the 8.3 name must be retrieved rathern than calclated
proc illegalname_fix {path} {
#don't add extra dos device path syntax protection-prefix if already done
if {[is_unc_path $path]} {
error "illegalname_fix called on UNC path $path - unable to process"
}
if {[is_dos_device_path $path]} {
#we may have appended
return $path
}
#\\servername\share theoretically maps to: \\?\UNC\servername\share in protected form. https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats
#NOTE: 2023-08 on windows 10 at least \\?\UNC\Server\share doesn't work - ie we can't use illegalname_fix on UNC paths such as \\Server\share
#(but mapped drive to same path will work)
#Note that test-path cmdlet in powershell is also flaky with regards to \\?\UNC\Server paths.
#It seems prudent for now to disallow \\?\ protection for UNC paths such as \\server\etc
if {[is_unc_path $path]} {
set err ""
append err "illegalname_fix doesn't currently support UNC paths (non dos device leading double slash or //?/UNC/...)"
append err \n " - because //?/UNC/Servername/share is not supported in Tcl (and only minimally even in powershell) as at 2023. (on windows use mapped drive instead)"
error $err
}
set strcopy_path [punk::objclone $path]
#Note: path could still have leading double slash if it is a Dos device path: e.g. //?/c:/etc
if {[file pathtype $path] eq "absolute"} {
if {$path eq "~"} {
# non-normalized ~ is classified as absolute
# tilde special meaning is a bit of a nuisance.. but as it's the entire path in this case.. presumably it should be kept that way
# leave for caller to interpret it - but it's not an illegal name whether it's interpreted with special meaning or not
# unlikely this fix will be called on a plain tilde anyway
return $path
} else {
set fullpath $path
}
} else {
#set fullpath [file normalize $path] ;#very slow on windows
#set fullpath [pwd]/$path ;#will keep ./ in middle of path - not valid for dos-device paths
if {[string range $strcopy_path 0 1] eq "./"} {
set strcopy_path [string range $strcopy_path 2 end]
}
set fullpath [file join [pwd] $strcopy_path]
}
#For file I/O, the "\\?\" prefix to a path string tells the Windows APIs to disable all string parsing
# and to send the string that follows it straight to the file system.
set protect "\\\\?\\" ;# value is: \\?\ prefix
set protect2 "//?/" ;#file normalize may do this - it still works
#don't use "//./" - not currently supported in Tcl - seems to work in powershell though.
#choose //?/ as normalized version - since likely 'file normalize' will do it anyway, and experimentall, the windows API accepts both REVIEW
set result ${protect2}$fullpath
file pathtype $result ;#make it return a path rep
return $result
}
#don't test for platform here - needs to be callable from any platform for potential passing to windows
#we can create files with windows illegal names by using //?/ dos device path syntax - but we need to detect when that is required.
#
# path int-rep preserving
proc illegalname_test {path} {
#https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file
#according to the above: Use any character in the current code page for a name, including Unicode characters and characters in the extended character set (128–255), except for the following:
set reserved [list < > : \" / \\ | ? *]
#we need to exclude things like path/.. path/.
foreach seg [file split $path] {
if {$seg in [list . ..]} {
#review - what if there is a folder or file that actually has a name such as . or .. ?
#unlikely in normal use - but could done deliberately for bad reasons?
#We are unable to check for it here anyway - as this command is intended for checking the path string - not the actual path on a filesystem.
#
#/./ /../ segments don't require protection - keep checking.
continue
}
#only check for actual space as other whitespace seems to work without being stripped
#trailing tab and trailing \n or \r seem to be creatable in windows with Tcl - map to some glyph
if {[string index $seg end] in [list " " "."]} {
#windows API doesn't handle trailing dots or spaces (silently strips) - even though such files can be created on NTFS systems (or seen via samba etc)
return 1
}
}
#glob chars '* ?' are probably illegal.. but although x*y.txt and x?y.txt don't display properly (* ? replaced with some other glyph)
#- they seem to be readable from cmd and tclsh as is.
# pipe symbol also has glyph substitution and behaves the same e.g a|b.txt
#(at least with encoding system utf-8)
#todo - determine what else constitutes an illegal name according to windows APIs and requires protection with dos device syntax
return 0
}
proc test_ntfs_tunneling {f1 f2 args} {
file mkdir $f1
puts stderr "waiting 15secs..."
after 5000 {puts -nonewline stderr .}
after 5000 {puts -nonewline stderr .}
after 5000 {puts -nonewline stderr .}
after 500 {puts stderr \n}
file mkdir $f2
puts stdout "$f1 [file stat $f1]"
puts stdout "$f2 [file stat $f2]"
file delete $f1
puts stdout "renaming $f2 to $f1"
file rename $f2 $f1
puts stdout "$f1 [file stat $f1]"
}
}
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
## Ready
package provide punk::winpath [namespace eval punk::winpath {
variable version
set version 0.1.0
}]
return

299
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/punkcheck-0.1.0.tm

@ -234,10 +234,12 @@ namespace eval punkcheck {
#related - installfile_begin
#call init before we know if we are going to run the operation vs skip
method targetset_init {operation targetset} {
set known_ops [list INSTALL MODIFY DELETE VIRTUAL]
set known_ops [list QUERY INSTALL MODIFY DELETE VIRTUAL]
if {[string toupper $operation] ni $known_ops} {
error "[self] add_target unknown operation '$operation'. Known operations $known_ops"
}
set o_operation [string toupper $operation]
if {$o_operation_start_ts ne ""} {
error "[self] targetset_tart $o_operation operation already in progress. Use targetset_finished or targetset_complete to finish."
}
@ -245,18 +247,24 @@ namespace eval punkcheck {
set seconds [expr {$o_operation_start_ts / 1000000}]
set tsiso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"]
set punkcheck_file [$o_installer get_checkfile]
set punkcheck_folder [file dirname $punkcheck_file]
set relativepath_targetset [list]
foreach p $targetset {
if {[file pathtype $p] eq "absolute"} {
lappend relativepath_targetset [punkcheck::lib::path_relative [file dirname $punkcheck_file] $p]
} else {
if {$o_operation eq "VIRTUAL"} {
foreach p $targetset {
lappend relativepath_targetset $p
}
} else {
foreach p $targetset {
if {[file pathtype $p] eq "absolute"} {
lappend relativepath_targetset [punkcheck::lib::path_relative $punkcheck_folder $p]
} else {
lappend relativepath_targetset $p
}
}
}
set o_operation $operation
set fields [list\
-tsiso $tsiso\
-ts $o_operation_start_ts\
@ -280,7 +288,7 @@ namespace eval punkcheck {
#-installer and -eventid keys are added here
set new_inprogress_record [dict create tag [string toupper $operation]-INPROGRESS {*}$fields -tempcontext [my as_record] body {}]
#set existing_body [dict_getwithdefault $o_fileset_record body [list]]
#todo - look for existing "-INPROGRESS" records - mark as failed?
#todo - look for existing "-INPROGRESS" records - mark as failed or incomplete?
dict lappend o_fileset_record body $new_inprogress_record
if {$isnew} {
@ -288,15 +296,36 @@ namespace eval punkcheck {
} else {
set record_list [linsert $record_list[unset record_list] $oldposition $o_fileset_record]
}
punkcheck::save_records_to_file $record_list $punkcheck_file
if {$o_operation ne "QUERY"} {
punkcheck::save_records_to_file $record_list $punkcheck_file
}
return $o_fileset_record
}
#operation has been started
#todo - upgrade .punkcheck format to hold more than just list of SOURCE entries in each record.
# - allow arbitrary targetset_startphase <name> targetset_endphase <name> calls to store timestamps and calculate elapsed time
method targetset_started {} {
set punkcheck_folder [file dirname [$o_installer get_checkfile]]
set o_fileset_record [punkcheck::installfile_started_install $punkcheck_folder $o_fileset_record]
if {$o_operation eq "QUERY"} {
set fileinfo_body [dict get $o_fileset_record body] ;#body of FILEINFO record
set installing_record [lindex $fileinfo_body end]
set ts_start [dict get $installing_record -ts]
set ts_now [clock microseconds]
set metadata_us [expr {$ts_now - $ts_start}]
dict set installing_record -metadata_us $metadata_us
dict set installing_record -ts_start_transfer $ts_now
lset fileinfo_body end $installing_record
return [dict set o_fileset_record body $fileinfo_body]
} else {
#legacy call
#saves to .punkcheck file
return [set o_fileset_record [punkcheck::installfile_started_install $punkcheck_folder $o_fileset_record]]
}
}
method targetset_end {status args} {
set defaults [dict create\
@ -311,7 +340,6 @@ namespace eval punkcheck {
dict unset opts -note
}
set status [string toupper $status]
set statusdict [dict create OK RECORD SKIPPED SKIPPED FAILED FAILED]
if {$o_operation_start_ts eq ""} {
@ -332,6 +360,7 @@ namespace eval punkcheck {
set file_record_body [dict get $o_fileset_record body]
set installing_record [lindex $file_record_body end]
set punkcheck_file [$o_installer get_checkfile]
set punkcheck_folder [file dirname $punkcheck_file]
set record_list [punkcheck::load_records_from_file $punkcheck_file]
if {[dict exists $installing_record -ts_start_transfer]} {
set ts_start_transfer [dict get $installing_record -ts_start_transfer]
@ -345,6 +374,23 @@ namespace eval punkcheck {
dict set installing_record -elapsed_us $elapsed_us
dict unset installing_record -tempcontext
dict set installing_record tag "${o_operation}-[dict get $statusdict $status]" ;# e.g INSTALL-RECORD, INSTALL-SKIPPED
if {$o_operation in [list INSTALL MODIFY] && [dict get $statusdict $status] eq "RECORD"} {
#only calculate and store post operation target cksums on successful INSTALL or MODIFY, doesn't make sense for DELETE or VIRTUAL operations
set new_targets_cksums [list] ;#ordered list of cksums matching targetset order
set cksum_all_opts "" ;#same cksum opts for each target so we store it once
set ts_begin_cksum [clock microseconds]
foreach p $o_targets {
set tgt_cksum_info [punk::mix::base::lib::cksum_path [file join $punkcheck_folder $p]]
lappend new_targets_cksums [dict get $tgt_cksum_info cksum]
if {$cksum_all_opts eq ""} {
set cksum_all_opts [dict get $tgt_cksum_info opts]
}
}
set cksum_us [expr {[clock microseconds] - $ts_begin_cksum}]
dict set installing_record -targets_cksums $new_targets_cksums
dict set installing_record -cksum_all_opts $cksum_all_opts
dict set installing_record -cksum_us $cksum_us
}
lset file_record_body end $installing_record
dict set o_fileset_record body $file_record_body
set o_fileset_record [punkcheck::recordlist::file_record_prune $o_fileset_record]
@ -356,7 +402,9 @@ namespace eval punkcheck {
} else {
lset record_list $old_posn $o_fileset_record
}
punkcheck::save_records_to_file $record_list $punkcheck_file
if {$o_operation ne "QUERY"} {
punkcheck::save_records_to_file $record_list $punkcheck_file
}
set o_operation_start_ts ""
set o_operation ""
return $o_fileset_record
@ -372,6 +420,20 @@ namespace eval punkcheck {
set o_fileset_record [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $rel_source_path $o_fileset_record]
}
method targetset_last_complete {} {
#retrieve last completed record for the fileset ie exclude SKIPPED,INSTALL-INPROGRESS,DELETE-INPROGRESS,MODIFY-INPROGRESS
set body [punkcheck::dict_getwithdefault $o_fileset_record body [list]]
set previous_records [lrange $body 0 end]
#get last that is tagged INSTALL-RECORD,MODIFY-RECORD,DELETE-RECORD
set revlist [lreverse $previous_records]
foreach rec $revlist {
if {[dict get $rec tag] in [list "INSTALL-RECORD" "MODIFY-RECORD" "DELETE-RECORD" "VIRTUAL-RECORD"]} {
return $rec
}
}
return [list]
}
method targetset_source_changes {} {
punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $o_fileset_record body] end]
@ -551,16 +613,6 @@ namespace eval punkcheck {
method get_event {} {
return $o_active_event
}
if 0 {
method unknown {args} {
puts "[self] unknown called with args:$args"
if {[llength $args]} {
} else {
}
}
}
}
}
proc start_installer_event {punkcheck_file installername from_fullpath to_fullpath config} {
@ -722,6 +774,10 @@ namespace eval punkcheck {
} else {
set cksum_opts ""
}
#todo - accept argument of cached source cksum info (for client calling multiple targets with same source in quick succession e.g when building .vfs kits with multiple runtimes)
#if same cksum_opts - then use cached data instead of checksumming here.
#allow nonexistant as a source
set fpath [file join $punkcheck_folder $source_relpath]
if {![file exists $fpath]} {
@ -857,7 +913,7 @@ namespace eval punkcheck {
set tsnow [clock microseconds]
set elapsed_us [expr {$tsnow - $ts_start}]
dict set installing_record -elapsed_us $elapsed_us
dict set installing_record tag "SKIPPED"
dict set installing_record tag "INSTALL-SKIPPED"
lset file_record_body end $installing_record
dict set file_record body $file_record_body
@ -879,13 +935,14 @@ namespace eval punkcheck {
#then: file_record_add_installrecord
namespace eval lib {
set pkg punkcheck
namespace path ::punkcheck
proc is_file_record_inprogress {file_record} {
if {[dict get $file_record tag] ne "FILEINFO"} {
return 0
}
set installing_record [lindex [dict_getwithdefault $file_record body [list]] end]
if {[dict_getwithdefault $installing_record tag [list]] ni [list INSTALL-INPROGRESS MODIFY-INPROGRESS DELETE-INPROGRESS VIRTUAL-INPROGRESS]} {
if {[dict_getwithdefault $installing_record tag [list]] ni [list QUERY-INPROGRESS INSTALL-INPROGRESS MODIFY-INPROGRESS DELETE-INPROGRESS VIRTUAL-INPROGRESS]} {
return 0
}
return 1
@ -1021,9 +1078,16 @@ namespace eval punkcheck {
}
proc install_non_tm_files {srcdir basedir args} {
#set keys [dict keys $args]
#adjust the default anti_glob_dir_core entries so that .fossil-custom, .fossil-settings are copied
set antiglob_dir_core [punkcheck::default_antiglob_dir_core]
set posn [lsearch $antiglob_dir_core ".fossil*"]
if {$posn >=0} {
set antiglob_dir_core [lreplace $antiglob_dir_core $posn $posn]
}
set defaults [list\
-glob *\
-antiglob_file [list "*.tm" "*-buildversion.txt" "*.exe"]\
-antiglob_dir_core $antiglob_dir_core\
-installer punkcheck::install_non_tm_files\
]
set opts [dict merge $defaults $args]
@ -1070,7 +1134,8 @@ namespace eval punkcheck {
# -overwrite newer-targets will copy files with older source timestamp over newer target timestamp and those missing at the target (a form of 'restore' operation)
# -overwrite older-targets will copy files with newer source timestamp over older target timestamp and those missing at the target
# -overwrite all-targets will copy regardless of timestamp at target
# -overwrite installedsourcechanged-targets
# -overwrite installedsourcechanged-targets will copy if the target doesn't exist or the source changed
# -overwrite synced-targets will copy if the target doesn't exist or the source changed and the target cksum is the same as the last INSTALL-RECORD targets_cksums entry
# review - timestamps unreliable
# - what about slightly mismatched system clocks and mounted filesystems? caller responsibility to verify first?
# if timestamp exactly equal - should we check content-hash? This is presumably only likely to occur deliberately(maliciously?)
@ -1091,11 +1156,13 @@ namespace eval punkcheck {
# -source_checksum compare|store|comparestore|false|true where true == comparestore
# -punkcheck_folder target|source|project|<absolutepath> target is default and is generally recommended
# -punkcheck_records empty string | parsed TDL records ie {tag xxx k v} structure
# install creates FILEINFO records with a single entry in the -targets field (it is legitimate to have a list of targets for an installation operation - the oo interface supports this)
proc install {srcdir tgtdir args} {
set defaults [list\
-call-depth-internal 0\
-max_depth 1000\
-subdirlist {}\
-createdir 0\
-glob *\
-antiglob_file_core "\uFFFF"\
-antiglob_file "" \
@ -1130,11 +1197,15 @@ namespace eval punkcheck {
set max_depth [dict get $opts -max_depth]
set subdirlist [dict get $opts -subdirlist] ;# generally should be same length as CALLDEPTH - but user could prefill
set fileglob [dict get $opts -glob]
set createdir [dict get $opts -createdir] ;#defaults to zero to help avoid mistakes with initial target dir - required target subdirs are created regardless of this setting
if {$CALLDEPTH == 0} {
#expensive to normalize but we need to do it at least once
set srcdir [file normalize $srcdir]
set tgtdir [file normalize $tgtdir]
if {$createdir} {
file mkdir $tgtdir
}
#now the values we build from these will be properly cased
}
# -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
@ -1157,7 +1228,7 @@ namespace eval punkcheck {
set opt_unpublish_paths [dict get $opts -unpublish_paths] ;#todo - combine with config file in source tree .punkcheckpublish (?)
set unpublish_paths_matched [list]
# -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
set known_whats [list no-targets newer-targets older-targets all-targets installedsourcechanged-targets]
set known_whats [list no-targets newer-targets older-targets all-targets installedsourcechanged-targets synced-targets]
set overwrite_what [string tolower [dict get $opts -overwrite]]; #accept any case for value to allow emphasis by caller e.g -overwrite NEWER-TARGETS
if {$overwrite_what ni $known_whats} {
error "punkcheck::install received unrecognised value for -overwrite. Received value '$overwrite_what' vs known values '$known_whats'"
@ -1246,9 +1317,9 @@ namespace eval punkcheck {
}
if {[string match *store* $opt_source_checksum]} {
set store_cksums 1
set store_source_cksums 1
} else {
set store_cksums 0
set store_source_cksums 0
}
@ -1280,7 +1351,7 @@ namespace eval punkcheck {
#puts "testing folder - globmatchpath $unpub $relative_source_dir"
if {[globmatchpath $unpub $relative_source_dir]} {
lappend unpublish_paths_matched $current_source_dir
return [list files_copied {} files_skipped {} sources_unchanged {} punkcheck_records $punkcheck_records unpublish_paths_matched $unpublish_paths_matched]
return [list files_copied {} files_skipped {} sources_unchanged {} punkcheck_records $punkcheck_records unpublish_paths_matched $unpublish_paths_matched srcdir $srcdir tgtdir $tgtdir punkcheck_folder $punkcheck_folder]
}
}
@ -1343,6 +1414,7 @@ namespace eval punkcheck {
#puts stdout "Current target dir: $current_target_dir"
foreach m $match_list {
set new_tgt_cksum_info [list]
set relative_target_path [file join $relative_target_dir $m]
set relative_source_path [file join $relative_source_dir $m]
set punkcheck_target_relpath [file join $target_relative_to_punkcheck_dir $m]
@ -1400,10 +1472,14 @@ namespace eval punkcheck {
#this hits the filesystem for the sourcepath - gets checksums/timestamps depending on config.
#It doesn't save to .punkcheck (the only punkcheck::installfile_ method which doesn't)
set filerec [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $relative_source_path $filerec]
#changeinfo comes from last record in body - which is the record we are working on and so will always exist
set changeinfo [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $filerec body] end]]
set changed [dict get $changeinfo changed]
set unchanged [dict get $changeinfo unchanged]
if {[llength $unchanged]} {
lappend sources_unchanged $current_source_dir/$m
}
@ -1415,6 +1491,7 @@ namespace eval punkcheck {
} else {
if {![file exists $current_target_dir/$m]} {
file copy $current_source_dir/$m $current_target_dir
set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
lappend files_copied $current_source_dir/$m
incr filecount_new
} else {
@ -1422,15 +1499,48 @@ namespace eval punkcheck {
if {[llength $changed]} {
#An unrecorded installation is considered a source change (from unknown/unrecorded source to recorded)
file copy -force $current_source_dir/$m $current_target_dir
set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
lappend files_copied $current_source_dir/$m
} else {
set is_skip 1
lappend files_skipped $current_source_dir/$m
}
} elseif {$overwrite_what eq "synced-targets"} {
if {[llength $changed]} {
#only overwrite if the target checksum equals the last installed checksum (ie target is in sync with source and so hasn't been customized)
set existing_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
set is_target_unmodified_since_install 0
set target_cksum_compare "unknown"
set latest_install_record [punkcheck::recordlist::file_record_latest_installrecord $filerec] ;#may be no such record - in which case we get an empty list
if {[dict exists $latest_install_record -targets_cksums]} {
set last_install_cksum [dict get $latest_install_record -targets_cksums] ;#in this case we know there is only one as 'install' always uses targetset size of 1. (FILEINFO record per file in source folder)
if {[dict get $existing_tgt_cksum_info cksum] eq $last_install_cksum} {
set is_target_unmodified_since_install 1
set target_cksum_compare "match"
} else {
set target_cksum_compare "nomatch"
}
} else {
set target_cksum_compare "norecord"
}
if {$is_target_unmodified_since_install} {
file copy -force $current_source_dir/$m $current_target_dir
set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
lappend files_copied $current_source_dir/$m
} else {
#either cksum is different or we were unable to verify the record. Either way we can't know if the target is in sync so we must skip it
set is_skip 1
puts stderr "Skipping file copy $m target $current_target_dir/$m - require synced_target to overwrite - current target cksum compared to previous install: $target_cksum_compare"
lappend files_skipped $current_source_dir/$m
}
} else {
set is_skip 1
lappend files_skipped $current_source_dir/$m
}
} else {
set is_skip 1
puts stderr "Skipping file copy $m target $current_target_dir/$m already exists (use -overwrite all-targets to overwrite)"
#TODO! implement newer-targets older-targets
#TODO! implement newer-targets older-targets? (note ctimes/mtimes are unreliable - may not be worth implementing)
lappend files_skipped $current_source_dir/$m
}
}
@ -1440,27 +1550,31 @@ namespace eval punkcheck {
set ts_now [clock microseconds]
set elapsed_us [expr {$ts_now - $ts_start}]
if {$store_cksums} {
#if {$store_source_cksums} {
#}
set install_records [dict get $filerec body]
set current_install_record [lindex $install_records end]
#change the tag from *-INPROGRESS to INSTALL-RECORD/SKIPPED
if {$is_skip} {
set tag INSTALL-SKIPPED
} else {
set tag INSTALL-RECORD
}
dict set current_install_record tag $tag
dict set current_install_record -elapsed_us $elapsed_us
lset install_records end $current_install_record
dict set filerec body $install_records
set filerec [punkcheck::recordlist::file_record_prune $filerec] ;#prune now that tag is finalized
if {!$has_filerec} {
#not found in original recordlist - append
lappend punkcheck_records $filerec
} else {
lset punkcheck_records $existing_filerec_posn $filerec
}
set install_records [dict get $filerec body]
set current_install_record [lindex $install_records end]
#change the tag from *-INPROGRESS to INSTALL-RECORD/SKIPPED
if {$is_skip} {
set tag INSTALL-SKIPPED
} else {
set tag INSTALL-RECORD
}
dict set current_install_record tag $tag
dict set current_install_record -elapsed_us $elapsed_us
if {[llength $new_tgt_cksum_info]} {
dict set current_install_record -targets_cksums [list [dict get $new_tgt_cksum_info cksum]]
dict set current_install_record -cksum_all_opts [dict get $new_tgt_cksum_info opts]
}
lset install_records end $current_install_record
dict set filerec body $install_records
set filerec [punkcheck::recordlist::file_record_prune $filerec] ;#prune now that tag is finalized
if {!$has_filerec} {
#not found in original recordlist - append
lappend punkcheck_records $filerec
} else {
lset punkcheck_records $existing_filerec_posn $filerec
}
}
@ -1536,11 +1650,10 @@ namespace eval punkcheck {
#puts "subdirlist: $subdirlist"
if {$CALLDEPTH == 0} {
if {[llength $files_copied] || [llength $files_skipped]} {
puts stdout ">>>>>>>>>>>>>>>>>>>"
#puts stdout ">>>>>>>>>>>>>>>>>>>"
set saveresult [punkcheck::save_records_to_file $punkcheck_records $punkcheck_file]
puts stdout "[dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file"
puts stdout "copied: [llength $files_copied] skipped: [llength $files_skipped]"
puts stdout ">>>>>>>>>>>>>>>>>>>"
puts stdout "punkcheck::install [dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file copied: [llength $files_copied] skipped: [llength $files_skipped]"
#puts stdout ">>>>>>>>>>>>>>>>>>>"
} else {
#todo - write db INSTALLER record if -debug true
@ -1551,10 +1664,30 @@ namespace eval punkcheck {
}
}
return [list files_copied $files_copied files_skipped $files_skipped sources_unchanged $sources_unchanged unpublish_paths_matched $unpublish_paths_matched punkcheck_records $punkcheck_records]
return [list files_copied $files_copied files_skipped $files_skipped sources_unchanged $sources_unchanged unpublish_paths_matched $unpublish_paths_matched punkcheck_records $punkcheck_records punkcheck_folder $punkcheck_folder srcdir $srcdir tgtdir $tgtdir]
}
proc summarize_install_resultdict {resultdict} {
set msg ""
if {[dict size $resultdict]} {
set copied [dict get $resultdict files_copied]
append msg "--------------------------" \n
append msg "[dict keys $resultdict]" \n
set tgtdir [dict get $resultdict tgtdir]
set checkfolder [dict get $resultdict punkcheck_folder]
append msg "Copied [llength $copied] files from [dict get $resultdict srcdir] to [dict get $resultdict tgtdir]" \n
foreach f $copied {
append msg "COPIED [punkcheck::lib::path_relative $checkfolder $f]" \n
append msg " TO $tgtdir" \n
}
append msg "[llength [dict get $resultdict sources_unchanged]] unchanged source files" \n
append msg "[llength [dict get $resultdict files_skipped]] skipped files" \n
append msg "--------------------------" \n
}
return $msg
}
namespace eval recordlist {
set pkg punkcheck
namespace path ::punkcheck
proc records_as_target_dict {record_list} {
@ -1590,8 +1723,8 @@ namespace eval punkcheck {
}
proc file_install_record_source_changes {install_record} {
#reject INSTALLFAILED items ?
if {[dict get $install_record tag] ni [list "INSTALL-RECORD" "SKIPPED" "INSTALL-INPROGRESS" "MODIFY-INPROGRESS" "MODIFY-RECORD" "VIRTUAL-INPROGRESS" "VIRTUAL-RECORD" "DELETE-RECORD" "DELETE-INPROGRESS"]} {
error "file_install_record_source_changes bad file->install record: tag not INSTALL-RECORD|SKIPPED|INSTALL-INPROGRESS|MODIFY-RECORD|MODIFY-INPROGRESS|VIRTUAL-RECORD|VIRTUAL-INPROGRESS|DELETE-RECORD|DELETE-INPROGRESS"
if {[dict get $install_record tag] ni [list "QUERY-INPROGRESS" "INSTALL-RECORD" "INSTALL-SKIPPED" "INSTALL-INPROGRESS" "MODIFY-INPROGRESS" "MODIFY-RECORD" "MODIFY-SKIPPED" "VIRTUAL-INPROGRESS" "VIRTUAL-RECORD" "VIRTUAL-SKIPPED" "DELETE-RECORD" "DELETE-INPROGRESS" "DELETE-SKIPPED"]} {
error "file_install_record_source_changes bad install record: tag '[dict get $install_record tag]' not INSTALL-RECORD|SKIPPED|INSTALL-INPROGRESS|MODIFY-RECORD|MODIFY-INPROGRESS|VIRTUAL-RECORD|VIRTUAL-INPROGRESS|DELETE-RECORD|DELETE-INPROGRESS"
}
set source_list [dict_getwithdefault $install_record body [list]]
set changed [list]
@ -1742,15 +1875,19 @@ namespace eval punkcheck {
return $installer_record
}
proc file_record_latest_installrecord {file_record} {
tailcall file_record_latest_operationrecord INSTALL $file_record
}
proc file_record_latest_operationrecord {operation file_record} {
set operation [string toupper $operation]
if {[dict get $file_record tag] ne "FILEINFO"} {
error "file_record_latest_installrecord bad file_record: tag not FILEINFO"
error "file_record_latest_operationrecord bad file_record: tag not FILEINFO"
}
if {![dict exists $file_record body]} {
return [list]
}
set body_items [dict get $file_record body]
foreach item [lreverse $body_items] {
if {[dict get $item tag] eq "INSTALL-RECORD"} {
if {[dict get $item tag] eq "$operation-RECORD"} {
return $item
}
}
@ -1758,47 +1895,6 @@ namespace eval punkcheck {
}
#dead code?
proc file_record_add_installrecordXXX {file_record install_record} {
if {[dict get $file_record tag] ne "FILEINFO"} {
error "file_record_add_installrecord bad file_record: tag not FILEINFO"
}
#disallow '-INPROGRESS' as it's not a final tag
if {[dict get $install_record tag] ni [list "INSTALL-RECORD" "SKIPPED"]} {
error "file_record_add_installrecord bad install_record: tag not INSTALL-RECORD"
}
set keep 3
if {[dict exists $file_record -keep_installrecords]} {
set keep [dict get $file_record -keep_installrecords]
}
if {[dict exists $file_record body]} {
set body_items [dict get $file_record body]
} else {
set body_items [list]
}
lappend body_items $install_record
set kept_body_items [list]
set kcount 0
foreach item [lreverse $body_items] {
if {[dict get $item tag] eq "INSTALL-RECORD"} {
incr kcount
if {$keep < 0 || $kcount <= $keep} {
lappend kept_body_items $item
}
} else {
lappend kept_body_items $item
}
}
set kept_body_items [lreverse $kept_body_items]
dict set file_record body $kept_body_items
return $file_record
}
proc file_record_set_defaults {file_record} {
if {[dict get $file_record tag] ne "FILEINFO"} {
error "file_record_set_defaults bad file_record: tag not FILEINFO"
@ -1881,6 +1977,7 @@ namespace eval punkcheck {
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
## Ready
package provide punkcheck [namespace eval punkcheck {
set pkg punkcheck
variable version
set version 0.1.0
}]

189
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/struct/set-2.2.3.tm

@ -0,0 +1,189 @@
#----------------------------------------------------------------------
#
# sets.tcl --
#
# Definitions for the processing of sets.
#
# Copyright (c) 2004-2008 by Andreas Kupries.
#
# See the file "license.terms" for information on usage and redistribution
# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
#
# RCS: @(#) $Id: sets.tcl,v 1.17 2008/03/09 04:24:37 andreas_kupries Exp $
#
#----------------------------------------------------------------------
# @mdgen EXCLUDE: sets_c.tcl
package require Tcl 8.5-
namespace eval ::struct::set {}
# ### ### ### ######### ######### #########
## Management of set implementations.
# ::struct::set::LoadAccelerator --
#
# Loads a named implementation, if possible.
#
# Arguments:
# key Name of the implementation to load.
#
# Results:
# A boolean flag. True if the implementation
# was successfully loaded; and False otherwise.
proc ::struct::set::LoadAccelerator {key} {
variable accel
set r 0
switch -exact -- $key {
critcl {
# Critcl implementation of set requires Tcl 8.4.
if {![package vsatisfies [package provide Tcl] 8.4]} {return 0}
if {[catch {package require tcllibc}]} {return 0}
set r [llength [info commands ::struct::set_critcl]]
}
tcl {
variable selfdir
source [file join $selfdir sets_tcl.tcl]
set r 1
}
default {
return -code error "invalid accelerator/impl. package $key:\
must be one of [join [KnownImplementations] {, }]"
}
}
set accel($key) $r
return $r
}
# ::struct::set::SwitchTo --
#
# Activates a loaded named implementation.
#
# Arguments:
# key Name of the implementation to activate.
#
# Results:
# None.
proc ::struct::set::SwitchTo {key} {
variable accel
variable loaded
if {[string equal $key $loaded]} {
# No change, nothing to do.
return
} elseif {![string equal $key ""]} {
# Validate the target implementation of the switch.
if {![info exists accel($key)]} {
return -code error "Unable to activate unknown implementation \"$key\""
} elseif {![info exists accel($key)] || !$accel($key)} {
return -code error "Unable to activate missing implementation \"$key\""
}
}
# Deactivate the previous implementation, if there was any.
if {![string equal $loaded ""]} {
rename ::struct::set ::struct::set_$loaded
}
# Activate the new implementation, if there is any.
if {![string equal $key ""]} {
rename ::struct::set_$key ::struct::set
}
# Remember the active implementation, for deactivation by future
# switches.
set loaded $key
return
}
proc ::struct::set::Loaded {} {
variable loaded
return $loaded
}
# ::struct::set::Implementations --
#
# Determines which implementations are
# present, i.e. loaded.
#
# Arguments:
# None.
#
# Results:
# A list of implementation keys.
proc ::struct::set::Implementations {} {
variable accel
set res {}
foreach n [array names accel] {
if {!$accel($n)} continue
lappend res $n
}
return $res
}
# ::struct::set::KnownImplementations --
#
# Determines which implementations are known
# as possible implementations.
#
# Arguments:
# None.
#
# Results:
# A list of implementation keys. In the order
# of preference, most prefered first.
proc ::struct::set::KnownImplementations {} {
return {critcl tcl}
}
proc ::struct::set::Names {} {
return {
critcl {tcllibc based}
tcl {pure Tcl}
}
}
# ### ### ### ######### ######### #########
## Initialization: Data structures.
namespace eval ::struct::set {
variable selfdir [file dirname [info script]]
variable accel
array set accel {tcl 0 critcl 0}
variable loaded {}
}
# ### ### ### ######### ######### #########
## Initialization: Choose an implementation,
## most prefered first. Loads only one of the
## possible implementations. And activates it.
namespace eval ::struct::set {
variable e
foreach e [KnownImplementations] {
if {[LoadAccelerator $e]} {
SwitchTo $e
break
}
}
unset e
}
# ### ### ### ######### ######### #########
## Ready
namespace eval ::struct {
# Export the constructor command.
namespace export set
}
package provide struct::set 2.2.3

189
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/struct/sets.tcl

@ -0,0 +1,189 @@
#----------------------------------------------------------------------
#
# sets.tcl --
#
# Definitions for the processing of sets.
#
# Copyright (c) 2004-2008 by Andreas Kupries.
#
# See the file "license.terms" for information on usage and redistribution
# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
#
# RCS: @(#) $Id: sets.tcl,v 1.17 2008/03/09 04:24:37 andreas_kupries Exp $
#
#----------------------------------------------------------------------
# @mdgen EXCLUDE: sets_c.tcl
package require Tcl 8.5-
namespace eval ::struct::set {}
# ### ### ### ######### ######### #########
## Management of set implementations.
# ::struct::set::LoadAccelerator --
#
# Loads a named implementation, if possible.
#
# Arguments:
# key Name of the implementation to load.
#
# Results:
# A boolean flag. True if the implementation
# was successfully loaded; and False otherwise.
proc ::struct::set::LoadAccelerator {key} {
variable accel
set r 0
switch -exact -- $key {
critcl {
# Critcl implementation of set requires Tcl 8.4.
if {![package vsatisfies [package provide Tcl] 8.4]} {return 0}
if {[catch {package require tcllibc}]} {return 0}
set r [llength [info commands ::struct::set_critcl]]
}
tcl {
variable selfdir
source [file join $selfdir sets_tcl.tcl]
set r 1
}
default {
return -code error "invalid accelerator/impl. package $key:\
must be one of [join [KnownImplementations] {, }]"
}
}
set accel($key) $r
return $r
}
# ::struct::set::SwitchTo --
#
# Activates a loaded named implementation.
#
# Arguments:
# key Name of the implementation to activate.
#
# Results:
# None.
proc ::struct::set::SwitchTo {key} {
variable accel
variable loaded
if {[string equal $key $loaded]} {
# No change, nothing to do.
return
} elseif {![string equal $key ""]} {
# Validate the target implementation of the switch.
if {![info exists accel($key)]} {
return -code error "Unable to activate unknown implementation \"$key\""
} elseif {![info exists accel($key)] || !$accel($key)} {
return -code error "Unable to activate missing implementation \"$key\""
}
}
# Deactivate the previous implementation, if there was any.
if {![string equal $loaded ""]} {
rename ::struct::set ::struct::set_$loaded
}
# Activate the new implementation, if there is any.
if {![string equal $key ""]} {
rename ::struct::set_$key ::struct::set
}
# Remember the active implementation, for deactivation by future
# switches.
set loaded $key
return
}
proc ::struct::set::Loaded {} {
variable loaded
return $loaded
}
# ::struct::set::Implementations --
#
# Determines which implementations are
# present, i.e. loaded.
#
# Arguments:
# None.
#
# Results:
# A list of implementation keys.
proc ::struct::set::Implementations {} {
variable accel
set res {}
foreach n [array names accel] {
if {!$accel($n)} continue
lappend res $n
}
return $res
}
# ::struct::set::KnownImplementations --
#
# Determines which implementations are known
# as possible implementations.
#
# Arguments:
# None.
#
# Results:
# A list of implementation keys. In the order
# of preference, most prefered first.
proc ::struct::set::KnownImplementations {} {
return {critcl tcl}
}
proc ::struct::set::Names {} {
return {
critcl {tcllibc based}
tcl {pure Tcl}
}
}
# ### ### ### ######### ######### #########
## Initialization: Data structures.
namespace eval ::struct::set {
variable selfdir [file dirname [info script]]
variable accel
array set accel {tcl 0 critcl 0}
variable loaded {}
}
# ### ### ### ######### ######### #########
## Initialization: Choose an implementation,
## most prefered first. Loads only one of the
## possible implementations. And activates it.
namespace eval ::struct::set {
variable e
foreach e [KnownImplementations] {
if {[LoadAccelerator $e]} {
SwitchTo $e
break
}
}
unset e
}
# ### ### ### ######### ######### #########
## Ready
namespace eval ::struct {
# Export the constructor command.
namespace export set
}
package provide struct::set 2.2.3

93
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/struct/sets_c.tcl

@ -0,0 +1,93 @@
#----------------------------------------------------------------------
#
# sets_tcl.tcl --
#
# Definitions for the processing of sets. C implementation.
#
# Copyright (c) 2007 by Andreas Kupries.
#
# See the file "license.terms" for information on usage and redistribution
# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
#
# RCS: @(#) $Id: sets_c.tcl,v 1.3 2008/03/25 07:15:34 andreas_kupries Exp $
#
#----------------------------------------------------------------------
package require critcl
# @sak notprovided struct_setc
package provide struct_setc 2.1.1
package require Tcl 8.5-
namespace eval ::struct {
# Supporting code for the main command.
catch {
#critcl::cheaders -g
#critcl::debug memory symbols
}
critcl::cheaders sets/*.h
critcl::csources sets/*.c
critcl::ccode {
/* -*- c -*- */
#include <m.h>
}
# Main command, set creation.
critcl::ccommand set_critcl {dummy interp objc objv} {
/* Syntax - dispatcher to the sub commands.
*/
static CONST char* methods [] = {
"add", "contains", "difference", "empty",
"equal","exclude", "include", "intersect",
"intersect3", "size", "subsetof", "subtract",
"symdiff", "union",
NULL
};
enum methods {
S_add, S_contains, S_difference, S_empty,
S_equal,S_exclude, S_include, S_intersect,
S_intersect3, S_size, S_subsetof, S_subtract,
S_symdiff, S_union
};
int m;
if (objc < 2) {
Tcl_WrongNumArgs (interp, objc, objv, "cmd ?arg ...?");
return TCL_ERROR;
} else if (Tcl_GetIndexFromObj (interp, objv [1], methods, "option",
0, &m) != TCL_OK) {
return TCL_ERROR;
}
/* Dispatch to methods. They check the #args in detail before performing
* the requested functionality
*/
switch (m) {
case S_add: return sm_ADD (NULL, interp, objc, objv);
case S_contains: return sm_CONTAINS (NULL, interp, objc, objv);
case S_difference: return sm_DIFFERENCE (NULL, interp, objc, objv);
case S_empty: return sm_EMPTY (NULL, interp, objc, objv);
case S_equal: return sm_EQUAL (NULL, interp, objc, objv);
case S_exclude: return sm_EXCLUDE (NULL, interp, objc, objv);
case S_include: return sm_INCLUDE (NULL, interp, objc, objv);
case S_intersect: return sm_INTERSECT (NULL, interp, objc, objv);
case S_intersect3: return sm_INTERSECT3 (NULL, interp, objc, objv);
case S_size: return sm_SIZE (NULL, interp, objc, objv);
case S_subsetof: return sm_SUBSETOF (NULL, interp, objc, objv);
case S_subtract: return sm_SUBTRACT (NULL, interp, objc, objv);
case S_symdiff: return sm_SYMDIFF (NULL, interp, objc, objv);
case S_union: return sm_UNION (NULL, interp, objc, objv);
}
/* Not coming to this place */
}
}
# ### ### ### ######### ######### #########
## Ready

452
src/modules/punk/mix/templates/layouts/project/src/bootsupport/modules/struct/sets_tcl.tcl

@ -0,0 +1,452 @@
#----------------------------------------------------------------------
#
# sets_tcl.tcl --
#
# Definitions for the processing of sets.
#
# Copyright (c) 2004-2008 by Andreas Kupries.
#
# See the file "license.terms" for information on usage and redistribution
# of this file, and for a DISCLAIMER OF ALL WARRANTIES.
#
# RCS: @(#) $Id: sets_tcl.tcl,v 1.4 2008/03/09 04:38:47 andreas_kupries Exp $
#
#----------------------------------------------------------------------
package require Tcl 8.5-
namespace eval ::struct::set {
# Only export one command, the one used to instantiate a new tree
namespace export set_tcl
}
##########################
# Public functions
# ::struct::set::set --
#
# Command that access all set commands.
#
# Arguments:
# cmd Name of the subcommand to dispatch to.
# args Arguments for the subcommand.
#
# Results:
# Whatever the result of the subcommand is.
proc ::struct::set::set_tcl {cmd args} {
# Do minimal args checks here
if { [llength [info level 0]] == 1 } {
return -code error "wrong # args: should be \"$cmd ?arg arg ...?\""
}
::set sub S_$cmd
if { [llength [info commands ::struct::set::$sub]] == 0 } {
::set optlist [info commands ::struct::set::S_*]
::set xlist {}
foreach p $optlist {
lappend xlist [string range $p 17 end]
}
return -code error \
"bad option \"$cmd\": must be [linsert [join [lsort $xlist] ", "] "end-1" "or"]"
}
return [uplevel 1 [linsert $args 0 ::struct::set::$sub]]
}
##########################
# Implementations of the functionality.
#
# ::struct::set::S_empty --
#
# Determines emptiness of the set
#
# Parameters:
# set -- The set to check for emptiness.
#
# Results:
# A boolean value. True indicates that the set is empty.
#
# Side effects:
# None.
#
# Notes:
proc ::struct::set::S_empty {set} {
return [expr {[llength $set] == 0}]
}
# ::struct::set::S_size --
#
# Computes the cardinality of the set.
#
# Parameters:
# set -- The set to inspect.
#
# Results:
# An integer greater than or equal to zero.
#
# Side effects:
# None.
proc ::struct::set::S_size {set} {
return [llength [Cleanup $set]]
}
# ::struct::set::S_contains --
#
# Determines if the item is in the set.
#
# Parameters:
# set -- The set to inspect.
# item -- The element to look for.
#
# Results:
# A boolean value. True indicates that the element is present.
#
# Side effects:
# None.
proc ::struct::set::S_contains {set item} {
return [expr {[lsearch -exact $set $item] >= 0}]
}
# ::struct::set::S_union --
#
# Computes the union of the arguments.
#
# Parameters:
# args -- List of sets to unify.
#
# Results:
# The union of the arguments.
#
# Side effects:
# None.
proc ::struct::set::S_union {args} {
switch -exact -- [llength $args] {
0 {return {}}
1 {return [lindex $args 0]}
}
foreach setX $args {
foreach x $setX {::set ($x) {}}
}
return [array names {}]
}
# ::struct::set::S_intersect --
#
# Computes the intersection of the arguments.
#
# Parameters:
# args -- List of sets to intersect.
#
# Results:
# The intersection of the arguments
#
# Side effects:
# None.
proc ::struct::set::S_intersect {args} {
switch -exact -- [llength $args] {
0 {return {}}
1 {return [lindex $args 0]}
}
::set res [lindex $args 0]
foreach set [lrange $args 1 end] {
if {[llength $res] && [llength $set]} {
::set res [Intersect $res $set]
} else {
# Squash 'res'. Otherwise we get the wrong result if res
# is not empty, but 'set' is.
::set res {}
break
}
}
return $res
}
proc ::struct::set::Intersect {A B} {
if {[llength $A] == 0} {return {}}
if {[llength $B] == 0} {return {}}
# This is slower than local vars, but more robust
if {[llength $B] > [llength $A]} {
::set res $A
::set A $B
::set B $res
}
::set res {}
foreach x $A {::set ($x) {}}
foreach x $B {
if {[info exists ($x)]} {
lappend res $x
}
}
return $res
}
# ::struct::set::S_difference --
#
# Compute difference of two sets.
#
# Parameters:
# A, B -- Sets to compute the difference for.
#
# Results:
# A - B
#
# Side effects:
# None.
proc ::struct::set::S_difference {A B} {
if {[llength $A] == 0} {return {}}
if {[llength $B] == 0} {return $A}
array set tmp {}
foreach x $A {::set tmp($x) .}
foreach x $B {catch {unset tmp($x)}}
return [array names tmp]
}
if {0} {
# Tcllib SF Bug 1002143. We cannot use the implementation below.
# It will treat set elements containing '(' and ')' as array
# elements, and this screws up the storage of elements as the name
# of local vars something fierce. No way around this. Disabling
# this code and always using the other implementation (s.a.) is
# the only possible fix.
if {[package vcompare [package provide Tcl] 8.4] < 0} {
# Tcl 8.[23]. Use explicit array to perform the operation.
} else {
# Tcl 8.4+, has 'unset -nocomplain'
proc ::struct::set::S_difference {A B} {
if {[llength $A] == 0} {return {}}
if {[llength $B] == 0} {return $A}
# Get the variable B out of the way, avoid collisions
# prepare for "pure list optimization"
::set ::struct::set::tmp [lreplace $B -1 -1 unset -nocomplain]
unset B
# unset A early: no local variables left
foreach [lindex [list $A [unset A]] 0] {.} {break}
eval $::struct::set::tmp
return [info locals]
}
}
}
# ::struct::set::S_symdiff --
#
# Compute symmetric difference of two sets.
#
# Parameters:
# A, B -- The sets to compute the s.difference for.
#
# Results:
# The symmetric difference of the two input sets.
#
# Side effects:
# None.
proc ::struct::set::S_symdiff {A B} {
# symdiff == (A-B) + (B-A) == (A+B)-(A*B)
if {[llength $A] == 0} {return $B}
if {[llength $B] == 0} {return $A}
return [S_union \
[S_difference $A $B] \
[S_difference $B $A]]
}
# ::struct::set::S_intersect3 --
#
# Return intersection and differences for two sets.
#
# Parameters:
# A, B -- The sets to inspect.
#
# Results:
# List containing A*B, A-B, and B-A
#
# Side effects:
# None.
proc ::struct::set::S_intersect3 {A B} {
return [list \
[S_intersect $A $B] \
[S_difference $A $B] \
[S_difference $B $A]]
}
# ::struct::set::S_equal --
#
# Compares two sets for equality.
#
# Parameters:
# a First set to compare.
# b Second set to compare.
#
# Results:
# A boolean. True if the lists are equal.
#
# Side effects:
# None.
proc ::struct::set::S_equal {A B} {
::set A [Cleanup $A]
::set B [Cleanup $B]
# Equal if of same cardinality and difference is empty.
if {[::llength $A] != [::llength $B]} {return 0}
return [expr {[llength [S_difference $A $B]] == 0}]
}
proc ::struct::set::Cleanup {A} {
# unset A to avoid collisions
if {[llength $A] < 2} {return $A}
# We cannot use variables to avoid an explicit array. The set
# elements may look like namespace vars (i.e. contain ::), and
# such elements break that, cannot be proc-local variables.
array set S {}
foreach item $A {set S($item) .}
return [array names S]
}
# ::struct::set::S_include --
#
# Add an element to a set.
#
# Parameters:
# Avar -- Reference to the set variable to extend.
# element -- The item to add to the set.
#
# Results:
# None.
#
# Side effects:
# The set in the variable referenced by Avar is extended
# by the element (if the element was not already present).
proc ::struct::set::S_include {Avar element} {
# Avar = Avar + {element}
upvar 1 $Avar A
if {![info exists A] || ![S_contains $A $element]} {
lappend A $element
}
return
}
# ::struct::set::S_exclude --
#
# Remove an element from a set.
#
# Parameters:
# Avar -- Reference to the set variable to shrink.
# element -- The item to remove from the set.
#
# Results:
# None.
#
# Side effects:
# The set in the variable referenced by Avar is shrunk,
# the element remove (if the element was actually present).
proc ::struct::set::S_exclude {Avar element} {
# Avar = Avar - {element}
upvar 1 $Avar A
if {![info exists A]} {return -code error "can't read \"$Avar\": no such variable"}
while {[::set pos [lsearch -exact $A $element]] >= 0} {
::set A [lreplace [K $A [::set A {}]] $pos $pos]
}
return
}
# ::struct::set::S_add --
#
# Add a set to a set. Similar to 'union', but the first argument
# is a variable.
#
# Parameters:
# Avar -- Reference to the set variable to extend.
# B -- The set to add to the set in Avar.
#
# Results:
# None.
#
# Side effects:
# The set in the variable referenced by Avar is extended
# by all the elements in B.
proc ::struct::set::S_add {Avar B} {
# Avar = Avar + B
upvar 1 $Avar A
if {![info exists A]} {set A {}}
::set A [S_union [K $A [::set A {}]] $B]
return
}
# ::struct::set::S_subtract --
#
# Remove a set from a set. Similar to 'difference', but the first argument
# is a variable.
#
# Parameters:
# Avar -- Reference to the set variable to shrink.
# B -- The set to remove from the set in Avar.
#
# Results:
# None.
#
# Side effects:
# The set in the variable referenced by Avar is shrunk,
# all elements of B are removed.
proc ::struct::set::S_subtract {Avar B} {
# Avar = Avar - B
upvar 1 $Avar A
if {![info exists A]} {return -code error "can't read \"$Avar\": no such variable"}
::set A [S_difference [K $A [::set A {}]] $B]
return
}
# ::struct::set::S_subsetof --
#
# A predicate checking if the first set is a subset
# or equal to the second set.
#
# Parameters:
# A -- The possible subset.
# B -- The set to compare to.
#
# Results:
# A boolean value, true if A is subset of or equal to B
#
# Side effects:
# None.
proc ::struct::set::S_subsetof {A B} {
# A subset|== B <=> (A == A*B)
return [S_equal $A [S_intersect $A $B]]
}
# ::struct::set::K --
# Performance helper command.
proc ::struct::set::K {x y} {::set x}
# ### ### ### ######### ######### #########
## Ready
namespace eval ::struct {
# Put 'set::set' into the general structure namespace
# for pickup by the main management.
namespace import -force set::set_tcl
}

31
src/modules/punk/mix/templates/layouts/project/src/doc/include/changes_0.1.inc

@ -1,25 +1,28 @@
[subsection {Changes for version 0.1}]
[vset punkshell_project https://www.gitea1.intx.com.au/jn/punkshell]
This release 0.1 of project %project%
[para] In detail:
[para] Summary
[list_begin enumerated]
[comment {- - -- --- ----- -------- ------------- ---------------------}]
[enum] %project% requires Tcl 8.6 or higher. Tcl 8.5 or less is not
supported.
[comment {- - -- --- ----- -------- ------------- ---------------------}]
[comment {Please consider retaining a link to PunkShell to support the project}]
[enum] This project uses [uri [vset punkshell_project] {PunkShell}] as a deployment management and documentation tool.
[comment {- - -- --- ----- -------- ------------- ---------------------}]
[enum] feature 1
[enum] feature 2
[list_end]
[para] Summary
[para] In detail:
[list_begin enumerated]
[enum] feature 1
[enum] feature 2
[comment {- - -- --- ----- -------- ------------- ---------------------}]
[enum] %project% requires Tcl 8.6 or higher. Tcl 8.5 or less is not
supported.
[comment {- - -- --- ----- -------- ------------- ---------------------}]
[enum]
[comment {- - -- --- ----- -------- ------------- ---------------------}]
[list_end]
[comment {- - -- --- ----- -------- ------------- ---------------------}]
[list_end]

183
src/modules/punk/mix/templates/layouts/project/src/make.tcl

@ -2,8 +2,6 @@
#
#make any tclkits and modules in src and place them and associated data files/scripts in the parent folder of src.
#e.g in 'bin' and 'modules' folders at same level as 'src' folder.
#It is assumed the src folder has been placed somewhere where appropriate
#(e.g not in /usr or c:/ - unless you intend it to directly make and place folders and files in those locations)
set hashline "# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###"
puts $hashline
@ -25,13 +23,18 @@ if {"::try" ni [info commands ::try]} {
#------------------------------------------------------------------------------
#Module loading from /src/bootsupport or src/*.vfs if script is within a .vfs folder
#------------------------------------------------------------------------------
#If the there is a folder directly under the current directory /src/bootsupport/modules which contains .tm files when the starts
#If the there is a folder under the current directory /src/bootsupport/modules which contains .tm files
# - then it will attempt to preference these modules
# This allows a source update via 'fossil update' 'git pull' etc to pull in support modules for the make script
# and load these in preference to ones that may have been in the interps tcl::tm::list or auto_path due to environment variables
# This allows a source update via 'fossil update' 'git pull' etc to pull in a minimal set of support modules for the make script
# and load these in preference to ones that may have been in the interp's tcl::tm::list or auto_path due to environment variables
set startdir [pwd]
set bootsupport_mod [file join $startdir src bootsupport modules]
set bootsupport_lib [file join $startdir src bootsupport lib]
if {[file exists [file join $startdir src bootsupport]]} {
set bootsupport_mod [file join $startdir src bootsupport modules]
set bootsupport_lib [file join $startdir src bootsupport lib]
} else {
set bootsupport_mod [file join $startdir bootsupport modules]
set bootsupport_lib [file join $startdir bootsupport lib]
}
if {[file exists $bootsupport_mod] || [file exists $bootsupport_lib]} {
set original_tm_list [tcl::tm::list]
@ -60,31 +63,16 @@ if {[file exists $bootsupport_mod] || [file exists $bootsupport_lib]} {
}
#todo - review usecase
if {[string match "*.vfs/*" [info script]]} {
#src/xxx.vfs/lib/app-punk/repl.tcl
#we assume if calling directly into .vfs that the user would prefer to use src/modules - so go up 4 levels
set modulefolder [file dirname [file dirname [file dirname [file dirname [info script]]]]]/modules
} else {
# .../bin/punkXX.exe look for ../modules (i.e modules folder at same level as bin folder)
set modulefolder [file dirname [file dirname [info nameofexecutable]]]/modules
}
if {[file exists $modulefolder]} {
tcl::tm::add $modulefolder
} else {
puts stderr "Warning unable to find module folder at: $modulefolder"
}
if {[file exists [pwd]/modules]} {
tcl::tm::add [pwd]/modules
}
#package require Thread
#These are strong dependencies
# - the repl requires Threading and punk,shellfilter,shellrun to call and display properly.
# - the full repl requires Threading and punk,shellfilter,shellrun to call and display properly.
# tm list already indexed - need 'package forget' to find modules based on current tcl::tm::list
#These are strong dependencies
package forget punk::mix
package require punk::mix
package forget punk::repo
@ -144,6 +132,8 @@ proc punkmake_gethelp {args} {
append h " $scriptname project ?-k?" \n
append h " - this is the literal word project - and confirms you want to run the project build" \n
append h " - the optional -k flag will terminate processes running as the executable being built (if applicable)" \n \n
append h " $scriptname bootsupport" \n
append h " - update the src/bootsupport modules as well as the mixtemplates/layouts/<layoutname>/src/bootsupport modules if the folder exists" \n \n
append h " $scriptname get-project-info" \n
append h " - show the name and base folder of the project to be built" \n
append h "" \n
@ -189,7 +179,7 @@ if {[llength $commands_found] != 1 } {
}
if {$do_help} {
puts stderr [punkmake_gethelp]
exit 1
exit 0
}
set ::punkmake::command [lindex $commands_found 0]
@ -224,6 +214,8 @@ if {![string length [set projectroot [punk::repo::find_project $scriptfolder]]]}
}
set sourcefolder $projectroot/src
if {$::punkmake::command eq "get-project-info"} {
puts stdout "- -- --- --- --- --- --- --- --- --- ---"
puts stdout "- -- get-project-info -- -"
@ -260,9 +252,128 @@ if {$::punkmake::command eq "shell"} {
}
if {$::punkmake::command eq "bootsupport"} {
puts "projectroot: $projectroot"
puts "script: [info script]"
#puts "-- [tcl::tm::list] --"
puts stdout "Updating bootsupport from local files"
proc bootsupport_localupdate {projectroot} {
set bootsupport_modules [list]
set bootsupport_config $projectroot/src/bootsupport/include_modules.config ;#
if {[file exists $bootsupport_config]} {
source $bootsupport_config ;#populate $bootsupport_modules with project-specific list
if {![llength $bootsupport_modules]} {
puts stderr "No local bootsupport modules configured for updating"
return
}
set targetroot $projectroot/src/bootsupport/modules
if {[catch {
#----------
set boot_installer [punkcheck::installtrack new make.tcl $projectroot/src/bootsupport/.punkcheck]
$boot_installer set_source_target $projectroot $projectroot/src/bootsupport
set boot_event [$boot_installer start_event {-make_step bootsupport}]
#----------
} errM]} {
puts stderr "Unable to use punkcheck for bootsupport error: $errM"
set boot_event ""
}
exit 1
foreach {relpath module} $bootsupport_modules {
set module [string trim $module :]
set module_subpath [string map [list :: /] [namespace qualifiers $module]]
set srclocation [file join $projectroot $relpath $module_subpath]
#puts stdout "$relpath $module $module_subpath $srclocation"
set pkgmatches [glob -nocomplain -dir $srclocation -tail [namespace tail $module]-*]
#lsort won't sort version numbers properly e.g with -dictionary 0.1.1 comes before 0.1
if {![llength $pkgmatches]} {
puts stderr "Missing source for bootsupport module $module - not found in $srclocation"
continue
}
set latestfile [lindex $pkgmatches 0]
set latestver [lindex [split [file rootname $latestfile] -] 1]
foreach m $pkgmatches {
lassign [split [file rootname $m] -] _pkg ver
#puts "comparing $ver vs $latestver"
if {[package vcompare $ver $latestver] == 1} {
set latestver $ver
set latestfile $m
}
}
set srcfile [file join $srclocation $latestfile]
set tgtfile [file join $targetroot $module_subpath $latestfile]
if {$boot_event ne ""} {
#----------
$boot_event targetset_init INSTALL $tgtfile
$boot_event targetset_addsource $srcfile
#----------
if {\
[llength [dict get [$boot_event targetset_source_changes] changed]]\
|| [llength [$boot_event get_targets_exist]] < [llength [$boot_event get_targets]]\
} {
$boot_event targetset_started
# -- --- --- --- --- ---
puts "BOOTSUPPORT update: $srcfile -> $tgtfile"
if {[catch {
file copy -force $srcfile $tgtfile
} errM]} {
$boot_event targetset_end FAILED
} else {
$boot_event targetset_end OK
}
# -- --- --- --- --- ---
} else {
puts -nonewline stderr "."
$boot_event targetset_end SKIPPED
}
$boot_event end
} else {
file copy -force $srcfile $tgtfile
}
}
if {$boot_event ne ""} {
puts \n
$boot_event destroy
$boot_installer destroy
}
}
}
bootsupport_localupdate $projectroot
#/modules/punk/mix/templates/layouts only applies if the project has it's own copy of the punk/mix modules. Generally this should only apply to the punkshell project itself.
set layout_bases [list\
$sourcefolder/mixtemplates/layouts\
$sourcefolder/modules/punk/mix/templates/layouts\
]
foreach project_layout_base $layout_bases {
if {[file exists $project_layout_base]} {
set project_layouts [glob -nocomplain -dir $project_layout_base -type d -tail *]
foreach layoutname $project_layouts {
if {[file exists [file join $project_layout_base $layoutname/src/bootsupport]]} {
set unpublish [list\
README.md\
]
set sourcemodules $projectroot/src/bootsupport/modules
set targetroot [file join $project_layout_base $layoutname/src/bootsupport/modules]
file mkdir $targetroot
puts stdout "BOOTSUPPORT layouts/$layoutname: copying from $sourcemodules to $targetroot (if source file changed)"
set resultdict [punkcheck::install $sourcemodules $targetroot -overwrite installedsourcechanged-targets -unpublish_paths $unpublish]
puts stdout [punkcheck::summarize_install_resultdict $resultdict]
flush stdout
}
}
} else {
puts stderr "No layout base at $project_layout_base"
}
}
puts stdout " bootsupport done "
flush stderr
flush stdout
#punk86 can hang if calling make.tcl via 'run' without this 'after' delay. punk87 unaffected. cause unknown.
#after 500
::exit 0
}
@ -273,7 +384,6 @@ if {$::punkmake::command ne "project"} {
}
set sourcefolder $projectroot/src
#only a single consolidated /modules folder used for target
set target_modules_base $projectroot/modules
@ -749,8 +859,14 @@ foreach vfs $vfs_folders {
} errMsg]} {
puts stderr "$killcmd returned an error:"
puts stderr $errMsg
puts stderr "(try '[info script] -k' option to force kill)"
exit 4
if {!$forcekill} {
puts stderr "(try '[info script] -k' option to force kill)"
}
#avoid exiting if the kill failure was because the task has already exited
#review - *no running instance* works with windows taskkill - "*No such process*" works with kill -9 on FreeBSD and linux - other platforms?
if {![string match "*no running instance*" $errMsg] && ![string match "*No such process*" $errMsg]} {
exit 4
}
} else {
puts stderr "$killcmd ran without error"
incr count_killed
@ -790,6 +906,9 @@ foreach vfs $vfs_folders {
$bin_installer set_source_target $buildfolder $deployment_folder
set bin_event [$bin_installer start_event {-make-step final_kit_install}]
$bin_event targetset_init INSTALL $deployment_folder/$targetkit
#todo - move final deployment step outside of the build vfs loop? (final deployment can fail and then isn't rerun even though _build and deployed versions differ, unless .vfs modified again)
#set last_completion [$bin_event targetset_last_complete]
$bin_event targetset_addsource $buildfolder/$targetkit
$bin_event targetset_started
# -- ----------
@ -802,7 +921,6 @@ foreach vfs $vfs_folders {
file delete $deployment_folder/$targetkit
} errMsg]} {
puts stderr "deletion of deployed version at $deployment_folder/$targetkit failed: $errMsg"
#exit 5
set delete_failed 1
}
}
@ -818,6 +936,7 @@ foreach vfs $vfs_folders {
# -- ----------
} else {
$bin_event targetset_end FAILED -note "could not delete"
exit 5
}
$bin_event destroy
$bin_installer destroy

24
src/modules/punk/mix/templates/layouts/project/src/mixtemplates/layouts/basic/src/bootsupport/modules/README.md

@ -0,0 +1,24 @@
This is primarily for tcl .tm modules required for your bootstrapping/make/build process.
It could include other files necessary for this process.
The .tm modules here may be required for your build script if it intended the installation operator uses an existing tclsh or other shell as opposed to a tclkit you may have for distribution which is more likely to include necessary libraries.
The modules here are loaded by your initialisation scripts and so can be a snapshot of different versions than those in your project src.
The modules can be your own, or 3rd party such as individual items from tcllib.
You can copy modules from a running punk shell to this location using the pmix command.
e.g
>pmix visible_lib_copy_to_modulefolder some::module::lib bootsupport
The pmix command will help you pick the latest version, and will create any necessary file structure matching the namespace of the package.
e.g the result might be a file such as
<projectname>/src/bootsupport/some/module/lib-0.1.tm
The originating library may not yet be in .tm form.
You can copy a pkgIndex.tcl based library that is composed of a single .tcl file the same way using the above process and it will automatically name and file it appropriately but you need to check that the library doesn't require/load additional files - and that it is Tcl script only.
Always verify that the library is copyable in this manner and test in a shell with tcl::tm::path pointed to ./bootsupport that it works.

4
src/modules/punk/mix/templates/layouts/project/src/sample.vfs/modules/shellthread-1.6.tm

@ -604,6 +604,10 @@ namespace eval shellthread::manager {
return $taginfo_list
}
#TODO - important.
#REVIEW!
#since moving to the unsubscribe mechansm - close_worker $source isn't being called
# - we need to set a limit to the number of free threads and shut down excess when detected during unsubscription
#instruction to shut-down the thread that has this source.
proc close_worker {source {timeout 2500}} {
variable workers

17
src/modules/punk/overlay-0.1.tm

@ -89,7 +89,11 @@ namespace eval ::punk::overlay {
#This won't stop the commandset being used with other cli/bases unless the import is done by looking up the callers namespace.
#The basic principle is that the commandset is loaded into the caller(s) with a prefix
#- but commandsets should explicitly package require if they have any backwards dependencies on cli/base (which they may or may not be loaded into)
proc import_commandset {prefix cmdnamespace} {
proc import_commandset {prefix separator cmdnamespace} {
set bad_seps [list "::"]
if {$separator in $bad_seps} {
error "import_commandset invalid separator '$separator'"
}
#namespace may or may not be a package
# allow with or without leading ::
if {[string range $cmdnamespace 0 1] eq "::"} {
@ -110,7 +114,7 @@ namespace eval ::punk::overlay {
} else {
set provinfo "(package $cmdpackage not present)"
}
error "punk::mix::base::lib::import_commandset supplied namespace '$cmdnamespace' doesn't exist. $provinfo Usage: import_commandset prefix namespace"
error "punk::overlay::import_commandset supplied namespace '$cmdnamespace' doesn't exist. $provinfo Usage: import_commandset prefix separator namespace"
}
}
@ -130,12 +134,17 @@ namespace eval ::punk::overlay {
if {[catch {
namespace eval ${nscaller}::temp_import [list namespace import ${cmdnamespace}::*]
foreach cmd [info commands ${nscaller}::temp_import::*] {
set import_as ${nscaller}::$prefix[namespace tail $cmd]
set cmdtail [namespace tail $cmd]
if {$cmdtail eq "_default"} {
set import_as ${nscaller}::${prefix}
} else {
set import_as ${nscaller}::${prefix}${separator}${cmdtail}
}
rename $cmd $import_as
lappend imported_commands $import_as
}
} errM]} {
puts stderr "Error loading commandset $prefix $cmdnamespace"
puts stderr "Error loading commandset $prefix $separator $cmdnamespace"
puts stderr "err: $errM"
}
return $imported_commands

3
src/modules/punk/repl-0.1.tm

@ -133,7 +133,8 @@ package require term::ansi::code::ctrl
if {$::tcl_platform(platform) eq "windows"} {
package require zzzload
zzzload::pkg_require twapi
after idle [list after 2000 {
after idle [list after 1000 {
#puts stdout "===============repl loading twapi==========="
zzzload::pkg_wait twapi
if {![catch {package require twapi}]} {

299
src/modules/punkcheck-0.1.0.tm

@ -234,10 +234,12 @@ namespace eval punkcheck {
#related - installfile_begin
#call init before we know if we are going to run the operation vs skip
method targetset_init {operation targetset} {
set known_ops [list INSTALL MODIFY DELETE VIRTUAL]
set known_ops [list QUERY INSTALL MODIFY DELETE VIRTUAL]
if {[string toupper $operation] ni $known_ops} {
error "[self] add_target unknown operation '$operation'. Known operations $known_ops"
}
set o_operation [string toupper $operation]
if {$o_operation_start_ts ne ""} {
error "[self] targetset_tart $o_operation operation already in progress. Use targetset_finished or targetset_complete to finish."
}
@ -245,18 +247,24 @@ namespace eval punkcheck {
set seconds [expr {$o_operation_start_ts / 1000000}]
set tsiso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"]
set punkcheck_file [$o_installer get_checkfile]
set punkcheck_folder [file dirname $punkcheck_file]
set relativepath_targetset [list]
foreach p $targetset {
if {[file pathtype $p] eq "absolute"} {
lappend relativepath_targetset [punkcheck::lib::path_relative [file dirname $punkcheck_file] $p]
} else {
if {$o_operation eq "VIRTUAL"} {
foreach p $targetset {
lappend relativepath_targetset $p
}
} else {
foreach p $targetset {
if {[file pathtype $p] eq "absolute"} {
lappend relativepath_targetset [punkcheck::lib::path_relative $punkcheck_folder $p]
} else {
lappend relativepath_targetset $p
}
}
}
set o_operation $operation
set fields [list\
-tsiso $tsiso\
-ts $o_operation_start_ts\
@ -280,7 +288,7 @@ namespace eval punkcheck {
#-installer and -eventid keys are added here
set new_inprogress_record [dict create tag [string toupper $operation]-INPROGRESS {*}$fields -tempcontext [my as_record] body {}]
#set existing_body [dict_getwithdefault $o_fileset_record body [list]]
#todo - look for existing "-INPROGRESS" records - mark as failed?
#todo - look for existing "-INPROGRESS" records - mark as failed or incomplete?
dict lappend o_fileset_record body $new_inprogress_record
if {$isnew} {
@ -288,15 +296,36 @@ namespace eval punkcheck {
} else {
set record_list [linsert $record_list[unset record_list] $oldposition $o_fileset_record]
}
punkcheck::save_records_to_file $record_list $punkcheck_file
if {$o_operation ne "QUERY"} {
punkcheck::save_records_to_file $record_list $punkcheck_file
}
return $o_fileset_record
}
#operation has been started
#todo - upgrade .punkcheck format to hold more than just list of SOURCE entries in each record.
# - allow arbitrary targetset_startphase <name> targetset_endphase <name> calls to store timestamps and calculate elapsed time
method targetset_started {} {
set punkcheck_folder [file dirname [$o_installer get_checkfile]]
set o_fileset_record [punkcheck::installfile_started_install $punkcheck_folder $o_fileset_record]
if {$o_operation eq "QUERY"} {
set fileinfo_body [dict get $o_fileset_record body] ;#body of FILEINFO record
set installing_record [lindex $fileinfo_body end]
set ts_start [dict get $installing_record -ts]
set ts_now [clock microseconds]
set metadata_us [expr {$ts_now - $ts_start}]
dict set installing_record -metadata_us $metadata_us
dict set installing_record -ts_start_transfer $ts_now
lset fileinfo_body end $installing_record
return [dict set o_fileset_record body $fileinfo_body]
} else {
#legacy call
#saves to .punkcheck file
return [set o_fileset_record [punkcheck::installfile_started_install $punkcheck_folder $o_fileset_record]]
}
}
method targetset_end {status args} {
set defaults [dict create\
@ -311,7 +340,6 @@ namespace eval punkcheck {
dict unset opts -note
}
set status [string toupper $status]
set statusdict [dict create OK RECORD SKIPPED SKIPPED FAILED FAILED]
if {$o_operation_start_ts eq ""} {
@ -332,6 +360,7 @@ namespace eval punkcheck {
set file_record_body [dict get $o_fileset_record body]
set installing_record [lindex $file_record_body end]
set punkcheck_file [$o_installer get_checkfile]
set punkcheck_folder [file dirname $punkcheck_file]
set record_list [punkcheck::load_records_from_file $punkcheck_file]
if {[dict exists $installing_record -ts_start_transfer]} {
set ts_start_transfer [dict get $installing_record -ts_start_transfer]
@ -345,6 +374,23 @@ namespace eval punkcheck {
dict set installing_record -elapsed_us $elapsed_us
dict unset installing_record -tempcontext
dict set installing_record tag "${o_operation}-[dict get $statusdict $status]" ;# e.g INSTALL-RECORD, INSTALL-SKIPPED
if {$o_operation in [list INSTALL MODIFY] && [dict get $statusdict $status] eq "RECORD"} {
#only calculate and store post operation target cksums on successful INSTALL or MODIFY, doesn't make sense for DELETE or VIRTUAL operations
set new_targets_cksums [list] ;#ordered list of cksums matching targetset order
set cksum_all_opts "" ;#same cksum opts for each target so we store it once
set ts_begin_cksum [clock microseconds]
foreach p $o_targets {
set tgt_cksum_info [punk::mix::base::lib::cksum_path [file join $punkcheck_folder $p]]
lappend new_targets_cksums [dict get $tgt_cksum_info cksum]
if {$cksum_all_opts eq ""} {
set cksum_all_opts [dict get $tgt_cksum_info opts]
}
}
set cksum_us [expr {[clock microseconds] - $ts_begin_cksum}]
dict set installing_record -targets_cksums $new_targets_cksums
dict set installing_record -cksum_all_opts $cksum_all_opts
dict set installing_record -cksum_us $cksum_us
}
lset file_record_body end $installing_record
dict set o_fileset_record body $file_record_body
set o_fileset_record [punkcheck::recordlist::file_record_prune $o_fileset_record]
@ -356,7 +402,9 @@ namespace eval punkcheck {
} else {
lset record_list $old_posn $o_fileset_record
}
punkcheck::save_records_to_file $record_list $punkcheck_file
if {$o_operation ne "QUERY"} {
punkcheck::save_records_to_file $record_list $punkcheck_file
}
set o_operation_start_ts ""
set o_operation ""
return $o_fileset_record
@ -372,6 +420,20 @@ namespace eval punkcheck {
set o_fileset_record [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $rel_source_path $o_fileset_record]
}
method targetset_last_complete {} {
#retrieve last completed record for the fileset ie exclude SKIPPED,INSTALL-INPROGRESS,DELETE-INPROGRESS,MODIFY-INPROGRESS
set body [punkcheck::dict_getwithdefault $o_fileset_record body [list]]
set previous_records [lrange $body 0 end]
#get last that is tagged INSTALL-RECORD,MODIFY-RECORD,DELETE-RECORD
set revlist [lreverse $previous_records]
foreach rec $revlist {
if {[dict get $rec tag] in [list "INSTALL-RECORD" "MODIFY-RECORD" "DELETE-RECORD" "VIRTUAL-RECORD"]} {
return $rec
}
}
return [list]
}
method targetset_source_changes {} {
punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $o_fileset_record body] end]
@ -551,16 +613,6 @@ namespace eval punkcheck {
method get_event {} {
return $o_active_event
}
if 0 {
method unknown {args} {
puts "[self] unknown called with args:$args"
if {[llength $args]} {
} else {
}
}
}
}
}
proc start_installer_event {punkcheck_file installername from_fullpath to_fullpath config} {
@ -722,6 +774,10 @@ namespace eval punkcheck {
} else {
set cksum_opts ""
}
#todo - accept argument of cached source cksum info (for client calling multiple targets with same source in quick succession e.g when building .vfs kits with multiple runtimes)
#if same cksum_opts - then use cached data instead of checksumming here.
#allow nonexistant as a source
set fpath [file join $punkcheck_folder $source_relpath]
if {![file exists $fpath]} {
@ -857,7 +913,7 @@ namespace eval punkcheck {
set tsnow [clock microseconds]
set elapsed_us [expr {$tsnow - $ts_start}]
dict set installing_record -elapsed_us $elapsed_us
dict set installing_record tag "SKIPPED"
dict set installing_record tag "INSTALL-SKIPPED"
lset file_record_body end $installing_record
dict set file_record body $file_record_body
@ -879,13 +935,14 @@ namespace eval punkcheck {
#then: file_record_add_installrecord
namespace eval lib {
set pkg punkcheck
namespace path ::punkcheck
proc is_file_record_inprogress {file_record} {
if {[dict get $file_record tag] ne "FILEINFO"} {
return 0
}
set installing_record [lindex [dict_getwithdefault $file_record body [list]] end]
if {[dict_getwithdefault $installing_record tag [list]] ni [list INSTALL-INPROGRESS MODIFY-INPROGRESS DELETE-INPROGRESS VIRTUAL-INPROGRESS]} {
if {[dict_getwithdefault $installing_record tag [list]] ni [list QUERY-INPROGRESS INSTALL-INPROGRESS MODIFY-INPROGRESS DELETE-INPROGRESS VIRTUAL-INPROGRESS]} {
return 0
}
return 1
@ -1021,9 +1078,16 @@ namespace eval punkcheck {
}
proc install_non_tm_files {srcdir basedir args} {
#set keys [dict keys $args]
#adjust the default anti_glob_dir_core entries so that .fossil-custom, .fossil-settings are copied
set antiglob_dir_core [punkcheck::default_antiglob_dir_core]
set posn [lsearch $antiglob_dir_core ".fossil*"]
if {$posn >=0} {
set antiglob_dir_core [lreplace $antiglob_dir_core $posn $posn]
}
set defaults [list\
-glob *\
-antiglob_file [list "*.tm" "*-buildversion.txt" "*.exe"]\
-antiglob_dir_core $antiglob_dir_core\
-installer punkcheck::install_non_tm_files\
]
set opts [dict merge $defaults $args]
@ -1070,7 +1134,8 @@ namespace eval punkcheck {
# -overwrite newer-targets will copy files with older source timestamp over newer target timestamp and those missing at the target (a form of 'restore' operation)
# -overwrite older-targets will copy files with newer source timestamp over older target timestamp and those missing at the target
# -overwrite all-targets will copy regardless of timestamp at target
# -overwrite installedsourcechanged-targets
# -overwrite installedsourcechanged-targets will copy if the target doesn't exist or the source changed
# -overwrite synced-targets will copy if the target doesn't exist or the source changed and the target cksum is the same as the last INSTALL-RECORD targets_cksums entry
# review - timestamps unreliable
# - what about slightly mismatched system clocks and mounted filesystems? caller responsibility to verify first?
# if timestamp exactly equal - should we check content-hash? This is presumably only likely to occur deliberately(maliciously?)
@ -1091,11 +1156,13 @@ namespace eval punkcheck {
# -source_checksum compare|store|comparestore|false|true where true == comparestore
# -punkcheck_folder target|source|project|<absolutepath> target is default and is generally recommended
# -punkcheck_records empty string | parsed TDL records ie {tag xxx k v} structure
# install creates FILEINFO records with a single entry in the -targets field (it is legitimate to have a list of targets for an installation operation - the oo interface supports this)
proc install {srcdir tgtdir args} {
set defaults [list\
-call-depth-internal 0\
-max_depth 1000\
-subdirlist {}\
-createdir 0\
-glob *\
-antiglob_file_core "\uFFFF"\
-antiglob_file "" \
@ -1130,11 +1197,15 @@ namespace eval punkcheck {
set max_depth [dict get $opts -max_depth]
set subdirlist [dict get $opts -subdirlist] ;# generally should be same length as CALLDEPTH - but user could prefill
set fileglob [dict get $opts -glob]
set createdir [dict get $opts -createdir] ;#defaults to zero to help avoid mistakes with initial target dir - required target subdirs are created regardless of this setting
if {$CALLDEPTH == 0} {
#expensive to normalize but we need to do it at least once
set srcdir [file normalize $srcdir]
set tgtdir [file normalize $tgtdir]
if {$createdir} {
file mkdir $tgtdir
}
#now the values we build from these will be properly cased
}
# -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
@ -1157,7 +1228,7 @@ namespace eval punkcheck {
set opt_unpublish_paths [dict get $opts -unpublish_paths] ;#todo - combine with config file in source tree .punkcheckpublish (?)
set unpublish_paths_matched [list]
# -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
set known_whats [list no-targets newer-targets older-targets all-targets installedsourcechanged-targets]
set known_whats [list no-targets newer-targets older-targets all-targets installedsourcechanged-targets synced-targets]
set overwrite_what [string tolower [dict get $opts -overwrite]]; #accept any case for value to allow emphasis by caller e.g -overwrite NEWER-TARGETS
if {$overwrite_what ni $known_whats} {
error "punkcheck::install received unrecognised value for -overwrite. Received value '$overwrite_what' vs known values '$known_whats'"
@ -1246,9 +1317,9 @@ namespace eval punkcheck {
}
if {[string match *store* $opt_source_checksum]} {
set store_cksums 1
set store_source_cksums 1
} else {
set store_cksums 0
set store_source_cksums 0
}
@ -1280,7 +1351,7 @@ namespace eval punkcheck {
#puts "testing folder - globmatchpath $unpub $relative_source_dir"
if {[globmatchpath $unpub $relative_source_dir]} {
lappend unpublish_paths_matched $current_source_dir
return [list files_copied {} files_skipped {} sources_unchanged {} punkcheck_records $punkcheck_records unpublish_paths_matched $unpublish_paths_matched]
return [list files_copied {} files_skipped {} sources_unchanged {} punkcheck_records $punkcheck_records unpublish_paths_matched $unpublish_paths_matched srcdir $srcdir tgtdir $tgtdir punkcheck_folder $punkcheck_folder]
}
}
@ -1343,6 +1414,7 @@ namespace eval punkcheck {
#puts stdout "Current target dir: $current_target_dir"
foreach m $match_list {
set new_tgt_cksum_info [list]
set relative_target_path [file join $relative_target_dir $m]
set relative_source_path [file join $relative_source_dir $m]
set punkcheck_target_relpath [file join $target_relative_to_punkcheck_dir $m]
@ -1400,10 +1472,14 @@ namespace eval punkcheck {
#this hits the filesystem for the sourcepath - gets checksums/timestamps depending on config.
#It doesn't save to .punkcheck (the only punkcheck::installfile_ method which doesn't)
set filerec [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $relative_source_path $filerec]
#changeinfo comes from last record in body - which is the record we are working on and so will always exist
set changeinfo [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $filerec body] end]]
set changed [dict get $changeinfo changed]
set unchanged [dict get $changeinfo unchanged]
if {[llength $unchanged]} {
lappend sources_unchanged $current_source_dir/$m
}
@ -1415,6 +1491,7 @@ namespace eval punkcheck {
} else {
if {![file exists $current_target_dir/$m]} {
file copy $current_source_dir/$m $current_target_dir
set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
lappend files_copied $current_source_dir/$m
incr filecount_new
} else {
@ -1422,15 +1499,48 @@ namespace eval punkcheck {
if {[llength $changed]} {
#An unrecorded installation is considered a source change (from unknown/unrecorded source to recorded)
file copy -force $current_source_dir/$m $current_target_dir
set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
lappend files_copied $current_source_dir/$m
} else {
set is_skip 1
lappend files_skipped $current_source_dir/$m
}
} elseif {$overwrite_what eq "synced-targets"} {
if {[llength $changed]} {
#only overwrite if the target checksum equals the last installed checksum (ie target is in sync with source and so hasn't been customized)
set existing_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
set is_target_unmodified_since_install 0
set target_cksum_compare "unknown"
set latest_install_record [punkcheck::recordlist::file_record_latest_installrecord $filerec] ;#may be no such record - in which case we get an empty list
if {[dict exists $latest_install_record -targets_cksums]} {
set last_install_cksum [dict get $latest_install_record -targets_cksums] ;#in this case we know there is only one as 'install' always uses targetset size of 1. (FILEINFO record per file in source folder)
if {[dict get $existing_tgt_cksum_info cksum] eq $last_install_cksum} {
set is_target_unmodified_since_install 1
set target_cksum_compare "match"
} else {
set target_cksum_compare "nomatch"
}
} else {
set target_cksum_compare "norecord"
}
if {$is_target_unmodified_since_install} {
file copy -force $current_source_dir/$m $current_target_dir
set new_tgt_cksum_info [punk::mix::base::lib::cksum_path $current_target_dir/$m]
lappend files_copied $current_source_dir/$m
} else {
#either cksum is different or we were unable to verify the record. Either way we can't know if the target is in sync so we must skip it
set is_skip 1
puts stderr "Skipping file copy $m target $current_target_dir/$m - require synced_target to overwrite - current target cksum compared to previous install: $target_cksum_compare"
lappend files_skipped $current_source_dir/$m
}
} else {
set is_skip 1
lappend files_skipped $current_source_dir/$m
}
} else {
set is_skip 1
puts stderr "Skipping file copy $m target $current_target_dir/$m already exists (use -overwrite all-targets to overwrite)"
#TODO! implement newer-targets older-targets
#TODO! implement newer-targets older-targets? (note ctimes/mtimes are unreliable - may not be worth implementing)
lappend files_skipped $current_source_dir/$m
}
}
@ -1440,27 +1550,31 @@ namespace eval punkcheck {
set ts_now [clock microseconds]
set elapsed_us [expr {$ts_now - $ts_start}]
if {$store_cksums} {
#if {$store_source_cksums} {
#}
set install_records [dict get $filerec body]
set current_install_record [lindex $install_records end]
#change the tag from *-INPROGRESS to INSTALL-RECORD/SKIPPED
if {$is_skip} {
set tag INSTALL-SKIPPED
} else {
set tag INSTALL-RECORD
}
dict set current_install_record tag $tag
dict set current_install_record -elapsed_us $elapsed_us
lset install_records end $current_install_record
dict set filerec body $install_records
set filerec [punkcheck::recordlist::file_record_prune $filerec] ;#prune now that tag is finalized
if {!$has_filerec} {
#not found in original recordlist - append
lappend punkcheck_records $filerec
} else {
lset punkcheck_records $existing_filerec_posn $filerec
}
set install_records [dict get $filerec body]
set current_install_record [lindex $install_records end]
#change the tag from *-INPROGRESS to INSTALL-RECORD/SKIPPED
if {$is_skip} {
set tag INSTALL-SKIPPED
} else {
set tag INSTALL-RECORD
}
dict set current_install_record tag $tag
dict set current_install_record -elapsed_us $elapsed_us
if {[llength $new_tgt_cksum_info]} {
dict set current_install_record -targets_cksums [list [dict get $new_tgt_cksum_info cksum]]
dict set current_install_record -cksum_all_opts [dict get $new_tgt_cksum_info opts]
}
lset install_records end $current_install_record
dict set filerec body $install_records
set filerec [punkcheck::recordlist::file_record_prune $filerec] ;#prune now that tag is finalized
if {!$has_filerec} {
#not found in original recordlist - append
lappend punkcheck_records $filerec
} else {
lset punkcheck_records $existing_filerec_posn $filerec
}
}
@ -1536,11 +1650,10 @@ namespace eval punkcheck {
#puts "subdirlist: $subdirlist"
if {$CALLDEPTH == 0} {
if {[llength $files_copied] || [llength $files_skipped]} {
puts stdout ">>>>>>>>>>>>>>>>>>>"
#puts stdout ">>>>>>>>>>>>>>>>>>>"
set saveresult [punkcheck::save_records_to_file $punkcheck_records $punkcheck_file]
puts stdout "[dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file"
puts stdout "copied: [llength $files_copied] skipped: [llength $files_skipped]"
puts stdout ">>>>>>>>>>>>>>>>>>>"
puts stdout "punkcheck::install [dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file copied: [llength $files_copied] skipped: [llength $files_skipped]"
#puts stdout ">>>>>>>>>>>>>>>>>>>"
} else {
#todo - write db INSTALLER record if -debug true
@ -1551,10 +1664,30 @@ namespace eval punkcheck {
}
}
return [list files_copied $files_copied files_skipped $files_skipped sources_unchanged $sources_unchanged unpublish_paths_matched $unpublish_paths_matched punkcheck_records $punkcheck_records]
return [list files_copied $files_copied files_skipped $files_skipped sources_unchanged $sources_unchanged unpublish_paths_matched $unpublish_paths_matched punkcheck_records $punkcheck_records punkcheck_folder $punkcheck_folder srcdir $srcdir tgtdir $tgtdir]
}
proc summarize_install_resultdict {resultdict} {
set msg ""
if {[dict size $resultdict]} {
set copied [dict get $resultdict files_copied]
append msg "--------------------------" \n
append msg "[dict keys $resultdict]" \n
set tgtdir [dict get $resultdict tgtdir]
set checkfolder [dict get $resultdict punkcheck_folder]
append msg "Copied [llength $copied] files from [dict get $resultdict srcdir] to [dict get $resultdict tgtdir]" \n
foreach f $copied {
append msg "COPIED [punkcheck::lib::path_relative $checkfolder $f]" \n
append msg " TO $tgtdir" \n
}
append msg "[llength [dict get $resultdict sources_unchanged]] unchanged source files" \n
append msg "[llength [dict get $resultdict files_skipped]] skipped files" \n
append msg "--------------------------" \n
}
return $msg
}
namespace eval recordlist {
set pkg punkcheck
namespace path ::punkcheck
proc records_as_target_dict {record_list} {
@ -1590,8 +1723,8 @@ namespace eval punkcheck {
}
proc file_install_record_source_changes {install_record} {
#reject INSTALLFAILED items ?
if {[dict get $install_record tag] ni [list "INSTALL-RECORD" "SKIPPED" "INSTALL-INPROGRESS" "MODIFY-INPROGRESS" "MODIFY-RECORD" "VIRTUAL-INPROGRESS" "VIRTUAL-RECORD" "DELETE-RECORD" "DELETE-INPROGRESS"]} {
error "file_install_record_source_changes bad file->install record: tag not INSTALL-RECORD|SKIPPED|INSTALL-INPROGRESS|MODIFY-RECORD|MODIFY-INPROGRESS|VIRTUAL-RECORD|VIRTUAL-INPROGRESS|DELETE-RECORD|DELETE-INPROGRESS"
if {[dict get $install_record tag] ni [list "QUERY-INPROGRESS" "INSTALL-RECORD" "INSTALL-SKIPPED" "INSTALL-INPROGRESS" "MODIFY-INPROGRESS" "MODIFY-RECORD" "MODIFY-SKIPPED" "VIRTUAL-INPROGRESS" "VIRTUAL-RECORD" "VIRTUAL-SKIPPED" "DELETE-RECORD" "DELETE-INPROGRESS" "DELETE-SKIPPED"]} {
error "file_install_record_source_changes bad install record: tag '[dict get $install_record tag]' not INSTALL-RECORD|SKIPPED|INSTALL-INPROGRESS|MODIFY-RECORD|MODIFY-INPROGRESS|VIRTUAL-RECORD|VIRTUAL-INPROGRESS|DELETE-RECORD|DELETE-INPROGRESS"
}
set source_list [dict_getwithdefault $install_record body [list]]
set changed [list]
@ -1742,15 +1875,19 @@ namespace eval punkcheck {
return $installer_record
}
proc file_record_latest_installrecord {file_record} {
tailcall file_record_latest_operationrecord INSTALL $file_record
}
proc file_record_latest_operationrecord {operation file_record} {
set operation [string toupper $operation]
if {[dict get $file_record tag] ne "FILEINFO"} {
error "file_record_latest_installrecord bad file_record: tag not FILEINFO"
error "file_record_latest_operationrecord bad file_record: tag not FILEINFO"
}
if {![dict exists $file_record body]} {
return [list]
}
set body_items [dict get $file_record body]
foreach item [lreverse $body_items] {
if {[dict get $item tag] eq "INSTALL-RECORD"} {
if {[dict get $item tag] eq "$operation-RECORD"} {
return $item
}
}
@ -1758,47 +1895,6 @@ namespace eval punkcheck {
}
#dead code?
proc file_record_add_installrecordXXX {file_record install_record} {
if {[dict get $file_record tag] ne "FILEINFO"} {
error "file_record_add_installrecord bad file_record: tag not FILEINFO"
}
#disallow '-INPROGRESS' as it's not a final tag
if {[dict get $install_record tag] ni [list "INSTALL-RECORD" "SKIPPED"]} {
error "file_record_add_installrecord bad install_record: tag not INSTALL-RECORD"
}
set keep 3
if {[dict exists $file_record -keep_installrecords]} {
set keep [dict get $file_record -keep_installrecords]
}
if {[dict exists $file_record body]} {
set body_items [dict get $file_record body]
} else {
set body_items [list]
}
lappend body_items $install_record
set kept_body_items [list]
set kcount 0
foreach item [lreverse $body_items] {
if {[dict get $item tag] eq "INSTALL-RECORD"} {
incr kcount
if {$keep < 0 || $kcount <= $keep} {
lappend kept_body_items $item
}
} else {
lappend kept_body_items $item
}
}
set kept_body_items [lreverse $kept_body_items]
dict set file_record body $kept_body_items
return $file_record
}
proc file_record_set_defaults {file_record} {
if {[dict get $file_record tag] ne "FILEINFO"} {
error "file_record_set_defaults bad file_record: tag not FILEINFO"
@ -1881,6 +1977,7 @@ namespace eval punkcheck {
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
## Ready
package provide punkcheck [namespace eval punkcheck {
set pkg punkcheck
variable version
set version 0.1.0
}]

69
src/modules/shellfilter-0.1.8.tm

@ -11,7 +11,7 @@
namespace eval shellfilter::log {
variable allow_adhoc_tags 0
variable allow_adhoc_tags 1
variable open_logs [dict create]
#'tag' is an identifier for the log source.
@ -22,6 +22,7 @@ namespace eval shellfilter::log {
if {![dict exists $settingsdict -tag]} {
dict set settingsdict -tag $tag
} else {
#review
if {$tag ne [dict get $settingsdict -tag]} {
error "shellfilter::log::open first argument tag: '$tag' does not match -tag '[dict get $settingsdict -tag]' omit -tag, or supply same value"
}
@ -36,6 +37,13 @@ namespace eval shellfilter::log {
return $worker_tid
}
proc write {tag msg} {
upvar ::shellfilter::sources sourcelist
variable allow_adhoc_tags
if {!$allow_adhoc_tags} {
if {$tag ni $sourcelist} {
error "shellfilter::log::write tag '$tag' hasn't been initialised with a call to shellfilter::log::open $tag <settings>, and allow_adhoc_tags has been set false. use shellfilter::log::require_open false to allow adhoc tags"
}
}
shellthread::manager::write_log $tag $msg
}
#write_sync - synchronous processing with logging thread, slower but potentially useful for debugging/testing or forcing delay til log written
@ -47,7 +55,8 @@ namespace eval shellfilter::log {
shellthread::manager::unsubscribe [list $tag]; #workertid will be added back to free list if no tags remain subscribed
}
#todo -implement
#review
#configure whether we can call shellfilter::log::write without having called open first
proc require_open {{is_open_required {}}} {
variable allow_adhoc_tags
if {![string length $is_open_required]} {
@ -131,7 +140,7 @@ namespace eval shellfilter::ansi2 {
overline 53 nooverline 55 frame 51 framecircle 52 noframe 54
}
variable SGR_colour_map {
black 30 red 31 green 32 yellow 33 blue 4 purple 35 cyan 36 white 37
black 30 red 31 green 32 yellow 33 blue 34 purple 35 cyan 36 white 37
Black 40 Red 41 Green 42 Yellow 43 Blue 44 Purple 45 Cyan 46 White 47
BLACK 100 RED 101 GREEN 102 YELLOW 103 BLUE 104 PURPLE 105 CYAN 106 WHITE 107
}
@ -1010,12 +1019,15 @@ namespace eval shellfilter::stack {
set pipecount [dict size $pipelines]
set tableprefix "$pipecount pipelines active\n"
foreach p [dict keys $pipelines] {
append tableprefix " " $p \n
}
package require overtype
#todo -verbose
set table ""
set ac1 [string repeat " " 15]
set ac2 [string repeat " " 32]
set ac3 [string repeat " " 80]
set ac2 [string repeat " " 42]
set ac3 [string repeat " " 70]
append table "[overtype::left $ac1 channel-ident] "
append table "[overtype::left $ac2 device-info] "
append table "[overtype::left $ac3 stack-info]"
@ -1028,10 +1040,11 @@ namespace eval shellfilter::stack {
foreach k [dict keys $pipelines $pipename] {
set lc [dict get $pipelines $k device localchan]
set tid [dict get $pipelines $k device workertid]
set col1 [overtype::left $ac1 $k]
set col2 [overtype::left $ac2 "localchan: $lc"]
set col2 [overtype::left $ac2 "localchan: $lc tid:$tid"]
set stack [dict get $pipelines $k stack]
if {![llength $stack]} {
@ -1148,15 +1161,24 @@ namespace eval shellfilter::stack {
dict set pipelines $pipename [list]
}
#todo
proc delete {pipename} {
set pipeinfo [dict get $pipename]
proc delete {pipename {wait 0}} {
variable pipelines
set pipeinfo [dict get $pipelines $pipename]
set deviceinfo [dict get $pipeinfo device]
set localchan [dict get $deviceinfo localchan]
unwind $pipename
#release associated thread
set tid [dict get $deviceinfo workertid]
if {$wait} {
thread::release -wait $tid
} else {
thread::release $tid
}
chan close $localchan
}
#review - proc name clarity is questionable. remove_stackitem?
proc remove {pipename remove_id} {
variable pipelines
if {![dict exists $pipelines $pipename]} {
@ -2087,8 +2109,14 @@ namespace eval shellfilter {
set worker_errorlist [list]
set tidied_sources [list]
set tidytag "logtidy"
set tid [::shellfilter::log::open $tidytag {-syslog 127.0.0.1:514}]
::shellfilter::log::write $tidytag " logtidyuptags '$tags'"
# opening a thread or writing to a log/syslog close to possible process exit is probably not a great idea.
# we should ensure the thread already exists early on if we really need logging here.
#
#set tid [::shellfilter::log::open $tidytag {-syslog 127.0.0.1:514}]
#::shellfilter::log::write $tidytag " logtidyuptags '$tags'"
foreach s $sources {
if {$s eq $tidytag} {
continue
@ -2112,7 +2140,10 @@ namespace eval shellfilter {
lappend remaining_sources $s
}
}
set sources [concat $remaining_sources $tidytag]
#set sources [concat $remaining_sources $tidytag]
set sources $remaining_sources
#shellfilter::stack::unwind stdout
#shellfilter::stack::unwind stderr
return [list tidied $tidied_sources errors $worker_errorlist]
@ -2705,7 +2736,7 @@ namespace eval shellfilter {
#} else {
# #false alarm, or ? we've reached eof with data but didn't get an empty blocking read just prior
# #Not known if this occurs
# #debugging output inline with data - don't leave enabled
# #debugging output inline with data - don't leave enabled
# puts $outchan "!!!prev read didn't block: instate b:[chan blocked $chan] eof:[chan eof $chan] pend:[chan pending output $chan] count:$countchunk"
#}
}
@ -2747,15 +2778,17 @@ namespace eval shellfilter {
}
} trap CHILDSTATUS {result options} {
set code [lindex [dict get $options -errorcode] 2]
set ::shellfilter::shellcommandvars($call_id,exitcode) $code
if {$debug} {
::shellfilter::log::write $debugname " CHILD PROCESS EXITED with code: $code"
}
set ::shellfilter::shellcommandvars($call_id,exitcode) $code
} trap CHILDKILLED {result options} {
#set code [lindex [dict get $options -errorcode] 2]
#set ::shellfilter::shellcommandvars(%id%,exitcode) $code
set ::shellfilter::shellcommandvars($call_id,exitcode) "childkilled"
::shellfilter::log::write $debugname " CHILD PROCESS EXITED with result:'$result' options:'$options'"
if {$debug} {
::shellfilter::log::write $debugname " CHILD PROCESS EXITED with result:'$result' options:'$options'"
}
} finally {
#puts stdout "HERE"
@ -2774,7 +2807,7 @@ namespace eval shellfilter {
#todo - add ability to detect activity/data-flow and change timeout to only apply for period with zero data
#e.g x hrs with no data(?)
#reset timeout when data detected.
after $timeout [string map [list %w $waitvar %id% $call_id %wrerr% $wrerr %rdout% $rdout %rderr% $rderr %debug% $debug %debugname% $debugname] {
after $timeout [string map [list %w% $waitvar %id% $call_id %wrerr% $wrerr %rdout% $rdout %rderr% $rderr %debug% $debug %debugname% $debugname] {
if {[info exists ::shellfilter::shellcommandvars(%id%,exitcode)]} {
if {[set ::shellfilter::shellcommandvars(%id%,exitcode)] ne ""} {
catch { chan close %wrerr% }
@ -2798,7 +2831,8 @@ namespace eval shellfilter {
set code [lindex [dict get $options -errorcode] 2]
#set code [dict get $options -code]
#set ::shellfilter::shellcommandvars(%id%,exitcode) $code
set ::shellfilter::shellcommandvars($call_id,exitcode) "childkilled-timeout"
#set ::shellfilter::shellcommandvars($call_id,exitcode) "childkilled-timeout"
set ::shellfilter::shellcommandvars(%id%,exitcode) "childkilled-timeout"
if {%debug%} {
::shellfilter::log::write %debugname% "(timeout) CHILDKILLED with code: $code"
::shellfilter::log::write %debugname% "(timeout) result:$result options:$options"
@ -2807,9 +2841,8 @@ namespace eval shellfilter {
}
catch { chan close %wrerr% }
catch { chan close %rderr%}
}
set %w "timeout"
set %w% "timeout"
}
}]

47
src/modules/shellrun-0.1.tm

@ -70,6 +70,7 @@ namespace eval shellrun {
#maintenance: similar used in punk::ns & punk::winrun
#todo - take runopts + aliases as args
#longopts must be passed as a single item ie --timeout=100 not --timeout 100
proc get_run_opts {arglist} {
if {[catch {
set callerinfo [info level -1]
@ -81,29 +82,49 @@ namespace eval shellrun {
#we provide -nonewline even for 'run' even though run doesn't deliver stderr or stdout to the tcl return value
#This is for compatibility with other runX commands, and the difference is also visible when calling from repl.
set known_runopts [list "-echo" "-e" "-nonewline" "-n" "-tcl"]
set aliases [list "-e" "-echo" "-echo" "-echo" "-n" "-nonewline" "-nonewline" "-nonewline" "-tcl" "-tcl"] ;#include map to self
set known_runopts [list "-echo" "-e" "-nonewline" "-n" "-tcl" "-debug"]
set known_longopts [list "--timeout"]
set known_longopts_msg ""
foreach lng $known_longopts {
append known_longopts_msg "${lng}=val "
}
set aliases [list "-e" "-echo" "-echo" "-echo" "-n" "-nonewline" "-nonewline" "-nonewline" "-tcl" "-tcl" "-debug" "-debug"] ;#include map to self
set runopts [list]
set runoptslong [list]
set cmdargs [list]
set idx_first_cmdarg [lsearch -not $arglist "-*"]
set runopts [lrange $arglist 0 $idx_first_cmdarg-1]
set allopts [lrange $arglist 0 $idx_first_cmdarg-1]
set cmdargs [lrange $arglist $idx_first_cmdarg end]
foreach o $runopts {
if {$o ni $known_runopts} {
error "$caller: Unknown runoption $o - known options $known_runopts"
foreach o $allopts {
if {[string match --* $o]} {
lassign [split $o =] flagpart valpart
if {$valpart eq ""} {
error "$caller: longopt $o seems to be missing a value - must be of form --option=value"
}
if {$flagpart ni $known_longopts} {
error "$caller: Unknown runoption $o - known options $known_runopts $known_longopts_msg"
}
lappend runoptslong $flagpart $valpart
} else {
if {$o ni $known_runopts} {
error "$caller: Unknown runoption $o - known options $known_runopts $known_longopts_msg"
}
lappend runopts [dict get $aliases $o]
}
}
set runopts [lmap o $runopts {dict get $aliases $o}]
return [list runopts $runopts cmdargs $cmdargs]
return [list runopts $runopts runoptslong $runoptslong cmdargs $cmdargs]
}
#todo - investigate cause of punk86 run hanging sometimes. An 'after 500' before exit in the called script fixes the issue. punk87 doesn't seem to be affected.
proc run {args} {
set_last_run_display [list]
set splitargs [get_run_opts $args]
set runopts [dict get $splitargs runopts]
set runoptslong [dict get $splitargs runoptslong]
set cmdargs [dict get $splitargs cmdargs]
if {"-nonewline" in $runopts} {
@ -115,7 +136,7 @@ namespace eval shellrun {
#we leave stdout without imposed ansi colouring - because the source may be colourised
#stderr might have source colouring - but it usually doesn't seem to, and the visual distiction of red stderr is very handy for the run command.
#A further enhancement could be to detect well-known options such as --color and/or use a configuration for specific commands that have useful colourised stderr,
#but defaulting stderr to red is a pretty reasonable compromise.
#but defaulting stderr to red is a compromise.
#Note that the other run commands, runout,runerr, runx don't emit in real-time - so for those commands there may be options to detect and/or post-process stdout and stderr.
#TODO - fix. This has no effect because the repl adds an ansiwrap transform
# what we probably want to do is 'aside' that transform for runxxx commands only.
@ -125,6 +146,12 @@ namespace eval shellrun {
if {"-tcl" in $runopts} {
dict set callopts -tclscript 1
}
if {"-debug" in $runopts} {
dict set callopts -debug 1
}
if {[dict exists $runoptslong --timeout]} {
dict set callopts -timeout [dict get $runoptslong --timeout] ;#convert to single dash
}
#---------------------------------------------------------------------------------------------
set exitinfo [shellfilter::run $cmdargs {*}$callopts -teehandle punk -inbuffering none -outbuffering none ]
#---------------------------------------------------------------------------------------------

66
src/modules/shellthread-1.6.tm

@ -352,13 +352,22 @@ namespace eval shellthread::worker {
#allow any client to terminate
proc terminate {tidclient} {
variable sock
variable fd
variable client_ids
if {$tidclient in $client_ids} {
catch {close $sock}
catch {close $fd}
set client_ids [list]
return 1
#review use of thread::release -wait
#docs indicate deprecated for regular use, and that we should use thread::join
#however.. how can we set a timeout on a thread::join ?
#by telling the thread to release itself - we can wait on the thread::send variable
# This needs review - because it's unclear that -wait even works on self
# (what does it mean to wait for the target thread to exit if the target is self??)
thread::release -wait
return [thread::id]
} else {
return 0
return ""
}
}
@ -369,6 +378,7 @@ namespace eval shellthread::worker {
namespace eval shellthread::manager {
variable workers [dict create]
variable worker_errors [list]
variable timeouts
variable free_threads [list]
#variable log_threads
@ -423,7 +433,7 @@ namespace eval shellthread::manager {
set free_threads [lassign $free_threads tidworker]
#todo - keep track of real ts_start of free threads... kill when too old
set winfo [dict create tid $tidworker list_client_tids [list $tidclient] ts_start $ts_start ts_end_list [list]]
puts stderr "shellfilter::new_worker Re-using free worker thread: $tidworker with tag $sourcetag"
#puts stderr "shellfilter::new_worker Re-using free worker thread: $tidworker with tag $sourcetag"
dict set workers $sourcetag $winfo
return $tidworker
}
@ -604,11 +614,58 @@ namespace eval shellthread::manager {
return $taginfo_list
}
#finalisation
proc shutdown_free_threads {{timeout 2500}} {
variable free_threads
if {![llength $free_threads]} {
return
}
upvar ::shellthread::manager::timeouts timeoutarr
if {[info exists timeoutarr(shutdown_free_threads)]} {
#already called
return false
}
#set timeoutarr(shutdown_free_threads) waiting
#after $timeout [list set timeoutarr(shutdown_free_threads) timed-out]
set ::shellthread::waitfor waiting
after $timeout [list set ::shellthread::waitfor]
set waiting_for [list]
set ended [list]
set timedout 0
foreach tid $free_threads {
if {[thread::exists $tid]} {
lappend waiting_for $tid
#thread::send -async $tid [list shellthread::worker::terminate [thread::id]] timeoutarr(shutdown_free_threads)
thread::send -async $tid [list shellthread::worker::terminate [thread::id]] ::shellthread::waitfor
}
}
if {[llength $waiting_for]} {
for {set i 0} {$i < [llength $waiting_for]} {incr i} {
vwait ::shellthread::waitfor
if {$::shellthread::waitfor eq "timed-out"} {
set timedout 1
break
} else {
lappend ended $::shellthread::waitfor
}
}
}
set free_threads [list]
return [dict create existed $waiting_for ended $ended timedout $timedout]
}
#TODO - important.
#REVIEW!
#since moving to the unsubscribe mechansm - close_worker $source isn't being called
# - we need to set a limit to the number of free threads and shut down excess when detected during unsubscription
#instruction to shut-down the thread that has this source.
#instruction to shut-down the thread that has this source.
proc close_worker {source {timeout 2500}} {
variable workers
variable worker_errors
variable free_threads
upvar ::shellthread::manager::timeouts timeoutarr
set ts_now [clock micros]
#puts stderr "close_worker $source"
if {[dict exists $workers $source]} {
@ -639,6 +696,8 @@ namespace eval shellthread::manager {
if {[thread::exists $tidworker]} {
#puts stderr "shellthread::manager::close_worker: thread $tidworker for source $source still running.. terminating"
#review - timeoutarr is local var (?)
set timeoutarr($source) 0
after $timeout [list set timeoutarr($source) 2]
@ -676,7 +735,6 @@ namespace eval shellthread::manager {
#worker errors only available for a source after close_worker called on that source
#It is possible for there to be multiple entries for a source because new_worker can be called multiple times with same sourcetag,
# e.g if a thread
proc get_and_clear_errors {source} {
variable worker_errors
set source_errors [lsearch -all -inline -index 0 $worker_errors $source]

40
src/modules/zzzload-999999.0a1.0.tm

@ -23,12 +23,40 @@ package require Thread
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
namespace eval zzzload {
variable loader_tid ;#thread id
set loader_tid [thread::create -preserved]
variable loader_tid "" ;#thread id
proc stacktrace {} {
set stack "Stack trace:\n"
for {set i 1} {$i < [info level]} {incr i} {
set lvl [info level -$i]
set pname [lindex $lvl 0]
append stack [string repeat " " $i]$pname
if {![catch {info args $pname} pargs]} {
foreach value [lrange $lvl 1 end] arg $pargs {
if {$value eq ""} {
if {$arg != 0} {
info default $pname $arg value
}
}
append stack " $arg='$value'"
}
} else {
append stack " !unknown vars for $pname"
}
append stack \n
}
return $stack
}
proc pkg_require {pkgname args} {
variable loader_tid
if {$loader_tid eq ""} {
set loader_tid [thread::create -joinable -preserved]
}
if {![tsv::exists zzzload_pkg $pkgname]} {
#puts stderr "zzzload pkg_require $pkgname"
#puts [stacktrace]
tsv::set zzzload_pkg $pkgname "loading"
tsv::set zzzload_pkg_mutex $pkgname [thread::mutex create]
set cond [thread::cond create]
@ -62,6 +90,14 @@ namespace eval zzzload {
return $pkgstate
}
}
proc shutdown {} {
variable loader_tid
if {[thread::exists $loader_tid]} {
thread::release $loader_tid
thread::join $loader_tid
set loader_tid ""
}
}
}

1
src/punk86.vfs/lib/app-punk/repl.tcl

@ -72,6 +72,7 @@ if {[llength $currentdir_modules]} {
#puts stdout "$::auto_path"
package require Thread
catch {package require tcllibc}
#These are strong dependencies
# - the repl requires Threading and punk,shellfilter,shellrun to call and display properly.
# tm list already indexed - need 'package forget' to find modules based on current tcl::tm::list

183
src/punk86.vfs/lib/app-shellspy/shellspy.tcl

@ -26,6 +26,21 @@
#
package provide app-shellspy 1.0
if 0 {
rename ::package ::package_orig
proc package {args} {
if {[lindex $args 0] eq "require"} {
if {[lindex $args 1] eq "twapi"} {
puts stderr "-------------------- loading twapi -------------"
} else {
puts stderr "-- loading [lindex $args 1] --"
}
}
tailcall ::package_orig {*}$args
}
}
#a test for windows
#fconfigure stdin -encoding utf-16le
@ -84,6 +99,7 @@ if {[file extension $arg1] in [list .tCl]} {
#lappend auto_path c:/tcl/lib/tcllib1.20
catch {package require tcllibc}
package require Thread
#NOTE: tm package index will probably already have been created so we must use 'package forget' to restrict to current tcl::tm::list path
@ -127,7 +143,9 @@ namespace eval shellspy {
#set id_ansistrip [shellfilter::stack::add stderr ansistrip -settings {}]
#set id_ansistrip [shellfilter::stack::add stdout ansistrip -settings {}]
lassign [shellfilter::redir_output_to_log "SPY"] id_stdout_redir id_stderr_redir
#redir on the shellfilter stack with no log or syslog specified acts to suppress output of stdout & stderr.
#todo - fix shellfilter code to make this noop more efficient (avoid creating corresponding logging thread and filter?)
lassign [shellfilter::redir_output_to_log "SUPPRESS"] id_stdout_redir id_stderr_redir
###
@ -326,6 +344,8 @@ namespace eval shellspy {
set params [dict merge $params [get_channel_config $::testconfig]] ;#working: 5 unbuffered, 6 linebuffered
dict set params -debug 1
dict set params -timeout 1000
#set exitinfo [shellfilter::run [list pwsh -nologo -noprofile -c {*}$args] -debug 1]
@ -392,11 +412,12 @@ namespace eval shellspy {
#shellfilter::stack::remove stdout $id_out
shellfilter::log::write $shellspy_status_log "do_in_cmdshell raw exitinfo: $exitinfo"
if {[lindex $exitinfo 0] eq "exitcode"} {
#exit [lindex $exitinfo 1]
shellfilter::log::write $shellspy_status_log "do_in_cmdshell returning $exitinfo"
#shellfilter::log::write $shellspy_status_log "do_in_cmdshell returning $exitinfo"
#puts stderr "do_in_cmdshell returning $exitinfo"
}
return $exitinfo
@ -514,21 +535,22 @@ namespace eval shellspy {
set id_err [shellfilter::stack::add stderr ansiwrap -action sink-locked -settings {-colour {red bold}}]
#set id_err [shellfilter::stack::add stderr ansiwrap -action sink-locked -settings {-colour {red bold}}]
#todo - use glob to check capitalisation of file tail (.TCL vs .tcl .Tcl etc)
set exitinfo [shellfilter::run [concat [auto_execok $scriptbin] $scriptpath $args] {*}$params]
shellfilter::log::write $shellspy_status_log "do_script_process exitinfo: $exitinfo"
shellfilter::stack::remove stderr $id_err
#shellfilter::stack::remove stderr $id_err
if {[lindex $exitinfo 0] eq "exitcode"} {
shellfilter::log::write $shellspy_status_log "do_script_process returning $exitinfo"
}
if {[dict exists $exitinfo errorCode]} {
exit [dict get $exitinfo $errorCode]
}
#if {[lindex $exitinfo 0] eq "exitcode"} {
# shellfilter::log::write $shellspy_status_log "do_script_process returning $exitinfo"
#}
#if {[dict exists $exitinfo errorCode]} {
# exit [dict get $exitinfo $errorCode]
#}
return $exitinfo
}
proc do_script {scriptname replwhen args} {
@ -604,7 +626,22 @@ source [file normalize $scriptname]
# shellfilter::log::write $shellspy_status_log "do_script returning $exitinfo"
#}
shellfilter::log::write $shellspy_status_log "do_script returning $exitinfo"
shellfilter::log::write $shellspy_status_log "do_script raw exitinfo: $exitinfo"
if {[dict exists $exitinfo errorInfo]} {
#strip out the irrelevant info from the errorInfo - we don't want info beyond 'invoked from within' as this is just plumbing related to the script sourcing
set stacktrace [string map [list \r\n \n] [dict get $exitinfo errorInfo]]
set output ""
set tracelines [split $stacktrace \n]
foreach ln $tracelines {
if {[string match "*invoked from within*" $ln]} {
break
}
append output $ln \n
}
set output [string trimright $output \n]
dict set exitinfo errorInfo $output
shellfilter::log::write $shellspy_status_log "do_script simplified exitinfo: $exitinfo"
}
return $exitinfo
}
@ -650,12 +687,9 @@ source [file normalize $scriptname]
dict set params -debug 0
set params [dict merge $params [get_channel_config $::testconfig]]
set id_out [shellfilter::stack::add stdout towindows -action sink-aside-locked -junction 1 -settings {}]
dict set params -teehandle shellspy ;#shellspyout shellspyerr must exist
set exitinfo [shellfilter::run [concat wsl -d $dist -e [shellescape $args]] {*}$params]
@ -792,15 +826,17 @@ source [file normalize $scriptname]
-values $::argv ]
set is_call_error 0
set arglist [list] ;#processed args result - contains dispatch info etc.
if {[catch {
set arglist [check_flags {*}$argdefinitions]
} errMsg]} {
} callError]} {
puts -nonewline stderr "|shellspy-stderr> ERROR during command dispatch\n"
puts -nonewline stderr "|shellspy-stderr> $errMsg\n"
puts -nonewline stderr "|shellspy-stderr> $callError\n"
puts -nonewline stderr "|shellspy-stderr> [set ::errorInfo]\n"
shellfilter::log::write $shellspy_status_log "check_flags error: $errMsg"
shellfilter::log::write $shellspy_status_log "check_flags error: $callError"
set is_call_error 1
} else {
shellfilter::log::write $shellspy_status_log "check_flags result: $arglist"
}
@ -820,28 +856,119 @@ source [file normalize $scriptname]
#don't open more logs..
#puts stdout ">$tidyinfo"
#lassign [shellfilter::redir_output_to_log "SUPPRESS"] id_stdout_redir id_stderr_redir
shellfilter::stack::remove stderr $id_stderr_redir
shellfilter::stack::remove stdout $id_stdout_redir
#shellfilter::log::write $shellspy_status_log "logtidyup -done- $tidyinfo"
set errorlist [dict get $tidyinfo errors]
if {[llength $errorlist]} {
foreach err $errorlist {
puts -nonewline stderr "|shellspy-final> worker-error-set $err\n"
}
catch {
set errorlist [dict get $tidyinfo errors]
if {[llength $errorlist]} {
foreach err $errorlist {
puts -nonewline stderr "|shellspy-final> worker-error-set $err\n"
}
}
}
puts stdout "shellspy -done-"
#shellfilter::log::write $shellspy_status_log "shellspy -done-"
flush stdout
#puts stdout "shellspy -done1-"
#flush stdout
#shellfilter::log::write $shellspy_status_log "shellspy -done-"
if {[catch {
shellfilter::logtidyup $shellspy_status_log
#puts stdout "shellspy logtidyup done"
#flush stdout
} errMsg]} {
puts stdout "shellspy logtidyup error $errMsg"
flush stdout
shellfilter::log::open shellspy-final {-tag shellspy-final -syslog 127.0.0.1:514}
shellfilter::log::write shellspy-final "FINAL logtidyup error $errMsg\n [set ::errorInfo]"
after 200
after 100
}
#puts [shellfilter::stack::status shellspyout]
#puts [shellfilter::stack::status shellspyerr]
#sample dispatch member of $arglist
#dispatch {
# tclscript {
# command {shellspy::do_script %matched% no_repl}
# matched stdout.tcl arguments {} raw {} dispatchtype raw
# asdispatched {shellspy::do_script stdout.tcl no_repl}
# result {result {}}
# error {}
# }
#}
# or
#dispatch {
# tclscript {
# command xxx
# matched error.tcl arguments {} raw {} dispatchtype raw
# asdispatched {shellspy::do_script error.tcl no_repl}
# result {
# error {This is the error}
# errorCode NONE
# errorInfo This\ is\ the\ error\n\ etc
# }
# error {}
# }
#}
shellfilter::stack::delete shellspyout
shellfilter::stack::delete shellspyerr
set free_info [shellthread::manager::shutdown_free_threads]
#puts stdout $free_info
#flush stdout
if {[package provide zzzload] ne ""} {
#puts "zzzload::loader_tid [set ::zzzload::loader_tid]"
zzzload::shutdown
}
#puts stdout "threads: [thread::names]"
#flush stdout
#puts stdout "calling release on remaining threads"
foreach tid [thread::names] {
thread::release $tid
}
#puts stdout "threads: [thread::names]"
#flush stdout
set colour ""; set reset ""
if {$is_call_error} {
catch {
set colour [punk::ansi::a+ yellow bold underline]; set reset [punk::ansi::a]
}
puts stderr $colour$callError$reset
flush stderr
exit 1
} else {
if {[dict exists $arglist dispatch tclscript result errorInfo]} {
catch {
set colour [punk::ansi::a+ yellow bold]; set reset [punk::ansi::a]
}
set err [dict get $arglist dispatch tclscript result errorInfo]
if {$err ne ""} {
puts stderr $colour$err$reset
flush stderr
exit 1
}
}
}
# -- --- ---
# -- a deadlock hack. after 500 is not enough, after 1000 seems to work.
#after 1000
#after 500
# -- --- ---
if {[dict exists $arglist errorCode]} {
exit [dict get $arglist errorCode]
}
#if we call exit - package require Tk scripts will exit prematurely
#review
#exit 0
}

16
src/runtime/mapvfs.config

@ -1,6 +1,10 @@
#single line per runtime executable. Name of runtime followed by list of .vfs folders with path relative to src folder.
#if runtime has no entry - it will only match a .vfs folder with a matching filename e.g runtime1.exe runtime1.vfs
tclkit86bi.exe punk86.vfs
tclkit87a5bawt.exe punk86.vfs
#tclkit86bi.exe vfs_windows/punk86win.vfs
#single line per runtime executable. Name of runtime followed by list of .vfs folders with path relative to src folder.
#if runtime has no entry - it will only match a .vfs folder with a matching filename e.g runtime1.exe runtime1.vfs
#Use a runtime with a name of dash (-) to build a .kit file from the .vfs folder using no runtime
#e.g
#- myproject.vfs
#- punk86.vfs
tclkit86bi.exe punk86.vfs
#tclkit87a5bawt.exe punk86.vfs
#tclkit86bi.exe vfs_windows/punk86win.vfs

Loading…
Cancel
Save