# tcl # #make any tclkits and modules in src and place them and associated data files/scripts in the parent folder of src. #e.g in 'bin' and 'modules' folders at same level as 'src' folder. #It is assumed the src folder has been placed somewhere where appropriate #(e.g not in /usr or c:/ - unless you intend it to directly make and place folders and files in those locations) set hashline "# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ###" puts $hashline puts " punkshell make script " puts $hashline\n namespace eval ::punkmake { variable scriptfolder [file normalize [file dirname [info script]]] variable foldername [file tail $scriptfolder] variable pkg_requirements [list]; variable pkg_missing [list];variable pkg_loaded [list] variable non_help_flags [list -k] variable help_flags [list -help --help /?] variable known_commands [list project get-project-info] } if {"::try" ni [info commands ::try]} { puts stderr "Tcl interpreter possibly too old - 'try' command not found - aborting" exit 1 } #------------------------------------------------------------------------------ #Module loading from /src/bootsupport or src/*.vfs if script is within a .vfs folder #------------------------------------------------------------------------------ #If the there is a folder directly under the current directory /src/bootsupport/modules which contains .tm files when the starts # - then it will attempt to preference these modules # This allows a source update via 'fossil update' 'git pull' etc to pull in support modules for the make script # and load these in preference to ones that may have been in the interps tcl::tm::list or auto_path due to environment variables set startdir [pwd] set bootsupport_mod [file join $startdir src bootsupport modules] set bootsupport_lib [file join $startdir src bootsupport lib] if {[file exists $bootsupport_mod] || [file exists $bootsupport_lib]} { set original_tm_list [tcl::tm::list] tcl::tm::remove {*}$original_tm_list set original_auto_path $::auto_path set ::auto_path [list $bootsupport_lib] set support_modules [glob -nocomplain -dir $bootsupport_mod -type f -tail *.tm] set tcl_core_packages [list tcl::zlib zlib tcl::oo TclOO tcl::tommath tcl::zipfs Tcl Tk] ;#packages we if {[llength $support_modules] || [llength [glob -nocomplain -dir $bootsupport_lib -tail *]]} { #only forget all *unloaded* package names foreach pkg [package names] { if {$pkg in $tcl_core_packages} { continue } if {![llength [package versions $pkg]]} { #puts stderr "Got no versions for pkg $pkg" continue } if {![string length [package provide $pkg]]} { #no returned version indicates it wasn't loaded - so we can forget its index package forget $pkg } } tcl::tm::add $bootsupport_mod } #todo - review usecase if {[string match "*.vfs/*" [info script]]} { #src/xxx.vfs/lib/app-punk/repl.tcl #we assume if calling directly into .vfs that the user would prefer to use src/modules - so go up 4 levels set modulefolder [file dirname [file dirname [file dirname [file dirname [info script]]]]]/modules } else { # .../bin/punkXX.exe look for ../modules (i.e modules folder at same level as bin folder) set modulefolder [file dirname [file dirname [info nameofexecutable]]]/modules } if {[file exists $modulefolder]} { tcl::tm::add $modulefolder } else { puts stderr "Warning unable to find module folder at: $modulefolder" } if {[file exists [pwd]/modules]} { tcl::tm::add [pwd]/modules } #package require Thread #These are strong dependencies # - the repl requires Threading and punk,shellfilter,shellrun to call and display properly. # tm list already indexed - need 'package forget' to find modules based on current tcl::tm::list package forget punk::mix package require punk::mix package forget punk::repo package require punk::repo package forget punkcheck package require punkcheck #restore module paths and auto_path in addition to the bootsupport ones set tm_list_now [tcl::tm::list] foreach p $original_tm_list { if {$p ni $tm_list_now} { tcl::tm::add $p } } set ::auto_path [list $bootsupport_lib {*}$original_auto_path] #------------------------------------------------------------------------------ } # ** *** *** *** *** *** *** *** *** *** *** *** #*temporarily* hijack package command # ** *** *** *** *** *** *** *** *** *** *** *** try { rename ::package ::punkmake::package_temp_aside proc ::package {args} { if {[lindex $args 0] eq "require"} { lappend ::punkmake::pkg_requirements [lindex $args 1] } } package require punk::mix package require punk::repo } finally { catch {rename ::package ""} catch {rename ::punkmake::package_temp_aside ::package} } # ** *** *** *** *** *** *** *** *** *** *** *** foreach pkg $::punkmake::pkg_requirements { if {[catch {package require $pkg} errM]} { puts stderr "missing pkg: $pkg" lappend ::punkmake::pkg_missing $pkg } else { lappend ::punkmake::pkg_loaded $pkg } } proc punkmake_gethelp {args} { set scriptname [file tail [info script]] append h "Usage:" \n append h "" \n append h " $scriptname -help or $scriptname --help or $scriptname /? or just $scriptname" \n append h " - This help." \n \n append h " $scriptname project ?-k?" \n append h " - this is the literal word project - and confirms you want to run the project build" \n append h " - the optional -k flag will terminate processes running as the executable being built (if applicable)" \n \n append h " $scriptname get-project-info" \n append h " - show the name and base folder of the project to be built" \n append h "" \n if {[llength $::punkmake::pkg_missing]} { append h "* ** NOTE ** ***" \n append h " punkmake has detected that the following packages could not be loaded:" \n append h " " [join $::punkmake::pkg_missing "\n "] \n append h "* ** *** *** ***" \n append h " These packages are required for punk make to function" \n \n append h "* ** *** *** ***" \n\n append h "Successfully Loaded packages:" \n append h " " [join $::punkmake::pkg_loaded "\n "] \n } return $h } set scriptargs $::argv set do_help 0 if {![llength $scriptargs]} { set do_help 1 } else { foreach h $::punkmake::help_flags { if {[lsearch $scriptargs $h] >= 0} { set do_help 1 break } } } set commands_found [list] foreach a $scriptargs { if {![string match -* $a]} { lappend commands_found $a } else { if {$a ni $::punkmake::non_help_flags} { set do_help 1 } } } if {[llength $commands_found] != 1 } { set do_help 1 } elseif {[lindex $commands_found 0] ni $::punkmake::known_commands} { puts stderr "Unknown command: [lindex $commands_found 0]\n\n" set do_help 1 } if {$do_help} { puts stderr [punkmake_gethelp] exit 1 } set ::punkmake::command [lindex $commands_found 0] if {[lsearch $::argv -k] >= 0} { set forcekill 1 } else { set forcekill 0 } #puts stdout "::argv $::argv" # ---------------------------------------- set scriptfolder $::punkmake::scriptfolder #first look for a project root (something under fossil or git revision control AND matches punk project folder structure) #If that fails - just look for a 'project shaped folder' ie meets minimum requirements of /src /src/lib /src/modules /lib /modules if {![string length [set projectroot [punk::repo::find_project $scriptfolder]]]} { if {![string length [set projectroot [punk::repo::find_candidate $scriptfolder]]]} { puts stderr "punkmake script unable to determine an approprite project root at or above the path '$scriptfolder' ensure the make script is within a project folder structure" puts stderr " -aborted- " exit 2 #todo? #ask user for a project name and create basic structure? #call punk::mix::cli::new $projectname on parent folder? } else { puts stderr "WARNING punkmake script operating in project space that is not under version control" } } else { } if {$::punkmake::command eq "get-project-info"} { puts stdout "- -- --- --- --- --- --- --- --- --- ---" puts stdout "- -- get-project-info -- -" puts stdout "- -- --- --- --- --- --- --- --- --- ---" puts stdout "- projectroot : $projectroot" if {[punk::repo::find_fossil $scriptfolder] eq $projectroot} { set vc "fossil" set rev [punk::repo::fossil_revision $scriptfolder] set rem [punk::repo::fossil_remote $scriptfolder] } elseif {[punk::repo::find_git $scriptfolder] eq $projectroot} { set vc "git" set rev [punk::repo::git_revision $scriptfolder] set rem [punk::repo::git_remote $scriptfolder] } else { set vc " - none found -" set rev "n/a" set remotes "n/a" } puts stdout "- version control : $vc" puts stdout "- revision : $rev" puts stdout "- remote : $rem" puts stdout "- -- --- --- --- --- --- --- --- --- ---" exit 0 } if {$::punkmake::command eq "shell"} { #package require pu } if {$::punkmake::command ne "project"} { puts stderr "Command $::punkmake::command not implemented - aborting." exit 1 } set sourcefolder $projectroot/src #only a single consolidated /modules folder used for target set target_modules_base $projectroot/modules file mkdir $target_modules_base #external libs and modules first - and any supporting files - no 'building' required if {[file exists $sourcefolder/vendorlib]} { #unpublish README.md from source folder - but on the root one set unpublish [list\ README.md\ ] set resultdict [punkcheck::install $sourcefolder/vendorlib $projectroot/lib -overwrite installedsourcechanged-targets -unpublish_paths $unpublish] set copied [dict get $resultdict files_copied] set sources_unchanged [dict get $resultdict sources_unchanged] puts stdout "--------------------------" puts stderr "Copied [llength $copied] vendor libs from src/vendorlib to $projectroot/lib" foreach f $copied { puts stdout "COPIED $f" } puts stdout "[llength $sources_unchanged] unchanged source files" puts stdout "--------------------------" } else { puts stderr "NOTE: No src/vendorlib folder found." } if {[file exists $sourcefolder/vendormodules]} { #install .tm *and other files* set resultdict [punkcheck::install $sourcefolder/vendormodules $target_modules_base -installer make.tcl -overwrite installedsourcechanged-targets -unpublish_paths {README.md}] set copied [dict get $resultdict files_copied] set sources_unchanged [dict get $resultdict sources_unchanged] puts stdout "--------------------------" puts stderr "Copied [llength $copied] vendor modules from src/vendormodules to $target_modules_base" foreach f $copied { puts stdout "COPIED $f" } puts stdout "[llength $sources_unchanged] unchanged source files" puts stdout "--------------------------" } else { puts stderr "NOTE: No src/vendormodules folder found." } #default source module folder is at projectroot/src/modules #There may be multiple other src module folders at same level (e.g folder not being other special-purpose folder and not matching name vendor* that contains at least one .tm file in its root) set source_module_folderlist [punk::mix::cli::lib::find_source_module_paths $projectroot] foreach src_module_dir $source_module_folderlist { puts stderr "Processing source module dir: $src_module_dir" set dirtail [file tail $src_module_dir] #modules and associated files belonging to this package/app set copied [punk::mix::cli::lib::build_modules_from_source_to_base $src_module_dir $target_modules_base -glob *.tm] ;#will only accept a glob ending in .tm #set copied [list] puts stdout "--------------------------" puts stderr "Copied [llength $copied] tm modules from src/$dirtail to $target_modules_base " puts stdout "--------------------------" set overwrite "installedsourcechanged-targets" #set overwrite "ALL-TARGETS" set resultdict [punkcheck::install_non_tm_files $src_module_dir $target_modules_base -installer make.tcl -overwrite $overwrite -unpublish_paths {README.md}] set copied [dict get $resultdict files_copied] set sources_unchanged [dict get $resultdict sources_unchanged] puts stdout "--------------------------" puts stderr "Copied [llength $copied] non-tm source files from $src_module_dir to $target_modules_base" puts stderr "[llength $sources_unchanged] unchanged source files" puts stdout "--------------------------" } # ---------------------------------------- set buildfolder [punk::mix::cli::lib::get_build_workdir $sourcefolder] if {$buildfolder ne "$sourcefolder/_build"} { puts stderr "$sourcefolder/_build doesn't match the project buildfolder $buildfolder - check project filestructure" puts stdout " -aborted- " exit 2 } #find runtimes set rtfolder $sourcefolder/runtime set runtimes [glob -nocomplain -dir $rtfolder -types {f x} -tail *] if {![llength $runtimes]} { puts stderr "No executable runtimes found in $rtfolder - unable to build any .vfs folders into executables." exit 2 } if {[catch {exec sdx help} errM]} { puts stderr "FAILED to find usable sdx command - check that sdx executable is on path" puts stderr "err: $errM" exit 1 } # -- --- --- --- --- --- --- --- --- --- #load mapvfs.config file (if any) in runtime folder to map runtimes to vfs folders. #build a dict keyed on runtime executable name. #If no mapfile (or no mapfile entry for that runtime) - the runtime will be paired with a matching .vfs folder in src folder. e.g punk.exe to src/punk.vfs #If vfs folders or runtime executables which are explicitly listed in the mapfile don't exist - warn on stderr - but continue. if such nonexistants found; prompt user for whether to continue or abort. set mapfile $rtfolder/mapvfs.config set runtime_vfs_map [dict create] set vfs_runtime_map [dict create] if {[file exists $mapfile]} { set fdmap [open $mapfile r] fconfigure $fdmap -translation binary set mapdata [read $fdmap] close $fdmap set mapdata [string map [list \r\n \n] $mapdata] set missing [list] foreach ln [split $mapdata \n] { set ln [string trim $ln] if {$ln eq "" || [string match #* $ln]} { continue } set vfspaths [lassign $ln runtime] if {[string match *.exe $runtime]} { #.exe is superfluous but allowed #drop windows .exe suffix so same config can work cross platform - extension will be re-added if necessary later set runtime [string range $runtime 0 end-4] } set runtime_test $runtime if {"windows" eq $::tcl_platform(platform)} { set runtime_test $runtime.exe } if {![file exists [file join $rtfolder $runtime_test]]} { puts stderr "WARNING: Missing runtime file $rtfolder/$runtime_test (line in mapvfs.config: $ln)" lappend missing $runtime } foreach vfs $vfspaths { if {![file isdirectory [file join $sourcefolder $vfs]]} { puts stderr "WARNNING: Missing vfs folder [file join $sourcefolder $vfs] specified in mapvfs.config for runtime $runtime" lappend missing $vfs } dict lappend vfs_runtime_map $vfs $runtime } if {[dict exists $runtime_vfs_map $runtime]} { puts stderr "CONFIG FILE ERROR. runtime: $runtime was specified more than once in $mapfile." exit 3 } dict set runtime_vfs_map $runtime $vfspaths } if {[llength $missing]} { puts stderr "WARNING [llength $missing] missing items from $mapfile. (TODO - prompt user to continue/abort)" foreach m $missing { puts stderr " $m" } puts stderr "continuing..." } } # -- --- --- --- --- --- --- --- --- --- set vfs_folders [glob -nocomplain -dir $sourcefolder -types d -tail *.vfs] #add any extra .vfs folders found in runtime/mapvfs.config file (e.g myotherruntimes/something.vfs) foreach vfs [dict keys $vfs_runtime_map] { if {$vfs ni $vfs_folders} { lappend vfs_folders $vfs } } if {![llength $vfs_folders]} { puts stdout "No .vfs folders found at '$sourcefolder' - no kits to build" puts stdout " -done- " exit 0 } set vfs_folder_changes [dict create] ;#cache whether each .vfs folder has changes so we don't re-run tests if building from same .vfs with multiple runtime executables set installername "make.tcl" # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- #set runtimefile [lindex $runtimes 0] foreach runtimefile $runtimes { #runtimefile e.g tclkit86bi.exe on windows tclkit86bi on other platforms #sdx *may* be pointed to use the runtime we use to build the kit, or the user may manually use this runtime if they don't have tclsh #sdx will complain if the same runtime is used for the shell as is used in the -runtime argument - so we make a copy (REVIEW) #if {![file exists $buildfolder/buildruntime.exe]} { # file copy $rtfolder/$runtimefile $buildfolder/buildruntime.exe #} set basedir $buildfolder set config [dict create\ -make-step copy_runtime\ ] lassign [punkcheck::start_installer_event $basedir/.punkcheck $installername $rtfolder $buildfolder $config] _eventid punkcheck_eventid _recordset record_list set target_relpath [punkcheck::lib::path_relative $basedir $buildfolder/build_$runtimefile] set file_record [punkcheck::installfile_begin $basedir $target_relpath $installername -eventid $punkcheck_eventid] # -- --- --- --- --- --- set source_relpath [punkcheck::lib::path_relative $basedir $rtfolder/$runtimefile] set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record] # -- --- --- --- --- --- set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]] if {[llength [dict get $changed_unchanged changed]] || ![file exists $buildfolder/build_$runtimefile]} { set file_record [punkcheck::installfile_started_install $basedir $file_record] # -- --- --- --- --- --- puts stdout "Copying runtime from $rtfolder/$runtimefile to $buildfolder/build_$runtimefile" file copy -force $rtfolder/$runtimefile $buildfolder/build_$runtimefile # -- --- --- --- --- --- set file_record [punkcheck::installfile_finished_install $basedir $file_record] } else { puts stderr "." set file_record [punkcheck::installfile_skipped_install $basedir $file_record] } } # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- # # loop over vfs_folders and for each one, loop over configured (or matching) runtimes - build with sdx if source .vfs or source runtime exe has changed. # we are using punkcheck to install result to buildfolder so we create a .punkcheck file at the target folder to store metadata. # punkcheck allows us to not rely purely on timestamps (which may be unreliable) # set startdir [pwd] puts stdout "Found [llength $vfs_folders] .vfs folders - building executable for each..." cd [file dirname $buildfolder] #root folder mtime is insufficient for change detection. Tree mtime of folders only is a barely passable mechanism for vfs change detection in some circumstances - e.g if files added/removed but never edited in place #a hash of full tree file & dir mtime may be more reasonable - but it remains to be seen if just tar & checksum is any/much slower. #Simply rebuilding all the time may be close the speed of detecting change anyway - and almost certainly much faster when there is a change. #Using first mtime encountered that is later than target is another option - but likely to be highly variable in speed. Last file in the tree could happen to be the latest, and this mechanism doesn't handle build on reversion to older source. set exe_names_seen [list] foreach vfs $vfs_folders { set vfsname [file rootname $vfs] puts stdout " Processing vfs $sourcefolder/$vfs" puts stdout " ------------------------------------" set skipped_vfs_build 0 # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- set basedir $buildfolder set config [dict create\ -make-step build_vfs\ ] set runtimes [list] if {[dict exists $vfs_runtime_map $vfs]} { set runtimes [dict get $vfs_runtime_map $vfs] ;#map dict is unsuffixed (.exe stripped or was not present) if {"windows" eq $::tcl_platform(platform)} { set runtimes_raw $runtimes set runtimes [list] foreach rt $runtimes_raw { if {![string match *.exe $rt]} { set rt $rt.exe } lappend runtimes $rt } } } else { #only match this vfs to a correspondingly named runtime if there was no explicit entry for that runtime set matchrt [file rootname [file tail $vfs]] ;#e.g project.vfs -> project if {![dict exists $runtime_vfs_map $matchrt]} { if {"windows" eq $::tcl_platform(platform)} { if {[file exists $rtfolder/$matchrt.exe]} { lappend runtimes $matchrt.exe } } else { lappend runtimes $matchrt } } } #assert $runtimes is a list of executable names suffixed with .exe if on windows - whether or not specified with .exe in the mapvfs.config foreach rtname $runtimes { #first configured runtime will be the one to use the same name as .vfs folder for output executable. Additional runtimes on this .vfs will need to suffix the runtime name to disambiguate. #review: This mechanism may not be great for multiplatform builds ? We may be better off consistently combining vfsname and rtname and letting a later platform-specific step choose ones to install in bin with simpler names. if {$::tcl_platform(platform) eq "windows"} { set targetexe ${vfsname}.exe } else { set targetexe $vfsname } if {$targetexe in $exe_names_seen} { #more than one runtime for this .vfs set targetexe ${vfsname}_$rtname } lappend exe_names_seen $targetexe lassign [punkcheck::start_installer_event $basedir/.punkcheck $installername $sourcefolder $buildfolder $config] _eventid punkcheck_eventid _recordset record_list set target_relpath [punkcheck::lib::path_relative $basedir $buildfolder/$targetexe] set file_record [punkcheck::installfile_begin $basedir $target_relpath $installername -eventid $punkcheck_eventid] # -- --- --- --- --- --- set source_relpath [punkcheck::lib::path_relative $basedir $sourcefolder/$vfs] set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record] # -- --- --- --- --- --- set source_relpath [punkcheck::lib::path_relative $basedir $buildfolder/build_$rtname] set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record] # -- --- --- --- --- --- set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]] if {[llength [dict get $changed_unchanged changed]] || ![file exists $buildfolder/$targetexe]} { #source .vfs folder has changes set file_record [punkcheck::installfile_started_install $basedir $file_record] # -- --- --- --- --- --- #use if {[file exists $buildfolder/$vfsname.new]} { puts stderr "deleting existing $buildfolder/$vfsname.new" file delete $buildfolder/$vfsname.new } puts stdout "building $vfsname with sdx.. vfsdir:$vfs cwd: [pwd]" if {[catch { exec sdx wrap $buildfolder/$vfsname.new -vfs $sourcefolder/$vfs -runtime $buildfolder/build_$rtname -verbose } result]} { puts stderr "sdx wrap $buildfolder/$vfsname.new -vfs $sourcefolder/$vfs -runtime $buildfolder/build_$rtname -verbose failed with msg: $result" } else { puts stdout "ok - finished sdx" set separator [string repeat = 40] puts stdout $separator puts stdout $result puts stdout $separator } if {![file exists $buildfolder/$vfsname.new]} { puts stderr "|err> build didn't seem to produce output at $sourcefolder/_build/$vfsname.new" exit 2 } # -- --- --- if {$::tcl_platform(platform) eq "windows"} { set pscmd "tasklist" } else { set pscmd "ps" } if {![catch { exec $pscmd | grep $vfsname } still_running]} { puts stdout "found $vfsname instances still running\n" set count_killed 0 foreach ln [split $still_running \n] { puts stdout " $ln" if {$::tcl_platform(platform) eq "windows"} { set pid [lindex $ln 1] if {$forcekill} { set killcmd [list taskkill /F /PID $pid] } else { set killcmd [list taskkill /PID $pid] } } else { set pid [lindex $ln 0] #review! if {$forcekill} { set killcmd [list kill -9 $pid] } else { set killcmd [list kill $pid] } } puts stdout " pid: $pid (attempting to kill now using '$killcmd')" if {[catch { exec {*}$killcmd } errMsg]} { puts stderr "$killcmd returned an error:" puts stderr $errMsg puts stderr "(try '[info script] -k' option to force kill)" exit 4 } else { puts stderr "$killcmd ran without error" incr count_killed } } if {$count_killed > 0} { puts stderr "\nKilled $count_killed processes. Waiting a short time before attempting to delete executable" after 1000 } } else { puts stderr "Ok.. no running '$vfsname' processes found" } if {[file exists $buildfolder/$targetexe]} { puts stderr "deleting existing $buildfolder/$targetexe" if {[catch { file delete $buildfolder/$targetexe } msg]} { puts stderr "Failed to delete $buildfolder/$targetexe" exit 4 } } file rename $buildfolder/$vfsname.new $buildfolder/$targetexe # -- --- --- --- --- --- set file_record [punkcheck::installfile_finished_install $basedir $file_record] after 200 set deployment_folder [file dirname $sourcefolder]/bin file mkdir $deployment_folder if {[file exists $deployment_folder/$targetexe]} { puts stderr "deleting existing deployed at $deployment_folder/$targetexe" if {[catch { file delete $deployment_folder/$targetexe } errMsg]} { puts stderr "deletion of deployed version at $deployment_folder/$targetexe failed: $errMsg" exit 5 } } puts stdout "copying.." puts stdout "$buildfolder/$targetexe" puts stdout "to:" puts stdout "$deployment_folder/$targetexe" after 500 file copy $buildfolder/$targetexe $deployment_folder/$targetexe } else { set skipped_vfs_build 1 puts stderr "." puts stdout "Skipping build for vfs $vfs - no change detected" set file_record [punkcheck::installfile_skipped_install $basedir $file_record] } } ;#end foreach rtname in runtimes # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- } cd $startdir puts stdout "done" exit 0