diff --git a/src/make.tcl b/src/make.tcl index 174c1f25..1ce41557 100644 --- a/src/make.tcl +++ b/src/make.tcl @@ -442,28 +442,32 @@ foreach runtimefile $runtimes { set config [dict create\ -make-step copy_runtime\ ] - lassign [punkcheck::start_installer_event $basedir/.punkcheck $installername $rtfolder $buildfolder $config] _eventid punkcheck_eventid _recordset record_list - - - set target_relpath [punkcheck::lib::path_relative $basedir $buildfolder/build_$runtimefile] - set file_record [punkcheck::installfile_begin $basedir $target_relpath $installername -eventid $punkcheck_eventid] - # -- --- --- --- --- --- - set source_relpath [punkcheck::lib::path_relative $basedir $rtfolder/$runtimefile] - set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record] - # -- --- --- --- --- --- - set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]] - if {[llength [dict get $changed_unchanged changed]] || ![file exists $buildfolder/build_$runtimefile]} { - set file_record [punkcheck::installfile_started_install $basedir $file_record] + #---------- + set installer [punkcheck::installtrack new $installername $basedir/.punkcheck] + $installer set_source_target $rtfolder $buildfolder + set event [$installer start_event $config] + $event targetset_init INSTALL $buildfolder/build_$runtimefile + $event targetset_addsource $rtfolder/$runtimefile + #---------- + + #set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]] + if {\ + [llength [dict get [$event targetset_source_changes] changed]]\ + || [llength [$event get_targets_exist]] < [llength [$event get_targets]]\ + } { + $event targetset_started # -- --- --- --- --- --- puts stdout "Copying runtime from $rtfolder/$runtimefile to $buildfolder/build_$runtimefile" file copy -force $rtfolder/$runtimefile $buildfolder/build_$runtimefile # -- --- --- --- --- --- - set file_record [punkcheck::installfile_finished_install $basedir $file_record] + #set file_record [punkcheck::installfile_finished_install $basedir $file_record] + $event targetset_finished } else { puts stderr "." - set file_record [punkcheck::installfile_skipped_install $basedir $file_record] + #set file_record [punkcheck::installfile_skipped_install $basedir $file_record] + $event targetset_skipped } - + $event end } @@ -539,8 +543,6 @@ foreach vfs $vfs_folders { lappend exe_names_seen $targetexe lassign [punkcheck::start_installer_event $basedir/.punkcheck $installername $sourcefolder $buildfolder $config] _eventid punkcheck_eventid _recordset record_list - - set target_relpath [punkcheck::lib::path_relative $basedir $buildfolder/$targetexe] set file_record [punkcheck::installfile_begin $basedir $target_relpath $installername -eventid $punkcheck_eventid] # -- --- --- --- --- --- @@ -551,7 +553,7 @@ foreach vfs $vfs_folders { set file_record [punkcheck::installfile_add_source_and_fetch_metadata $basedir $source_relpath $file_record] # -- --- --- --- --- --- set changed_unchanged [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $file_record body] end]] - if {[llength [dict get $changed_unchanged changed]] || ![file exists $buildfolder/$targetexe]} { + if {[llength [dict get $changed_unchanged changed]] || ![file exists $buildfolder/$targetexe]} { #source .vfs folder has changes set file_record [punkcheck::installfile_started_install $basedir $file_record] # -- --- --- --- --- --- @@ -644,31 +646,52 @@ foreach vfs $vfs_folders { exit 4 } } - + #WINDOWS filesystem 'tunneling' (file replacement within 15secs) could cause targetexe to copy ctime & shortname metadata from previous file! + #This is probably harmless - but worth being aware of. file rename $buildfolder/$vfsname.new $buildfolder/$targetexe # -- --- --- --- --- --- set file_record [punkcheck::installfile_finished_install $basedir $file_record] + after 200 set deployment_folder [file dirname $sourcefolder]/bin file mkdir $deployment_folder + # -- ---------- + set installer [punkcheck::installtrack new "make.tcl" $deployment_folder/.punkcheck] + $installer set_source_target $buildfolder $deployment_folder + set event [$installer start_event {-make-step final_exe_install}] + $event targetset_init INSTALL $deployment_folder/$targetexe + $event targetset_addsource $buildfolder/$targetexe + $event targetset_started + # -- ---------- + + + set delete_failed 0 if {[file exists $deployment_folder/$targetexe]} { puts stderr "deleting existing deployed at $deployment_folder/$targetexe" if {[catch { file delete $deployment_folder/$targetexe } errMsg]} { puts stderr "deletion of deployed version at $deployment_folder/$targetexe failed: $errMsg" - exit 5 + #exit 5 + set delete_failed 1 } } - - puts stdout "copying.." - puts stdout "$buildfolder/$targetexe" - puts stdout "to:" - puts stdout "$deployment_folder/$targetexe" - after 500 - file copy $buildfolder/$targetexe $deployment_folder/$targetexe + if {!$delete_failed} { + puts stdout "copying.." + puts stdout "$buildfolder/$targetexe" + puts stdout "to:" + puts stdout "$deployment_folder/$targetexe" + after 300 + file copy $buildfolder/$targetexe $deployment_folder/$targetexe + # -- ---------- + $event targetset_finished + # -- ---------- + } else { + #todo - targetset_failed + $event targetset_skipped + } } else { set skipped_vfs_build 1 diff --git a/src/modules/oolib-0.1.1.tm b/src/modules/oolib-0.1.1.tm index a9214267..ecf2cca9 100644 --- a/src/modules/oolib-0.1.1.tm +++ b/src/modules/oolib-0.1.1.tm @@ -168,7 +168,10 @@ namespace eval oolib { set o_data [dict create] return } - method reverse {} { + method reverse_the_collection {} { + #named slightly obtusely because reversing the data when there may be references held is a potential source of bugs + #the name reverse_the_collection should make it clear that the object is being modified in place as opposed to simply 'reverse' which may imply a view/copy. + #todo - consider implementing a get_reverse which provides an interface to the same collection without affecting original references, yet both allowing delete/edit operations. set dictnew [dict create] foreach k [lreverse [dict keys $o_data]] { dict set dictnew $k [dict get $o_data $k] diff --git a/src/modules/punkcheck-0.1.0.tm b/src/modules/punkcheck-0.1.0.tm index c2c39567..80607768 100644 --- a/src/modules/punkcheck-0.1.0.tm +++ b/src/modules/punkcheck-0.1.0.tm @@ -94,8 +94,330 @@ namespace eval punkcheck { set objname [namespace current]::installtrack if {$objname ni [info commands $objname]} { package require oolib + + #FILEINFO record - target fileset with body records: INSTALLRECORD,INSTALL-INPROGRESS,SKIPPED,DELETERECORD,DELETE-INPROGRESS,MODIFY-INPROGRESS,MODIFYRECORD + #each FILEINFO body being a list of SOURCE records + oo::class create targetset { + variable o_targets + variable o_keep_installrecords + variable o_keep_skipped + variable o_keep_inprogress + variable o_records + constructor {args} { + #set o_records [oolib::collection create [namespace current]::recordcollection] + set o_records [list] + + } + + method as_record {} { + + set fields [list\ + -targets $o_targets\ + -keep_installrecords $o_keep_installrecords\ + -keep_skipped $o_keep_skipped\ + -keep_inprogress $o_keep_inprogress\ + body $o_records\ + ] + + set record [dict create tag FILEINFO {*}$fields] + } + + #retrieve last completed record for the fileset ie exclude SKIPPED,INSTALL-INPROGRESS,DELETE-INPROGRESS,MODIFY-INPROGRESS + method get_last_record {fileset_record} { + set body [dict_getwithdefault $fileset_record body [list]] + set previous_records [lrange $body 0 end-1] + #get last previous that is tagged INSTALLRECORD,MODIFYRECORD,DELETERECORD + set revlist [lreverse $previous_records] + foreach rec $revlist { + if {[dict get $rec tag] in [list "INSTALLRECORD" "MODIFYRECORD" "DELETERECORD"]} { + return $rec + } + } + return [list] + } + } + + oo::class create installevent { + variable o_id + variable o_rel_sourceroot + variable o_rel_targetroot + variable o_ts_begin + variable o_ts_end + variable o_types + variable o_configdict + variable o_targets + variable o_operation + variable o_operation_start_ts + variable o_fileset_record + variable o_installer ;#parent object + constructor {installer rel_sourceroot rel_targetroot args} { + set o_installer $installer + set o_operation_start_ts "" + set o_operation "" + set defaults [dict create\ + -id ""\ + -tsbegin ""\ + -config [list]\ + -tsend ""\ + -types [list]\ + ] + set opts [dict merge $defaults $args] + if {[dict get $opts -id] eq ""} { + set o_id [punkcheck::uuid] + } else { + set o_id [dict get $opts -id] + } + if {[dict get $opts -tsbegin] eq ""} { + set o_ts_begin [clock microseconds] + } else { + set o_ts_begin [dict get $opts -tsbegin] + } + set o_ts_end [dict get $opts -tsend] + set o_types [dict get $opts -types] + set o_configdict [dict get $opts -config] + + set o_rel_sourceroot $rel_sourceroot + set o_rel_targetroot $rel_targetroot + } + destructor { + puts "[self] destructor called" + } + method as_record {} { + set begin_seconds [expr {$o_ts_begin / 1000000}] + set tsiso_begin [clock format $begin_seconds -format "%Y-%m-%dT%H:%M:%S"] + if {$o_ts_end ne ""} { + set end_seconds [expr {$o_ts_end / 1000000}] + set tsiso_end [clock format $end_seconds -format "%Y-%m-%dT%H:%M:%S"] + } else { + set tsiso_end "" + } + set fields [list\ + -tsiso_begin $tsiso_begin\ + -ts_begin $o_ts_begin\ + -tsiso_end $tsiso_end\ + -ts_end $o_ts_end\ + -id $o_id\ + -source $o_rel_sourceroot\ + -targets $o_rel_targetroot\ + -types $o_types\ + -config $o_configdict\ + ] + + set record [dict create tag EVENT {*}$fields] + } + method get_id {} { + return $o_id + } + method get_operation {} { + return $o_operation + } + method get_targets {} { + return $o_targets + } + method get_targets_exist {} { + set punkcheck_folder [file dirname [$o_installer get_checkfile]] + set existing [list] + foreach t $o_targets { + if {[file exists [file join $puncheck_folder $t]]} { + lappend existing $t + } + } + return $existing + } + method end {} { + set o_ts_end [clock microseconds] + } + method targetset_dict {} { + punk::records_as_target_dict [$o_installer get_recordlist] + } + + #related - installfile_begin + #call init before we know if we are going to run the operation vs skip + method targetset_init {operation targetset} { + set known_ops [list INSTALL MODIFY DELETE VIRTUAL] + if {[string toupper $operation] ni $known_ops} { + error "[self] add_target unknown operation '$operation'. Known operations $known_ops" + } + if {$o_operation_start_ts ne ""} { + error "[self] targetset_tart $o_operation operation already in progress. Use targetset_finished or targetset_complete to finish." + } + set o_operation_start_ts [clock microseconds] + set seconds [expr {$o_operation_start_ts / 1000000}] + set tsiso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"] + set punkcheck_file [$o_installer get_checkfile] + + set relativepath_targetset [list] + foreach p $targetset { + if {[file pathtype $p] eq "absolute"} { + lappend relativepath_targetset [punkcheck::lib::path_relative [file dirname $punkcheck_file] $p] + } else { + lappend relativepath_targetset $p + } + } + + + set o_operation $operation + set fields [list\ + -tsiso $tsiso\ + -ts $o_operation_start_ts\ + -installer [$o_installer get_name]\ + -eventid $o_id\ + ] + + set o_targets [lsort -dictionary -increasing $relativepath_targetset] ;#exact sort order not critical - but must be consistent + + #set targetdict [my targetset_dict] + + set record_list [punkcheck::load_records_from_file $punkcheck_file] + set extractioninfo [punkcheck::recordlist::extract_or_create_fileset_record $o_targets $record_list] + set o_fileset_record [dict get $extractioninfo record] + set record_list [dict get $extractioninfo recordset] + set isnew [dict get $extractioninfo isnew] + set oldposition [dict get $extractioninfo oldposition] + unset extractioninfo + + #INSTALL-INPROGRESS will become INSTALLRECORD or INSTALLFAILED or SKIPPED upon finalisation + #-installer and -eventid keys are added here + set new_inprogress_record [dict create tag [string toupper $operation]-INPROGRESS {*}$fields -tempcontext [my as_record] body {}] + #set existing_body [dict_getwithdefault $o_fileset_record body [list]] + #todo - look for existing "-INPROGRESS" records - mark as failed? + dict lappend o_fileset_record body $new_inprogress_record + + if {$isnew} { + lappend record_list $o_fileset_record + } else { + set record_list [linsert $record_list[unset record_list] $oldposition $o_fileset_record] + } + + punkcheck::save_records_to_file $record_list $punkcheck_file + return $o_fileset_record + + } + #operation has been started + method targetset_started {} { + set punkcheck_folder [file dirname [$o_installer get_checkfile]] + set o_fileset_record [punkcheck::installfile_started_install $punkcheck_folder $o_fileset_record] + } + method targetset_finished {} { + if {$o_operation_start_ts eq ""} { + error "[self] targetset_finished - no current operation - call targetset_start first" + } + set operation_end_ts [clock microseconds] + set elapsed [expr {$o_operation_start_ts - $operation_end_ts}] + + set punkcheck_file [$o_installer get_checkfile] + set record_list [punkcheck::load_records_from_file $punkcheck_file] + + + + if {![punkcheck::lib::is_file_record_inprogress $o_fileset_record]} { + error "targetset_finished_install error: bad fileset_record - expected FILEINFO with last body element *-INPROGRESS" + } + + set file_record_body [dict get $o_fileset_record body] + set targetlist [dict get $o_fileset_record -targets] + if {$targetlist ne $o_targets} { + error "targetset_finished error. targetlist mismatch between file : $targetlist vs $o_targets" + } + set installing_record [lindex $file_record_body end] + + set ts_start [dict get $installing_record -ts] + set ts_start_transfer [dict get $installing_record -ts_start_transfer] + set ts_now [clock microseconds] + set elapsed_us [expr {$ts_now - $ts_start}] + set transfer_us [expr {$ts_now - $ts_start_transfer}] + dict set installing_record -transfer_us $transfer_us + dict set installing_record -elapsed_us $elapsed_us + dict unset installing_record -tempcontext + dict set installing_record tag "${o_operation}RECORD" + + lset file_record_body end $installing_record + dict set o_fileset_record body $file_record_body + + set o_fileset_record [punkcheck::recordlist::file_record_prune $o_fileset_record] + + set oldrecordinfo [punkcheck::recordlist::get_file_record $targetlist $record_list] + set old_posn [dict get $oldrecordinfo position] + if {$old_posn == -1} { + lappend record_list $o_fileset_record + } else { + lset record_list $old_posn $o_fileset_record + } + + punkcheck::save_records_to_file $record_list $punkcheck_file + set o_operation_start_ts "" + set o_operation "" + return $o_fileset_record + + } + method targetset_skipped {} { + if {$o_operation_start_ts eq ""} { + error "[self] targetset_skipped - no current operation - call targetset_start first" + } + set operation_end_ts [clock microseconds] + set elapsed [expr {$o_operation_start_ts - $operation_end_ts}] + set o_operation_start_ts "" + set o_operation "" + if {![punkcheck::lib::is_file_record_inprogress $o_fileset_record]} { + set msg "targetset_skipped_install error: bad fileset_record - expected FILEINFO with last body element *-INPROGRESS" + append msg \n "received:" + append msg \n $o_fileset_record + error $msg + } + set punkcheck_file [$o_installer get_checkfile] + set record_list [punkcheck::load_records_from_file $punkcheck_file] + + set file_record_body [dict get $o_fileset_record body] + set targetlist [dict get $o_fileset_record -targets] + set inprogress_record [lindex $file_record_body end] + + set ts_start [dict get $inprogress_record -ts] + set tsnow [clock microseconds] + set elapsed_us [expr {$tsnow - $ts_start}] + dict set inprogress_record -elapsed_us $elapsed_us + dict set inprogress_record tag "SKIPPED" + + lset file_record_body end $inprogress_record + dict set o_fileset_record body $file_record_body + + set o_fileset_record [punkcheck::recordlist::file_record_prune $o_fileset_record] + + set getresult [punkcheck::recordlist::get_file_record $targetlist $record_list] + set old_posn [dict get $getresult position] + if {$old_posn == -1} { + lappend record_list $o_fileset_record + } else { + lset record_list $old_posn $o_fileset_record + } + + punkcheck::save_records_to_file $record_list $punkcheck_file + return $o_fileset_record + + } + method targetset_addsource {source_path} { + set punkcheck_file [$o_installer get_checkfile] + set punkcheck_folder [file dirname $punkcheck_file] + if {[file pathtype $source_path] eq "absolute"} { + set rel_source_path [punkcheck::lib::path_relative $punkcheck_folder $source_path] + } else { + set rel_source_path $source_path + } + + set o_fileset_record [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $rel_source_path $o_fileset_record] + + } + method targetset_source_changes {} { + punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $o_fileset_record body] end] + } + + } + + oo::class create installtrack { variable o_name + variable o_tsiso + variable o_ts + variable o_keep_events variable o_checkfile variable o_sourceroot variable o_rel_sourceroot @@ -104,51 +426,111 @@ namespace eval punkcheck { variable o_record_list variable o_active_event variable o_events - constructor {installername punkcheck_file sourceroot targetroot} { + constructor {installername punkcheck_file} { set o_active_event "" - puts "constructor [self]" set o_name $installername + set o_checkfile [file normalize $punkcheck_file] + set o_sourceroot "" + set o_targetroot "" + set o_rel_sourceroot "" + set o_rel_targetroot "" #todo - validate punkcheck file location further?? set punkcheck_folder [file dirname $o_checkfile] if {![file isdirectory $punkcheck_folder]} { error "[self] constructor error. Folder for punkcheck_file not found - $o_checkfile" } - if {[file pathtype $sourceroot] ne "absolute"} { - error "[self] constructor error: sourceroot must be absolute path. Received '$sourceroot'" - } - if {[file pathtype $targetroot] ne "absolute"} { - error "[self] constructor error: targetroot must be absolute path. Received '$targetroot'" - } - set o_sourceroot $sourceroot - set o_targetroot $targetroot - set o_rel_sourceroot [punkcheck::lib::path_relative $punkcheck_folder $sourceroot] - set o_rel_targetroot [punkcheck::lib::path_relative $punkcheck_folder $targetroot] my load_all_records set resultinfo [punkcheck::recordlist::get_installer_record $o_name $o_record_list] set existing_header_posn [dict get $resultinfo position] if {$existing_header_posn == -1} { set this_installer_record [punkcheck::recordlist::new_installer_record $o_name] + set o_record_list [linsert $o_record_list 0 $this_installer_record] } else { set this_installer_record [dict get $resultinfo record] } - set o_events [oolib::collection new] + set o_tsiso [dict get $this_installer_record -tsiso] + set o_ts [dict get $this_installer_record -ts] + set o_keep_events [dict get $this_installer_record -keep_events] + + set o_events [oolib::collection create [namespace current]::eventcollection] set eventlist [punkcheck::dict_getwithdefault $this_installer_record body [list]] foreach e $eventlist { - $o_events add $e [dict get $e -id] + set eobj [punkcheck::installevent create [namespace current]::event_[my events count] [self] [dict get $e -source] [dict get $e -targets] {*}$e] + #$o_events add $e [dict get $e -id] + $o_events add $eobj [dict get $e -id] } + } destructor { puts "[self] destructor called" } + method test {} { + return [self] + } + method get_name {} { + return $o_name + } + method get_checkfile {} { + return $o_checkfile + } + + #call set_source_target before calling start_event/end_event + #each event can have different source->target pairs - but may often have same, so set on installtrack as defaults. Only persisted in event records. + method set_source_target {sourceroot targetroot} { + if {[file pathtype $sourceroot] ne "absolute"} { + error "[self] set_source_target error: sourceroot must be absolute path. Received '$sourceroot'" + } + if {[file pathtype $targetroot] ne "absolute"} { + error "[self] set_source_target error: targetroot must be absolute path. Received '$targetroot'" + } + set punkcheck_folder [file dirname $o_checkfile] + set o_sourceroot $sourceroot + set o_targetroot $targetroot + set o_rel_sourceroot [punkcheck::lib::path_relative $punkcheck_folder $sourceroot] + set o_rel_targetroot [punkcheck::lib::path_relative $punkcheck_folder $targetroot] + return [list $o_rel_sourceroot $o_rel_targetroot] + } #review/fix to allow multiple installtrack objects on same punkcheck file. method load_all_records {} { set o_record_list [punkcheck::load_records_from_file $o_checkfile] } - #todo - open file and save only own records + + #does not include associated FILEINFO records - as a targetset (FILEINFO record) can be associated with events from multiple installers over time. + #e.g a logfile common to installers, or a separate installer that updates a previous output. + method as_record {} { + set eventrecords [list] + foreach eobj [my events items] { + lappend eventrecords [$eobj as_record] + } + set fields [list\ + -tsiso $o_tsiso\ + -ts $o_ts\ + -name $o_name\ + -keep_events $o_keep_events\ + body $eventrecords\ + ] + set record [dict create tag INSTALLER {*}$fields] + } + #open file and save only own records method save_all_records {} { - punkcheck::save_records_to_file $o_record_list $o_checkfile + my save_installer_record + #todo - save FILEINFO targetset records + } + method save_installer_record {} { + set file_records [punkcheck::load_records_from_file $o_checkfile] + + set this_installer_record [my as_record] + + set persistedinfo [punkcheck::recordlist::get_installer_record $o_name $file_records] + set existing_header_posn [dict get $persistedinfo position] + if {$existing_header_posn == -1} { + set file_records [linsert $file_records 0 $this_installer_record] + } else { + lset file_records $existing_header_posn $this_installer_record + } + punkcheck::save_records_to_file $file_records $o_checkfile } method events {args} { tailcall $o_events {*}$args @@ -157,46 +539,50 @@ namespace eval punkcheck { if {$o_active_event ne ""} { error "[self] start_event error - event already started: $o_active_event" } + if {$o_rel_sourceroot eq "" || $o_rel_targetroot eq ""} { + error "[self] No configured sourceroot or targetroot. Call [self] set_source_target first" + } + if {[llength $configdict] %2 != 0} { error "[self] new_event configdict must have an even number of elements" } - set eventid [punkcheck::uuid] set resultinfo [punkcheck::recordlist::get_installer_record $o_name $o_record_list] set existing_header_posn [dict get $resultinfo position] if {$existing_header_posn == -1} { - set this_installer_record [punkcheck::recordlist::new_installer_record $o_name] + error "[self] start_event - installer record missing. installer: $o_name" } else { set this_installer_record [dict get $resultinfo record] } - set event_record [punkcheck::recordlist::new_installer_event_record install\ - -id $eventid\ - -source $o_rel_sourceroot\ - -target $o_rel_targetroot\ - -config $configdict\ - ] + set eventobj [punkcheck::installevent create [namespace current]::event_[my events count] [self] $o_rel_sourceroot $o_rel_targetroot -config $configdict] + set eventid [$eventobj get_id] + set event_record [$eventobj as_record] set this_installer_record [punkcheck::recordlist::installer_record_add_event $this_installer_record $event_record] set this_installer_record [punkcheck::recordlist::installer_record_pruneevents $this_installer_record $o_record_list] - if {$existing_header_posn == -1} { - #not found - prepend - set o_record_list [linsert $o_record_list 0 $this_installer_record] - } else { - #replace - lset o_record_list $existing_header_posn $this_installer_record - } + #replace + lset o_record_list $existing_header_posn $this_installer_record punkcheck::save_records_to_file $o_record_list $o_checkfile - set o_active_event $eventid - my events add $event_record $eventid - + set o_active_event $eventobj + my events add $eventobj $eventid + return $eventobj } - method get_event {} { - return [my events item $o_active_event] + method installer_record_from_file {} { + set resultinfo [punkcheck::recordlist::get_installer_record $o_name $o_record_list] } - method add_target {targetpath} { - + method get_recordlist {} { + return $o_recordlist + } + method end_event {} { + if {$o_active_event eq ""} { + error "[self] end_event error - no active event" + } + $o_active_event end + } + method get_event {} { + return $o_active_event } if 0 { method unknown {args} { @@ -235,7 +621,7 @@ namespace eval punkcheck { set event_record [punkcheck::recordlist::new_installer_event_record install\ -id $eventid\ -source $rel_source\ - -target $rel_target\ + -targets $rel_target\ -config $config\ ] @@ -273,6 +659,7 @@ namespace eval punkcheck { if {[llength $args] %2 !=0} { error "punkcheck installfile_begin args must be name-value pairs" } + set target_relpath [lsort -dictionary -increasing $target_relpath] ;#exact sort order not critical - but must be consistent set ts [clock microseconds] set seconds [expr {$ts / 1000000}] set tsiso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"] @@ -307,18 +694,18 @@ namespace eval punkcheck { } - set extractioninfo [recordlist::extract_or_create_file_record $target_relpath $record_list] + set extractioninfo [punkcheck::recordlist::extract_or_create_fileset_record $target_relpath $record_list] set file_record [dict get $extractioninfo record] set record_list [dict get $extractioninfo recordset] set isnew [dict get $extractioninfo isnew] set oldposition [dict get $extractioninfo oldposition] unset extractioninfo - #INSTALLING will become INSTALLRECORD or INSTALLFAILED or SKIPPED upon finalisation + #INSTALL-INPROGRESS will become INSTALLRECORD or INSTALLFAILED or SKIPPED upon finalisation #-installer and -eventid keys are added here - set new_installing_record [dict create tag INSTALLING {*}$opts -tempcontext $active_event body {}] + set new_installing_record [dict create tag INSTALL-INPROGRESS {*}$opts -tempcontext $active_event body {}] #set existing_body [dict_getwithdefault $file_record body [list]] - #todo - look for existing "INSTALLING" records - mark as failed? + #todo - look for existing "INSTALL-INPROGRESS" records - mark as failed? dict lappend file_record body $new_installing_record if {$isnew} { @@ -335,8 +722,8 @@ namespace eval punkcheck { #e.g previous installrecord had 2 source records - but we now only depend on one. #The files we depended on for the previous record haven't changed themselves - but the list of files has (reduced by one) proc installfile_add_source_and_fetch_metadata {punkcheck_folder source_relpath file_record} { - if {![lib::is_file_record_installing $file_record]} { - error "installfile_add_source_and_fetch_metdata error: bad file_record - expected FILEINFO with last body element INSTALLING" + if {![lib::is_file_record_inprogress $file_record]} { + error "installfile_add_source_and_fetch_metdata error: bad file_record - expected FILEINFO with last body element *-INPROGRESS ($file_record)" } set ts_start [clock microseconds] set last_installrecord [lib::file_record_get_last_installrecord $file_record] @@ -353,7 +740,7 @@ namespace eval punkcheck { } } } - #check that this relpath not already added as child of INSTALLING + #check that this relpath not already added as child of *-INPROGRESS set file_record_body [dict_getwithdefault $file_record body [list]] ;#new file_record may have no body set installing_record [lindex $file_record_body end] set already_present_record [lib::install_record_get_matching_source_record $installing_record $source_relpath] @@ -403,14 +790,14 @@ namespace eval punkcheck { #write back to punkcheck - don't accept recordset - invalid to update anything other than the installing_record at this time proc installfile_started_install {punkcheck_folder file_record} { - if {![lib::is_file_record_installing $file_record]} { - error "installfile_started_install error: bad file_record - expected FILEINFO with last body element INSTALLING" + if {![lib::is_file_record_inprogress $file_record]} { + error "installfile_started_install error: bad file_record - expected FILEINFO with last body element *-INPROGRESS" } set punkcheck_file [file join $punkcheck_folder/.punkcheck] set record_list [load_records_from_file $punkcheck_file] set file_record_body [dict get $file_record body] - set target [dict get $file_record -target] + set targetlist [dict get $file_record -targets] set installing_record [lindex $file_record_body end] set ts_start [dict get $installing_record -ts] @@ -425,7 +812,7 @@ namespace eval punkcheck { dict set file_record body $file_record_body - set getresult [punkcheck::recordlist::get_file_record $target $record_list] + set getresult [punkcheck::recordlist::get_file_record $targetlist $record_list] set old_posn [dict get $getresult position] if {$old_posn == -1} { lappend record_list $file_record @@ -437,14 +824,14 @@ namespace eval punkcheck { return $file_record } proc installfile_finished_install {punkcheck_folder file_record} { - if {![lib::is_file_record_installing $file_record]} { - error "installfile_finished_install error: bad file_record - expected FILEINFO with last body element INSTALLING" + if {![lib::is_file_record_inprogress $file_record]} { + error "installfile_finished_install error: bad file_record - expected FILEINFO with last body element *-INPROGRESS" } set punkcheck_file [file join $punkcheck_folder/.punkcheck] set record_list [load_records_from_file $punkcheck_file] set file_record_body [dict get $file_record body] - set target [dict get $file_record -target] + set targetlist [dict get $file_record -targets] set installing_record [lindex $file_record_body end] set ts_start [dict get $installing_record -ts] @@ -462,7 +849,7 @@ namespace eval punkcheck { set file_record [punkcheck::recordlist::file_record_prune $file_record] - set oldrecordinfo [punkcheck::recordlist::get_file_record $target $record_list] + set oldrecordinfo [punkcheck::recordlist::get_file_record $targetlist $record_list] set old_posn [dict get $oldrecordinfo position] if {$old_posn == -1} { lappend record_list $file_record @@ -474,8 +861,8 @@ namespace eval punkcheck { return $file_record } proc installfile_skipped_install {punkcheck_folder file_record} { - if {![lib::is_file_record_installing $file_record]} { - set msg "installfile_skipped_install error: bad file_record - expected FILEINFO with last body element INSTALLING" + if {![lib::is_file_record_inprogress $file_record]} { + set msg "installfile_skipped_install error: bad file_record - expected FILEINFO with last body element *-INPROGRESS" append msg \n "received:" append msg \n $file_record error $msg @@ -484,7 +871,7 @@ namespace eval punkcheck { set record_list [load_records_from_file $punkcheck_file] set file_record_body [dict get $file_record body] - set target [dict get $file_record -target] + set targetlist [dict get $file_record -targets] set installing_record [lindex $file_record_body end] set ts_start [dict get $installing_record -ts] @@ -498,7 +885,7 @@ namespace eval punkcheck { set file_record [punkcheck::recordlist::file_record_prune $file_record] - set getresult [punkcheck::recordlist::get_file_record $target $record_list] + set getresult [punkcheck::recordlist::get_file_record $targetlist $record_list] set old_posn [dict get $getresult position] if {$old_posn == -1} { lappend record_list $file_record @@ -514,6 +901,16 @@ namespace eval punkcheck { namespace eval lib { namespace path ::punkcheck + proc is_file_record_inprogress {file_record} { + if {[dict get $file_record tag] ne "FILEINFO"} { + return 0 + } + set installing_record [lindex [dict_getwithdefault $file_record body [list]] end] + if {[dict_getwithdefault $installing_record tag [list]] ni [list INSTALL-INPROGRESS MODIFY-INPROGRESS DELETE-INPROGRESS VIRTUAL-INPROGRESS]} { + return 0 + } + return 1 + } proc is_file_record_installing {file_record} { if {[dict get $file_record tag] ne "FILEINFO"} { return 0 @@ -527,17 +924,18 @@ namespace eval punkcheck { proc file_record_get_last_installrecord {file_record} { set body [dict_getwithdefault $file_record body [list]] set previous_install_records [lrange $body 0 end-1] - #get last previous that is tagged INSTALLRECORD + #get last previous that is tagged INSTALLRECORD,MODIFYRECORD,VIRTUALRECORD + #REVIEW DELETERECORD ??? set revlist [lreverse $previous_install_records] foreach rec $revlist { - if {[dict get $rec tag] eq "INSTALLRECORD"} { + if {[dict get $rec tag] in [list "INSTALLRECORD" "MODIFYRECORD" "VIRTUALRECORD"]} { return $rec } } return [list] } - #should work on INSTALLING or INSTALL record - don't restrict tag to INSTALL + #should work on *-INPROGRESS or INSTALL(etc) record - don't restrict tag to INSTALL proc install_record_get_matching_source_record {install_record source_relpath} { set body [dict_getwithdefault $install_record body [list]] foreach src $body { @@ -694,7 +1092,8 @@ namespace eval punkcheck { # -overwrite older-targets will copy files with newer source timestamp over older target timestamp and those missing at the target # -overwrite all-targets will copy regardless of timestamp at target # -overwrite installedsourcechanged-targets - # review - what about slightly mismatched system clocks and mounted filesystems? caller responsibility to verify first? + # review - timestamps unreliable + # - what about slightly mismatched system clocks and mounted filesystems? caller responsibility to verify first? # if timestamp exactly equal - should we check content-hash? This is presumably only likely to occur deliberately(maliciously?) # e.g some process that digitally signs or otherwise modifies a file and preserves update timestmp? # if such a content-mismatch - what default behaviour and what options would make sense? @@ -928,10 +1327,10 @@ namespace eval punkcheck { } #sample .punkcheck file record (raw form) to make the code clearer - #punk::tdl converts to dict form e.g: tag FILEINFO -target filename body sublist - #Valid installrecord types are INSTALLRECORD SKIPPED INSTALLING + #punk::tdl converts to dict form e.g: tag FILEINFO -targets filename body sublist + #Valid installrecord types are INSTALLRECORD SKIPPED INSTALL-INPROGRESS, MODIFYRECORD MODIFY-INPROGRESS DELETERECORD DELETE-INPROGRESS # - #FILEINFO -target jjjetc-0.1.0.tm -keep_installrecords 2 -keep_skipped 1 -keep_installing 2 { + #FILEINFO -targets jjjetc-0.1.0.tm -keep_installrecords 2 -keep_skipped 1 -keep_inprogress 2 { # INSTALLRECORD -tsiso 2023-09-20T07:30:30 -ts 1695159030266610 -installer punk::mix::cli::build_modules_from_source_to_base -metadata_us 18426 -ts_start_transfer 1695159030285036 -transfer_us 10194 -elapsed_us 28620 { # SOURCE -type file -path ../src/modules/jjjetc-buildversion.txt -cksum c7c71839c36b3d21c8370fed106192fcd659eca9 -cksum_all_opts {-cksum_content 1 -cksum_meta 0 -cksum_acls 0 -cksum_usetar 0 -cksum_algorithm sha1} -changed 1 -metadata_us 3423 # SOURCE -type file -path ../src/modules/jjjetc-999999.0a1.0.tm -cksum b646fc2ee88cbd068d2e946fe929b7aea96bd39d -cksum_all_opts {-cksum_content 1 -cksum_meta 0 -cksum_acls 0 -cksum_usetar 0 -cksum_algorithm sha1} -changed 1 -metadata_us 3413 @@ -968,12 +1367,12 @@ namespace eval punkcheck { #puts stdout " rel_target: $relative_target_path" set fetch_filerec_result [punkcheck::recordlist::get_file_record $relative_target_path $punkcheck_records] - #change to use extract_or_create_file_record ? + #change to use extract_or_create_fileset_record ? set existing_filerec_posn [dict get $fetch_filerec_result position] if {$existing_filerec_posn == -1} { puts stdout "NO existing record for $relative_target_path" set has_filerec 0 - set new_filerec [dict create tag FILEINFO -target $relative_target_path] + set new_filerec [dict create tag FILEINFO -targets $relative_target_path] set filerec $new_filerec } else { set has_filerec 1 @@ -983,9 +1382,9 @@ namespace eval punkcheck { } set filerec [punkcheck::recordlist::file_record_set_defaults $filerec] - #new INSTALLREC must be tagged as INSTALLING to use recordlist::installfile_ method - set new_install_record [dict create tag INSTALLING -tsiso $ts_start_iso -ts $ts_start -installer $opt_installer -eventid $punkcheck_eventid] - dict lappend filerec body $new_install_record ;#can't use recordlist::file_record_add_installrecord as 'INSTALLING' isn't a final tag - so pruning would be mucked up. No need to prune now anyway. + #new INSTALLREC must be tagged as INSTALL-INPROGRESS to use recordlist::installfile_ method + set new_install_record [dict create tag INSTALL-INPROGRESS -tsiso $ts_start_iso -ts $ts_start -installer $opt_installer -eventid $punkcheck_eventid] + dict lappend filerec body $new_install_record ;#can't use recordlist::file_record_add_installrecord as '*-INPROGRESS' isn't a final tag - so pruning would be mucked up. No need to prune now anyway. unset new_install_record @@ -1045,7 +1444,7 @@ namespace eval punkcheck { set install_records [dict get $filerec body] set current_install_record [lindex $install_records end] - #change the tag from INSTALLING to INSTALLRECORD/SKIPPED + #change the tag from *-INPROGRESS to INSTALLRECORD/SKIPPED if {$is_skip} { set tag SKIPPED } else { @@ -1151,8 +1550,8 @@ namespace eval punkcheck { set result [dict create] foreach rec $record_list { if {[dict get $rec tag] eq "FILEINFO"} { - set tgt [dict get $rec -target] - dict set result $tgt $rec + set tgtlist [dict get $rec -targets] + dict set result $tgtlist $rec } } return $result @@ -1161,14 +1560,14 @@ namespace eval punkcheck { - #will only match if same base was used.. - proc get_file_record {targetpath record_list} { + #will only match if same base was used.. and same targetlist + proc get_file_record {targetlist record_list} { set posn 0 set found_posn -1 set record "" foreach rec $record_list { if {[dict get $rec tag] eq "FILEINFO"} { - if {[dict get $rec -target] eq $targetpath} { + if {[dict get $rec -targets] eq $targetlist} { set found_posn $posn set record $rec break @@ -1180,8 +1579,8 @@ namespace eval punkcheck { } proc file_install_record_source_changes {install_record} { #reject INSTALLFAILED items ? - if {[dict get $install_record tag] ni [list "INSTALLRECORD" "SKIPPED" "INSTALLING"]} { - error "file_install_record_source_changes bad file->install record: tag not INSTALLRECORD|SKIPPED|INSTALLING" + if {[dict get $install_record tag] ni [list "INSTALLRECORD" "SKIPPED" "INSTALL-INPROGRESS" "MODIFY-INPROGRESS" "MODIFYRECORD" "VIRTUAL-INPROGRESS" "VIRTUALRECORD" "DELETERECORD" "DELETE-INPROGRESS"]} { + error "file_install_record_source_changes bad file->install record: tag not INSTALLRECORD|SKIPPED|INSTALL-INPROGRESS|MODIFYRECORD|MODIFY-INPROGRESS|VIRTUALRECORD|VIRTUAL-INPROGRESS|DELETERECORD|DELETE-INPROGRESS" } set source_list [dict_getwithdefault $install_record body [list]] set changed [list] @@ -1191,7 +1590,7 @@ namespace eval punkcheck { if {[dict get $src -changed] !=0} { lappend changed [dict get $src -path] } else { - lappend unchnaged [dict get $src -path] + lappend unchanged [dict get $src -path] } } else { lappend changed [dict get $src -path] @@ -1350,7 +1749,7 @@ namespace eval punkcheck { if {[dict get $file_record tag] ne "FILEINFO"} { error "file_record_add_installrecord bad file_record: tag not FILEINFO" } - #disallow 'INSTALLING' as it's not a final tag + #disallow '-INPROGRESS' as it's not a final tag if {[dict get $install_record tag] ni [list "INSTALLRECORD" "SKIPPED"]} { error "file_record_add_installrecord bad install_record: tag not INSTALLRECORD" } @@ -1390,7 +1789,7 @@ namespace eval punkcheck { if {[dict get $file_record tag] ne "FILEINFO"} { error "file_record_set_defaults bad file_record: tag not FILEINFO" } - set defaults [list -keep_installrecords 2 -keep_skipped 1 -keep_installing 2] + set defaults [list -keep_installrecords 3 -keep_skipped 1 -keep_inprogress 2] dict for {k v} $defaults { if {![dict exists $file_record $k]} { dict set file_record $k $v @@ -1405,7 +1804,7 @@ namespace eval punkcheck { error "file_record_prune bad file_record: tag not FILEINFO" } set file_record [file_record_set_defaults $file_record] - set kmap [list -keep_installrecords INSTALLRECORD -keep_skipped SKIPPED -keep_installing INSTALLING] + set kmap [list -keep_installrecords *RECORD -keep_skipped SKIPPED -keep_inprogress *-INPROGRESS] foreach {key rtype} $kmap { set keep [dict get $file_record $key] if {[dict exists $file_record body]} { @@ -1416,7 +1815,7 @@ namespace eval punkcheck { set kept_body_items [list] set kcount 0 foreach item [lreverse $body_items] { - if {[dict get $item tag] eq $rtype} { + if {[string match $rtype [dict get $item tag]]} { incr kcount if {$keep < 0 || $kcount <= $keep} { lappend kept_body_items $item @@ -1433,19 +1832,19 @@ namespace eval punkcheck { #extract new or existing filerecord for path given #review - locking/concurrency - proc extract_or_create_file_record {relative_target_path recordset} { - set fetch_record_result [punkcheck::recordlist::get_file_record $relative_target_path $recordset] + proc extract_or_create_fileset_record {relative_target_paths recordset} { + set fetch_record_result [punkcheck::recordlist::get_file_record $relative_target_paths $recordset] set existing_posn [dict get $fetch_record_result position] if {$existing_posn == -1} { - #puts stdout "NO existing record for $relative_target_path" + #puts stdout "NO existing record for $relative_target_paths" set isnew 1 - set file_record [dict create tag FILEINFO -target $relative_target_path body {}] + set fileset_record [dict create tag FILEINFO -targets $relative_target_paths body {}] } else { set recordset [lreplace $recordset[unset recordset] $existing_posn $existing_posn] set isnew 0 - set file_record [dict get $fetch_record_result record] + set fileset_record [dict get $fetch_record_result record] } - return [list record $file_record recordset $recordset isnew $isnew oldposition $existing_posn] + return [list record $fileset_record recordset $recordset isnew $isnew oldposition $existing_posn] } } diff --git a/src/modules/punkcheck/cli-999999.0a1.0.tm b/src/modules/punkcheck/cli-999999.0a1.0.tm index 5eb8dfa6..87a83b30 100644 --- a/src/modules/punkcheck/cli-999999.0a1.0.tm +++ b/src/modules/punkcheck/cli-999999.0a1.0.tm @@ -35,9 +35,112 @@ namespace eval punkcheck::cli { } return $bottom_to_top } + #todo! - group by fileset proc status {{path {}}} { if {$path eq {}} { set path [pwd] } set fullpath [file normalize $path] + + set ftype [file type $fullpath] + + + set files [list] + if {$ftype eq "file"} { + set container [file dirname $fullpath] + lappend files $fullpath + } else { + set container $fullpath + #vfs can mask mounted files - so we can't just use 'file type' or glob with -type f + ##set files [glob -nocomplain -dir $fullpath -type f *] + set folderinfo [punk::dirfiles_dict $fullpath] + set files [concat [dict get $folderinfo files] [dict get $folderinfo underlayfiles]] + } + set punkcheck_files [paths $container] + set repodict [punk::repo::find_repo $container] + + if {![llength $punkcheck_files]} { + puts stderr "No .punkcheck files found at or above this folder" + } + + set table "" + set files_with_records [list] + foreach p $punkcheck_files { + set basedir [file dirname $p] + set recordlist [punkcheck::load_records_from_file $p] + set tgt_dict [punkcheck::recordlist::records_as_target_dict $recordlist] + + foreach f $files { + set relpath [punkcheck::lib::path_relative $basedir $f] + + if {[dict exists $tgt_dict $relpath]} { + set filerec [dict get $tgt_dict $relpath] + set records [punkcheck::dict_getwithdefault $filerec body [list]] + if {$ftype eq "file"} { + if {![llength $records]} { + set pcheck "(has file record but no installation entries)" + } else { + set pcheck \n + foreach irec $records { + append pcheck [punk::tdl::prettyprint [list $irec] 1] \n + #append pcheck " $irec" \n + } + } + } else { + if {![llength $records]} { + set pcheck "(has file record but no installation entries)" + } else { + set display_records [list] + set pcheck \n + set latest_install_record [punkcheck::recordlist::file_record_latest_installrecord $filerec] + lappend display_records $latest_install_record + if {$latest_install_record ne [lindex $records end]} { + lappend display_records [lindex $records end] + } + foreach irec $display_records { + append pcheck "[format %-14s [dict get $irec tag]] [punkcheck::dict_getwithdefault $irec -tsiso "no-timestamp"]" + set bodyrecords [punkcheck::dict_getwithdefault $irec body [list]] + set source_files [list] + set source_files_changed [list] + set source_folders [list] + set source_folders_changed [list] + foreach r $bodyrecords { + if {[dict get $r tag] eq "SOURCE"} { + set path [dict get $r -path] + set changed [dict get $r -changed] + if {[dict get $r -type] eq "file"} { + lappend source_files $path + if {$changed} { + lappend source_files_changed $path + } + } elseif {[dict get $r -type] eq "directory"} { + lappend source_folders $path + if {$changed} { + lappend source_folders_changed $path + } + } + } + } + if {[llength $source_files]} { + append pcheck " source files : [llength $source_files] (changed [llength $source_files_changed])" + } + if {[llength $source_folders]} { + append pcheck " source folders: [llength $source_folders] (changed [llength $source_folders_changed])" + } + append pcheck \n + + #append pcheck [punk::tdl::prettyprint [list $irec] 1] \n + #append pcheck " $irec" \n + } + } + } + append table "$f $pcheck" \n + } + } + } + return $table + } + proc status_by_file {{path {}}} { + if {$path eq {}} { set path [pwd] } + set fullpath [file normalize $path] set ftype [file type $fullpath] set files [list] if {$ftype eq "file"} { @@ -62,6 +165,7 @@ namespace eval punkcheck::cli { set tgt_dict [punkcheck::recordlist::records_as_target_dict $recordlist] foreach f $files { set relpath [punkcheck::lib::path_relative $basedir $f] + if {[dict exists $tgt_dict $relpath]} { set filerec [dict get $tgt_dict $relpath] set records [punkcheck::dict_getwithdefault $filerec body [list]]