# -*- tcl -*-
# Maintenance Instruction: leave the 999999.xxx.x as is and use 'pmix make' or src/make.tcl to update from <pkg>-buildversion.txt
#
# Please consider using a BSD or MIT style license for greatest compatibility with the Tcl ecosystem.
# Code using preferred Tcl licenses can be eligible for inclusion in Tcllib, Tklib and the punk package repository.
# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
# (C) 2023
#
# @@ Meta Begin
# Application punkcheck 0.1.0
# Meta platform     tcl
# Meta license      <unspecified>
# @@ Meta End



# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
##  Requirements
##e.g package require frobz

package require punk::tdl
package require punk::repo
package require punk::mix::util



# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
# Punkcheck uses the TDL format which is a list of lists in Tcl format
# It is intended primarily for source build/distribution tracking within a punk project or single filesystem - with relative paths.
namespace eval punkcheck {
        variable default_antiglob_dir_core  [list "#*" "_aside" ".git" ".fossil*"]
        variable default_antiglob_file_core ""
        proc uuid {} {
            set has_twapi 0
            if {"windows" eq $::tcl_platform(platform)} {
                if {![catch {package require twapi}]} {
                    set has_twapi 1
                }
            }
            if {!$has_twapi} {
                if {[catch {package require uuid} errM]} {
                    error "Unable to load a package for uuid on this platform. Try tcllib's uuid (any platform) - twapi for windows"
                }
                return [uuid::uuid generate]
            } else {
                return [twapi::new_uuid]
            }
        }

        proc default_antiglob_dir_core {} {
            variable default_antiglob_dir_core
            return $default_antiglob_dir_core
        }
        proc default_antiglob_file_core {} {
            variable default_antiglob_file_core
            if {$default_antiglob_file_core eq ""} {
                set default_antiglob_file_core [list "*.swp" "*[punk::mix::util::magic_tm_version]*" "*-buildversion.txt" ".punkcheck"]
            }
            return $default_antiglob_file_core
        }


        proc load_records_from_file {punkcheck_file} { 
            set record_list [list]
            if {[file exists $punkcheck_file]} {
                set tdlscript [punk::mix::util::fcat $punkcheck_file]
                set record_list [punk::tdl::prettyparse $tdlscript]
            }
            return $record_list
        }
        proc save_records_to_file {recordlist punkcheck_file} {
            set newtdl [punk::tdl::prettyprint $recordlist]
            set linecount [llength [split $newtdl \n]]
            #puts stdout $newtdl
            set fd [open $punkcheck_file w]
            fconfigure $fd -translation binary
            puts -nonewline $fd $newtdl
            close $fd
            return [list recordcount [llength $recordlist] linecount $linecount]
        }
        proc start_installer_event {punkcheck_file installername from_fullpath to_fullpath config} {
            #set eventid [punkcheck::start_installer_event $punkcheck_file $opt_installer $from $to $config]
            set eventid [punkcheck::uuid]
            if {[file pathtype $from_fullpath] ne "absolute"} {
                error "start_installer_event error: from_fullpath must be absolute path. Received '$from_fullpath'"
            }
            if {[file pathtype $to_fullpath] ne "absolute"} {
                error "start_installer_event error: to_fullpath must be absolute path. Received '$to_fullpath'"
            }
            set punkcheck_folder [file dirname $punkcheck_file]
            set rel_source [punkcheck::lib::path_relative $punkcheck_folder $from_fullpath]
            set rel_target [punkcheck::lib::path_relative $punkcheck_folder $to_fullpath]


            set record_list [punkcheck::load_records_from_file $punkcheck_file]
            set resultinfo [punkcheck::recordlist::get_installer_record $installername $record_list]
            set existing_header_posn [dict get $resultinfo position]
            if {$existing_header_posn == -1} {
                set this_installer_record [punkcheck::recordlist::new_installer_record $installername]
            } else {
                set this_installer_record [dict get $resultinfo record]
            }

            set event_record [punkcheck::recordlist::new_installer_event_record install\
                -id $eventid\
                -source $rel_source\
                -target $rel_target\
                -config $config\
            ]

            set this_installer_record [punkcheck::recordlist::installer_record_add_event $this_installer_record $event_record]
            set this_installer_record [punkcheck::recordlist::installer_record_pruneevents $this_installer_record $record_list]

            if {$existing_header_posn == -1} {
                #not found - prepend
                set record_list [linsert $record_list 0 $this_installer_record]
            } else {
                #replace
                lset record_list $existing_header_posn $this_installer_record
            }

            punkcheck::save_records_to_file $record_list $punkcheck_file
            return [list eventid $eventid recordset $record_list]
        }
        #-----------------------------------------------
        proc installfile_help {} {
            set msg     "Call in order:" \n
            append msg  "  start_installer_event (get dict with eventid and recordset keys)"
            append msg  "  installfile_begin  (to return a new INSTALLING record) - must pass in a valid eventid"  \n
            append msg  "  installfile_add_source_and_fetch_metadata (1+ times to update SOURCE record with checksum/timestamp info from source)" \n 
            append msg  "     ( - possibly with same algorithm as previous installrecord)" \n
            append msg  "     ( - todo - search/load metadata for this source from other FILEINFO records for same installer)" \n
            append msg  "Finalize by calling:" \n
            append msg  "  installfile_started_install" \n
            append msg  "  (install the file e.g file copy)" \n
            append msg  "  installfile_finished_install" \n
            append msg  "  OR" \n
            append msg  "  installfile_skipped_install" \n
        }
        proc installfile_begin {punkcheck_folder target_relpath installername args} {
            if {[llength $args] %2 !=0} {
                error "punkcheck installfile_begin args must be name-value pairs"
            }
            set ts [clock microseconds]
            set seconds [expr {$ts / 1000000}]
            set tsiso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"]
            set defaults [list\
                -tsiso $tsiso\
                -ts $ts\
                -installer $installername\
                -eventid unspecified\
                ]
            set opts [dict merge $defaults $args]
            set opt_eventid [dict get $opts -eventid]

            set punkcheck_file [file join $punkcheck_folder/.punkcheck]
            set record_list [load_records_from_file $punkcheck_file]
           
            set resultinfo [punkcheck::recordlist::get_installer_record $installername $record_list]
            set installer_record_position [dict get $resultinfo position]
            if {$installer_record_position == -1} {
                error "installfile_begin error: Failed to retrieve installer record for installer name:'$installername' - ensure start_installer_event has been called with same installer, and -eventid is passed to installfile_begin"
            }
            set this_installer_record [dict get $resultinfo record]
            set events [dict get $this_installer_record body]
            set active_event [list]
            foreach evt [lreverse $events] {
                if {[dict get $evt -id] eq $opt_eventid} {
                    set active_event $evt
                    break
                }
            }
            if {![llength $active_event]} {
                error "installfile_begin error: eventid $opt_eventid not found for installer '$installername' - aborting"
            }


            set extractioninfo [recordlist::extract_or_create_file_record $target_relpath $record_list]
            set file_record [dict get $extractioninfo record]
            set record_list [dict get $extractioninfo recordset]
            set isnew       [dict get $extractioninfo isnew]
            set oldposition [dict get $extractioninfo oldposition]
            unset extractioninfo

            #INSTALLING will become INSTALLRECORD or INSTALLFAILED or SKIPPED upon finalisation
            #-installer and -eventid keys are added here
            set new_installing_record [dict create tag INSTALLING {*}$opts -tempcontext $active_event body {}]
            #set existing_body [dict_getwithdefault $file_record body [list]]
            #todo - look for existing "INSTALLING" records - mark as failed?
            dict lappend file_record body $new_installing_record

            if {$isnew} {
                lappend record_list $file_record
            } else {
                set record_list [linsert $record_list[unset record_list] $oldposition $file_record]
            }

            save_records_to_file $record_list $punkcheck_file
            return $file_record
        }

        proc installfile_add_source_and_fetch_metadata {punkcheck_folder source_relpath file_record} {
            if {![lib::is_file_record_installing $file_record]} {
                error "installfile_add_source_and_fetch_metdata error: bad file_record - expected FILEINFO with last body element INSTALLING"
            }
            set ts_start [clock microseconds]
            set last_installrecord [lib::file_record_get_last_installrecord $file_record]
            set prev_cksum ""
            set prev_cksum_opts ""
            if {[llength $last_installrecord]} {
                set src [lib::install_record_get_matching_source_record $last_installrecord $source_relpath]
                if {[llength $src]} {
                    if {[dict_getwithdefault $src -path ""] eq $source_relpath} {
                        set prev_ftype [dict_getwithdefault $src -type ""]
                        set prev_cksum [dict_getwithdefault $src -cksum ""]
                        set prev_cksum_opts [dict_getwithdefault $src -cksum_all_opts ""]
                    }
                }
            }
            #check that this relpath not already added as child of INSTALLING 
            set file_record_body [dict_getwithdefault $file_record body [list]] ;#new file_record may have no body
            set installing_record [lindex $file_record_body end]
            set already_present_record [lib::install_record_get_matching_source_record $installing_record $source_relpath]
            if {[llength $already_present_record]} {
                error "installfile_add_source_and_fetch_metadata error: source path $source_relpath already exists in the file_record - cannot add again"
            }

            if {$prev_cksum_opts ne ""} {
                set cksum_opts $prev_cksum_opts
            } else {
                set cksum_opts ""
            }

            set ftype [file type [file join $punkcheck_folder/$source_relpath]]
            if {$ftype eq "directory"} {
                set source_cksum_info [punk::mix::base::lib::get_relativecksum_from_base $punkcheck_folder $source_relpath {*}$cksum_opts]
            } else {
                #todo - optionally use mtime instead of cksum (for files only)
                set source_cksum_info [punk::mix::base::lib::get_relativecksum_from_base $punkcheck_folder $source_relpath {*}$cksum_opts]
            }


            lassign $source_cksum_info pathkey ckinfo
            if {$pathkey ne $source_relpath} {
                error "installfile_add_source_and_fetch_metadata error: cksum returned wrong path info '$pathkey' expected '$source_relpath'"
            }
            set cksum [dict get $ckinfo cksum]
            set cksum_all_opts [dict get $ckinfo cksum_all_opts]
            if {$cksum ne $prev_cksum || $ftype ne $prev_ftype} {
                set changed 1
            } else {
                set changed 0
            }
            set installing_record_sources [dict_getwithdefault $installing_record body [list]]
            set ts_now [clock microseconds] ;#gathering metadata - especially checsums on folder can take some time - calc and store elapsed us for time taken to gather metadata
            set metadata_us [expr {$ts_now - $ts_start}]
            set this_source_record [dict create tag SOURCE -type $ftype -path $source_relpath -cksum $cksum -cksum_all_opts $cksum_all_opts -changed $changed -metadata_us $metadata_us]
            lappend installing_record_sources $this_source_record
            dict set installing_record body $installing_record_sources

            lset file_record_body end $installing_record

            dict set file_record body $file_record_body
            return $file_record
        }

        #write back to punkcheck - don't accept recordset - invalid to update anything other than the installing_record at this time
        proc installfile_started_install {punkcheck_folder file_record} {
            if {![lib::is_file_record_installing $file_record]} {
                error "installfile_started_install error: bad file_record - expected FILEINFO with last body element INSTALLING"
            }
            set punkcheck_file [file join $punkcheck_folder/.punkcheck]
            set record_list [load_records_from_file $punkcheck_file]

            set file_record_body [dict get $file_record body]
            set target [dict get $file_record -target]
            set installing_record [lindex $file_record_body end]

            set ts_start [dict get $installing_record -ts]
            set ts_now [clock microseconds] 
            set metadata_us [expr {$ts_now - $ts_start}]
            
            dict set installing_record -metadata_us $metadata_us
            dict set installing_record -ts_start_transfer $ts_now

            lset file_record_body end $installing_record
            
            dict set file_record body $file_record_body


            set getresult [punkcheck::recordlist::get_file_record $target $record_list]
            set old_posn [dict get $getresult position]
            if {$old_posn == -1} {
                lappend record_list $file_record
            } else {
                lset record_list $old_posn $file_record
            }

            save_records_to_file $record_list $punkcheck_file
            return $file_record
        }
        proc installfile_finished_install {punkcheck_folder file_record} {
            if {![lib::is_file_record_installing $file_record]} {
                error "installfile_finished_install error: bad file_record - expected FILEINFO with last body element INSTALLING"
            }
            set punkcheck_file [file join $punkcheck_folder/.punkcheck]
            set record_list [load_records_from_file $punkcheck_file]

            set file_record_body [dict get $file_record body]
            set target [dict get $file_record -target]
            set installing_record [lindex $file_record_body end]

            set ts_start [dict get $installing_record -ts]
            set ts_start_transfer [dict get $installing_record -ts_start_transfer]
            set ts_now [clock microseconds]
            set elapsed_us [expr {$ts_now - $ts_start}]
            set transfer_us [expr {$ts_now - $ts_start_transfer}]
            dict set installing_record  -transfer_us $transfer_us
            dict set installing_record -elapsed_us $elapsed_us
            dict unset installing_record -tempcontext
            dict set installing_record tag "INSTALLRECORD"

            lset file_record_body end $installing_record
            dict set file_record body $file_record_body 

            set file_record [punkcheck::recordlist::file_record_prune $file_record]

            set oldrecordinfo [punkcheck::recordlist::get_file_record $target $record_list]
            set old_posn [dict get $oldrecordinfo position]
            if {$old_posn == -1} {
                lappend record_list $file_record
            } else {
                lset record_list $old_posn $file_record
            }

            save_records_to_file $record_list $punkcheck_file
            return $file_record
        }
        proc installfile_skipped_install {punkcheck_folder file_record} {
            if {![lib::is_file_record_installing $file_record]} {
                set msg "installfile_skipped_install error: bad file_record - expected FILEINFO with last body element INSTALLING"
                append msg \n "received:"
                append msg \n $file_record
                error $msg
            }
            set punkcheck_file [file join $punkcheck_folder/.punkcheck]
            set record_list [load_records_from_file $punkcheck_file]

            set file_record_body [dict get $file_record body]
            set target [dict get $file_record -target]
            set installing_record [lindex $file_record_body end]

            set ts_start [dict get $installing_record -ts]
            set tsnow [clock microseconds]
            set elapsed_us [expr {$tsnow - $ts_start}]
            dict set installing_record -elapsed_us $elapsed_us
            dict set installing_record tag "SKIPPED" 
             
            lset file_record_body end $installing_record
            dict set file_record body $file_record_body

            set file_record [punkcheck::recordlist::file_record_prune $file_record]

            set getresult [punkcheck::recordlist::get_file_record $target $record_list]
            set old_posn [dict get $getresult position]
            if {$old_posn == -1} {
                lappend record_list $file_record
            } else {
                lset record_list $old_posn $file_record
            }

            save_records_to_file $record_list $punkcheck_file
            return $file_record
        }
        #-----------------------------------------------
        #then: file_record_add_installrecord

        namespace eval lib {
            namespace path ::punkcheck
            proc is_file_record_installing {file_record} {
                if {[dict get $file_record tag] ne "FILEINFO"} {
                    return 0
                }
                set installing_record [lindex [dict_getwithdefault $file_record body [list]] end]
                if {[dict_getwithdefault $installing_record tag [list]] ne "INSTALLING"} {
                    return 0
                }
                return 1
            }
            proc file_record_get_last_installrecord {file_record} {
                set body [dict_getwithdefault $file_record body [list]]
                set previous_install_records [lrange $body 0 end-1]
                #get last previous that is tagged INSTALLRECORD
                set revlist [lreverse $previous_install_records]
                foreach rec $revlist {
                    if {[dict get $rec tag] eq "INSTALLRECORD"} {
                        return $rec
                    }
                }
                return [list]
            }

            #should work on INSTALLING or INSTALL record - don't restrict tag to INSTALL
            proc install_record_get_matching_source_record {install_record source_relpath} {
                set body [dict_getwithdefault $install_record body [list]] 
                foreach src $body {
                    if {[dict get $src tag] eq "SOURCE"} {
                        if {[dict_getwithdefault $src -path ""] eq $source_relpath} {
                            return $src
                        }
                    }
                }
                return [list]
            }



            #maint warning - also in punk::mix::util
            proc path_relative {base dst} {
                #see also kettle
                # Modified copy of ::fileutil::relative (tcllib)
                # Adapted to 8.5 ({*}).
                #
                #    Taking two _directory_ paths, a base and a destination, computes the path
                #    of the destination relative to the base.
                #
                # Arguments:
                #    base    The path to make the destination relative to.
                #    dst     The destination path
                #
                # Results:
                #    The path of the destination, relative to the base.

                # Ensure that the link to directory 'dst' is properly done relative to
                # the directory 'base'.

                #review - check volume info on windows.. UNC paths?
                if {[file pathtype $base] ne [file pathtype $dst]} {
                    return -code error "Unable to compute relation for paths of different pathtypes: [file pathtype $base] vs. [file pathtype $dst], ($base vs. $dst)"
                }

                #avoid normalizing if possible - at least for relative paths which we are likely to loop on (file normalize *very* expensive on windows)
                set do_normalize 0
                if {[file pathtype $base] eq "relative"} {
                    #if base is relative so is dst
                    if {[regexp {[.]{2}} [list $base $dst]]} {
                        set do_normalize 1
                    }
                    if {[regexp {[.]/} [list $base $dst]]} {
                        set do_normalize 1
                    }
                } else {
                    #case differences in volumes is common on windows 
                    set do_normalize 1
                }
                if {$do_normalize} {
                    set base [file normalize $base]
                    set dst  [file normalize $dst]
                }

                set save $dst
                set base [file split $base]
                set dst  [file split $dst]

                while {[lindex $dst 0] eq [lindex $base 0]} {
                    set dst  [lrange $dst  1 end]
                    set base [lrange $base 1 end]
                    if {![llength $dst]} {break}
                }

                set dstlen  [llength $dst]
                set baselen [llength $base]

                if {($dstlen == 0) && ($baselen == 0)} {
                    # Cases:
                    # (a) base == dst

                    set dst .
                    } else {
                    # Cases:
                    # (b) base is: base/sub = sub
                    #     dst  is: base     = {}

                    # (c) base is: base     = {}
                    #     dst  is: base/sub = sub

                    while {$baselen > 0} {
                        set dst [linsert $dst 0 ..]
                        incr baselen -1
                    }
                    set dst [file join {*}$dst]
                }

                return $dst
            }
        }
        #skip writing punkcheck during checksum/timestamp checks

        proc install_tm_files {srcdir basedir args} {
            set defaults [list\
                -glob *.tm\
                -antiglob_file [list "*[punk::mix::util::magic_tm_version]*"]\
                -installer punkcheck::install_tm_files\
                ]
            set opts [dict merge $defaults $args]
            punkcheck::install $srcdir $basedir {*}$opts
        }
        proc install_non_tm_files {srcdir basedir args} {
            #set keys [dict keys $args]
            set defaults [list\
                -glob *\
                -antiglob_file [list "*.tm" "*-buildversion.txt" "*.exe"]\
                -installer punkcheck::install_non_tm_files\
                ]
            set opts [dict merge $defaults $args]
            punkcheck::install $srcdir $basedir {*}$opts
        }

        #for tcl8.6 - tcl8.7+ has dict getwithdefault
        proc dict_getwithdefault {dictValue args} {
            if {[llength $args] < 2} {
                error {wrong # args: should be "dict_getwithdefault dictValue ?key ...? key default"}
            }
            set default [lindex $args end]
            set keys [lrange $args 0 end-1]

            if {[dict exists $dictValue {*}$keys]} {
                return [dict get $dictValue {*}$keys]
            } else {
                return $default
            }
        }
        proc pathglob_as_re {glob} {
            #any segment that is not just * must match exactly one segment in the path
            set pats [list]
            foreach seg [file split $glob] {
                if {$seg eq "*"} {
                    lappend pats {[^/]*}
                } elseif {$seg eq "**"} {
                    lappend pats {.*}
                } else {
                    set seg [string map [list  . {[.]}] $seg]
                    if {[regexp {[*?]} $seg]} {
                        set pat [string map [list * {[^/]*} ? {[^/]}] $seg]
                        lappend pats "$pat"
                    } else {
                        lappend pats "$seg"
                    }
                }
            }
            return  "^[join $pats /]\$"
        }
        proc globmatchpath {glob path} {
            return [regexp [pathglob_as_re $glob] $path]
        }
        ## unidirectional file transfer to possibly non empty folder
        #default of -overwrite no-targets will only copy files that are missing at the target
        #           -overwrite newer-targets will copy files with older source timestamp over newer target timestamp and those missing at the target (a form of 'restore' operation)
        #           -overwrite older-targets will copy files with newer source timestamp over older target timestamp and those missing at the target
        #           -overwrite all-targets will copy regardless of timestamp at target
        #           -overwrite installedsourcechanged-targets 
        #           review - what about slightly mismatched system clocks and mounted filesystems? caller responsibility to verify first? 
        #           if timestamp exactly equal -  should we check content-hash? This is presumably only likely to occur deliberately(maliciously?) 
        #           e.g some process that digitally signs or otherwise modifies a file and preserves update timestmp?
        #           if such a content-mismatch - what default behaviour and what options would make sense?
        #           probably it's reasonable that only all-targets would overwrite such files.
        #           consider -source_fudge_seconds  +-X  ?,  -source_override_timestamp ts  ???  etc which only adjust timestamp for calculation purposes? Define a specific need/usecase when reviewing.
        #
        # valid filetypes for src tgt
        #  src dir      tgt dir
        #  todo - review and consider enabling symlink src and dst
        #  no need for src file  - as we use -glob with no glob characters to match one  source file file
        #  no ability to target file with different name - keep it simpler and caller will have to use an intermediate folder/file if they need to rename something?
        #
        #  todo - determine what happens if mismatch between file type of a src vs target e.g target has dir matching filename at source
        #  A pre-scan to determine no such conflict - before attempting to copy anything might provide the most integrity at slight cost in speed.
        #  REVIEW  we should only expect dirs to be created as necessary to hold files? i.e target folder won't be created if no source file matches for that folder
        #  -source_checksum  compare|store|comparestore|false|true  where true == comparestore
        #  -punkcheck_folder target|source|project|<absolutepath>  target is default and is generally recommended
        #  -punkcheck_records empty string | parsed TDL records ie {tag xxx k v}  structure
        proc install {srcdir tgtdir args} {
            set defaults [list\
                -call-depth-internal 0\
                -max_depth 1000\
                -subdirlist {}\
                -glob *\
                -antiglob_file_core "\uFFFF"\
                -antiglob_file "" \
                -antiglob_dir_core "\uFFFF"\
                -antiglob_dir {}\
                -unpublish_paths {}\
                -overwrite no-targets\
                -source_checksum comparestore\
                -punkcheck_folder target\
                -punkcheck_eventid "\uFFFF"\
                -punkcheck_records ""\
                -installer punkcheck::install\
                ]

            set opts [dict merge $defaults $args]
            if {([llength $args] %2) != 0} {
                error "punkcheck::install requires option-style arguments to be in pairs. Received args: $args"
            }
            foreach k [dict keys $args] {
                if {$k ni [dict keys $defaults]} {
                    error "punkcheck::install  unrecognised option '$k' known options: '[dict keys $defaults]'"
                }
            }

            #The choice to recurse using the original values of srcdir & tgtdir, and passing the subpath down as a list in -subdirlist seems an odd one.
            #(as opposed to a more 'standard' mechanism of adjusting srcdir & tgtdir as we move down the tree)
            #It comes from build_modules_from_source_to_base where we need to keep track of position relative to our targetdir starting point to handle submodules e.g pkg::something::mypkg-0.1.tm
            #It could have been handled with some other parameter such as -depth, but this -subdirlist mechanism, whilst perhaps not beautiful, is straightforward enough
            #and may be less error prone than doing slightly more opaue path manipulations at each recursion level to determine where we started
            #For consistency - we'll use the same mechanism in various recursive directory walking procedures such as this one.
            set CALLDEPTH                               [dict get $opts -call-depth-internal] ;#added for extra debug/sanity checking - clearer test for initial function call ie CALLDPEPTH = 0
            set max_depth                               [dict get $opts -max_depth]
            set subdirlist                              [dict get $opts -subdirlist] ;# generally should be same length as CALLDEPTH - but user could prefill
            set fileglob                                [dict get $opts -glob]

            if {$CALLDEPTH == 0} {
                #expensive to normalize  but we need to do it at least once
                set srcdir [file normalize $srcdir]
                set tgtdir [file normalize $tgtdir]
                #now the values we build from these will be properly cased
            }
            # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
            set opt_antiglob_file_core                  [dict get $opts -antiglob_file_core]
            if {$opt_antiglob_file_core eq "\uFFFF"} {
                set opt_antiglob_file_core [default_antiglob_file_core]
            }
            # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
            set opt_antiglob_file                       [dict get $opts -antiglob_file]
            # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
            set opt_antiglob_dir_core                   [dict get $opts -antiglob_dir_core]
            if {$opt_antiglob_dir_core eq "\uFFFF"} {
                set opt_antiglob_dir_core [default_antiglob_dir_core]
            }
            # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
            set opt_antiglob_dir                        [dict get $opts -antiglob_dir]
            # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
            set opt_unpublish_paths                     [dict get $opts -unpublish_paths] ;#todo - combine with config file in source tree .punkcheckpublish (?)
            set unpublish_paths_matched [list]
            # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
            set known_whats [list no-targets newer-targets older-targets all-targets installedsourcechanged-targets]
            set overwrite_what                          [string tolower [dict get $opts -overwrite]]; #accept any case for value to allow emphasis by caller e.g -overwrite NEWER-TARGETS
            if {$overwrite_what ni $known_whats} {
                error "punkcheck::install received unrecognised value for -overwrite. Received value '$overwrite_what' vs known values '$known_whats'"
            }
            if {$overwrite_what in [list newer-targets older-targets]} {
                error "punkcheck::install newer-target, older-targets not implemented - sorry"
                #TODO - check crossplatform availability of ctime  (on windows it still seems to be creation time, but on bsd/linux it's last attribute mod time)
                # external pkg?  use twapi and ctime only on other platforms?
            }
            # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
            set opt_source_checksum                     [dict get $opts -source_checksum]
            if {[string is boolean $opt_source_checksum]} {
                if {$opt_source_checksum} {
                    set opt_source_checksum "comparestore"
                } else {
                    set opt_source_checksum 0
                }
            }
            # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
            set opt_punkcheck_folder                    [dict get $opts -punkcheck_folder]
            if {$opt_punkcheck_folder eq "target"} {
                set punkcheck_folder $tgtdir
            } elseif {$opt_punkcheck_folder eq "source"} {
                set punkcheck_folder $srcdir
            } elseif {$opt_punkcheck_folder eq "project"} {
                set sourceprojectinfo [punk::repo::find_repos $srcdir]
                set targetprojectinfo [punk::repo::find_repos $tgtdir]
                set srcproj [lindex [dict get $sourceprojectinfo project] 0]
                set tgtproj [lindex [dict get $targetprojectinfo project] 0]
                if {$srcproj eq $tgtproj} {
                    set punkcheck_folder $tgtproj
                } else {
                    error "copy_files_from_source_to_target error: Unable to find common project dir for source and target folder - use absolutepath for -punkcheck_folder if source and target are not within same project"
                }
            } else {
                set punkcheck_folder $opt_punkcheck_folder
            }
            if {$punkcheck_folder ne ""} {
                if {[file pathtype $punkcheck_folder] ne "absolute"} {
                    error "copy_files_from_source_to_target error: -punkcheck_folder '$punkcheck_folder' must be an absolute path, or one of: target|source|project"
                }
                if {![file isdirectory $punkcheck_folder]} {
                    error "copy_files_from_source_to_target error: -punkcheck_folder '$punkcheck_folder' not found"
                }
            } else {
                #review - leave empty? use pwd?
            }
            # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
            set punkcheck_records                       [dict get $opts -punkcheck_records]
            set punkcheck_records_init $punkcheck_records ;#change-detection
            # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
            set opt_installer                           [dict get $opts -installer]
            # -- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---
            set opt_punkcheck_eventid                             [dict get $opts -punkcheck_eventid]



            set punkcheck_file [file join $punkcheck_folder/.punkcheck]

            if {$CALLDEPTH == 0} {
                set punkcheck_eventid "<invalid>"
                if {$punkcheck_folder ne ""} {
                    set config [dict create\
                        -glob $fileglob\
                        -antiglob_file_core $opt_antiglob_file_core\
                        -antiglob_file $opt_antiglob_file\
                        -antiglob_dir_core $opt_antiglob_dir_core\
                        -antiglob_dir $opt_antiglob_dir\
                    ]
                    lassign [punkcheck::start_installer_event $punkcheck_file $opt_installer $srcdir $tgtdir $config] _eventid punkcheck_eventid _recordset punkcheck_records
                }
            } else {
                set punkcheck_eventid $opt_punkcheck_eventid
            }



            if {$opt_source_checksum != 0} {
                #we need to read the file even if only set to store (or we would overwrite entries)
                set compare_cksums 1
            } else {
                set compare_cksums 0
            }

            if {[string match *store* $opt_source_checksum]} {
                set store_cksums 1
            } else {
                set store_cksums 0
            }
            




            if {[llength $subdirlist] == 0} {
                set current_source_dir $srcdir
                set current_target_dir $tgtdir
            } else {
                set current_source_dir $srcdir/[file join {*}$subdirlist]
                set current_target_dir $tgtdir/[file join {*}$subdirlist]
            }


            #normalize?  review/test
            foreach unpub $opt_unpublish_paths {
                if {[globmatchpath $unpub $current_source_dir]} {
                    lappend unpublish_paths_matched $current_source_dir
                    return [list files_copied {} files_skipped {} sources_unchanged {} punkcheck_records $punkcheck_records]
                }
            }


            if {![file exists $current_source_dir]} {
                error "copy_files_from_source_to_target current source dir:'$current_source_dir' doesn't exist (srcdir:$srcdir tgtdir:$tgtdir args:'$args')"
            }
            if {![file exists $current_target_dir]} {
                error "copy_files_from_source_to_target current target dir:'$current_target_dir' doesn't exist (srcdir:$srcdir tgtdir:$tgtdir args:'$args')"
            }
            if {([file type $current_source_dir] ni [list directory]) || ([file type $current_target_dir] ni [list directory])} {
                error "copy_files_from_source_to_target requires source and target dirs to be of type 'directory' type current source: [file type $current_source_dir] type current target: [file type $current_target_dir]"
            }

            set files_copied [list]
            set files_skipped [list]
            set sources_unchanged [list]


            set candidate_list [glob -nocomplain -dir $current_source_dir -type f -tail $fileglob]
            set hidden_candidate_list [glob -nocomplain -dir $current_source_dir -types {hidden f} -tail $fileglob]
            foreach h $hidden_candidate_list {
                if {$h ni $candidate_list} {
                    lappend candidate_list $h
                }
            }
            set match_list [list]
            foreach m $candidate_list {
                set suppress 0
                foreach anti [concat $opt_antiglob_file_core $opt_antiglob_file] {
                    if {[string match $anti $m]} {
                        #puts stderr "anti: $anti  vs m:$m"
                        set suppress 1
                        break
                    }
                }
                if {$suppress == 0} {
                    lappend match_list $m
                } 
            }

            #sample .punkcheck file record (raw form) to make the code clearer
            #punk::tdl converts to dict form e.g: tag FILEINFO -target filename body sublist
            #Valid installrecord types are INSTALLRECORD SKIPPED INSTALLING
            #
            #FILEINFO -target jjjetc-0.1.0.tm -keep_installrecords 2 -keep_skipped 1 -keep_installing 2 {
            #    INSTALLRECORD -tsiso 2023-09-20T07:30:30 -ts 1695159030266610 -installer punk::mix::cli::build_modules_from_source_to_base -metadata_us 18426 -ts_start_transfer 1695159030285036 -transfer_us 10194 -elapsed_us 28620 {
            #        SOURCE -type file -path ../src/modules/jjjetc-buildversion.txt -cksum c7c71839c36b3d21c8370fed106192fcd659eca9 -cksum_all_opts {-cksum_content 1 -cksum_meta 0 -cksum_acls 0 -cksum_usetar 0 -cksum_algorithm sha1} -changed 1 -metadata_us 3423
            #        SOURCE -type file -path ../src/modules/jjjetc-999999.0a1.0.tm -cksum b646fc2ee88cbd068d2e946fe929b7aea96bd39d -cksum_all_opts {-cksum_content 1 -cksum_meta 0 -cksum_acls 0 -cksum_usetar 0 -cksum_algorithm sha1} -changed 1 -metadata_us 3413
            #    }
            #    SKIPPED -tsiso 2023-09-20T08:14:26 -ts 1695161666087880 -installer punk::mix::cli::build_modules_from_source_to_base -elapsed_us 18914 {
            #        SOURCE -type file -path ../src/modules/jjjetc-buildversion.txt -cksum c7c71839c36b3d21c8370fed106192fcd659eca9 -cksum_all_opts {-cksum_content 1 -cksum_meta 0 -cksum_acls 0 -cksum_usetar 0 -cksum_algorithm sha1} -changed 0 -metadata_us 3435
            #        SOURCE -type file -path ../src/modules/jjjetc-999999.0a1.0.tm -cksum b646fc2ee88cbd068d2e946fe929b7aea96bd39d -cksum_all_opts {-cksum_content 1 -cksum_meta 0 -cksum_acls 0 -cksum_usetar 0 -cksum_algorithm sha1} -changed 0 -metadata_us 3338
            #    }
            #}
            
            
            #proc get_relativecksum_from_base_and_fullpath {base fullpath args}
          

            #puts stdout     "Current target dir: $current_target_dir"
            foreach m $match_list {
                set is_unpublished 0
                foreach unpub $opt_unpublish_paths {
                    if {[globmatchpath $unpub $current_source_dir/$m]} {
                        lappend unpublish_paths_matched $current_source_dir
                        set is_unpublished 1
                        break
                    }
                }
                if {$is_unpublished} {
                    continue
                }
                #puts stdout "    checking file : $current_source_dir/$m"
                set ts_start [clock microseconds]
                set seconds [expr {$ts_start / 1000000}]
                set ts_start_iso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"]

                set relative_target_path [lib::path_relative $punkcheck_folder $current_target_dir/$m]
                #puts stdout "  rel_target: $relative_target_path"
                
                set fetch_filerec_result [punkcheck::recordlist::get_file_record $relative_target_path $punkcheck_records]
                #change to use extract_or_create_file_record ?
                set existing_filerec_posn [dict get $fetch_filerec_result position]
                if {$existing_filerec_posn == -1} {
                    puts stdout "NO existing record for $relative_target_path"
                    set has_filerec 0
                    set new_filerec [dict create tag FILEINFO -target $relative_target_path]
                    set filerec $new_filerec
                } else {
                    set has_filerec 1
                    #puts stdout "  TDL existing FILEINFO record for $relative_target_path"
                    #puts stdout "  $existing_install_record"
                    set filerec [dict get $fetch_filerec_result record]
                }
                set filerec [punkcheck::recordlist::file_record_set_defaults $filerec]
             
                #new INSTALLREC must be tagged as INSTALLING to use recordlist::installfile_ method
                set new_install_record   [dict create tag INSTALLING -tsiso $ts_start_iso -ts $ts_start -installer $opt_installer -eventid $punkcheck_eventid]
                dict lappend filerec body $new_install_record ;#can't use recordlist::file_record_add_installrecord as 'INSTALLING' isn't a final tag - so pruning would be mucked up. No need to prune now anyway.
                unset new_install_record





                set relative_source_path [lib::path_relative $punkcheck_folder $current_source_dir/$m]
                #puts stdout "  rel_source: $relative_source_path"
                if {[file pathtype $relative_source_path] ne "relative"} {
                    #different volume or root
                }
                #Note this isn't a recordlist function - so it doesn't purely operate on the records
                #this hits the filesystem for the sourcepath - gets checksums/timestamps depending on config. 
                #It doesn't save to .punkcheck (the only punkcheck::installfile_ method which doesn't)
                set filerec [punkcheck::installfile_add_source_and_fetch_metadata $punkcheck_folder $relative_source_path $filerec]
                
                set changeinfo [punkcheck::recordlist::file_install_record_source_changes [lindex [dict get $filerec body] end]]
                set changed [dict get $changeinfo changed]
                set unchanged [dict get $changeinfo unchanged]
                if {[llength $unchanged]} {
                    lappend sources_unchanged $current_source_dir/$m
                }

                set is_skip 0
                if {$overwrite_what eq "all-targets"} {
                    file copy -force $current_source_dir/$m $current_target_dir
                    lappend files_copied $current_source_dir/$m
                } else {
                    if {![file exists $current_target_dir/$m]} {
                        file copy $current_source_dir/$m $current_target_dir
                        lappend files_copied $current_source_dir/$m
                        incr filecount_new
                    } else {
                        if {$overwrite_what eq "installedsourcechanged-targets"} {
                            if {[llength $changed]} {
                                #An unrecorded installation is considered a source change (from unknown/unrecorded source to recorded)
                                file copy -force $current_source_dir/$m $current_target_dir
                                lappend files_copied $current_source_dir/$m
                            } else {
                                set is_skip 1
                                lappend files_skipped $current_source_dir/$m
                            }
                        } else {
                            set is_skip 1
                            puts stderr "Skipping file copy $m target $current_target_dir/$m already exists (use -overwrite all-targets  to overwrite)"
                            #TODO! implement newer-targets older-targets
                            lappend files_skipped $current_source_dir/$m
                        }
                    }
                }


                set ts_now    [clock microseconds]
                set elapsed_us [expr {$ts_now - $ts_start}]

                if {$store_cksums} {

                    set install_records [dict get $filerec body] 
                    set current_install_record [lindex $install_records end]
                    #change the tag from INSTALLING to INSTALLRECORD/SKIPPED
                    if {$is_skip} {
                        set tag SKIPPED
                    } else {
                        set tag INSTALLRECORD
                    }
                    dict set current_install_record tag $tag
                    dict set current_install_record -elapsed_us $elapsed_us
                    lset install_records end $current_install_record
                    dict set filerec body $install_records
                    set filerec [punkcheck::recordlist::file_record_prune $filerec] ;#prune now that tag is finalized
                    if {!$has_filerec} {
                        #not found in original recordlist - append
                        lappend punkcheck_records $filerec
                    } else {
                        lset punkcheck_records $existing_filerec_posn $filerec
                    }
                }

            }

            if {$CALLDEPTH >= $max_depth} {
                #don't process any more subdirs
                set subdirs [list]
            } else {
                set subdirs [glob -nocomplain -dir $current_source_dir -type d -tail *]
                set hiddensubdirs [glob -nocomplain -dir $current_source_dir -type {hidden d} -tail *]
                foreach h $hiddensubdirs {
                    if {$h in [list "." ".."]} {
                        continue
                    }
                    if {$h ni $subdirs} {
                        lappend subdirs $h
                    }
                }
            }
            #puts stderr "subdirs: $subdirs"
            foreach d $subdirs {
                set skipd 0
                foreach dg [concat $opt_antiglob_dir_core $opt_antiglob_dir] {
                    if {[string match $dg $d]} {
                        puts stdout "SKIPPING FOLDER $d due to antiglob_dir-match: $dg "
                        set skipd 1
                        break
                    }
                }
                if {$skipd} {
                    continue
                }


                if {![file exists $current_target_dir/$d]} {
                    file mkdir $current_target_dir/$d
                }
                
                set sub_result [punkcheck::install $srcdir $tgtdir\
                    -call-depth-internal [expr {$CALLDEPTH + 1}]\
                    -subdirlist [list {*}$subdirlist $d]\
                    -glob $fileglob\
                    -antiglob_file_core $opt_antiglob_file_core\
                    -antiglob_file $opt_antiglob_file\
                    -antiglob_dir_core  $opt_antiglob_dir_core\
                    -antiglob_dir $opt_antiglob_dir\
                    -overwrite $overwrite_what\
                    -source_checksum $opt_source_checksum\
                    -punkcheck_folder $punkcheck_folder\
                    -punkcheck_eventid $punkcheck_eventid\
                    -punkcheck_records $punkcheck_records\
                    -installer $opt_installer\
                    ]

                lappend files_copied {*}[dict get $sub_result files_copied]
                lappend files_skipped {*}[dict get $sub_result files_skipped]
                lappend sources_unchanged {*}[dict get $sub_result sources_unchanged]
                lappend unpublish_paths_matched {*}[dict get $sub_result unpublish_paths_matched]
                set punkcheck_records [dict get $sub_result punkcheck_records]
            }
            
            if {[string match *store* $opt_source_checksum]} {
                #puts "subdirlist: $subdirlist"
                if {$CALLDEPTH == 0} {
                    if {[llength $files_copied]} {
                        puts stdout ">>>>>>>>>>>>>>>>>>>"
                        set saveresult [punkcheck::save_records_to_file $punkcheck_records $punkcheck_file]
                        puts stdout "[dict get $saveresult recordcount] records saved as [dict get $saveresult linecount] lines to $punkcheck_file"
                        puts stdout ">>>>>>>>>>>>>>>>>>>"
                    } else {
                        #todo - write db INSTALLER record if -debug true
                        
                    }
                    #puts stdout "sources_unchanged"
                    #puts stdout "$sources_unchanged"
                    #puts stdout "- -- --- --- --- ---"
                }
            }

            return [list files_copied $files_copied files_skipped $files_skipped sources_unchanged $sources_unchanged unpublish_paths_matched $unpublish_paths_matched punkcheck_records $punkcheck_records]
        }

        namespace eval recordlist {
            namespace path ::punkcheck

            proc records_as_target_dict {record_list} {
                set result [dict create]
                foreach rec $record_list {
                    if {[dict get $rec tag] eq "FILEINFO"} {
                        set tgt [dict get $rec -target]
                        dict set result $tgt $rec
                    }
                }
                return $result
            }




            #will only match if same base was used.. 
            proc get_file_record {targetpath record_list} {
                set posn 0
                set found_posn -1
                set record ""
                foreach rec $record_list {
                    if {[dict get $rec tag] eq "FILEINFO"} {
                        if {[dict get $rec -target] eq $targetpath} {
                            set found_posn $posn
                            set record $rec
                            break
                        }
                    }
                    incr posn
                }
                return [list position $found_posn record $record]
            }
            proc file_install_record_source_changes {install_record} {
                #reject INSTALLFAILED items ?
                if {[dict get $install_record tag] ni [list "INSTALLRECORD" "SKIPPED" "INSTALLING"]} {
                    error "file_install_record_source_changes bad file->install record: tag not INSTALLRECORD|SKIPPED|INSTALLING"
                }
                set source_list [dict_getwithdefault $install_record body [list]]
                set changed [list]
                set unchanged [list]
                foreach src $source_list {
                    if {[dict exists $src -changed]} {
                        if {[dict get $src -changed] !=0} {
                            lappend changed [dict get $src -path]
                        } else {
                            lappend unchnaged [dict get $src -path]
                        }
                    } else {
                        lappend changed [dict get $src -path]
                    }
                }
                return [dict create changed $changed unchanged $unchanged]
            }

            #assume only one for name - use first encountered
            proc get_installer_record {name record_list} {
                set posn 0
                set found_posn -1
                set record ""
                #puts ">>>> checking [llength $record_list] punkcheck records"
                foreach rec $record_list {
                    if {[dict get $rec tag] eq "INSTALLER"} {
                        if {[dict get $rec -name] eq $name} {
                            set found_posn $posn
                            set record $rec
                            break
                        }
                    }
                    incr posn
                }
                return [list position $found_posn record $record]
            }

            proc new_installer_record {name args} {
                if {[llength $args] %2 !=0} {
                    error "punkcheck new_installer_record args must be name-value pairs"
                }
                set ts [clock microseconds]
                set seconds [expr {$ts / 1000000}]
                set tsiso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"]

                #put -tsiso first so it lines up with -tsiso in event records
                set defaults [list\
                    -tsiso $tsiso\
                    -ts $ts\
                    -name $name\
                    -keep_events 5\
                    ]
                set opts [dict merge $defaults $args]

                #set this_installer_record_list [punk::tdl::prettyparse [list INSTALLER name $opt_installer ts $ts tsiso $tsiso keep_events 5 {}]]
                #set this_installer_record [lindex $this_installer_record_list 0]

                set record [dict create tag INSTALLER {*}$opts body {}]


                return $record
            }
            proc new_installer_event_record {type args} {
                if {[llength $args] %2 !=0} {
                    error "punkcheck new_installer_event_record args must be name-value pairs"
                }
                set ts [clock microseconds]
                set seconds [expr {$ts / 1000000}]
                set tsiso [clock format $seconds -format "%Y-%m-%dT%H:%M:%S"]
                set defaults [list\
                    -tsiso $tsiso\
                    -ts $ts\
                    -type $type\
                    ]
                set opts [dict merge $defaults $args]

                set record [dict create tag EVENT {*}$opts]
            }
            #need to scan entire set if filerecords to check if event is still referenced
            proc installer_record_pruneevents {installer_record record_list} {
                set keep 5
                if {[dict exists $installer_record -keep_events]} {
                    set keep [dict get $installer_record -keep_events]
                }

                if {[dict exists $installer_record body]} {
                    set body_items [dict get $installer_record body]
                } else {
                    set body_items [list]
                }
                set kept_body_items [list]
                set kcount 0
                foreach item [lreverse $body_items] {
                    if {[dict get $item tag] eq "EVENT"} {
                        incr kcount
                        if {$keep < 0 || $kcount <= $keep} {
                            lappend kept_body_items $item
                        } else {
                            set eventid ""
                            if {[dict exists $item -id]} {
                                set eventid [dict get $item -id]
                            }
                            if {$eventid ne "" && $eventid ne "unspecified"} {
                                #keep if referenced, discard if not, or if eventid empty/unspecified
                                set is_referenced 0
                                foreach rec $record_list {
                                    if {[dict get $rec tag] eq "FILEINFO"} {
                                        if {[dict exists $rec body]} {
                                            foreach install [dict get $rec body] {
                                                if {[dict exists $install -eventid] && [dict get $install -eventid] eq $eventid} {
                                                    set is_referenced 1
                                                    break
                                                }
                                            }
                                        }
                                    }
                                    if {$is_referenced} {
                                        break
                                    }
                                }
                                if {$is_referenced} {
                                    lappend kept_body_items $item
                                }
                            }
                        }
                    } else {
                        lappend kept_body_items $item
                    }
                }
                set kept_body_items [lreverse $kept_body_items]
                dict set installer_record body $kept_body_items
                return $installer_record
            }
            proc installer_record_add_event {installer_record event} {
                if {[dict get $installer_record tag] ne "INSTALLER"} {
                    error "installer_record_add_event bad installer record: tag not INSTALLER"
                }
                if {[dict get $event tag] ne "EVENT"} {
                    error "installer_record_add_event bad event record: tag not EVENT"
                }
                if {[dict exists $installer_record body]} {
                    set body_items [dict get $installer_record body]
                } else {
                    set body_items [list]
                }
                lappend body_items $event
                dict set installer_record body $body_items
                return $installer_record
            }
            proc file_record_add_installrecord {file_record install_record} {
                if {[dict get $file_record tag] ne "FILEINFO"} {
                    error "file_record_add_installrecord bad file_record: tag not FILEINFO"
                }
                #disallow 'INSTALLING' as it's not a final tag
                if {[dict get $install_record tag] ni [list "INSTALLRECORD" "SKIPPED"]} {
                    error "file_record_add_installrecord bad install_record: tag not INSTALLRECORD"
                }
                set keep 3
                if {[dict exists $file_record -keep_installrecords]} {
                    set keep [dict get $file_record -keep_installrecords]
                }

                if {[dict exists $file_record body]} {
                    set body_items [dict get $file_record body]
                } else {
                    set body_items [list]
                }
                lappend body_items $install_record
                set kept_body_items [list]
                set kcount 0
                foreach item [lreverse $body_items] {
                    if {[dict get $item tag] eq "INSTALLRECORD"} {
                        incr kcount
                        if {$keep < 0 || $kcount <= $keep} {
                            lappend kept_body_items $item
                        }
                    } else {
                        lappend kept_body_items $item
                    }
                }
                set kept_body_items [lreverse $kept_body_items]

                dict set file_record body $kept_body_items
                return $file_record


            }


            proc file_record_set_defaults {file_record} {
                if {[dict get $file_record tag] ne "FILEINFO"} {
                    error "file_record_set_defaults bad file_record: tag not FILEINFO"
                }
                set defaults [list -keep_installrecords 2 -keep_skipped 1 -keep_installing 2] 
                dict for {k v} $defaults {
                    if {![dict exists $file_record $k]} {
                        dict set file_record $k $v
                    }
                }
                return $file_record
            }

            #negative keep_ value will keep all
            proc file_record_prune {file_record} {
                if {[dict get $file_record tag] ne "FILEINFO"} {
                    error "file_record_prune bad file_record: tag not FILEINFO"
                }
                set file_record [file_record_set_defaults $file_record]
                set kmap [list -keep_installrecords INSTALLRECORD -keep_skipped SKIPPED -keep_installing INSTALLING]
                foreach {key rtype} $kmap {
                    set keep [dict get $file_record $key]
                    if {[dict exists $file_record body]} {
                        set body_items [dict get $file_record body]
                    } else {
                        set body_items [list]
                    }
                    set kept_body_items [list]
                    set kcount 0
                    foreach item [lreverse $body_items] {
                        if {[dict get $item tag] eq $rtype} {
                            incr kcount
                            if {$keep < 0 || $kcount <= $keep} {
                                lappend kept_body_items $item
                            }
                        } else {
                            lappend kept_body_items $item
                        }
                    }
                    set kept_body_items [lreverse $kept_body_items]
                    dict set file_record body $kept_body_items
                }
                return $file_record
            }

            #extract new or existing filerecord for path given
            #review - locking/concurrency
            proc extract_or_create_file_record {relative_target_path recordset} {
                set fetch_record_result [punkcheck::recordlist::get_file_record $relative_target_path $recordset]
                set existing_posn [dict get $fetch_record_result position]
                if {$existing_posn == -1} {
                    #puts stdout "NO existing record for $relative_target_path"
                    set isnew 1
                    set file_record [dict create tag FILEINFO -target $relative_target_path body {}]
                } else {
                    set recordset [lreplace $recordset[unset recordset] $existing_posn $existing_posn]
                    set isnew 0
                    set file_record [dict get $fetch_record_result record]
                }
                return [list record $file_record recordset $recordset isnew $isnew oldposition $existing_posn]
            }

        }

}














# ++ +++ +++ +++ +++ +++ +++ +++ +++ +++ +++
##  Ready   
package provide punkcheck [namespace eval punkcheck {
    variable version
    set version 0.1.0 
}]
return