From f90cd52561d93545ff9414ff826b93eb76867046 Mon Sep 17 00:00:00 2001 From: "@royashbrook" Date: Mon, 9 Nov 2026 15:45:51 -0500 Subject: [PATCH 1/3] sent: send each row once, 0.12.0 (fix #36) --- DataAgent/CAPABILITIES.md | 4 +- DataAgent/DataAgent.psd1 | 2 +- DataAgent/DataAgent.psm1 | 28 ++++++++ README.md | 4 +- examples/job.ps1 | 2 +- testing/DataAgent.Test/DataAgent.Test.psd1 | 4 +- tests/acceptance.ps1 | 3 +- tests/sent.ps1 | 84 ++++++++++++++++++++++ 8 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 tests/sent.ps1 diff --git a/DataAgent/CAPABILITIES.md b/DataAgent/CAPABILITIES.md index ce2d329..b015b47 100644 --- a/DataAgent/CAPABILITIES.md +++ b/DataAgent/CAPABILITIES.md @@ -16,6 +16,8 @@ the runner loads Add-PrefixForLogging, optionally calls Clear-Files with `purgef `directory`, if present, is the path the run works in: its log, relative adapter paths and output all resolve there. give an absolute path; a relative one resolves against the caller's current location, not the job. leave it out and the run works in the calling script's directory. set it when another module wraps the runner and makes the call, since that module's own file would otherwise be the caller. +`sent`, if present, sends each row once. `{ key, path, keep_days }`: `key` is a column or a list of columns, and with no key the whole row is the key, so an edited row goes again. rows whose key a receipt already names are dropped after the source, and the log says how many. if none are left, that is `No data available`. a receipt is one csv line per key, written to `path` (default `sent`, in the job directory) as `yyyyMMddHHmmss.txt`, only after every destination returned. a failed send or a format-only run marks nothing. `keep_days` removes a receipt once the date in its name is older than that many days, and its rows can go again. without it, receipts are kept. a key column the source does not return stops the run, since it would make every row the same row. receipts are files, so a job that runs on fresh checkouts has to commit them back. + `file_format`, if present, is formatted with the run's current date and overrides `fmt.args.Path`. otherwise supply Path yourself. the configured filename goes to the formatter and destinations, not a FileInfo object. built-ins write directly and overwrite. use a dedicated job directory without overlapping workers. ## built-ins @@ -81,7 +83,7 @@ adapters must not change location. the runner moves PowerShell's location, not t the runner does not install dependencies. every helper is imported at a pinned version, the one the tests ran against: Add-PrefixForLogging 1.0.0.3, Clear-Files 1.2.0.0, SqlServer 22.4.5.1, Send-FileViaEmail 2.1.0.0, ImportExcel 7.8.10, Posh-SSH 4.0.0. install exactly those versions on the runner, only for the helpers the job uses; a newer install beside them changes nothing, a missing pinned version stops the run. -no cache/deduplication, acknowledgment tracking, retry engine, receipts, scheduler or conditional routing. cleanup remains Clear-Files behavior; choose deliberate patterns in a dedicated output directory. provider limits still apply. before adopting a feed, prove its output/layout, runner identity, dependencies, cleanup and scoped delivery behavior. this brief is not a live-provider test. +no cache beyond `sent` receipts, no acknowledgment tracking, retry engine, scheduler or conditional routing. a receipt means a destination returned, not that the receiver processed the file. cleanup remains Clear-Files behavior; choose deliberate patterns in a dedicated output directory. provider limits still apply. before adopting a feed, prove its output/layout, runner identity, dependencies, cleanup and scoped delivery behavior. this brief is not a live-provider test. 0.4 replaces the old Pipeline/Extract/Transform/Deliver API with Config and src/fmt/dst. installation does not migrate jobs. optional synthetic exercises are in DataAgent.Test, not production mock modes. diff --git a/DataAgent/DataAgent.psd1 b/DataAgent/DataAgent.psd1 index 9b61430..8bdb8e4 100644 --- a/DataAgent/DataAgent.psd1 +++ b/DataAgent/DataAgent.psd1 @@ -1,6 +1,6 @@ @{ RootModule = 'DataAgent.psm1' - ModuleVersion = '0.11.0' + ModuleVersion = '0.12.0' GUID = '32662c0b-0a5d-49ac-8c62-8f7ae05f79c0' Author = 'Roy Ashbrook' Copyright = '(c) 2026 Roy Ashbrook. MIT License.' diff --git a/DataAgent/DataAgent.psm1 b/DataAgent/DataAgent.psm1 index 05e7d9b..425c8a5 100644 --- a/DataAgent/DataAgent.psm1 +++ b/DataAgent/DataAgent.psm1 @@ -62,6 +62,27 @@ function Invoke-DataAgent { } l 'Get Data' $data = @(& ($adapters.src[0]) -Data @() -Options $Config.src.args | Where-Object { $null -ne $_ }) + if ($data.Count -and $Config.sent) { + $sent = $Config.sent + $receipts = if ($sent.path) { $sent.path } else { 'sent' } + $columns = if ($data[0] -is [Data.DataRow]) { $data[0].Table.Columns.ColumnName } else { $data[0].PSObject.Properties.Name } + $fields = if ($sent.key) { @($sent.key) } else { @($columns) } + # a missing key column would make every row the same row, and after one send nothing goes again + foreach ($field in $fields) { if ($field -notin $columns) { throw "sent.key '$field' is not a column of the source." } } + $went = [Collections.Generic.HashSet[string]]::new() + foreach ($receipt in Get-ChildItem -LiteralPath $receipts -Filter *.txt -File -ErrorAction Ignore) { + # the name dates a receipt, not its mtime: a fresh checkout stamps every file now + if ($sent.keep_days -and [datetime]::ParseExact($receipt.Name.Substring(0, 8), 'yyyyMMdd', $null) -lt (Get-Date).Date.AddDays(-[int]$sent.keep_days)) { + Remove-Item -LiteralPath $receipt.FullName; continue + } + # read back through csv so a value with a line break or other quoting compares the same + foreach ($line in Import-Csv -LiteralPath $receipt.FullName -Header $fields | ConvertTo-Csv -NoHeader -UseQuotes AsNeeded) { $null = $went.Add($line) } + } + $lines = @($data | Select-Object $fields | ConvertTo-Csv -NoHeader -UseQuotes AsNeeded) + $new = @(for ($i = 0; $i -lt $data.Count; $i++) { if (!$went.Contains($lines[$i])) { $i } }) + l "Sent before: $($data.Count - $new.Count) of $($data.Count) row(s) skipped" + $data = @(foreach ($i in $new) { $data[$i] }); $lines = @(foreach ($i in $new) { $lines[$i] }) + } if (!$data.Count) { l 'No data available'; return } l 'Data Found. Formatting File.' $options = if ($Config.fmt.args) { @{} + $Config.fmt.args } else { @{} } @@ -74,6 +95,13 @@ function Invoke-DataAgent { } l 'Use Data' for ($i = 0; $i -lt $adapters.dst.Count; $i++) { & ($adapters.dst[$i]) -Data $file -Options @($Config.dst)[$i].args } + # only after every destination returned; a format-only run sent nothing, so it marks nothing + if ($Config.sent -and $adapters.dst.Count) { + $null = New-Item -ItemType Directory -Force -Path $receipts + $receipt = Join-Path $receipts ('{0:yyyyMMddHHmmss}.txt' -f (Get-Date)) + $lines | Sort-Object -Unique | Set-Content -LiteralPath $receipt + l "Receipt: $receipt" + } l 'End' } *>&1 | Tee-Object -Append $log } catch { diff --git a/README.md b/README.md index 9eed727..cb1ead8 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,13 @@ the repeated part of a feed job: get data, format it, send it. use the tools you already use. ```powershell -Install-Module DataAgent -RequiredVersion 0.11.0 -Scope CurrentUser +Install-Module DataAgent -RequiredVersion 0.12.0 -Scope CurrentUser ``` ## the job ```powershell -Import-Module DataAgent -RequiredVersion 0.11.0 +Import-Module DataAgent -RequiredVersion 0.12.0 Invoke-DataAgent "$PSScriptRoot/settings.json" ``` diff --git a/examples/job.ps1 b/examples/job.ps1 index c1cf34f..62c4119 100644 --- a/examples/job.ps1 +++ b/examples/job.ps1 @@ -1,2 +1,2 @@ -Import-Module DataAgent -RequiredVersion 0.11.0 -ErrorAction Stop +Import-Module DataAgent -RequiredVersion 0.12.0 -ErrorAction Stop Invoke-DataAgent "$PSScriptRoot/settings.json" diff --git a/testing/DataAgent.Test/DataAgent.Test.psd1 b/testing/DataAgent.Test/DataAgent.Test.psd1 index 39d7f72..93bc6a1 100644 --- a/testing/DataAgent.Test/DataAgent.Test.psd1 +++ b/testing/DataAgent.Test/DataAgent.Test.psd1 @@ -1,6 +1,6 @@ @{ RootModule = 'DataAgent.Test.psm1' - ModuleVersion = '0.11.0' + ModuleVersion = '0.12.0' GUID = '3d16fcb0-a991-4c65-a6e4-3bfcd360688d' Author = 'Roy Ashbrook' Copyright = '(c) 2026 Roy Ashbrook. MIT License.' @@ -10,7 +10,7 @@ CmdletsToExport = @() VariablesToExport = @() AliasesToExport = @() - RequiredModules = @(@{ ModuleName = 'DataAgent'; RequiredVersion = '0.11.0' }) + RequiredModules = @(@{ ModuleName = 'DataAgent'; RequiredVersion = '0.12.0' }) FileList = @('DataAgent.Test.psd1', 'DataAgent.Test.psm1', 'LICENSE', 'synthetic.csv') PrivateData = @{ PSData = @{ Tags = @('ETL', 'DataAgent'); ProjectUri = 'https://github.com/royashbrook/DataAgent'; LicenseUri = 'https://github.com/royashbrook/DataAgent/blob/main/LICENSE' } } } diff --git a/tests/acceptance.ps1 b/tests/acceptance.ps1 index 3bf998e..0acb5bf 100644 --- a/tests/acceptance.ps1 +++ b/tests/acceptance.ps1 @@ -375,6 +375,7 @@ Assert ((Get-Content "$root/list/login.txt") -eq 'PSCredential|synthetic user|li Remove-Item Env:DATAAGENT_TEST_SOURCE, Env:DATAAGENT_TEST_USER . "$PSScriptRoot/transfers.ps1" +. "$PSScriptRoot/sent.ps1" . "$PSScriptRoot/cosmos.ps1" & "$PSScriptRoot/invoice-formats.ps1" & "$PSScriptRoot/partner-options.ps1" @@ -388,7 +389,7 @@ foreach ($package in @('DataAgent','testing/DataAgent.Test')) { } Remove-Module DataAgent -Force $env:PSModulePath = (@("$root/staged", $priorModules) -join [IO.Path]::PathSeparator) -Import-Module DataAgent.Test -RequiredVersion 0.11.0 +Import-Module DataAgent.Test -RequiredVersion 0.12.0 $result = @(Test-DataAgent) Assert (@($result | Where-Object { $_ -is [IO.FileInfo] -and $_.Name -eq 'output.csv' }).Count -eq 1) 'staged optional test package exercises bundled formatter' $env:PSModulePath = $priorModules diff --git a/tests/sent.ps1 b/tests/sent.ps1 new file mode 100644 index 0000000..174c6b5 --- /dev/null +++ b/tests/sent.ps1 @@ -0,0 +1,84 @@ +# sent: rows a receipt already names are skipped, and a receipt is written only after every destination +# returned. a stub destination counts what it was handed and fails when told to. +$dst = 'param($Data,$Options); if (Test-Path fail.flag) { throw "send refused" }; (Import-Csv $Data).Count | Add-Content sends.txt' +function Sends($Name) { @(Get-Content "$root/$Name/sends.txt" -ErrorAction Ignore) } +function Receipts($Name) { @(Get-ChildItem "$root/$Name/sent" -Filter *.txt -ErrorAction Ignore) } + +$cfg = Config 'sent-key' +$dst | Set-Content "$root/sent-key/dst.ps1" +$cfg.dst = @{ adapter = './dst.ps1'; args = @{} } +$cfg.sent = @{ key = 'Id'; keep_days = 14 } +$null = Run 'sent-key' $cfg +Assert ((Sends 'sent-key') -join ',' -eq '5' -and (Receipts 'sent-key').Count -eq 1) 'first run sends every row and writes one receipt' +Assert ((Get-Content (Receipts 'sent-key')[0].FullName) -join ',' -eq '1,2,3,4,5') 'the receipt names each key once' +$log = Run 'sent-key' $cfg +Assert ((Sends 'sent-key').Count -eq 1 -and ($log -match 'No data available')) 'a second run sends nothing' + +# one new row goes, the rest stay skipped +Import-Csv "$repo/testing/DataAgent.Test/synthetic.csv" | Select-Object -First 2 | Export-Csv "$root/sent-key/more.csv" +[pscustomobject]@{ Id = '6'; Description = 'new'; Quantity = '1' } | Export-Csv "$root/sent-key/more.csv" -Append +$cfg.src.args.LiteralPath = "$root/sent-key/more.csv" +Start-Sleep 1 +$null = Run 'sent-key' $cfg +Assert ((Sends 'sent-key')[-1] -eq '1' -and (Receipts 'sent-key').Count -eq 2) 'only the new row goes' + +# a receipt past keep_days is removed by its name's date, and its rows go again +Rename-Item (Receipts 'sent-key')[0].FullName ('20000101000000.txt') +$cfg.src.args.LiteralPath = "$repo/testing/DataAgent.Test/synthetic.csv" +Start-Sleep 1 +$null = Run 'sent-key' $cfg +Assert ((Sends 'sent-key')[-1] -eq '5' -and !(Test-Path "$root/sent-key/sent/20000101000000.txt")) 'an expired receipt is dropped and its rows go again' + +# a failed destination marks nothing sent +$cfg = Config 'sent-fail' +$dst | Set-Content "$root/sent-fail/dst.ps1" +'' | Set-Content "$root/sent-fail/fail.flag" +$cfg.dst = @{ adapter = './dst.ps1'; args = @{} } +$cfg.sent = @{ key = 'Id'; path = 'receipts' } +Refuses { Run 'sent-fail' $cfg } 'send refused' +Assert (!(Test-Path "$root/sent-fail/receipts")) 'a failed send writes no receipt' +Remove-Item "$root/sent-fail/fail.flag" +$null = Run 'sent-fail' $cfg +Assert ((Sends 'sent-fail') -join ',' -eq '5' -and @(Get-ChildItem "$root/sent-fail/receipts").Count -eq 1) 'the next run sends them, receipts in the named path' + +# format-only filters but marks nothing +$cfg = Config 'sent-format' +$cfg.sent = @{ key = 'Id' } +$null = Run 'sent-format' $cfg +Assert ((Test-Path "$root/sent-format/output.csv") -and !(Test-Path "$root/sent-format/sent")) 'a format-only run writes no receipt' + +# whole row: an edited row is a new row. a value with a line break and quotes compares the same. +$cfg = Config 'sent-row' +$dst | Set-Content "$root/sent-row/dst.ps1" +$cfg.dst = @{ adapter = './dst.ps1'; args = @{} } +$cfg.sent = @{} +$cfg.sent.path = 'sent' +$null = Run 'sent-row' $cfg +$null = Run 'sent-row' $cfg +Assert ((Sends 'sent-row') -join ',' -eq '5') 'whole rows, the multi-line and quoted ones too, are skipped on the rerun' +$rows = Import-Csv "$repo/testing/DataAgent.Test/synthetic.csv" +$rows[0].Quantity = '13' +$rows | Export-Csv "$root/sent-row/edited.csv" +$cfg.src.args.LiteralPath = "$root/sent-row/edited.csv" +Start-Sleep 1 +$null = Run 'sent-row' $cfg +Assert ((Sends 'sent-row')[-1] -eq '1') 'an edited row goes again' + +# a list of fields is one key +$cfg = Config 'sent-pair' +$dst | Set-Content "$root/sent-pair/dst.ps1" +$cfg.dst = @{ adapter = './dst.ps1'; args = @{} } +$cfg.sent = @{ key = @('Id', 'Quantity') } +$null = Run 'sent-pair' $cfg +$rows = Import-Csv "$repo/testing/DataAgent.Test/synthetic.csv" +$rows[0].Description = 'renamed'; $rows[1].Quantity = '7' +$rows | Export-Csv "$root/sent-pair/changed.csv" +$cfg.src.args.LiteralPath = "$root/sent-pair/changed.csv" +Start-Sleep 1 +$null = Run 'sent-pair' $cfg +Assert ((Sends 'sent-pair')[-1] -eq '1') 'only a change to a key field makes a row new' + +# a key that is not a column would skip everything after one send, so it stops the run +$cfg = Config 'sent-missing' +$cfg.sent = @{ key = 'InvoiceNumber' } +Refuses { Run 'sent-missing' $cfg } "sent.key 'InvoiceNumber' is not a column" From 0ac23541a31f7fe57ed3504dbe881b0a41ea4753 Mon Sep 17 00:00:00 2001 From: "@royashbrook" Date: Mon, 9 Nov 2026 15:54:05 -0500 Subject: [PATCH 2/3] sent: append a same-second receipt, keep case-distinct keys (refs #36) --- DataAgent/DataAgent.psm1 | 4 +++- tests/sent.ps1 | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/DataAgent/DataAgent.psm1 b/DataAgent/DataAgent.psm1 index 425c8a5..c45043c 100644 --- a/DataAgent/DataAgent.psm1 +++ b/DataAgent/DataAgent.psm1 @@ -99,7 +99,9 @@ function Invoke-DataAgent { if ($Config.sent -and $adapters.dst.Count) { $null = New-Item -ItemType Directory -Force -Path $receipts $receipt = Join-Path $receipts ('{0:yyyyMMddHHmmss}.txt' -f (Get-Date)) - $lines | Sort-Object -Unique | Set-Content -LiteralPath $receipt + # append: a run in the same second must not replace the receipt before it. case-sensitive, + # like the compare, or a key that differs only by case is dropped here and sent again + $lines | Sort-Object -Unique -CaseSensitive | Add-Content -LiteralPath $receipt l "Receipt: $receipt" } l 'End' diff --git a/tests/sent.ps1 b/tests/sent.ps1 index 174c6b5..c4b0cd9 100644 --- a/tests/sent.ps1 +++ b/tests/sent.ps1 @@ -78,6 +78,28 @@ Start-Sleep 1 $null = Run 'sent-pair' $cfg Assert ((Sends 'sent-pair')[-1] -eq '1') 'only a change to a key field makes a row new' +# a run in the same second as an earlier receipt keeps that receipt's keys +$cfg = Config 'sent-second' +$dst | Set-Content "$root/sent-second/dst.ps1" +$cfg.dst = @{ adapter = './dst.ps1'; args = @{} } +$cfg.sent = @{ key = 'Id' } +$null = New-Item -ItemType Directory "$root/sent-second/sent" +$now = Get-Date +foreach ($s in 0..30) { 'earlier' | Set-Content ("$root/sent-second/sent/{0:yyyyMMddHHmmss}.txt" -f $now.AddSeconds($s)) } +$null = Run 'sent-second' $cfg +Assert (@(Get-ChildItem "$root/sent-second/sent" | Where-Object { @(Get-Content $_.FullName)[0] -ne 'earlier' }).Count -eq 0) 'a same-second receipt is appended to, not replaced' + +# keys that differ only by case are different keys, both recorded and both skipped after +$cfg = Config 'sent-case' +$dst | Set-Content "$root/sent-case/dst.ps1" +@([pscustomobject]@{ Id = 'abc'; Description = 'lower' }, [pscustomobject]@{ Id = 'ABC'; Description = 'upper' }) | Export-Csv "$root/sent-case/case.csv" +$cfg.src.args.LiteralPath = "$root/sent-case/case.csv" +$cfg.dst = @{ adapter = './dst.ps1'; args = @{} } +$cfg.sent = @{ key = 'Id' } +$null = Run 'sent-case' $cfg +$null = Run 'sent-case' $cfg +Assert ((Sends 'sent-case') -join ',' -eq '2' -and @(Get-Content (Receipts 'sent-case')[0].FullName).Count -eq 2) 'case-distinct keys both recorded, nothing sent twice' + # a key that is not a column would skip everything after one send, so it stops the run $cfg = Config 'sent-missing' $cfg.sent = @{ key = 'InvoiceNumber' } From 18a61e758c9cf542be430d50248c8f9c4537bfb5 Mon Sep 17 00:00:00 2001 From: "@royashbrook" Date: Mon, 9 Nov 2026 16:00:23 -0500 Subject: [PATCH 3/3] sent: ordinal distinct for receipt keys (refs #36) --- DataAgent/DataAgent.psm1 | 6 +++--- tests/sent.ps1 | 12 +++++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/DataAgent/DataAgent.psm1 b/DataAgent/DataAgent.psm1 index c45043c..00bc89f 100644 --- a/DataAgent/DataAgent.psm1 +++ b/DataAgent/DataAgent.psm1 @@ -99,9 +99,9 @@ function Invoke-DataAgent { if ($Config.sent -and $adapters.dst.Count) { $null = New-Item -ItemType Directory -Force -Path $receipts $receipt = Join-Path $receipts ('{0:yyyyMMddHHmmss}.txt' -f (Get-Date)) - # append: a run in the same second must not replace the receipt before it. case-sensitive, - # like the compare, or a key that differs only by case is dropped here and sent again - $lines | Sort-Object -Unique -CaseSensitive | Add-Content -LiteralPath $receipt + # append: a run in the same second must not replace the receipt before it. distinct is + # ordinal, like the compare: a culture or case fold here drops a key that then goes again + [Linq.Enumerable]::Distinct([string[]]$lines) | Add-Content -LiteralPath $receipt l "Receipt: $receipt" } l 'End' diff --git a/tests/sent.ps1 b/tests/sent.ps1 index c4b0cd9..11e4939 100644 --- a/tests/sent.ps1 +++ b/tests/sent.ps1 @@ -89,16 +89,22 @@ foreach ($s in 0..30) { 'earlier' | Set-Content ("$root/sent-second/sent/{0:yyyy $null = Run 'sent-second' $cfg Assert (@(Get-ChildItem "$root/sent-second/sent" | Where-Object { @(Get-Content $_.FullName)[0] -ne 'earlier' }).Count -eq 0) 'a same-second receipt is appended to, not replaced' -# keys that differ only by case are different keys, both recorded and both skipped after +# keys that differ only by case, or by unicode form (e-acute vs e + combining acute), are different +# keys, all recorded and all skipped after $cfg = Config 'sent-case' $dst | Set-Content "$root/sent-case/dst.ps1" -@([pscustomobject]@{ Id = 'abc'; Description = 'lower' }, [pscustomobject]@{ Id = 'ABC'; Description = 'upper' }) | Export-Csv "$root/sent-case/case.csv" +@( + [pscustomobject]@{ Id = 'abc'; Description = 'lower' } + [pscustomobject]@{ Id = 'ABC'; Description = 'upper' } + [pscustomobject]@{ Id = "caf$([char]0x00E9)"; Description = 'composed' } + [pscustomobject]@{ Id = "cafe$([char]0x0301)"; Description = 'decomposed' } +) | Export-Csv "$root/sent-case/case.csv" $cfg.src.args.LiteralPath = "$root/sent-case/case.csv" $cfg.dst = @{ adapter = './dst.ps1'; args = @{} } $cfg.sent = @{ key = 'Id' } $null = Run 'sent-case' $cfg $null = Run 'sent-case' $cfg -Assert ((Sends 'sent-case') -join ',' -eq '2' -and @(Get-Content (Receipts 'sent-case')[0].FullName).Count -eq 2) 'case-distinct keys both recorded, nothing sent twice' +Assert ((Sends 'sent-case') -join ',' -eq '4' -and @(Get-Content (Receipts 'sent-case')[0].FullName).Count -eq 4) 'case and unicode-form distinct keys all recorded, nothing sent twice' # a key that is not a column would skip everything after one send, so it stops the run $cfg = Config 'sent-missing'