diff --git a/.github/workflows/integration-candidate.yml b/.github/workflows/integration-candidate.yml index 4a575168d7..45c8465e8b 100644 --- a/.github/workflows/integration-candidate.yml +++ b/.github/workflows/integration-candidate.yml @@ -255,6 +255,10 @@ jobs: run: | go build -o /dev/null ./cmd/crux (cd crux && GOWORK=off go build -o /dev/null ./cmd/crux) + - name: Durable Crux lane and Hive participant integration + run: | + (cd crux && GOWORK=off go test -race -timeout 3m ./lanes ./api ./serve) + GOWORK=off go test -race -timeout 3m ./pkg/laneparticipant - name: Crux module smoke (crux serve over a unix socket) working-directory: src/crux run: GOWORK=off bash scripts/smoke.sh diff --git a/.github/workflows/integration-technical-v2-ci.yml b/.github/workflows/integration-technical-v2-ci.yml index 44fe2bdd66..7c7ebd7a06 100644 --- a/.github/workflows/integration-technical-v2-ci.yml +++ b/.github/workflows/integration-technical-v2-ci.yml @@ -121,6 +121,10 @@ jobs: run: | go build -o /dev/null ./cmd/crux (cd crux && GOWORK=off go build -o /dev/null ./cmd/crux) + - name: Durable Crux lane and Hive participant integration + run: | + (cd crux && GOWORK=off go test -race -timeout 3m ./lanes ./api ./serve) + GOWORK=off go test -race -timeout 3m ./pkg/laneparticipant - name: Crux module smoke (crux serve over a unix socket) working-directory: src/crux run: GOWORK=off bash scripts/smoke.sh diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index 31cd94ba3e..deb415b91c 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -197,6 +197,14 @@ jobs: go build -o /dev/null ./cmd/crux (cd crux && GOWORK=off go build -o /dev/null ./cmd/crux) + # Real embedded JetStream only: no live broker, cluster or deployment. + # Keep the standalone module boundary honest as well as the Hive adapter. + # Normal v2-tests shards also collect coverage for these packages. + - name: Durable Crux lane and Hive participant integration + run: | + (cd crux && GOWORK=off go test -race -timeout 3m ./lanes ./api ./serve) + GOWORK=off go test -race -timeout 3m ./pkg/laneparticipant + # crux/scripts/smoke.sh (`make -C crux smoke` locally; the self-hosted # runners have no make) starts `crux serve` on a private unix socket # with an empty runtime, drives it with `crux --server … ps` and GET diff --git a/NOTICE b/NOTICE index b43064542a..faff4b0a18 100644 --- a/NOTICE +++ b/NOTICE @@ -1471,12 +1471,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ----------------------------------------------------------------------------- -Package: github.com/ledongthuc/pdf -Version: v0.0.0-20250511090121-5959a4027728 -License: BSD-3-Clause -Source: https://github.com/ledongthuc/pdf/blob/5959a4027728/LICENSE +Package: github.com/klauspost/compress +Version: v1.20.3 +License: Apache-2.0 +Source: https://github.com/klauspost/compress/blob/v1.20.3/LICENSE -Copyright (c) 2009 The Go Authors. All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2019 Klaus Post. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1504,48 +1505,219 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ +------------------ -Package: github.com/lucasb-eyer/go-colorful -Version: v1.4.1 -License: MIT -Source: https://github.com/lucasb-eyer/go-colorful/blob/v1.4.1/LICENSE +Files: gzhttp/* -Copyright (c) 2013 Lucas Beyer + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + 1. Definitions. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. ------------------------------------------------------------------------------ + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. -Package: github.com/mattn/go-isatty -Version: v0.0.20 -License: MIT -Source: https://github.com/mattn/go-isatty/blob/v0.0.20/LICENSE + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. -Copyright (c) Yasuhiro MATSUMOTO + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. -MIT License (Expat) + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. ------------------------------------------------------------------------------ + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." -Package: github.com/mattn/go-runewidth -Version: v0.0.27 -License: MIT -Source: https://github.com/mattn/go-runewidth/blob/v0.0.27/LICENSE + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2017 The New York Times Company + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------ + +Files: s2/cmd/internal/readahead/* The MIT License (MIT) -Copyright (c) 2016 Yasuhiro Matsumoto +Copyright (c) 2015 Klaus Post Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -1554,103 +1726,870 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +--------------------- +Files: snappy/* +Files: internal/snapref/* + +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +----------------- + +Files: s2/cmd/internal/filepathx/* + +Copyright 2016 The filepathx Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +----------------------------------------------------------------------------- + +Package: github.com/ledongthuc/pdf +Version: v0.0.0-20250511090121-5959a4027728 +License: BSD-3-Clause +Source: https://github.com/ledongthuc/pdf/blob/5959a4027728/LICENSE + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +----------------------------------------------------------------------------- + +Package: github.com/lucasb-eyer/go-colorful +Version: v1.4.1 +License: MIT +Source: https://github.com/lucasb-eyer/go-colorful/blob/v1.4.1/LICENSE + +Copyright (c) 2013 Lucas Beyer + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +----------------------------------------------------------------------------- + +Package: github.com/mattn/go-isatty +Version: v0.0.20 +License: MIT +Source: https://github.com/mattn/go-isatty/blob/v0.0.20/LICENSE + +Copyright (c) Yasuhiro MATSUMOTO + +MIT License (Expat) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +----------------------------------------------------------------------------- + +Package: github.com/mattn/go-runewidth +Version: v0.0.27 +License: MIT +Source: https://github.com/mattn/go-runewidth/blob/v0.0.27/LICENSE + +The MIT License (MIT) + +Copyright (c) 2016 Yasuhiro Matsumoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +----------------------------------------------------------------------------- + +Package: github.com/muesli/ansi +Version: v0.0.0-20230316100256-276c6243b2f6 +License: MIT +Source: https://github.com/muesli/ansi/blob/276c6243b2f6/LICENSE + +MIT License + +Copyright (c) 2021 Christian Muehlhaeuser + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +----------------------------------------------------------------------------- + +Package: github.com/muesli/cancelreader +Version: v0.2.2 +License: MIT +Source: https://github.com/muesli/cancelreader/blob/v0.2.2/LICENSE + +MIT License + +Copyright (c) 2022 Erik Geiser and Christian Muehlhaeuser + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +----------------------------------------------------------------------------- + +Package: github.com/muesli/termenv +Version: v0.16.0 +License: MIT +Source: https://github.com/muesli/termenv/blob/v0.16.0/LICENSE + +MIT License + +Copyright (c) 2019 Christian Muehlhaeuser + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +----------------------------------------------------------------------------- + +Package: github.com/nats-io/nats.go +Version: v1.55.0 +License: Apache-2.0 +Source: https://github.com/nats-io/nats.go/blob/v1.55.0/LICENSE + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +----------------------------------------------------------------------------- + +Package: github.com/nats-io/nkeys +Version: v0.4.16 +License: Apache-2.0 +Source: https://github.com/nats-io/nkeys/blob/v0.4.16/LICENSE + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +----------------------------------------------------------------------------- + +Package: github.com/nats-io/nuid +Version: v1.0.1 +License: Apache-2.0 +Source: https://github.com/nats-io/nuid/blob/v1.0.1/LICENSE + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. ------------------------------------------------------------------------------ + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." -Package: github.com/muesli/ansi -Version: v0.0.0-20230316100256-276c6243b2f6 -License: MIT -Source: https://github.com/muesli/ansi/blob/276c6243b2f6/LICENSE + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. -MIT License + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. -Copyright (c) 2021 Christian Muehlhaeuser + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and ------------------------------------------------------------------------------ + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and -Package: github.com/muesli/cancelreader -Version: v0.2.2 -License: MIT -Source: https://github.com/muesli/cancelreader/blob/v0.2.2/LICENSE + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. -MIT License + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. -Copyright (c) 2022 Erik Geiser and Christian Muehlhaeuser + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. ------------------------------------------------------------------------------ + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. -Package: github.com/muesli/termenv -Version: v0.16.0 -License: MIT -Source: https://github.com/muesli/termenv/blob/v0.16.0/LICENSE + END OF TERMS AND CONDITIONS -MIT License + APPENDIX: How to apply the Apache License to your work. -Copyright (c) 2019 Christian Muehlhaeuser + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + Copyright [yyyy] [name of copyright owner] -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. ----------------------------------------------------------------------------- @@ -4842,6 +5781,41 @@ limitations under the License. ----------------------------------------------------------------------------- +Package: golang.org/x/crypto +Version: v0.58.0 +License: BSD-3-Clause +Source: https://cs.opensource.google/go/x/crypto/+/v0.58.0:LICENSE + +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +----------------------------------------------------------------------------- + Package: golang.org/x/exp Version: v0.0.0-20240823005443-9b4947da3948 License: BSD-3-Clause diff --git a/changelog.d/added-crux-durable-lanes.md b/changelog.d/added-crux-durable-lanes.md new file mode 100644 index 0000000000..5f41130d13 --- /dev/null +++ b/changelog.d/added-crux-durable-lanes.md @@ -0,0 +1 @@ +- Add opt-in, forge-neutral Crux durable event/work lanes backed by NATS JetStream, project-scoped API credentials, verified publish generations, replay, fenced leases, authorized audited expiration recovery, persistent fair retry rotation and reusable reserved delivery slots, with Hive participating through a read-only governor adapter. diff --git a/changelog.d/fixed-crux-lane-diagnostics.md b/changelog.d/fixed-crux-lane-diagnostics.md new file mode 100644 index 0000000000..50ec05114a --- /dev/null +++ b/changelog.d/fixed-crux-lane-diagnostics.md @@ -0,0 +1 @@ +- Distinguish durable lane broker initialization conflicts from unavailability without exposing backend details, and expose stable lane error-code constants while preserving existing HTTP responses. diff --git a/src/cmd/hive/lanes.go b/src/cmd/hive/lanes.go new file mode 100644 index 0000000000..73bfa59823 --- /dev/null +++ b/src/cmd/hive/lanes.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "errors" + "os" + "strings" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/pkg/governor" + "github.com/hivecommons/hive/pkg/laneparticipant" +) + +// HIVE_LANES_ADDRESS is the opt-in gate. No broker or forge permissions are +// inferred from being Hive; its token is an ordinary configured Crux principal. +func (b *boot) installLaneParticipant() { + address := os.Getenv("HIVE_LANES_ADDRESS") + if address == "" { + return + } + lane, group, token := os.Getenv("HIVE_LANES_LANE"), os.Getenv("HIVE_LANES_GROUP"), os.Getenv("HIVE_LANES_TOKEN") + if lane == "" || group == "" || token == "" { + b.logger.Error("Hive lane participant disabled: incomplete configuration") + return + } + client, err := api.NewClient(address, token) + if err != nil { + b.logger.Error("Hive lane participant disabled: invalid Crux address") + return + } + participant, err := laneparticipant.New(client, lane, group, b.logger) + if err != nil { + b.logger.Error("Hive lane participant disabled: invalid configuration") + return + } + // Snapshot work is optional and read-only. No configured publisher means + // no registered handler, not a wildcard grant. + if publishers := os.Getenv("HIVE_LANES_SNAPSHOT_PROJECTS"); publishers != "" { + if err := participant.RegisterSnapshot(strings.Split(publishers, ","), b.gov.GetState); err != nil { + b.logger.Error("Hive lane participant disabled: invalid snapshot publisher configuration") + return + } + } + ctx := b.ctx + if ctx == nil { + ctx = context.Background() + } + laneCtx, cancel := context.WithCancel(ctx) + b.cleanup.push(cancel) + // Keep the existing hook observer; durable publication is an additional + // post-commit observer, not a replacement for local hooks. + b.gov.AddModeChangeObserver(func(change governor.ModeChange) { participant.ObserveMode(laneCtx, change) }) + go func() { + if err := participant.Run(laneCtx); err != nil && !errors.Is(err, context.Canceled) { + b.logger.Error("Hive lane participant stopped", "error", err) + } + }() +} diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 29183a6dea..39a994c32b 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -3932,6 +3932,7 @@ func (b *boot) bootWatchersWith(deps bootWatchersDeps) { // firing, so a later config reload that arms or disarms hooks is picked up // without re-registering. installGovernorModeChangeEmitter(b.gov) + b.installLaneParticipant() installAgentPauseEmitter(b.agentMgr) // Register custom GHE hostnames with the proxy allowlist so mode diff --git a/src/crux/README.md b/src/crux/README.md index a81750f297..1d93579916 100644 --- a/src/crux/README.md +++ b/src/crux/README.md @@ -11,6 +11,8 @@ and depends on nothing else in the Hive repository. The design, the boundary rules and the roadmap are in [`src/docs/design/crux.md`](../docs/design/crux.md); the operator pages are [`crux-cli.md`](../docs/crux-cli.md) and [`crux-serve.md`](../docs/crux-serve.md). +Forge-neutral durable event/work lanes are described in +[`crux-lanes.md`](../docs/crux-lanes.md). ## What is in the module @@ -22,6 +24,7 @@ The design, the boundary rules and the roadmap are in | `crux/cli` | the `crux` command: `run`, `ps`, `logs`, `kick`, `stop`, `serve`, `agy-turn` (the pane-side agy subprocess), and `--server` remote mode | | `crux/api`, `crux/serve` | the wire contract of `crux serve` (stdlib-only client and types, `api.RuntimeRoutes`: every `crux.Runtime` method → route) and the server | | `crux/api/client` | `client.Runtime`: `crux.Runtime` over the API; what `crux/runtime.New` returns for `CRUX_MODE=sidecar` | +| `crux/lanes` | opt-in JetStream durable lanes: neutral project ACLs, event replay, claimable delivery groups and fenced receipts; Hive is an ordinary participant | | `crux/api/callback`, `crux/hivecap` | the hive callback contract (capabilities hive serves the runtime: scoped tokens, agent mint, Bob credential, PRs, Linear, scanners, conditions, host policy, inference routes, effects, usage) and the runtime-side adapters `crux serve --hive-callback` installs; `api.HostSetters` is the table that accounts for every `crux.Host` setter (route / callback / config key / in-process-only) | | `crux/runtime` | `runtime.New` (mode selection: `CRUX_MODE=inprocess` default, `sidecar`) and `runtime.Supervisor`, the child supervisor hive uses for `crux serve` when `CRUX_SUPERVISE=child` | | `crux/defs` | the file-based agent-definition loader the module's binary uses | diff --git a/src/crux/api/client.go b/src/crux/api/client.go index 59b3805b04..6066734102 100644 --- a/src/crux/api/client.go +++ b/src/crux/api/client.go @@ -279,7 +279,7 @@ func (c *Client) do(ctx context.Context, method, path string, in, out any) error if err := checkStatus(resp); err != nil { return err } - if out == nil { + if out == nil || resp.StatusCode == http.StatusNoContent { return nil } if err := json.NewDecoder(resp.Body).Decode(out); err != nil { diff --git a/src/crux/api/lanes.go b/src/crux/api/lanes.go new file mode 100644 index 0000000000..16ffc5f254 --- /dev/null +++ b/src/crux/api/lanes.go @@ -0,0 +1,134 @@ +package api + +import "time" + +const PathLanes = "/" + Version + "/lanes" + +// Lane error codes retain the same meaning across HTTP handlers and clients. +const ( + CodeLaneUnavailable = "lane_unavailable" + CodeLaneForbidden = "lane_forbidden" + CodeLaneInvalid = "lane_invalid" + CodeLaneConflict = "lane_conflict" + CodeLaneCursor = "lane_cursor" + CodeLaneStale = "lane_stale" + CodeLaneCapacity = "lane_capacity" + CodeLaneTimeout = "lane_timeout" + CodeLaneCanceled = "lane_canceled" +) + +// Lane IDs are plain subject-safe identifiers, not forge coordinates. +type Lane struct { + ID string `json:"id"` + Owner string `json:"owner"` + Publishers []string `json:"publishers"` + Subscribers []string `json:"subscribers"` + Groups map[string][]string `json:"groups"` + Recoverers map[string][]string `json:"recoverers,omitempty"` +} + +const ( + LaneEvent = "event" + LaneWork = "work" +) + +// LanePublish deliberately has no caller-supplied publisher. Authentication +// supplies the project identity; provenance is optional, untrusted metadata. +type LanePublish struct { + ID string `json:"id"` + Kind string `json:"kind"` + Schema string `json:"schema"` + Provenance string `json:"provenance,omitempty"` + Payload []byte `json:"payload"` +} + +type LaneEnvelope struct { + LanePublish + Lane string `json:"lane"` + Project string `json:"project"` + CreatedAt time.Time `json:"created_at"` +} + +type LanePublished struct { + Sequence uint64 `json:"sequence"` + Duplicate bool `json:"duplicate"` +} + +// A cursor is the last examined lane stream sequence, including work records +// skipped by event replay. Zero explicitly opts into the earliest retained data. +type LaneReplay struct { + Events []LaneRecord `json:"events"` + Cursor uint64 `json:"cursor"` +} + +type LaneRecord struct { + Sequence uint64 `json:"sequence"` + Envelope LaneEnvelope `json:"envelope"` +} + +type LaneClaim struct { + LaneRecord + Receipt string `json:"receipt"` + Expires time.Time `json:"expires"` + Attempt uint64 `json:"attempt"` +} + +type LaneReceipt struct { + Sequence uint64 `json:"sequence"` + Receipt string `json:"receipt"` +} + +type LaneRenewed struct { + Expires time.Time `json:"expires"` +} + +type LaneGroupState struct { + Revision uint64 `json:"revision"` + Cursor uint64 `json:"cursor"` + FirstRetained uint64 `json:"first_retained"` + LastSequence uint64 `json:"last_sequence"` + Recoverable []LaneReceipt `json:"recoverable"` + Recoveries []LaneRecovered `json:"recoveries"` + Retries []LaneRetry `json:"retries"` + MissingThrough uint64 `json:"missing_through"` + Missing []LaneExpiration `json:"missing"` + Slots int `json:"slots"` +} + +// LaneExpiration describes missing, lease-expired work without exposing its +// ownership receipt or representing successful execution. +type LaneExpiration struct { + Sequence uint64 `json:"sequence"` + Project string `json:"project"` + Attempt uint64 `json:"attempt"` +} + +type LaneRetry struct { + Sequence uint64 `json:"sequence"` + Attempt uint64 `json:"attempt"` + AvailableAt time.Time `json:"available_at"` +} + +// Recovery explicitly accepts an expired prefix and listed missing deliveries. +// Missing needs the configured group recovery capability, not a foreign receipt. +// Expired optionally supplies the caller's original receipts for extra fencing. +// Revision fences the entire delivery-group state. +type LaneRecover struct { + ID string `json:"id"` + Revision uint64 `json:"revision"` + CursorThrough uint64 `json:"cursor_through"` + Expired []LaneReceipt `json:"expired"` + Missing []uint64 `json:"missing"` +} + +type LaneRecovered struct { + ID string `json:"id"` + Project string `json:"project"` + Cursor uint64 `json:"cursor"` + GapFrom uint64 `json:"gap_from,omitempty"` + GapThrough uint64 `json:"gap_through,omitempty"` + MissingThrough uint64 `json:"missing_through"` + Retired []uint64 `json:"retired"` + Expirations []LaneExpiration `json:"expirations"` + RecoveredAt time.Time `json:"recovered_at"` +} diff --git a/src/crux/api/lanes_client.go b/src/crux/api/lanes_client.go new file mode 100644 index 0000000000..45f0c10a4d --- /dev/null +++ b/src/crux/api/lanes_client.go @@ -0,0 +1,59 @@ +package api + +import ( + "context" + "fmt" + "net/http" + "net/url" +) + +func lanePath(lane string) string { return PathLanes + "/" + url.PathEscape(lane) } +func laneGroupPath(lane, group string) string { + return lanePath(lane) + "/groups/" + url.PathEscape(group) +} + +func (c *Client) LaneRegistry(ctx context.Context) ([]Lane, error) { + var out []Lane + err := c.Do(ctx, http.MethodGet, PathLanes, nil, &out) + return out, err +} +func (c *Client) PublishLane(ctx context.Context, lane string, in LanePublish) (LanePublished, error) { + var out LanePublished + err := c.Do(ctx, http.MethodPost, lanePath(lane)+"/publish", in, &out) + return out, err +} +func (c *Client) ReplayLane(ctx context.Context, lane string, cursor uint64, limit int) (LaneReplay, error) { + var out LaneReplay + err := c.Do(ctx, http.MethodGet, lanePath(lane)+fmt.Sprintf("/events?cursor=%d&limit=%d", cursor, limit), nil, &out) + return out, err +} + +// ClaimLane returns nil, nil when no work is currently claimable (HTTP 204). +func (c *Client) ClaimLane(ctx context.Context, lane, group string) (*LaneClaim, error) { + var out *LaneClaim + err := c.Do(ctx, http.MethodPost, laneGroupPath(lane, group)+"/claim", nil, &out) + return out, err +} +func (c *Client) AckLane(ctx context.Context, lane, group string, in LaneReceipt) error { + return c.Do(ctx, http.MethodPost, laneGroupPath(lane, group)+"/ack", in, nil) +} +func (c *Client) NackLane(ctx context.Context, lane, group string, in LaneReceipt) error { + return c.Do(ctx, http.MethodPost, laneGroupPath(lane, group)+"/nack", in, nil) +} +func (c *Client) RenewLane(ctx context.Context, lane, group string, in LaneReceipt) (LaneRenewed, error) { + var out LaneRenewed + err := c.Do(ctx, http.MethodPost, laneGroupPath(lane, group)+"/renew", in, &out) + return out, err +} + +func (c *Client) InspectLaneGroup(ctx context.Context, lane, group string) (LaneGroupState, error) { + var out LaneGroupState + err := c.Do(ctx, http.MethodGet, laneGroupPath(lane, group), nil, &out) + return out, err +} + +func (c *Client) RecoverLaneGroup(ctx context.Context, lane, group string, in LaneRecover) (LaneRecovered, error) { + var out LaneRecovered + err := c.Do(ctx, http.MethodPost, laneGroupPath(lane, group)+"/recover", in, &out) + return out, err +} diff --git a/src/crux/api/lanes_test.go b/src/crux/api/lanes_test.go new file mode 100644 index 0000000000..3ed51f1390 --- /dev/null +++ b/src/crux/api/lanes_test.go @@ -0,0 +1,26 @@ +package api + +import "testing" + +func TestLaneErrorCodesPreserveWireValues(t *testing.T) { + for _, test := range []struct { + code string + wire string + }{ + {CodeLaneUnavailable, "lane_unavailable"}, + {CodeLaneForbidden, "lane_forbidden"}, + {CodeLaneInvalid, "lane_invalid"}, + {CodeLaneConflict, "lane_conflict"}, + {CodeLaneCursor, "lane_cursor"}, + {CodeLaneStale, "lane_stale"}, + {CodeLaneCapacity, "lane_capacity"}, + {CodeLaneTimeout, "lane_timeout"}, + {CodeLaneCanceled, "lane_canceled"}, + } { + t.Run(test.wire, func(t *testing.T) { + if test.code != test.wire { + t.Fatalf("code = %q, want %q", test.code, test.wire) + } + }) + } +} diff --git a/src/crux/cli/serve.go b/src/crux/cli/serve.go index 70ee837561..5549ed9496 100644 --- a/src/crux/cli/serve.go +++ b/src/crux/cli/serve.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "fmt" "log/slog" "os" @@ -9,7 +10,9 @@ import ( "github.com/hivecommons/hive/crux/api/callback" "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/lanes" "github.com/hivecommons/hive/crux/serve" + "github.com/nats-io/nats.go" ) // cmdServe is `crux serve`: build the runtime from --config and expose it @@ -29,12 +32,14 @@ func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) i fs.StringVar(&cfg.Listen, "listen", "", "also serve on this loopback TCP host:port (plain http); requires a token") fs.StringVar(&cfg.Token, "token", "", "bearer token every request must carry (default $"+config.ServeTokenEnvVar+")") var hiveAddr string + var laneConfig string + fs.StringVar(&laneConfig, "lanes-config", "", "opt-in durable lane configuration file (default $CRUX_LANES_CONFIG)") fs.StringVar(&hiveAddr, "hive-callback", "", "hive capability endpoint to dial (default $"+callback.SocketEnvVar+"; none when unset)") if code, ok := parseSub(fs, args); !ok { return code } if fs.NArg() != 0 { - fmt.Fprintln(e.Stderr, "usage: crux serve [-socket PATH] [-listen HOST:PORT] [-token T] [-hive-callback ADDR]") + fmt.Fprintln(e.Stderr, "usage: crux serve [-socket PATH] [-listen HOST:PORT] [-token T] [-hive-callback ADDR] [-lanes-config PATH]") return ExitUsage } if cfg.Token == "" { @@ -66,6 +71,36 @@ func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) i srv := serve.New(rt) srv.Token = cfg.Token srv.Logger = slog.New(slog.NewTextHandler(e.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) + if laneConfig == "" { + laneConfig = os.Getenv("CRUX_LANES_CONFIG") + } + if laneConfig != "" { + lc, err := lanes.LoadConfig(laneConfig) + if err != nil { + fmt.Fprintln(e.Stderr, "crux: lanes:", err) + return ExitUsage + } + auth, err := lanes.Bind(lc, os.Getenv, cfg.Token) + if err != nil { + fmt.Fprintln(e.Stderr, "crux: lanes:", err) + return ExitUsage + } + var options []nats.Option + if lc.CredentialsEnv != "" { + path := os.Getenv(lc.CredentialsEnv) + if path == "" { + fmt.Fprintln(e.Stderr, "crux: lanes: missing broker credential file binding") + return ExitUsage + } + options = append(options, nats.UserCredentials(path)) + } + broker, err := lanes.Open(ctx, lc, options...) + if err != nil { + return e.laneBrokerFailure(err) + } + defer broker.Close() + srv.Lanes, srv.LaneAuth = broker, auth + } if hive != nil { hive.install(ctx, rt, srv, e.Stdout) hive.watchPolicy(ctx, rt, e.Stdout, e.Stderr) @@ -84,3 +119,16 @@ func (e *Env) cmdServe(ctx context.Context, opts globalOptions, args []string) i fmt.Fprintln(e.Stdout, "serve stopped") return ExitOK } + +func (e *Env) laneBrokerFailure(err error) int { + category := "initialization error" + switch { + case errors.Is(err, lanes.ErrConflict): + category = "state conflict" + case errors.Is(err, lanes.ErrUnavailable): + category = "broker unavailable" + } + // Backend errors can contain addresses or credentials; only emit a typed category. + fmt.Fprintf(e.Stderr, "crux: lanes: cannot initialize durable broker (%s)\n", category) + return ExitError +} diff --git a/src/crux/cli/serve_lanes_test.go b/src/crux/cli/serve_lanes_test.go new file mode 100644 index 0000000000..da42c013fe --- /dev/null +++ b/src/crux/cli/serve_lanes_test.go @@ -0,0 +1,44 @@ +package cli + +import ( + "bytes" + "errors" + "fmt" + "net/url" + "testing" + + "github.com/hivecommons/hive/crux/lanes" +) + +func TestLaneBrokerFailureIsTypedAndRedacted(t *testing.T) { + addressError := &url.Error{ + Op: "dial", + URL: "tls://dummy-user:dummy-password@broker.example:4222", + Err: errors.New("dummy-backend-detail"), + } + for _, test := range []struct { + name string + err error + category string + }{ + {"conflict", lanes.ErrConflict, "state conflict"}, + {"unavailable", lanes.ErrUnavailable, "broker unavailable"}, + {"wrapped conflict", fmt.Errorf("%w: %w", lanes.ErrConflict, addressError), "state conflict"}, + {"wrapped unavailable", fmt.Errorf("%w: %w", lanes.ErrUnavailable, addressError), "broker unavailable"}, + {"address credentials", addressError, "initialization error"}, + {"arbitrary error", errors.New("dummy-arbitrary-secret"), "initialization error"}, + {"untyped conflict text", errors.New("lane state conflict"), "initialization error"}, + } { + t.Run(test.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + e := &Env{Stdout: &stdout, Stderr: &stderr} + if code := e.laneBrokerFailure(test.err); code != ExitError { + t.Fatalf("exit = %d, want %d", code, ExitError) + } + want := "crux: lanes: cannot initialize durable broker (" + test.category + ")\n" + if stderr.String() != want || stdout.Len() != 0 { + t.Fatalf("stdout = %q, stderr = %q; want only %q", stdout.String(), stderr.String(), want) + } + }) + } +} diff --git a/src/crux/go.mod b/src/crux/go.mod index ff02559201..aa0e140b0b 100644 --- a/src/crux/go.mod +++ b/src/crux/go.mod @@ -4,6 +4,8 @@ go 1.26.10 require ( github.com/google/cel-go v0.31.0 + github.com/nats-io/nats-server/v2 v2.15.2 + github.com/nats-io/nats.go v1.55.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e go.opentelemetry.io/otel v1.47.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 @@ -16,27 +18,33 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) -require ( - github.com/kr/text v0.2.0 // indirect - go.opentelemetry.io/otel/log v1.47.0 // indirect -) +require go.opentelemetry.io/otel/log v1.47.0 // indirect require ( cel.dev/expr v0.25.2 // indirect + github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/go-tpm v0.9.8 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect + github.com/klauspost/compress v1.20.3 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.2 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.58.0 // indirect golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect golang.org/x/net v0.62.0 // indirect golang.org/x/text v0.43.0 // indirect + golang.org/x/time v0.16.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect google.golang.org/grpc v1.83.2 // indirect diff --git a/src/crux/go.sum b/src/crux/go.sum index 5b6ed28d81..8bac4b5df6 100644 --- a/src/crux/go.sum +++ b/src/crux/go.sum @@ -1,12 +1,13 @@ cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= +github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -18,14 +19,30 @@ github.com/google/cel-go v0.31.0 h1:H0bhpFTqOvmHrBGrWKp7ZlhBm5Hh8PYUEXnwxT1LL7A= github.com/google/cel-go v0.31.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= +github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= +github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc= +github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc= +github.com/nats-io/nats-server/v2 v2.15.2 h1:yLsgt2kslgymvxvjFq1WGWck5hkat2YYfbXvQ145tzU= +github.com/nats-io/nats-server/v2 v2.15.2/go.mod h1:VXHCZgDglV5wO2B8ZE58yG2HzqxQxYUbQqibUh5Xcr8= +github.com/nats-io/nats.go v1.55.0 h1:Ng8QKHdvdHIWoaiq0+d/aH1BWkxvcJxWJ0Y/9BPKx3M= +github.com/nats-io/nats.go v1.55.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= @@ -60,14 +77,19 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= diff --git a/src/crux/lanes/auth.go b/src/crux/lanes/auth.go new file mode 100644 index 0000000000..51d31849c6 --- /dev/null +++ b/src/crux/lanes/auth.go @@ -0,0 +1,47 @@ +package lanes + +import ( + "crypto/subtle" + "fmt" + "strings" +) + +const ( + minTokenBytes = 32 + maxTokenBytes = 4096 +) + +type Authenticator struct{ tokens map[string]string } + +// Bind resolves administrator-selected environment bindings, not anything in +// an envelope or project header. Tokens must be distinct, including from the +// existing agent-control token. +func Bind(cfg Config, getenv func(string) string, adminToken string) (*Authenticator, error) { + a := &Authenticator{tokens: map[string]string{}} + for _, p := range cfg.Principals { + token := getenv(p.TokenEnv) + if len(token) < minTokenBytes || len(token) > maxTokenBytes || strings.ContainsAny(token, "\r\n\t ") || token == adminToken { + return nil, fmt.Errorf("%w: missing, short or overlapping project credential", ErrInvalid) + } + for _, existing := range a.tokens { + if existing == token { + return nil, fmt.Errorf("%w: duplicate project credential", ErrInvalid) + } + } + a.tokens[p.Project] = token + } + return a, nil +} + +func (a *Authenticator) Project(token string) (string, bool) { + if a == nil { + return "", false + } + project := "" + for p, expected := range a.tokens { + if subtle.ConstantTimeCompare([]byte(token), []byte(expected)) == 1 { + project = p + } + } + return project, project != "" +} diff --git a/src/crux/lanes/auth_test.go b/src/crux/lanes/auth_test.go new file mode 100644 index 0000000000..e808cc6515 --- /dev/null +++ b/src/crux/lanes/auth_test.go @@ -0,0 +1,116 @@ +package lanes + +import ( + "context" + "errors" + "os" + "reflect" + "strings" + "testing" + "testing/iotest" +) + +func TestPrincipalBindingsAreFailClosed(t *testing.T) { + cfg := testConfig("nats://127.0.0.1:4222") + secrets := map[string]string{"ATLAS_TOKEN": strings.Repeat("a", minTokenBytes), "BEACON_TOKEN": strings.Repeat("b", minTokenBytes)} + getenv := func(env string) string { return secrets[env] } + auth, err := Bind(cfg, getenv, "admin") + if err != nil { + t.Fatal(err) + } + if p, ok := auth.Project(secrets["ATLAS_TOKEN"]); !ok || p != "atlas" { + t.Fatal("wrong bound principal") + } + for _, token := range []string{"", "admin", "atlas", strings.Repeat("a", minTokenBytes-1)} { + if _, ok := auth.Project(token); ok { + t.Fatal("forged token accepted") + } + } + var absent *Authenticator + if _, ok := absent.Project("any"); ok { + t.Fatal("nil authenticator accepted") + } + if _, err := Bind(cfg, getenv, secrets["ATLAS_TOKEN"]); !errors.Is(err, ErrInvalid) { + t.Fatal("admin-token overlap accepted", err) + } + secrets["BEACON_TOKEN"] = secrets["ATLAS_TOKEN"] + if _, err := Bind(cfg, getenv, "admin"); !errors.Is(err, ErrInvalid) { + t.Fatal("duplicate token accepted", err) + } + delete(secrets, "BEACON_TOKEN") + if _, err := Bind(cfg, getenv, "admin"); !errors.Is(err, ErrInvalid) { + t.Fatal("missing token accepted", err) + } +} + +func TestLoadConfigStrictAndBounded(t *testing.T) { + if _, err := LoadConfig("testdata/missing-config.json"); !errors.Is(err, ErrInvalid) { + t.Fatal("missing config", err) + } + loaded, err := LoadConfig("testdata/config.json") + if err != nil || !reflect.DeepEqual(loaded, testConfig("nats://127.0.0.1:4222")) { + t.Fatalf("load: %+v %v", loaded, err) + } + if _, err := LoadConfig("testdata/unknown-config.json"); !errors.Is(err, ErrInvalid) { + t.Fatal("unknown config field accepted", err) + } +} + +func TestConfigReaderStrictAndBounded(t *testing.T) { + data, err := os.ReadFile("testdata/config.json") + if err != nil { + t.Fatal(err) + } + atLimit := string(data) + strings.Repeat(" ", maxConfigBytes-len(data)) + loaded, err := loadConfig(strings.NewReader(atLimit)) + if err != nil || !reflect.DeepEqual(loaded, testConfig("nats://127.0.0.1:4222")) { + t.Fatalf("exact byte limit: %+v %v", loaded, err) + } + for _, test := range []struct { + name string + body string + }{ + {"unknown field", `{"unknown":true}`}, + {"unknown nested field", `{"principals":[{"project":"atlas","token_env":"ATLAS_TOKEN","token":"dummy"}]}`}, + {"trailing object", string(data) + `{}`}, + {"trailing invalid JSON", string(data) + `not-json`}, + {"malformed JSON", `not-json`}, + {"empty input", ""}, + {"invalid configuration", `{}`}, + {"oversized whitespace", strings.Repeat(" ", maxConfigBytes+1)}, + {"oversized valid prefix", atLimit + " "}, + } { + t.Run(test.name, func(t *testing.T) { + if _, err := loadConfig(strings.NewReader(test.body)); !errors.Is(err, ErrInvalid) { + t.Fatal("invalid config accepted", err) + } + }) + } + if _, err := loadConfig(iotest.ErrReader(errors.New("dummy-read-detail"))); !errors.Is(err, ErrInvalid) { + t.Fatal("failed read accepted", err) + } +} + +func TestRegistryAndConfigurationAccess(t *testing.T) { + b := testBroker(t, nil) + if b.Config().Namespace != DefaultNamespace { + t.Fatal("wrong configuration") + } + for _, project := range []string{"atlas", "beacon"} { + if len(b.Registry(project)) != 1 { + t.Fatal("member cannot see lane") + } + } + if len(b.Registry("outsider")) != 0 { + t.Fatal("outsider can see lane") + } + if _, err := b.authorize("atlas", "shared", "admin", ""); !errors.Is(err, ErrForbidden) { + t.Fatal("implicit ownership admin capability", err) + } + if _, err := b.authorize("atlas", "unknown", "publish", ""); !errors.Is(err, ErrForbidden) { + t.Fatal("unknown lane", err) + } + if _, err := b.Replay(context.Background(), "atlas", "shared", 0, 0); !errors.Is(err, ErrInvalid) { + t.Fatal("invalid replay batch", err) + } +} diff --git a/src/crux/lanes/broker.go b/src/crux/lanes/broker.go new file mode 100644 index 0000000000..269b8c7774 --- /dev/null +++ b/src/crux/lanes/broker.go @@ -0,0 +1,387 @@ +package lanes + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +type Broker struct { + cfg Config + nc *nats.Conn + js jetstream.JetStream + state jetstream.KeyValue + dedup jetstream.KeyValue + streams map[string]jetstream.Stream + registry map[string]api.Lane + now func() time.Time +} + +// Open owns a bounded NATS connection. The caller owns Close. Existing broker +// resources must exactly match configuration; Open never rewrites retention or +// membership underneath another running service. +func Open(ctx context.Context, cfg Config, options ...nats.Option) (*Broker, error) { + cfg = cfg.clone() + cfg.Defaults() + if err := cfg.Validate(); err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(ctx, cfg.Timeout) + defer cancel() + if err := ctx.Err(); err != nil { + return nil, err + } + options = append(options, nats.Name("crux-lanes"), nats.Timeout(cfg.Timeout), + nats.MaxReconnects(-1), nats.ReconnectWait(time.Second), nats.ReconnectBufSize(0)) + nc, err := nats.Connect(cfg.URL, options...) + if err != nil { + return nil, ErrUnavailable + } + js, err := jetstream.New(nc) + if err != nil { + nc.Close() + return nil, ErrUnavailable + } + b := &Broker{cfg: cfg, nc: nc, js: js, streams: map[string]jetstream.Stream{}, registry: map[string]api.Lane{}, now: time.Now} + if err := b.initialize(ctx); err != nil { + nc.Close() + return nil, err + } + return b, nil +} + +func (b *Broker) Close() { b.nc.Close() } +func (b *Broker) Config() Config { return b.cfg.clone() } + +func backend(err error) error { + if err == nil { + return nil + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + return fmt.Errorf("%w: %w", ErrUnavailable, err) +} + +func (b *Broker) initialize(ctx context.Context) error { + var err error + b.state, err = b.bucket(ctx, b.cfg.Namespace+"_state", 0) + if err != nil { + return err + } + b.dedup, err = b.bucket(ctx, b.cfg.Namespace+"_dedup", b.cfg.Retention) + if err != nil { + return err + } + // Registry and bounds are an immutable namespace-wide generation. Changing + // membership requires an explicit new namespace, not racing deployments. + snapshot := struct { + Lanes []api.Lane + Retention, Dedup, Lease, AckRetention, RetryDelay, MaxRetryDelay time.Duration + MaxPayload, MaxLedger, MaxPending, Replicas int + MaxMessages, MaxBytes, MaxKVBytes int64 + StateFormat int + }{b.cfg.Lanes, b.cfg.Retention, b.cfg.Dedup, b.cfg.Lease, b.cfg.AckRetention, b.cfg.RetryDelay, b.cfg.MaxRetryDelay, + b.cfg.MaxPayload, b.cfg.MaxLedger, b.cfg.MaxPending, b.cfg.Replicas, + b.cfg.MaxMessages, b.cfg.MaxBytes, b.cfg.MaxKVBytes, groupStateFormat} + data, err := json.Marshal(snapshot) + if err != nil { + return err + } + if len(data) > maxStateBytes { + return ErrCapacity + } + if _, err = b.state.Create(ctx, "registry", data); err != nil { + if !errors.Is(err, jetstream.ErrKeyExists) { + return backend(err) + } + entry, err := b.state.Get(ctx, "registry") + if err != nil { + return backend(err) + } + if !bytes.Equal(entry.Value(), data) { + return fmt.Errorf("%w: namespace registry differs", ErrConflict) + } + } + for _, l := range b.cfg.Lanes { + sc := jetstream.StreamConfig{Name: b.streamName(l.ID), Subjects: []string{b.subject(l.ID) + "*"}, + Storage: jetstream.FileStorage, Retention: jetstream.LimitsPolicy, + Discard: jetstream.DiscardNew, MaxAge: b.cfg.Retention, + MaxMsgs: b.cfg.MaxMessages, MaxBytes: b.cfg.MaxBytes, + MaxMsgSize: int32(base64.StdEncoding.EncodedLen(b.cfg.MaxPayload) + envelopeOverhead), Replicas: b.cfg.Replicas, + Duplicates: b.cfg.Dedup, DenyDelete: true, DenyPurge: true} + s, err := b.js.CreateStream(ctx, sc) + if errors.Is(err, jetstream.ErrStreamNameAlreadyInUse) { + s, err = b.js.Stream(ctx, sc.Name) + if err == nil { + info, e := s.Info(ctx) + if e != nil { + return backend(e) + } + got := info.Config + if got.MaxAge != sc.MaxAge || got.MaxMsgs != sc.MaxMsgs || got.MaxBytes != sc.MaxBytes || + got.Storage != sc.Storage || got.Retention != sc.Retention || got.Discard != sc.Discard || + got.Replicas != sc.Replicas || got.MaxMsgSize != sc.MaxMsgSize || got.Duplicates != sc.Duplicates || + !got.DenyDelete || !got.DenyPurge || len(got.Subjects) != 1 || got.Subjects[0] != sc.Subjects[0] { + return fmt.Errorf("%w: stream configuration differs", ErrConflict) + } + } + } + if err != nil { + return backend(err) + } + b.streams[l.ID], b.registry[l.ID] = s, l + } + return nil +} + +const envelopeOverhead = 8 << 10 + +func (b *Broker) bucket(ctx context.Context, name string, ttl time.Duration) (jetstream.KeyValue, error) { + kv, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: name, Storage: jetstream.FileStorage, History: 1, TTL: ttl, + MaxBytes: b.cfg.MaxKVBytes, Replicas: b.cfg.Replicas}) + if errors.Is(err, jetstream.ErrBucketExists) { + kv, err = b.js.KeyValue(ctx, name) + } + if err != nil { + return nil, backend(err) + } + s, err := b.js.Stream(ctx, "KV_"+name) + if err != nil { + return nil, backend(err) + } + info, err := s.Info(ctx) + if err != nil { + return nil, backend(err) + } + if info.Config.MaxAge != ttl || info.Config.MaxBytes != b.cfg.MaxKVBytes || + info.Config.Storage != jetstream.FileStorage || info.Config.Replicas != b.cfg.Replicas || + info.Config.MaxMsgsPerSubject != 1 { + return nil, fmt.Errorf("%w: key-value configuration differs", ErrConflict) + } + return kv, nil +} + +func (b *Broker) streamName(lane string) string { + return "crux_lane_" + digest([]byte(b.cfg.Namespace+"\x00"+lane)) +} +func (b *Broker) subject(lane string) string { return b.cfg.Namespace + ".lane." + lane + "." } +func digest(data []byte) string { sum := sha256.Sum256(data); return hex.EncodeToString(sum[:]) } + +type publishReservation struct { + Hash string `json:"hash"` + Created time.Time `json:"created"` + Generation string `json:"generation"` +} + +func (b *Broker) Publish(ctx context.Context, project, lane string, in api.LanePublish) (api.LanePublished, error) { + ctx, cancel := context.WithTimeout(ctx, b.cfg.Timeout) + defer cancel() + if _, err := b.authorize(project, lane, "publish", ""); err != nil { + return api.LanePublished{}, err + } + if !ValidID(in.ID) || !ValidID(in.Schema) || (in.Kind != api.LaneEvent && in.Kind != api.LaneWork) || + len(in.Payload) == 0 || len(in.Payload) > b.cfg.MaxPayload || len(in.Provenance) > MaxProvenanceLength { + return api.LanePublished{}, ErrInvalid + } + if in.Provenance != "" { + u, err := url.Parse(in.Provenance) + if err != nil || !u.IsAbs() || u.User != nil { + return api.LanePublished{}, ErrInvalid + } + } + body, err := json.Marshal(in) + if err != nil { + return api.LanePublished{}, err + } + key := lane + "." + project + "." + in.ID + generation, err := newReceipt() + if err != nil { + return api.LanePublished{}, err + } + res := publishReservation{Hash: digest(body), Created: b.now().UTC(), Generation: generation} + data, err := json.Marshal(res) + if err != nil { + return api.LanePublished{}, err + } + if _, err := b.dedup.Create(ctx, key, data); err != nil { + if !errors.Is(err, jetstream.ErrKeyExists) { + return api.LanePublished{}, backend(err) + } + entry, err := b.dedup.Get(ctx, key) + if err != nil { + return api.LanePublished{}, backend(err) + } + if err := json.Unmarshal(entry.Value(), &res); err != nil { + return api.LanePublished{}, backend(err) + } + if res.Hash != digest(body) { + return api.LanePublished{}, ErrConflict + } + } + if !validReceipt(res.Generation) { + return api.LanePublished{}, ErrConflict + } + subject := b.subject(lane) + digest([]byte(key)) + stream := b.streams[lane] + if msg, err := stream.GetLastMsgForSubject(ctx, subject); err == nil { + var envelope api.LaneEnvelope + if err := json.Unmarshal(msg.Data, &envelope); err != nil { + return api.LanePublished{}, backend(err) + } + existing, err := json.Marshal(envelope.LanePublish) + if err != nil { + return api.LanePublished{}, err + } + if !bytes.Equal(existing, body) || envelope.Project != project || envelope.Lane != lane { + return api.LanePublished{}, ErrConflict + } + if err := b.verifyReservation(ctx, key, res); err != nil { + return api.LanePublished{}, err + } + return api.LanePublished{Sequence: msg.Sequence, Duplicate: true}, nil + } else if !errors.Is(err, jetstream.ErrMsgNotFound) { + return api.LanePublished{}, backend(err) + } + if b.now().Sub(res.Created) >= b.cfg.Retention { + return api.LanePublished{}, ErrCursor + } + envelope := api.LaneEnvelope{LanePublish: in, Lane: lane, Project: project, CreatedAt: res.Created} + data, err = json.Marshal(envelope) + if err != nil { + return api.LanePublished{}, err + } + if err := b.verifyReservation(ctx, key, res); err != nil { + return api.LanePublished{}, err + } + // Subject CAS closes the crash-between-reservation-and-publish gap even + // after JetStream's finite message-ID deduplication window has elapsed. + msg := &nats.Msg{Subject: subject, Data: data, Header: nats.Header{reservationHeader: []string{res.Generation}}} + ack, err := b.js.PublishMsg(ctx, msg, + jetstream.WithMsgID(key+"."+res.Generation), jetstream.WithExpectLastSequencePerSubject(0)) + if err != nil { + // A concurrent identical publisher may have won subject CAS. Only a + // persisted, verified envelope is accepted as a successful duplicate. + msg, readErr := stream.GetLastMsgForSubject(ctx, subject) + if readErr == nil { + var got api.LaneEnvelope + if e := json.Unmarshal(msg.Data, &got); e != nil { + return api.LanePublished{}, backend(e) + } + encoded, e := json.Marshal(got.LanePublish) + if e != nil { + return api.LanePublished{}, e + } + if bytes.Equal(encoded, body) && got.Project == project && got.Lane == lane && msg.Header.Get(reservationHeader) == res.Generation { + if e := b.verifyReservation(ctx, key, res); e != nil { + return api.LanePublished{}, e + } + return api.LanePublished{Sequence: msg.Sequence, Duplicate: true}, nil + } + return api.LanePublished{}, ErrConflict + } + return api.LanePublished{}, backend(err) + } + // JetStream deduplicates message IDs before evaluating subject CAS. Never + // equate a broker duplicate ACK with this request's content or generation. + persisted, err := stream.GetMsg(ctx, ack.Sequence) + if errors.Is(err, jetstream.ErrMsgNotFound) { + return api.LanePublished{}, ErrCursor + } + if err != nil { + return api.LanePublished{}, backend(err) + } + if persisted.Subject != subject || persisted.Header.Get(reservationHeader) != res.Generation || !bytes.Equal(persisted.Data, data) { + return api.LanePublished{}, ErrConflict + } + if err := b.verifyReservation(ctx, key, res); err != nil { + return api.LanePublished{}, err + } + return api.LanePublished{Sequence: ack.Sequence, Duplicate: ack.Duplicate}, nil +} + +const reservationHeader = "Crux-Reservation-Generation" + +func (b *Broker) verifyReservation(ctx context.Context, key string, expected publishReservation) error { + entry, err := b.dedup.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) || errors.Is(err, jetstream.ErrKeyDeleted) { + return ErrStale + } + if err != nil { + return backend(err) + } + var current publishReservation + if err := json.Unmarshal(entry.Value(), ¤t); err != nil { + return backend(err) + } + if current.Generation != expected.Generation || current.Hash != expected.Hash || !current.Created.Equal(expected.Created) { + return ErrConflict + } + if b.now().Sub(current.Created) >= b.cfg.Retention { + return ErrStale + } + return nil +} + +func (b *Broker) Replay(ctx context.Context, project, lane string, cursor uint64, limit int) (api.LaneReplay, error) { + ctx, cancel := context.WithTimeout(ctx, b.cfg.Timeout) + defer cancel() + if _, err := b.authorize(project, lane, "subscribe", ""); err != nil { + return api.LaneReplay{}, err + } + if limit < 1 || limit > MaxReplayBatch { + return api.LaneReplay{}, ErrInvalid + } + return b.scan(ctx, lane, cursor, limit, api.LaneEvent) +} + +func (b *Broker) scan(ctx context.Context, lane string, cursor uint64, limit int, kind string) (api.LaneReplay, error) { + s := b.streams[lane] + info, err := s.Info(ctx) + if err != nil { + return api.LaneReplay{}, backend(err) + } + first, last := info.State.FirstSeq, info.State.LastSeq + if cursor > last || (cursor != 0 && first > 0 && cursor < first-1) { + return api.LaneReplay{}, ErrCursor + } + next := cursor + 1 + if cursor == 0 && first > next { + next = first + } + out := api.LaneReplay{Events: []api.LaneRecord{}, Cursor: cursor} + for count := 0; next <= last && count < MaxScanBatch && len(out.Events) < limit; count++ { + msg, err := s.GetMsg(ctx, next) + if errors.Is(err, jetstream.ErrMsgNotFound) { + return api.LaneReplay{}, ErrCursor + } + if err != nil { + return api.LaneReplay{}, backend(err) + } + var envelope api.LaneEnvelope + if err := json.Unmarshal(msg.Data, &envelope); err != nil { + return api.LaneReplay{}, backend(err) + } + if envelope.Lane != lane { + return api.LaneReplay{}, ErrConflict + } + out.Cursor = next + if envelope.Kind == kind { + out.Events = append(out.Events, api.LaneRecord{Sequence: next, Envelope: envelope}) + } + next++ + } + return out, nil +} diff --git a/src/crux/lanes/broker_test.go b/src/crux/lanes/broker_test.go new file mode 100644 index 0000000000..4b5b6fa17f --- /dev/null +++ b/src/crux/lanes/broker_test.go @@ -0,0 +1,390 @@ +package lanes + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/nats-io/nats-server/v2/server" +) + +const ( + testWait = 10 * time.Second + fixturePoll = 25 * time.Millisecond +) + +func testServer(t *testing.T) (*server.Server, *server.Options) { + t.Helper() + dir := filepath.Join(".", fmt.Sprintf(".lane-fixture-%s-%d", strings.ReplaceAll(t.Name(), "/", "-"), time.Now().UnixNano())) + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }) + opts := &server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: dir, NoLog: true, NoSigs: true} + s := startServer(t, opts) + opts.Port = s.Addr().(*net.TCPAddr).Port + t.Cleanup(func() { + s.Shutdown() + s.WaitForShutdown() + }) + return s, opts +} + +func startServer(t *testing.T, opts *server.Options) *server.Server { + t.Helper() + s, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(testWait) { + s.Shutdown() + t.Fatal("broker not ready") + } + return s +} + +func testConfig(url string) Config { + c := Config{URL: url, Principals: []Principal{{Project: "atlas", TokenEnv: "ATLAS_TOKEN"}, {Project: "beacon", TokenEnv: "BEACON_TOKEN"}}, + Lanes: []api.Lane{{ID: "shared", Owner: "atlas", Publishers: []string{"atlas", "beacon"}, Subscribers: []string{"atlas", "beacon"}, + Groups: map[string][]string{"workers": {"atlas", "beacon"}, "other": {"beacon"}}, + Recoverers: map[string][]string{"workers": {"atlas", "beacon"}}}}} + c.Defaults() + return c +} + +func testBroker(t *testing.T, change func(*Config)) *Broker { + t.Helper() + s, _ := testServer(t) + cfg := testConfig(s.ClientURL()) + if change != nil { + change(&cfg) + } + b, err := Open(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(b.Close) + return b +} + +func publish(t *testing.T, b *Broker, id, kind string) api.LanePublished { + t.Helper() + out, err := b.Publish(context.Background(), "atlas", "shared", api.LanePublish{ID: id, Kind: kind, Schema: "sample-v1", Payload: []byte(`{"value":1}`)}) + if err != nil { + t.Fatal(err) + } + return out +} + +func claim(t *testing.T, b *Broker, project, group string) api.LaneClaim { + t.Helper() + out, err := b.Claim(context.Background(), project, "shared", group) + if err != nil { + t.Fatal(err) + } + return out +} + +func receipt(c api.LaneClaim) api.LaneReceipt { + return api.LaneReceipt{Sequence: c.Sequence, Receipt: c.Receipt} +} + +func TestPublishFanoutReplayAndIdempotency(t *testing.T) { + b := testBroker(t, nil) + first := publish(t, b, "first", api.LaneEvent) + duplicate := publish(t, b, "first", api.LaneEvent) + if !duplicate.Duplicate || duplicate.Sequence != first.Sequence { + t.Fatalf("duplicate %+v", duplicate) + } + publish(t, b, "job", api.LaneWork) + last := publish(t, b, "last", api.LaneEvent) + for _, project := range []string{"atlas", "beacon"} { + page, err := b.Replay(context.Background(), project, "shared", 0, 1) + if err != nil || len(page.Events) != 1 || page.Events[0].Envelope.Project != "atlas" { + t.Fatalf("%s: %+v %v", project, page, err) + } + page, err = b.Replay(context.Background(), project, "shared", page.Cursor, MaxReplayBatch) + if err != nil || len(page.Events) != 1 || page.Cursor != last.Sequence { + t.Fatalf("cursor %+v %v", page, err) + } + } + _, err := b.Publish(context.Background(), "atlas", "shared", api.LanePublish{ID: "first", Kind: api.LaneEvent, Schema: "sample-v1", Payload: []byte("different")}) + if !errors.Is(err, ErrConflict) { + t.Fatalf("mismatched payload: %v", err) + } + _, err = b.Replay(context.Background(), "atlas", "shared", last.Sequence+1, 1) + if !errors.Is(err, ErrCursor) { + t.Fatalf("future cursor: %v", err) + } +} + +func TestWorkCompetitionGroupsFencingAndAcknowledgements(t *testing.T) { + b := testBroker(t, nil) + publish(t, b, "job", api.LaneWork) + a := claim(t, b, "atlas", "workers") + if _, err := b.Claim(context.Background(), "beacon", "shared", "workers"); !errors.Is(err, ErrEmpty) { + t.Fatalf("competing claim: %v", err) + } + other := claim(t, b, "beacon", "other") + if other.Sequence != a.Sequence { + t.Fatal("other delivery group did not get its own work") + } + if err := b.Ack(context.Background(), "beacon", "shared", "workers", receipt(a)); !errors.Is(err, ErrStale) { + t.Fatalf("cross-principal ack: %v", err) + } + if err := b.Ack(context.Background(), "atlas", "shared", "other", receipt(a)); !errors.Is(err, ErrForbidden) { + t.Fatalf("forged group: %v", err) + } + renewed, err := b.Renew(context.Background(), "atlas", "shared", "workers", receipt(a)) + if err != nil || renewed.Before(a.Expires) { + t.Fatalf("renew: %v %v", renewed, err) + } + if err := b.Nack(context.Background(), "atlas", "shared", "workers", receipt(a)); err != nil { + t.Fatal(err) + } + if err := b.Nack(context.Background(), "atlas", "shared", "workers", receipt(a)); err != nil { + t.Fatal("duplicate nack", err) + } + makeRetriesDue(t, b) + reassigned := claim(t, b, "beacon", "workers") + if reassigned.Attempt != 2 || reassigned.Receipt == a.Receipt { + t.Fatalf("redelivery %+v", reassigned) + } + for _, operation := range []string{"ack", "nack", "renew"} { + _, err := b.receipt(context.Background(), "atlas", "shared", "workers", operation, receipt(a)) + if !errors.Is(err, ErrStale) { + t.Fatalf("stale %s: %v", operation, err) + } + } + for range 2 { + if err := b.Ack(context.Background(), "beacon", "shared", "workers", receipt(reassigned)); err != nil { + t.Fatal("idempotent ack", err) + } + } + if _, err := b.Claim(context.Background(), "beacon", "shared", "workers"); !errors.Is(err, ErrEmpty) { + t.Fatalf("acked redelivered: %v", err) + } +} + +func TestExpiryCapacityCancellationAndIsolation(t *testing.T) { + b := testBroker(t, func(c *Config) { c.MaxPending, c.MaxLedger = 1, 1 }) + publish(t, b, "job", api.LaneWork) + publish(t, b, "next", api.LaneWork) + c := claim(t, b, "atlas", "workers") + if _, err := b.Claim(context.Background(), "atlas", "shared", "workers"); !errors.Is(err, ErrCapacity) { + t.Fatalf("pending bound: %v", err) + } + future := b.now().Add(b.cfg.Lease + time.Second) + b.now = func() time.Time { return future } + if err := b.Ack(context.Background(), "atlas", "shared", "workers", receipt(c)); !errors.Is(err, ErrStale) { + t.Fatalf("expired ack: %v", err) + } + next := claim(t, b, "beacon", "workers") + if next.Sequence != c.Sequence || next.Attempt != 2 { + t.Fatal("expiry did not redeliver") + } + if err := b.Ack(context.Background(), "beacon", "shared", "workers", receipt(next)); err != nil { + t.Fatal(err) + } + if _, err := b.Claim(context.Background(), "atlas", "shared", "workers"); !errors.Is(err, ErrCapacity) { + t.Fatalf("ack ledger bound: %v", err) + } + future = future.Add(b.cfg.AckRetention + time.Second) + if c := claim(t, b, "atlas", "workers"); c.Sequence <= next.Sequence { + t.Fatal("ledger pruning did not allow next work") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := b.Replay(ctx, "atlas", "shared", 0, 1); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation: %v", err) + } + if _, err := b.Publish(context.Background(), "outsider", "shared", api.LanePublish{}); !errors.Is(err, ErrForbidden) { + t.Fatalf("unauthenticated: %v", err) + } + if _, err := b.Claim(context.Background(), "atlas", "shared", "*"); !errors.Is(err, ErrForbidden) { + t.Fatalf("wildcard: %v", err) + } +} + +func TestConcurrentPublishAndClaims(t *testing.T) { + b := testBroker(t, nil) + const writers = 8 + var wg sync.WaitGroup + results := make(chan api.LanePublished, writers) + errs := make(chan error, writers) + for range writers { + wg.Add(1) + go func() { + defer wg.Done() + out, err := b.Publish(context.Background(), "atlas", "shared", api.LanePublish{ID: "same", Kind: api.LaneWork, Schema: "sample-v1", Payload: []byte("work")}) + if err != nil { + errs <- err + } else { + results <- out + } + }() + } + wg.Wait() + close(results) + close(errs) + for err := range errs { + t.Fatal(err) + } + var seq uint64 + for out := range results { + if seq != 0 && seq != out.Sequence { + t.Fatal("duplicate stream messages") + } + seq = out.Sequence + } + claims := make(chan api.LaneClaim, writers) + errs = make(chan error, writers) + for range writers { + wg.Add(1) + go func() { + defer wg.Done() + c, err := b.Claim(context.Background(), "atlas", "shared", "workers") + if err == nil { + claims <- c + } else if !errors.Is(err, ErrEmpty) { + errs <- err + } + }() + } + wg.Wait() + close(claims) + close(errs) + for err := range errs { + t.Fatal(err) + } + if len(claims) != 1 { + t.Fatalf("owners: %d", len(claims)) + } +} + +func TestServiceAndBrokerRestart(t *testing.T) { + s, opts := testServer(t) + cfg := testConfig(s.ClientURL()) + b, err := Open(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + published := publish(t, b, "event", api.LaneEvent) + publish(t, b, "work", api.LaneWork) + c := claim(t, b, "atlas", "workers") + b.Close() + s.Shutdown() + s.WaitForShutdown() + s = startServer(t, opts) + defer func() { s.Shutdown(); s.WaitForShutdown() }() + b, err = Open(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + defer b.Close() + duplicate := publish(t, b, "event", api.LaneEvent) + if duplicate.Sequence != published.Sequence || !duplicate.Duplicate { + t.Fatal("restart lost dedup") + } + page, err := b.Replay(context.Background(), "beacon", "shared", 0, 1) + if err != nil || len(page.Events) != 1 { + t.Fatalf("restart lost replay: %+v %v", page, err) + } + if _, err := b.Claim(context.Background(), "beacon", "shared", "workers"); !errors.Is(err, ErrEmpty) { + t.Fatalf("restart lost lease: %v", err) + } + if err := b.Ack(context.Background(), "atlas", "shared", "workers", receipt(c)); err != nil { + t.Fatal("restart lost receipt", err) + } +} + +func awaitCondition(t *testing.T, check func() (bool, error)) { + t.Helper() + awaitConditionWithin(t, testWait, check) +} + +func awaitConditionWithin(t *testing.T, timeout time.Duration, check func() (bool, error)) { + t.Helper() + ticker := time.NewTicker(fixturePoll) + defer ticker.Stop() + deadline := time.NewTimer(timeout) + defer deadline.Stop() + for { + done, err := check() + if err != nil { + t.Fatal(err) + } + if done { + return + } + select { + case <-ticker.C: + case <-deadline.C: + t.Fatal("broker condition not reached") + } + } +} + +func makeRetriesDue(t *testing.T, b *Broker) { + t.Helper() + state, _, err := b.readGroup(context.Background(), groupKey("shared", "workers")) + if err != nil { + t.Fatal(err) + } + now := b.now() + for _, l := range state.Leases { + if l.NextAttempt.After(now) { + now = l.NextAttempt + } + } + now = now.Add(time.Nanosecond) + b.now = func() time.Time { return now } +} + +func TestConfigurationAndInputBounds(t *testing.T) { + cfg := testConfig("nats://127.0.0.1:4222") + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + for _, id := range []string{"", "a.b", "*", ">", "a/b", strings.Repeat("a", MaxIDLength+1)} { + if ValidID(id) { + t.Fatalf("unsafe identifier %q", id) + } + } + for _, mutate := range []func(*Config){ + func(c *Config) { c.Namespace = "*" }, + func(c *Config) { c.Lease = MaxLease + 1 }, + func(c *Config) { c.URL = "nats://user:secret@localhost:4222" }, + func(c *Config) { c.Lanes[0].Owner = "outsider" }, + } { + c := testConfig("nats://127.0.0.1:4222") + mutate(&c) + if !errors.Is(c.Validate(), ErrInvalid) { + t.Fatal("invalid configuration accepted") + } + } + b := testBroker(t, nil) + for _, in := range []api.LanePublish{ + {ID: "x", Kind: "other", Schema: "v1", Payload: []byte("x")}, + {ID: "*", Kind: api.LaneEvent, Schema: "v1", Payload: []byte("x")}, + {ID: "x", Kind: api.LaneEvent, Schema: "v1", Payload: make([]byte, b.cfg.MaxPayload+1)}, + {ID: "x", Kind: api.LaneEvent, Schema: "v1", Payload: []byte("x"), Provenance: "relative/path"}, + } { + if _, err := b.Publish(context.Background(), "atlas", "shared", in); !errors.Is(err, ErrInvalid) { + t.Fatalf("invalid envelope: %v", err) + } + } +} diff --git a/src/crux/lanes/config.go b/src/crux/lanes/config.go new file mode 100644 index 0000000000..6c40be394a --- /dev/null +++ b/src/crux/lanes/config.go @@ -0,0 +1,275 @@ +// Package lanes owns forge-neutral durable events and work. Streams and all +// mutable delivery state live in JetStream; no process-local journal is used. +package lanes + +import ( + "errors" + "fmt" + "net/url" + "regexp" + "slices" + "time" + + "github.com/hivecommons/hive/crux/api" +) + +const ( + DefaultNamespace = "crux" + DefaultRetention = 7 * 24 * time.Hour + DefaultDedup = 2 * time.Minute + DefaultLease = time.Minute + DefaultAckRetention = time.Hour + DefaultRetryDelay = time.Second + DefaultMaxRetryDelay = time.Minute + MinRetryDelay = 100 * time.Millisecond + MaxRetryDelay = time.Hour + DefaultTimeout = 5 * time.Second + DefaultMaxPayload = 64 << 10 + DefaultMaxMessages = 10000 + DefaultMaxBytes = 64 << 20 + DefaultMaxKVBytes = 64 << 20 + DefaultMaxLedger = 1024 + DefaultMaxPending = 128 + DefaultMaxLanes = 64 + DefaultMaxGroups = 64 + DefaultMaxMembers = 64 + MaxLaneMessages = 1000000 + MaxStorageBytes = 1 << 30 + MaxLedgerEntries = 10000 + MaxIDLength = 64 + MaxProvenanceLength = 2048 + MaxReplayBatch = 128 + MaxRecoveryBatch = 16 + MaxScanBatch = 512 + MinLease = time.Second + MinDedup = 100 * time.Millisecond + MaxLease = time.Hour + MinRetention = time.Second + MaxRetention = 30 * 24 * time.Hour + MaxTimeout = time.Minute + MaxPayload = 1 << 20 + MaxReplicas = 5 +) + +var ( + ErrInvalid = errors.New("invalid lane request") + ErrForbidden = errors.New("lane capability denied") + ErrConflict = errors.New("lane state conflict") + ErrCursor = errors.New("invalid or expired lane cursor") + ErrStale = errors.New("stale lane receipt") + ErrCapacity = errors.New("lane capacity exhausted") + ErrEmpty = errors.New("no claimable work") + ErrUnavailable = errors.New("lane broker unavailable") + idPattern = regexp.MustCompile(`^[a-z][a-z0-9_-]*$`) + envPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) +) + +type Principal struct { + Project string `json:"project"` + TokenEnv string `json:"token_env"` +} + +// Config is administrator-owned configuration, never an API request. Secrets +// are referenced by environment variable name and never serialized into it. +type Config struct { + URL string `json:"url"` + CredentialsEnv string `json:"credentials_env,omitempty"` + Namespace string `json:"namespace"` + Retention time.Duration `json:"retention"` + Dedup time.Duration `json:"dedup"` + Lease time.Duration `json:"lease"` + AckRetention time.Duration `json:"ack_retention"` + RetryDelay time.Duration `json:"retry_delay"` + MaxRetryDelay time.Duration `json:"max_retry_delay"` + Timeout time.Duration `json:"timeout"` + MaxPayload int `json:"max_payload"` + MaxMessages int64 `json:"max_messages"` + MaxBytes int64 `json:"max_bytes"` + MaxKVBytes int64 `json:"max_kv_bytes"` + MaxLedger int `json:"max_ledger"` + MaxPending int `json:"max_pending"` + Replicas int `json:"replicas"` + Principals []Principal `json:"principals"` + Lanes []api.Lane `json:"lanes"` +} + +func ValidID(id string) bool { return len(id) <= MaxIDLength && idPattern.MatchString(id) } + +func (c *Config) Defaults() { + if c.Namespace == "" { + c.Namespace = DefaultNamespace + } + if c.Retention == 0 { + c.Retention = DefaultRetention + } + if c.Dedup == 0 { + c.Dedup = DefaultDedup + } + if c.Lease == 0 { + c.Lease = DefaultLease + } + if c.AckRetention == 0 { + c.AckRetention = DefaultAckRetention + } + if c.RetryDelay == 0 { + c.RetryDelay = DefaultRetryDelay + } + if c.MaxRetryDelay == 0 { + c.MaxRetryDelay = DefaultMaxRetryDelay + } + if c.Timeout == 0 { + c.Timeout = DefaultTimeout + } + if c.MaxPayload == 0 { + c.MaxPayload = DefaultMaxPayload + } + if c.MaxMessages == 0 { + c.MaxMessages = DefaultMaxMessages + } + if c.MaxBytes == 0 { + c.MaxBytes = DefaultMaxBytes + } + if c.MaxKVBytes == 0 { + c.MaxKVBytes = DefaultMaxKVBytes + } + if c.MaxLedger == 0 { + c.MaxLedger = DefaultMaxLedger + } + if c.MaxPending == 0 { + c.MaxPending = DefaultMaxPending + } + if c.Replicas == 0 { + c.Replicas = 1 + } +} + +func (c Config) Validate() error { + u, err := url.Parse(c.URL) + if err != nil || (u.Scheme != "nats" && u.Scheme != "tls") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return fmt.Errorf("%w: broker URL must be nats:// or tls:// without inline credentials", ErrInvalid) + } + if !ValidID(c.Namespace) || c.Retention < MinRetention || c.Retention > MaxRetention || + c.Dedup < MinDedup || c.Dedup > c.Retention || c.Lease < MinLease || c.Lease > MaxLease || + c.AckRetention < c.Lease || c.AckRetention > c.Retention || c.Timeout <= 0 || c.Timeout > MaxTimeout || + c.RetryDelay < MinRetryDelay || c.MaxRetryDelay < c.RetryDelay || c.MaxRetryDelay > MaxRetryDelay || + c.MaxPayload <= 0 || c.MaxPayload > MaxPayload || c.MaxMessages <= 0 || c.MaxMessages > MaxLaneMessages || + c.MaxBytes < int64(c.MaxPayload) || c.MaxBytes > MaxStorageBytes || c.MaxKVBytes <= 0 || c.MaxKVBytes > MaxStorageBytes || + c.MaxPending <= 0 || c.MaxLedger < c.MaxPending || c.MaxLedger > MaxLedgerEntries || + c.Replicas < 1 || c.Replicas > MaxReplicas || len(c.Lanes) > DefaultMaxLanes || + len(c.Principals) == 0 || len(c.Principals) > DefaultMaxMembers { + return fmt.Errorf("%w: lane configuration bounds", ErrInvalid) + } + if c.CredentialsEnv != "" && !validEnv(c.CredentialsEnv) { + return fmt.Errorf("%w: broker credential binding", ErrInvalid) + } + projects := map[string]bool{} + envs := map[string]bool{} + for _, p := range c.Principals { + if !ValidID(p.Project) || !validEnv(p.TokenEnv) || projects[p.Project] || envs[p.TokenEnv] { + return fmt.Errorf("%w: principal configuration", ErrInvalid) + } + + projects[p.Project], envs[p.TokenEnv] = true, true + } + ids := map[string]bool{} + for _, l := range c.Lanes { + if !ValidID(l.ID) || !projects[l.Owner] || ids[l.ID] || len(l.Groups) > DefaultMaxGroups { + return fmt.Errorf("%w: lane registry", ErrInvalid) + } + ids[l.ID] = true + members := append(slices.Clone(l.Publishers), l.Subscribers...) + for group, m := range l.Groups { + if !ValidID(group) || len(m) == 0 || len(m) > DefaultMaxMembers { + return fmt.Errorf("%w: delivery group", ErrInvalid) + } + members = append(members, m...) + } + for group, recoverers := range l.Recoverers { + if !ValidID(group) || len(recoverers) == 0 || len(recoverers) > DefaultMaxMembers { + return fmt.Errorf("%w: recovery capability", ErrInvalid) + } + for _, project := range recoverers { + if !allowed(l.Groups[group], project) { + return fmt.Errorf("%w: recoverer must be a group member", ErrInvalid) + } + } + } + if len(l.Publishers) > DefaultMaxMembers || len(l.Subscribers) > DefaultMaxMembers { + return fmt.Errorf("%w: membership bounds", ErrInvalid) + } + for _, p := range members { + if !projects[p] { + return fmt.Errorf("%w: unknown member", ErrInvalid) + } + } + } + return nil +} + +func validEnv(name string) bool { + return len(name) <= MaxIDLength && envPattern.MatchString(name) +} + +func allowed(members []string, project string) bool { return slices.Contains(members, project) } + +func cloneLane(l api.Lane) api.Lane { + l.Publishers = slices.Clone(l.Publishers) + l.Subscribers = slices.Clone(l.Subscribers) + groups := make(map[string][]string, len(l.Groups)) + for group, members := range l.Groups { + groups[group] = slices.Clone(members) + } + l.Groups = groups + recoverers := make(map[string][]string, len(l.Recoverers)) + for group, members := range l.Recoverers { + recoverers[group] = slices.Clone(members) + } + l.Recoverers = recoverers + return l +} + +func (c Config) clone() Config { + c.Principals = slices.Clone(c.Principals) + c.Lanes = slices.Clone(c.Lanes) + for i := range c.Lanes { + c.Lanes[i] = cloneLane(c.Lanes[i]) + } + return c +} + +func (b *Broker) authorize(project, lane, capability, group string) (api.Lane, error) { + l, ok := b.registry[lane] + if !ok { + return api.Lane{}, ErrForbidden + } + switch capability { + case "publish": + ok = allowed(l.Publishers, project) + case "subscribe": + ok = allowed(l.Subscribers, project) + case "work": + ok = allowed(l.Groups[group], project) + case "recover": + ok = allowed(l.Groups[group], project) && allowed(l.Recoverers[group], project) + default: + ok = false + } + if !ok { + return api.Lane{}, ErrForbidden + } + return l, nil +} + +func (b *Broker) Registry(project string) []api.Lane { + out := []api.Lane{} + for _, l := range b.cfg.Lanes { + member := l.Owner == project || allowed(l.Publishers, project) || allowed(l.Subscribers, project) + for _, m := range l.Groups { + member = member || allowed(m, project) + } + if member { + out = append(out, cloneLane(l)) + } + } + return out +} diff --git a/src/crux/lanes/config_recoverers_test.go b/src/crux/lanes/config_recoverers_test.go new file mode 100644 index 0000000000..63473f4715 --- /dev/null +++ b/src/crux/lanes/config_recoverers_test.go @@ -0,0 +1,115 @@ +package lanes + +import ( + "errors" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/api" +) + +func TestRecoverersValidationDoesNotDependOnGroupIteration(t *testing.T) { + repeatedMembers := make([]string, DefaultMaxMembers) + for i := range repeatedMembers { + repeatedMembers[i] = "atlas" + } + tests := []struct { + name string + groups map[string][]string + recoverers map[string][]string + valid bool + }{ + {name: "nil groups and recoverers", valid: true}, + {name: "empty groups and recoverers", groups: map[string][]string{}, recoverers: map[string][]string{}, valid: true}, + {name: "nil groups with otherwise valid recovery identifiers", recoverers: map[string][]string{"workers": {"atlas"}}}, + {name: "empty groups with otherwise valid recovery identifiers", groups: map[string][]string{}, recoverers: map[string][]string{"workers": {"atlas"}}}, + {name: "empty groups with malformed recovery group", groups: map[string][]string{}, recoverers: map[string][]string{"bad.*": {"atlas"}}}, + {name: "empty groups with whitespace recovery group", groups: map[string][]string{}, recoverers: map[string][]string{"workers ": {"atlas"}}}, + {name: "empty groups with whitespace recovery member", groups: map[string][]string{}, recoverers: map[string][]string{"workers": {"atlas "}}}, + {name: "empty groups with empty recovery members", groups: map[string][]string{}, recoverers: map[string][]string{"workers": {}}}, + {name: "empty groups with duplicate recovery members", groups: map[string][]string{}, recoverers: map[string][]string{"workers": {"atlas", "atlas"}}}, + {name: "empty groups with excessive recovery members", groups: map[string][]string{}, recoverers: map[string][]string{"workers": append(append([]string(nil), repeatedMembers...), "atlas")}}, + {name: "empty groups with oversized recovery group", groups: map[string][]string{}, recoverers: map[string][]string{strings.Repeat("a", MaxIDLength+1): {"atlas"}}}, + {name: "registered group without recovery grant", groups: map[string][]string{"workers": {"atlas"}}, valid: true}, + {name: "registered group with empty recovery map", groups: map[string][]string{"workers": {"atlas"}}, recoverers: map[string][]string{}, valid: true}, + {name: "registered group with valid recovery grant", groups: map[string][]string{"workers": {"atlas"}}, recoverers: map[string][]string{"workers": {"atlas"}}, valid: true}, + {name: "duplicate members retain membership semantics", groups: map[string][]string{"workers": {"atlas", "atlas"}}, recoverers: map[string][]string{"workers": {"atlas", "atlas"}}, valid: true}, + {name: "recovery member bound permits existing duplicate semantics", groups: map[string][]string{"workers": {"atlas"}}, recoverers: map[string][]string{"workers": repeatedMembers}, valid: true}, + {name: "bound group rejects whitespace member", groups: map[string][]string{"workers": {"atlas"}}, recoverers: map[string][]string{"workers": {" atlas"}}}, + {name: "bound group rejects excessive recovery members", groups: map[string][]string{"workers": {"atlas"}}, recoverers: map[string][]string{"workers": append(append([]string(nil), repeatedMembers...), "atlas")}}, + {name: "registered principal is not necessarily a group member", groups: map[string][]string{"workers": {"atlas"}}, recoverers: map[string][]string{"workers": {"beacon"}}}, + {name: "recovery group must exist", groups: map[string][]string{"workers": {"atlas"}}, recoverers: map[string][]string{"other": {"atlas"}}}, + {name: "membership in another group grants no recovery", groups: map[string][]string{"workers": {"atlas"}, "other": {"beacon"}}, recoverers: map[string][]string{"workers": {"beacon"}}}, + {name: "unknown principal cannot recover", groups: map[string][]string{"workers": {"atlas"}}, recoverers: map[string][]string{"workers": {"outsider"}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := testConfig("nats://127.0.0.1:4222") + cfg.Lanes[0].Groups = tt.groups + cfg.Lanes[0].Recoverers = tt.recoverers + cfg.Defaults() + err := cfg.Validate() + if tt.valid { + if err != nil { + t.Fatal("valid membership configuration rejected", err) + } + } else if !errors.Is(err, ErrInvalid) { + t.Fatal("invalid recovery capability accepted", err) + } + }) + } +} + +func TestRecoveryConfigurationDefaultsPreserveEmptyLaneRegistry(t *testing.T) { + cfg := Config{ + URL: "nats://127.0.0.1:4222", + Principals: []Principal{{Project: "atlas", TokenEnv: "ATLAS_TOKEN"}}, + } + cfg.Defaults() + if err := cfg.Validate(); err != nil { + t.Fatal("defaulted empty lane registry rejected", err) + } +} + +func TestRecoveryAuthorizationRemainsProjectLaneAndGroupScoped(t *testing.T) { + cfg := testConfig("nats://127.0.0.1:4222") + cfg.Lanes[0].Recoverers = map[string][]string{"workers": {"atlas", "atlas"}} + cfg.Lanes = append(cfg.Lanes, api.Lane{ + ID: "separate", Owner: "beacon", + Groups: map[string][]string{"workers": {"beacon"}}, + Recoverers: map[string][]string{"workers": {"beacon"}}, + }) + if err := cfg.Validate(); err != nil { + t.Fatal("valid recovery authorization fixture rejected", err) + } + b := &Broker{registry: map[string]api.Lane{}} + for _, lane := range cfg.Lanes { + b.registry[lane.ID] = lane + } + tests := []struct { + project, lane, group string + allowed bool + }{ + {"atlas", "shared", "workers", true}, + {"beacon", "shared", "workers", false}, + {"atlas", "shared", "other", false}, + {"beacon", "shared", "other", false}, + {"beacon", "separate", "workers", true}, + {"atlas", "separate", "workers", false}, + {"outsider", "shared", "workers", false}, + {"atlas", "unknown", "workers", false}, + {"atlas", "shared", "unknown", false}, + } + for _, tt := range tests { + t.Run(tt.project+"/"+tt.lane+"/"+tt.group, func(t *testing.T) { + _, err := b.authorize(tt.project, tt.lane, "recover", tt.group) + if tt.allowed { + if err != nil { + t.Fatal("authorized recovery rejected", err) + } + } else if !errors.Is(err, ErrForbidden) { + t.Fatal("foreign recovery capability accepted", err) + } + }) + } +} diff --git a/src/crux/lanes/coverage_failures_test.go b/src/crux/lanes/coverage_failures_test.go new file mode 100644 index 0000000000..4377ef2664 --- /dev/null +++ b/src/crux/lanes/coverage_failures_test.go @@ -0,0 +1,190 @@ +package lanes + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/nats-io/nats.go/jetstream" +) + +var errFixtureBroker = errors.New("injected broker response failure") + +// Delegate reads and successful operations to the embedded broker; intercept +// only the response whose failure contract the caller must preserve. +type faultResponseKV struct { + jetstream.KeyValue + getErr, createErr, updateErr error + updates int +} + +func (k *faultResponseKV) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) { + if k.getErr != nil { + return nil, k.getErr + } + return k.KeyValue.Get(ctx, key) +} + +func (k *faultResponseKV) Create(ctx context.Context, key string, data []byte, opts ...jetstream.KVCreateOpt) (uint64, error) { + if k.createErr != nil { + return 0, k.createErr + } + return k.KeyValue.Create(ctx, key, data, opts...) +} + +func (k *faultResponseKV) Update(ctx context.Context, key string, data []byte, revision uint64) (uint64, error) { + k.updates++ + if k.updateErr != nil { + return 0, k.updateErr + } + return k.KeyValue.Update(ctx, key, data, revision) +} + +func TestFailedGroupCASCannotClaimOrAcknowledgeOwnership(t *testing.T) { + b := testBroker(t, nil) + ctx := context.Background() + publish(t, b, "cas-work", api.LaneWork) + actual := b.state + fault := &faultResponseKV{KeyValue: actual, createErr: jetstream.ErrKeyRevisionMismatch} + b.state = fault + if out, err := b.Claim(ctx, "atlas", "shared", "workers"); !errors.Is(err, ErrConflict) || out.Sequence != 0 { + t.Fatal("repeated admission CAS conflict produced ownership", out, err) + } + if _, err := actual.Get(ctx, groupKey("shared", "workers")); !errors.Is(err, jetstream.ErrKeyNotFound) { + t.Fatal("failed admission persisted a group", err) + } + fault.createErr = errFixtureBroker + if _, err := b.Claim(ctx, "atlas", "shared", "workers"); !errors.Is(err, ErrUnavailable) || !errors.Is(err, errFixtureBroker) { + t.Fatal("failed admission lost broker failure", err) + } + b.state = actual + c := claim(t, b, "atlas", "workers") + fault.createErr, fault.updateErr = nil, jetstream.ErrKeyRevisionMismatch + b.state = fault + if err := b.Ack(ctx, "atlas", "shared", "workers", receipt(c)); !errors.Is(err, ErrConflict) || fault.updates != maxCASAttempts { + t.Fatal("repeated receipt CAS conflict falsely acknowledged", err, fault.updates) + } + fault.updates = 0 + if _, err := b.Claim(ctx, "atlas", "shared", "workers"); !errors.Is(err, ErrConflict) || fault.updates != maxCASAttempts { + t.Fatal("empty-cursor CAS retries were not bounded", err, fault.updates) + } + fault.updateErr = errFixtureBroker + if err := b.Ack(ctx, "atlas", "shared", "workers", receipt(c)); !errors.Is(err, ErrUnavailable) { + t.Fatal("failed receipt CAS falsely acknowledged", err) + } + if _, err := b.Claim(ctx, "atlas", "shared", "workers"); !errors.Is(err, ErrUnavailable) { + t.Fatal("empty-cursor checkpoint hid broker failure", err) + } + b.state = actual + state, _, err := b.readGroup(ctx, groupKey("shared", "workers")) + if err != nil || state.Leases[c.Sequence].Done { + t.Fatal("failed CAS changed durable ownership", err) + } + if err := b.Ack(ctx, "atlas", "shared", "workers", receipt(c)); err != nil { + t.Fatal("accepted receipt could not complete after broker recovery", err) + } + canceled, cancel := context.WithCancel(ctx) + cancel() + if _, err := b.Claim(canceled, "atlas", "shared", "workers"); !errors.Is(err, context.Canceled) { + t.Fatal("canceled claim lost cancellation", err) + } + if _, err := b.Renew(canceled, "atlas", "shared", "workers", receipt(c)); !errors.Is(err, context.Canceled) { + t.Fatal("canceled receipt lost cancellation", err) + } +} + +func TestStoredGroupByteAndReservationBoundsFailClosed(t *testing.T) { + b := testBroker(t, func(c *Config) { + c.MaxLedger, c.MaxPending = minGroupSlots, minGroupSlots + }) + ctx := context.Background() + key := groupKey("shared", "workers") + if _, err := b.state.Put(ctx, key, bytes.Repeat([]byte("x"), maxStateBytes+1)); err != nil { + t.Fatal(err) + } + if _, _, err := b.readGroup(ctx, key); !errors.Is(err, ErrCapacity) { + t.Fatal("oversized stored value escaped the read bound", err) + } + state := groupState{Slots: minGroupSlots, Leases: map[uint64]lease{}} + data, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if _, err := b.state.Put(ctx, key, data); err != nil { + t.Fatal(err) + } + if _, _, err := b.readGroup(ctx, key); !errors.Is(err, ErrConflict) { + t.Fatal("unreserved stored state accepted", err) + } + state.Slots = minGroupSlots + 1 + if err := b.saveGroup(ctx, key, state, 0); !errors.Is(err, ErrCapacity) { + t.Fatal("configured ledger bound ignored at write", err) + } +} + +func TestRecoveryInvalidInputsHistoryAndReadFailuresAreExplicit(t *testing.T) { + b := testBroker(t, nil) + ctx := context.Background() + publish(t, b, "recover-work", api.LaneWork) + c := claim(t, b, "atlas", "workers") + key := groupKey("shared", "workers") + state, revision, err := b.readGroup(ctx, key) + if err != nil { + t.Fatal(err) + } + tests := []struct { + name string + in api.LaneRecover + want error + }{ + {"invalid request", api.LaneRecover{ID: "bad.*", Revision: revision}, ErrInvalid}, + {"missing revision", api.LaneRecover{ID: "missing-revision"}, ErrInvalid}, + {"stale revision", api.LaneRecover{ID: "stale-revision", Revision: revision + 1}, ErrConflict}, + {"invalid receipt", api.LaneRecover{ID: "bad-receipt", Revision: revision, Expired: []api.LaneReceipt{{Sequence: c.Sequence, Receipt: "invalid"}}}, ErrInvalid}, + {"zero missing sequence", api.LaneRecover{ID: "zero", Revision: revision, Missing: []uint64{0}}, ErrInvalid}, + {"no accepted loss", api.LaneRecover{ID: "empty", Revision: revision, CursorThrough: state.Cursor}, ErrInvalid}, + {"retained cursor", api.LaneRecover{ID: "cursor", Revision: revision, CursorThrough: state.Cursor + 1}, ErrCursor}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.in.CursorThrough = max(tt.in.CursorThrough, state.Cursor) + if _, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", tt.in); !errors.Is(err, tt.want) { + t.Fatal("recovery input lost typed rejection", err) + } + }) + } + in := api.LaneRecover{ID: "expired-history", Revision: revision, CursorThrough: state.Cursor} + request, err := json.Marshal(in) + if err != nil { + t.Fatal(err) + } + state.Recoveries = map[string]recoveryRecord{in.ID: { + RequestHash: digest(request), + Result: api.LaneRecovered{ID: in.ID, Project: "atlas", Cursor: state.Cursor, + RecoveredAt: time.Now().UTC().Add(-b.cfg.AckRetention)}, + }} + if err := b.saveGroup(ctx, key, state, revision); err != nil { + t.Fatal(err) + } + if _, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in); !errors.Is(err, ErrStale) { + t.Fatal("expired idempotent recovery response reported success", err) + } + actual := b.state + b.state = &faultResponseKV{KeyValue: actual, getErr: errFixtureBroker} + if _, err := b.InspectGroup(ctx, "atlas", "shared", "workers"); !errors.Is(err, ErrUnavailable) { + t.Fatal("inspection hid durable-state failure", err) + } + if _, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in); !errors.Is(err, ErrUnavailable) { + t.Fatal("recovery hid durable-state failure", err) + } + b.state = actual + corrupt := strings.Repeat("a", receiptBytes*2) + if err := b.Ack(ctx, "beacon", "shared", "workers", api.LaneReceipt{Sequence: c.Sequence, Receipt: corrupt}); !errors.Is(err, ErrStale) { + t.Fatal("foreign receipt became an acknowledgement", err) + } +} diff --git a/src/crux/lanes/coverage_publish_test.go b/src/crux/lanes/coverage_publish_test.go new file mode 100644 index 0000000000..44ec4d4b56 --- /dev/null +++ b/src/crux/lanes/coverage_publish_test.go @@ -0,0 +1,106 @@ +package lanes + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/api" +) + +func TestPublicationReservationFailuresNeverReturnSuccess(t *testing.T) { + b := testBroker(t, nil) + ctx := context.Background() + invalid := b.Config() + invalid.Namespace = "bad.*" + if opened, err := Open(ctx, invalid); !errors.Is(err, ErrInvalid) || opened != nil { + t.Fatal("invalid configuration reached broker initialization", err) + } + if _, err := b.Replay(ctx, "outsider", "shared", 0, 1); !errors.Is(err, ErrForbidden) { + t.Fatal("unregistered project replayed lane data", err) + } + in := api.LanePublish{ID: "reserved", Kind: api.LaneEvent, Schema: "v1", Payload: []byte("value")} + body, err := json.Marshal(in) + if err != nil { + t.Fatal(err) + } + key := "shared.atlas." + in.ID + res := publishReservation{Hash: digest(body), Created: b.now().UTC(), Generation: strings.Repeat("a", receiptBytes*2)} + if err := b.verifyReservation(ctx, key+".missing", res); !errors.Is(err, ErrStale) { + t.Fatal("missing reservation accepted", err) + } + put := func(value publishReservation) { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + if _, err := b.dedup.Put(ctx, key, data); err != nil { + t.Fatal(err) + } + } + put(res) + actual := b.dedup + b.dedup = &faultResponseKV{KeyValue: actual, getErr: errFixtureBroker} + if out, err := b.Publish(ctx, "atlas", "shared", in); !errors.Is(err, ErrUnavailable) || out.Sequence != 0 { + t.Fatal("unreadable reservation produced a publication", out, err) + } + if err := b.verifyReservation(ctx, key, res); !errors.Is(err, ErrUnavailable) { + t.Fatal("reservation verification hid broker failure", err) + } + b.dedup = actual + if _, err := b.dedup.Put(ctx, key, []byte("invalid-json")); err != nil { + t.Fatal(err) + } + if err := b.verifyReservation(ctx, key, res); !errors.Is(err, ErrUnavailable) { + t.Fatal("malformed active reservation accepted", err) + } + invalidGeneration := res + invalidGeneration.Generation = "invalid" + put(invalidGeneration) + if out, err := b.Publish(ctx, "atlas", "shared", in); !errors.Is(err, ErrConflict) || out.Sequence != 0 { + t.Fatal("invalid reservation generation accepted", out, err) + } + expired := res + expired.Created = expired.Created.Add(-b.cfg.Retention) + put(expired) + if out, err := b.Publish(ctx, "atlas", "shared", in); !errors.Is(err, ErrCursor) || out.Sequence != 0 { + t.Fatal("expired unpublished reservation accepted", out, err) + } + if err := b.verifyReservation(ctx, key, expired); !errors.Is(err, ErrStale) { + t.Fatal("expired active generation accepted", err) + } + put(res) + subject := b.subject("shared") + digest([]byte(key)) + if _, err := b.js.Publish(ctx, subject, []byte("invalid-json")); err != nil { + t.Fatal(err) + } + if out, err := b.Publish(ctx, "atlas", "shared", in); !errors.Is(err, ErrUnavailable) || out.Sequence != 0 { + t.Fatal("malformed persisted envelope accepted", out, err) + } + envelope := api.LaneEnvelope{LanePublish: in, Project: "beacon", Lane: "shared", CreatedAt: res.Created} + data, err := json.Marshal(envelope) + if err != nil { + t.Fatal(err) + } + if _, err := b.js.Publish(ctx, subject, data); err != nil { + t.Fatal(err) + } + if out, err := b.Publish(ctx, "atlas", "shared", in); !errors.Is(err, ErrConflict) || out.Sequence != 0 { + t.Fatal("foreign persisted envelope reported as duplicate success", out, err) + } + envelope.Project = "atlas" + data, err = json.Marshal(envelope) + if err != nil { + t.Fatal(err) + } + if _, err := b.js.Publish(ctx, subject, data); err != nil { + t.Fatal(err) + } + put(expired) + if out, err := b.Publish(ctx, "atlas", "shared", in); !errors.Is(err, ErrStale) || out.Sequence != 0 { + t.Fatal("retained envelope bypassed expired active reservation", out, err) + } +} diff --git a/src/crux/lanes/coverage_validation_test.go b/src/crux/lanes/coverage_validation_test.go new file mode 100644 index 0000000000..b507306464 --- /dev/null +++ b/src/crux/lanes/coverage_validation_test.go @@ -0,0 +1,103 @@ +package lanes + +import ( + "errors" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" +) + +func TestConfigurationRejectsInvalidMembershipAndCredentialBindings(t *testing.T) { + tests := []struct { + name string + change func(*Config) + }{ + {"credential binding", func(c *Config) { c.CredentialsEnv = "not-an-env" }}, + {"duplicate principal", func(c *Config) { c.Principals = append(c.Principals, c.Principals[0]) }}, + {"invalid group", func(c *Config) { c.Lanes[0].Groups["bad.*"] = []string{"atlas"} }}, + {"invalid recoverer group", func(c *Config) { c.Lanes[0].Recoverers["bad.*"] = []string{"atlas"} }}, + {"recoverer not a member", func(c *Config) { c.Lanes[0].Recoverers["workers"] = []string{"outsider"} }}, + {"publisher limit", func(c *Config) { + c.Lanes[0].Publishers = make([]string, DefaultMaxMembers+1) + for i := range c.Lanes[0].Publishers { + c.Lanes[0].Publishers[i] = "atlas" + } + }}, + {"unknown subscriber", func(c *Config) { c.Lanes[0].Subscribers = []string{"outsider"} }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := testConfig("nats://127.0.0.1:4222") + tt.change(&c) + if err := c.Validate(); !errors.Is(err, ErrInvalid) { + t.Fatal("invalid administrator configuration accepted", err) + } + }) + } +} + +func TestGroupBudgetRejectsInvalidHistoryReservationsAndMissingLedger(t *testing.T) { + state, _, err := maximumGroupShape() + if err != nil { + t.Fatal(err) + } + state.Slots = minGroupSlots + for _, slots := range []int{-minGroupSlots, MaxLedgerEntries + minGroupSlots} { + if _, err := groupReservation(slots); !errors.Is(err, ErrCapacity) { + t.Fatal("invalid reservation count accepted", slots, err) + } + } + for id, record := range state.Recoveries { + original := record + record.Result.Project = "" + state.Recoveries[id] = record + if err := validateGroup(state); !errors.Is(err, ErrConflict) { + t.Fatal("invalid recovery identity accepted", err) + } + record = original + record.Result.Expirations = append([]api.LaneExpiration(nil), original.Result.Expirations...) + record.Result.Expirations[0].Attempt = 0 + state.Recoveries[id] = record + if err := validateGroup(state); !errors.Is(err, ErrConflict) { + t.Fatal("invalid expiration history accepted", err) + } + state.Recoveries[id] = original + break + } + state.Leases = nil + if err := validateGroup(state); !errors.Is(err, ErrConflict) { + t.Fatal("missing durable ledger accepted", err) + } +} + +func TestPruningPreservesSlotsAndLiveHistoryWhileExpiringTerminalRecords(t *testing.T) { + b := &Broker{cfg: Config{AckRetention: MinLease}} + now := time.Now().UTC() + old := now.Add(-b.cfg.AckRetention) + token := strings.Repeat("a", receiptBytes*2) + const slots = 3 + state := groupState{Slots: slots, Leases: map[uint64]lease{ + 1: {Receipt: token, Project: "atlas", Expires: now, Attempt: 1, Done: true, AckedAt: old}, + 2: {Receipt: token, Project: "atlas", Expires: now, Attempt: 1, Expired: true, RetiredAt: old}, + 3: {Receipt: token, Project: "atlas", Expires: now.Add(MinLease), Attempt: 1}, + }, Recoveries: map[string]recoveryRecord{ + "old": {RequestHash: digest([]byte("old")), Result: api.LaneRecovered{ID: "old", Project: "atlas", RecoveredAt: old}}, + "live": {RequestHash: digest([]byte("live")), Result: api.LaneRecovered{ID: "live", Project: "atlas", RecoveredAt: now}}, + }} + if err := validateGroup(state); err != nil { + t.Fatal("terminal-history fixture is not otherwise valid", err) + } + b.pruneGroup(&state, now) + if len(state.Leases) != 1 || state.Leases[3].Project != "atlas" || state.Slots != slots { + t.Fatal("pruning lost a live lease or shrank reusable slots", state) + } + if len(state.Recoveries) != 1 || state.Recoveries["live"].Result.RecoveredAt != now { + t.Fatal("pruning lost live recovery response history", state.Recoveries) + } + b.cfg.RetryDelay, b.cfg.MaxRetryDelay = MinRetryDelay, MinRetryDelay+MinRetryDelay/2 + if delay := b.retryDelay(^uint64(0)); delay != b.cfg.MaxRetryDelay { + t.Fatal("non-power-of-two retry ceiling exceeded", delay) + } +} diff --git a/src/crux/lanes/failure_test.go b/src/crux/lanes/failure_test.go new file mode 100644 index 0000000000..26712ba37b --- /dev/null +++ b/src/crux/lanes/failure_test.go @@ -0,0 +1,196 @@ +package lanes + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +func TestClosedBrokerAndCanceledOpenFail(t *testing.T) { + b := testBroker(t, nil) + b.Close() + ctx := context.Background() + in := api.LanePublish{ID: "x", Kind: api.LaneEvent, Schema: "v1", Payload: []byte("value")} + if _, err := b.Publish(ctx, "atlas", "shared", in); err == nil { + t.Fatal("closed broker publish") + } + if _, err := b.Replay(ctx, "atlas", "shared", 0, 1); err == nil { + t.Fatal("closed broker replay") + } + if _, err := b.Claim(ctx, "atlas", "shared", "workers"); err == nil { + t.Fatal("closed broker claim") + } + if err := b.Ack(ctx, "atlas", "shared", "workers", api.LaneReceipt{Sequence: 1, Receipt: strings.Repeat("a", receiptBytes*2)}); err == nil { + t.Fatal("closed broker ack") + } + canceled, cancel := context.WithCancel(ctx) + cancel() + if _, err := Open(canceled, b.Config()); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled open: %v", err) + } + if backend(nil) != nil || !errors.Is(backend(context.DeadlineExceeded), context.DeadlineExceeded) { + t.Fatal("backend context errors lost") + } +} + +func TestMalformedDurableStateFailsInsteadOfDefaulting(t *testing.T) { + b := testBroker(t, nil) + ctx := context.Background() + key := groupKey("shared", "workers") + for _, data := range []string{"invalid-json", `{}`, `{"cursor":0,"leases":{}}`} { + if _, err := b.state.Put(ctx, key, []byte(data)); err != nil { + t.Fatal(err) + } + if _, err := b.Claim(ctx, "atlas", "shared", "workers"); err == nil { + t.Fatal("malformed group state defaulted") + } + } + if _, err := b.dedup.Put(ctx, "shared.atlas.bad", []byte("invalid-json")); err != nil { + t.Fatal(err) + } + if _, err := b.Publish(ctx, "atlas", "shared", api.LanePublish{ID: "bad", Kind: api.LaneEvent, Schema: "v1", Payload: []byte("x")}); err == nil { + t.Fatal("malformed dedup state defaulted") + } + if _, err := b.js.Publish(ctx, b.subject("shared")+"bad", []byte("invalid-json")); err != nil { + t.Fatal(err) + } + if _, err := b.Replay(ctx, "atlas", "shared", 0, 1); err == nil { + t.Fatal("malformed envelope ignored") + } +} + +func TestReceiptTerminalAndInvalidStates(t *testing.T) { + b := testBroker(t, nil) + ctx := context.Background() + publish(t, b, "job", api.LaneWork) + c := claim(t, b, "atlas", "workers") + if _, err := b.receipt(ctx, "atlas", "shared", "workers", "invalid", receipt(c)); !errors.Is(err, ErrInvalid) { + t.Fatal("invalid receipt operation", err) + } + if err := b.Nack(ctx, "atlas", "shared", "workers", receipt(c)); err != nil { + t.Fatal(err) + } + if _, err := b.Renew(ctx, "atlas", "shared", "workers", receipt(c)); !errors.Is(err, ErrStale) { + t.Fatal("released lease renewed", err) + } + if err := b.Ack(ctx, "atlas", "shared", "workers", receipt(c)); !errors.Is(err, ErrStale) { + t.Fatal("released lease acked", err) + } + makeRetriesDue(t, b) + c = claim(t, b, "atlas", "workers") + if err := b.Ack(ctx, "atlas", "shared", "workers", receipt(c)); err != nil { + t.Fatal(err) + } + if _, err := b.Renew(ctx, "atlas", "shared", "workers", receipt(c)); !errors.Is(err, ErrStale) { + t.Fatal("completed lease renewed", err) + } + if err := b.Nack(ctx, "atlas", "shared", "workers", receipt(c)); !errors.Is(err, ErrStale) { + t.Fatal("completed lease nacked", err) + } + for _, r := range []api.LaneReceipt{{}, {Sequence: 1, Receipt: "invalid"}, {Sequence: c.Sequence + 1, Receipt: c.Receipt}} { + if err := b.Ack(ctx, "atlas", "shared", "workers", r); err == nil { + t.Fatal("invalid/unknown receipt accepted") + } + } + data, delivery, err := maximumGroupShape() + if err != nil { + t.Fatal(err) + } + base, perDelivery, err := completionBudget() + if err != nil { + t.Fatal(err) + } + data.Slots = (maxStateBytes-base)/perDelivery + minGroupSlots + for seq := uint64(1); len(data.Leases) < data.Slots; seq++ { + data.Leases[seq] = delivery + } + if err := validateGroup(data); err != nil { + t.Fatal("capacity fixture must satisfy the durable state contract", err) + } + if reserved, err := groupReservation(data.Slots); err != nil || reserved <= maxStateBytes { + t.Fatal("capacity fixture must exceed the byte reservation cap", reserved, err) + } + if err := b.saveGroup(ctx, "oversized", data, 0); !errors.Is(err, ErrCapacity) { + t.Fatal("oversized durable ledger accepted", err) + } +} + +func TestMalformedDurableLedgerFailsConflictBeforeCapacity(t *testing.T) { + data := groupState{Leases: map[uint64]lease{1: {Receipt: strings.Repeat("a", maxStateBytes)}}} + if _, err := encodeGroup(data, 0); !errors.Is(err, ErrConflict) { + t.Fatal("malformed oversized state must fail its typed contract first", err) + } +} + +func TestPublishCommittedBeforeResponseRecovery(t *testing.T) { + b := testBroker(t, func(c *Config) { c.Dedup = MinDedup }) + ctx := context.Background() + in := api.LanePublish{ID: "lost-response", Kind: api.LaneWork, Schema: "v1", Payload: []byte("work")} + body, err := json.Marshal(in) + if err != nil { + t.Fatal(err) + } + res := publishReservation{Hash: digest(body), Created: time.Now().UTC(), Generation: strings.Repeat("a", receiptBytes*2)} + data, err := json.Marshal(res) + if err != nil { + t.Fatal(err) + } + key := "shared.atlas.lost-response" + if _, err := b.dedup.Create(ctx, key, data); err != nil { + t.Fatal(err) + } + data, err = json.Marshal(api.LaneEnvelope{LanePublish: in, Lane: "shared", Project: "atlas", CreatedAt: res.Created}) + if err != nil { + t.Fatal(err) + } + ack, err := b.js.PublishMsg(ctx, &nats.Msg{Subject: b.subject("shared") + digest([]byte(key)), Data: data, + Header: nats.Header{reservationHeader: []string{res.Generation}}}, + jetstream.WithMsgID(key+"."+res.Generation)) + if err != nil { + t.Fatal(err) + } + cfg := b.Config() + b.Close() + timer := time.NewTimer(MinDedup) + <-timer.C + b, err = Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer b.Close() + out, err := b.Publish(ctx, "atlas", "shared", in) + if err != nil || !out.Duplicate || out.Sequence != ack.Sequence { + t.Fatalf("committed response recovery: %+v %v", out, err) + } +} + +func TestConfigurationSnapshotsCannotMutateAuthorization(t *testing.T) { + b := testBroker(t, nil) + cfg := b.Config() + cfg.Lanes[0].Publishers[0] = "outsider" + cfg.Lanes[0].Groups["workers"][0] = "outsider" + registry := b.Registry("atlas") + registry[0].Publishers[0] = "outsider" + registry[0].Groups["workers"][0] = "outsider" + if _, err := b.authorize("atlas", "shared", "publish", ""); err != nil { + t.Fatal("configuration mutated", err) + } + if _, err := b.authorize("outsider", "shared", "work", "workers"); !errors.Is(err, ErrForbidden) { + t.Fatal("registry granted a capability", err) + } + cfg = b.Config() + cfg.MaxKVBytes++ + if modified, err := Open(context.Background(), cfg); !errors.Is(err, ErrConflict) { + if modified != nil { + modified.Close() + } + t.Fatal("incompatible KV bounds accepted", err) + } +} diff --git a/src/crux/lanes/load.go b/src/crux/lanes/load.go new file mode 100644 index 0000000000..2de7c72d37 --- /dev/null +++ b/src/crux/lanes/load.go @@ -0,0 +1,43 @@ +package lanes + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" +) + +const maxConfigBytes = 1 << 20 + +func LoadConfig(path string) (Config, error) { + f, err := os.Open(path) + if err != nil { + return Config{}, fmt.Errorf("%w: cannot open lane configuration", ErrInvalid) + } + defer f.Close() + stat, err := f.Stat() + if err != nil || stat.Size() > maxConfigBytes { + return Config{}, fmt.Errorf("%w: lane configuration size", ErrInvalid) + } + return loadConfig(f) +} + +func loadConfig(r io.Reader) (Config, error) { + data, err := io.ReadAll(io.LimitReader(r, maxConfigBytes+1)) + if err != nil || len(data) > maxConfigBytes { + return Config{}, fmt.Errorf("%w: lane configuration size", ErrInvalid) + } + d := json.NewDecoder(bytes.NewReader(data)) + d.DisallowUnknownFields() + var cfg Config + if err := d.Decode(&cfg); err != nil { + return Config{}, fmt.Errorf("%w: cannot decode lane configuration", ErrInvalid) + } + var extra any + if err := d.Decode(&extra); err != io.EOF { + return Config{}, fmt.Errorf("%w: trailing lane configuration", ErrInvalid) + } + cfg.Defaults() + return cfg, cfg.Validate() +} diff --git a/src/crux/lanes/process_retry_test.go b/src/crux/lanes/process_retry_test.go new file mode 100644 index 0000000000..5370182b72 --- /dev/null +++ b/src/crux/lanes/process_retry_test.go @@ -0,0 +1,43 @@ +package lanes + +import ( + "context" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" +) + +func TestRetryFairnessAcrossIndependentCruxWorkerProcesses(t *testing.T) { + b := testBroker(t, func(c *Config) { c.RetryDelay, c.MaxRetryDelay = MinRetryDelay, MinRetryDelay }) + poison := publish(t, b, "poison", api.LaneWork) + first := startPublisherProcess(t, b.Config(), "", false, processWorkActionEnv+"=nack", processProjectEnv+"=atlas") + rejected := processOutcome(t, first) + if rejected.Error != "" || rejected.ClaimSequence != poison.Sequence || rejected.Attempt != 1 || rejected.AvailableAt.IsZero() { + t.Fatalf("first worker: %+v", rejected) + } + fresh := publish(t, b, "valid", api.LaneWork) + timer := time.NewTimer(time.Until(rejected.AvailableAt.Add(DefaultPollForRetryTest))) + <-timer.C + second := startPublisherProcess(t, b.Config(), "", false, processWorkActionEnv+"=nack", processProjectEnv+"=atlas") + retried := processOutcome(t, second) + if first.pid == second.pid || retried.Error != "" || retried.ClaimSequence != poison.Sequence || retried.Attempt != 2 { + t.Fatalf("second worker: %+v", retried) + } + timer = time.NewTimer(time.Until(retried.AvailableAt.Add(DefaultPollForRetryTest))) + <-timer.C + third := startPublisherProcess(t, b.Config(), "", false, processWorkActionEnv+"=ack", processProjectEnv+"=beacon") + completed := processOutcome(t, third) + if third.pid == second.pid || completed.Error != "" || completed.ClaimSequence != fresh.Sequence || completed.Attempt != 1 { + t.Fatalf("fresh work starved across worker restarts: %+v", completed) + } + status, err := b.InspectGroup(context.Background(), "atlas", "shared", "workers") + if err != nil || len(status.Retries) != 1 || status.Retries[0].Sequence != poison.Sequence { + t.Fatal("rejected work disappeared", err) + } + fourth := startPublisherProcess(t, b.Config(), "", false, processWorkActionEnv+"=nack", processProjectEnv+"=atlas") + remaining := processOutcome(t, fourth) + if remaining.Error != "" || remaining.ClaimSequence != poison.Sequence || remaining.Attempt != 3 { + t.Fatalf("rejected work was silently ACKed/dropped: %+v", remaining) + } +} diff --git a/src/crux/lanes/publish_generation_test.go b/src/crux/lanes/publish_generation_test.go new file mode 100644 index 0000000000..7fc1d18f1f --- /dev/null +++ b/src/crux/lanes/publish_generation_test.go @@ -0,0 +1,337 @@ +package lanes + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +const ( + processConfigEnv = "CRUX_LANES_TEST_PROCESS_CONFIG" + processPayloadEnv = "CRUX_LANES_TEST_PROCESS_PAYLOAD" + processDelayEnv = "CRUX_LANES_TEST_PROCESS_DELAY" + processWorkActionEnv = "CRUX_LANES_TEST_PROCESS_WORK_ACTION" + processProjectEnv = "CRUX_LANES_TEST_PROCESS_PROJECT" + processReady = "LANE_READY" + processResult = "LANE_RESULT " +) + +type delayedProcessPublisher struct{ jetstream.JetStream } + +func (d *delayedProcessPublisher) PublishMsg(ctx context.Context, msg *nats.Msg, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + fmt.Fprintln(os.Stdout, processReady) + if _, err := bufio.NewReader(os.Stdin).ReadByte(); err != nil { + return nil, err + } + return d.JetStream.PublishMsg(ctx, msg, opts...) +} + +type publisherProcessResult struct { + Published api.LanePublished `json:"published"` + Error string `json:"error,omitempty"` + ClaimSequence uint64 `json:"claim_sequence,omitempty"` + Attempt uint64 `json:"attempt,omitempty"` + AvailableAt time.Time `json:"available_at,omitempty"` +} + +// The same CI test executable acts as two isolated Crux publisher processes; +// pipes pause A after reservation validation but before broker publication. +func TestLanePublisherProcessHelper(t *testing.T) { + raw := os.Getenv(processConfigEnv) + if raw == "" { + return + } + var cfg Config + if err := json.Unmarshal([]byte(raw), &cfg); err != nil { + t.Fatal(err) + } + b, err := Open(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + defer b.Close() + if os.Getenv(processDelayEnv) == "yes" { + b.js = &delayedProcessPublisher{JetStream: b.js} + } + result := publisherProcessResult{} + if action := os.Getenv(processWorkActionEnv); action != "" { + project := os.Getenv(processProjectEnv) + var claim api.LaneClaim + claim, err = b.Claim(context.Background(), project, "shared", "workers") + if err == nil { + result.ClaimSequence, result.Attempt = claim.Sequence, claim.Attempt + switch action { + case "nack": + err = b.Nack(context.Background(), project, "shared", "workers", receipt(claim)) + if err == nil { + var status api.LaneGroupState + status, err = b.InspectGroup(context.Background(), project, "shared", "workers") + for _, retry := range status.Retries { + if retry.Sequence == claim.Sequence { + result.AvailableAt = retry.AvailableAt + } + } + } + case "ack": + err = b.Ack(context.Background(), project, "shared", "workers", receipt(claim)) + default: + err = ErrInvalid + } + } + } else { + result.Published, err = b.Publish(context.Background(), "atlas", "shared", api.LanePublish{ID: "shared-key", Kind: api.LaneEvent, Schema: "v1", Payload: []byte(os.Getenv(processPayloadEnv))}) + } + switch { + case errors.Is(err, ErrConflict): + result.Error = "conflict" + case errors.Is(err, ErrStale): + result.Error = "stale" + case err != nil: + result.Error = err.Error() + } + data, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + fmt.Fprintln(os.Stdout, processResult+string(data)) +} + +type publisherProcess struct { + pid int + input io.WriteCloser + ready chan struct{} + results chan publisherProcessResult + finished chan struct{} + waitErr error +} + +func startPublisherProcess(t *testing.T, cfg Config, payload string, delay bool, extraEnv ...string) *publisherProcess { + t.Helper() + data, err := json.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), testWait) + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestLanePublisherProcessHelper$", "-test.timeout="+testWait.String()) + cmd.Env = append(os.Environ(), processConfigEnv+"="+string(data), processPayloadEnv+"="+payload, processDelayEnv+"=no", processWorkActionEnv+"=") + cmd.Env = append(cmd.Env, extraEnv...) + if delay { + cmd.Env = append(cmd.Env, processDelayEnv+"=yes") + } + out, err := cmd.StdoutPipe() + if err != nil { + cancel() + t.Fatal(err) + } + cmd.Stderr = cmd.Stdout + input, err := cmd.StdinPipe() + if err != nil { + cancel() + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + cancel() + input.Close() + t.Fatal(err) + } + p := &publisherProcess{pid: cmd.Process.Pid, input: input, ready: make(chan struct{}, 1), results: make(chan publisherProcessResult, 1), finished: make(chan struct{})} + go func() { + scanner := bufio.NewScanner(out) + for scanner.Scan() { + line := scanner.Text() + if line == processReady { + p.ready <- struct{}{} + } + if strings.HasPrefix(line, processResult) { + var result publisherProcessResult + if err := json.Unmarshal([]byte(strings.TrimPrefix(line, processResult)), &result); err != nil { + p.waitErr = err + } else { + p.results <- result + } + } + } + scanErr := scanner.Err() + p.waitErr = errors.Join(p.waitErr, scanErr, cmd.Wait()) + close(p.finished) + }() + t.Cleanup(func() { cancel(); input.Close(); <-p.finished }) + return p +} + +func processOutcome(t *testing.T, p *publisherProcess) publisherProcessResult { + t.Helper() + result := processResponse(t, p) + waitPublisherProcess(t, p) + return result +} + +func waitPublisherProcess(t *testing.T, p *publisherProcess) { + t.Helper() + select { + case <-p.finished: + case <-time.After(testWait): + t.Fatal("publisher process did not exit after its result") + } + if p.waitErr != nil { + t.Fatalf("publisher process failed after result: %v", p.waitErr) + } +} + +func processResponse(t *testing.T, p *publisherProcess) publisherProcessResult { + t.Helper() + select { + case result := <-p.results: + return result + case <-p.finished: + if p.waitErr != nil { + t.Fatalf("publisher process failed: %v", p.waitErr) + } + select { + case result := <-p.results: + return result + default: + t.Fatalf("publisher process failed: %v", p.waitErr) + } + case <-time.After(testWait): + t.Fatal("publisher process did not finish") + } + return publisherProcessResult{} +} + +func TestTwoCruxProcessesFenceDelayedExpiredPublicationGeneration(t *testing.T) { + b := testBroker(t, shortRetention) + a := startPublisherProcess(t, b.Config(), "payload-A", true) + select { + case <-a.ready: + case <-a.finished: + t.Fatalf("A failed before pause: %v", a.waitErr) + case <-time.After(testWait): + t.Fatal("A never reserved") + } + key := "shared.atlas.shared-key" + entry, err := b.dedup.Get(context.Background(), key) + if err != nil { + t.Fatal(err) + } + var first publishReservation + if err := json.Unmarshal(entry.Value(), &first); err != nil { + t.Fatal(err) + } + awaitCondition(t, func() (bool, error) { + _, err := b.dedup.Get(context.Background(), key) + if errors.Is(err, jetstream.ErrKeyNotFound) || errors.Is(err, jetstream.ErrKeyDeleted) { + return true, nil + } + return false, err + }) + second := startPublisherProcess(t, b.Config(), "payload-B", false) + if a.pid == second.pid { + t.Fatal("publishers were not separate processes") + } + // Read the durable response before waiting for process exit: the race + // detector's exit delay alone exceeds this stream's one-second retention. + winner := processResponse(t, second) + if winner.Error != "" || winner.Published.Sequence == 0 { + t.Fatalf("B did not publish: %+v", winner) + } + if _, err := a.input.Write([]byte("r")); err != nil { + t.Fatal(err) + } + loser := processResponse(t, a) + if loser.Error != "conflict" && loser.Error != "stale" { + t.Fatalf("delayed A falsely succeeded or only timed out: %+v", loser) + } + if loser.Published.Sequence != 0 { + t.Fatal("A received success-shaped result for B") + } + record, err := b.streams["shared"].GetMsg(context.Background(), winner.Published.Sequence) + if err != nil { + t.Fatal(err) + } + var envelope api.LaneEnvelope + if err := json.Unmarshal(record.Data, &envelope); err != nil { + t.Fatal(err) + } + if string(envelope.Payload) != "payload-B" || record.Header.Get(reservationHeader) == first.Generation { + t.Fatal("A replaced B or shared expired generation") + } + waitPublisherProcess(t, second) + waitPublisherProcess(t, a) +} + +type wrongAckPublisher struct { + jetstream.JetStream + sequence uint64 +} + +func (w wrongAckPublisher) PublishMsg(context.Context, *nats.Msg, ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + return &jetstream.PubAck{Sequence: w.sequence, Duplicate: true}, nil +} + +func TestDuplicateAckMustResolveToMatchingRetainedEnvelope(t *testing.T) { + b := testBroker(t, nil) + other := publish(t, b, "other", api.LaneEvent) + b.js = wrongAckPublisher{JetStream: b.js, sequence: other.Sequence} + in := api.LanePublish{ID: "not-other", Kind: api.LaneEvent, Schema: "v1", Payload: []byte("unpublished")} + if out, err := b.Publish(context.Background(), "atlas", "shared", in); !errors.Is(err, ErrConflict) || out.Sequence != 0 { + t.Fatalf("wrong duplicate ACK accepted: %+v %v", out, err) + } + b.js = wrongAckPublisher{JetStream: b.js, sequence: other.Sequence + 1} + if out, err := b.Publish(context.Background(), "atlas", "shared", in); !errors.Is(err, ErrCursor) || out.Sequence != 0 { + t.Fatalf("missing duplicate metadata accepted: %+v %v", out, err) + } +} + +type replaceAfterAckPublisher struct { + jetstream.JetStream + kv jetstream.KeyValue +} + +func (r replaceAfterAckPublisher) PublishMsg(ctx context.Context, msg *nats.Msg, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + ack, err := r.JetStream.PublishMsg(ctx, msg, opts...) + if err != nil { + return nil, err + } + entry, err := r.kv.Get(ctx, "shared.atlas.normal") + if err != nil { + return nil, err + } + var res publishReservation + if err := json.Unmarshal(entry.Value(), &res); err != nil { + return nil, err + } + res.Generation, err = newReceipt() + if err != nil { + return nil, err + } + data, err := json.Marshal(res) + if err != nil { + return nil, err + } + if _, err := r.kv.Update(ctx, entry.Key(), data, entry.Revision()); err != nil { + return nil, err + } + return ack, nil +} + +func TestNormalAckAlsoFencesReplacedReservation(t *testing.T) { + b := testBroker(t, nil) + b.js = replaceAfterAckPublisher{JetStream: b.js, kv: b.dedup} + out, err := b.Publish(context.Background(), "atlas", "shared", api.LanePublish{ID: "normal", Kind: api.LaneEvent, Schema: "v1", Payload: []byte("persisted-but-fenced")}) + if !errors.Is(err, ErrConflict) || out.Sequence != 0 { + t.Fatalf("replaced reservation got successful normal ACK: %+v %v", out, err) + } +} diff --git a/src/crux/lanes/publish_response_test.go b/src/crux/lanes/publish_response_test.go new file mode 100644 index 0000000000..cc20fe2a53 --- /dev/null +++ b/src/crux/lanes/publish_response_test.go @@ -0,0 +1,26 @@ +package lanes + +import ( + "testing" + + "github.com/hivecommons/hive/crux/api" +) + +func TestPublisherResponseDoesNotWaitForProcessExit(t *testing.T) { + const publishedSequence = 1 + p := &publisherProcess{ + results: make(chan publisherProcessResult, 1), + finished: make(chan struct{}), + } + p.results <- publisherProcessResult{Published: api.LanePublished{Sequence: publishedSequence}} + if result := processResponse(t, p); result.Published.Sequence != publishedSequence { + t.Fatal("publication response was lost before process exit", result) + } + select { + case <-p.finished: + t.Fatal("response required process exit") + default: + } + close(p.finished) + waitPublisherProcess(t, p) +} diff --git a/src/crux/lanes/recovery_test.go b/src/crux/lanes/recovery_test.go new file mode 100644 index 0000000000..5e0131b043 --- /dev/null +++ b/src/crux/lanes/recovery_test.go @@ -0,0 +1,130 @@ +package lanes + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/nats-io/nats.go" +) + +func TestExpiredRetentionFailsExplicitly(t *testing.T) { + b := testBroker(t, func(c *Config) { + c.Retention, c.Lease, c.AckRetention = time.Second, time.Second, time.Second + c.Dedup = MinDedup + }) + first := publish(t, b, "first", api.LaneEvent) + publish(t, b, "second", api.LaneEvent) + publish(t, b, "work", api.LaneWork) + claim(t, b, "atlas", "workers") + awaitCondition(t, func() (bool, error) { + info, err := b.streams["shared"].Info(context.Background()) + if err != nil { + return false, err + } + return info.State.Msgs == 0, nil + }) + if _, err := b.Replay(context.Background(), "beacon", "shared", first.Sequence, 1); !errors.Is(err, ErrCursor) { + t.Fatalf("expired cursor: %v", err) + } + if _, err := b.Claim(context.Background(), "beacon", "shared", "workers"); !errors.Is(err, ErrCursor) { + t.Fatalf("expired pending work: %v", err) + } + page, err := b.Replay(context.Background(), "beacon", "shared", 0, 1) + if err != nil || len(page.Events) != 0 { + t.Fatalf("explicit earliest-retained replay: %+v %v", page, err) + } +} + +func TestPublishReservationCrashRecoveryAfterDedupWindow(t *testing.T) { + b := testBroker(t, func(c *Config) { c.Dedup = MinDedup }) + in := api.LanePublish{ID: "reserved", Kind: api.LaneWork, Schema: "v1", Payload: []byte("work")} + body, err := json.Marshal(in) + if err != nil { + t.Fatal(err) + } + key := "shared.atlas.reserved" + data, err := json.Marshal(publishReservation{Hash: digest(body), Created: time.Now().Add(-time.Second), Generation: strings.Repeat("a", receiptBytes*2)}) + if err != nil { + t.Fatal(err) + } + if _, err := b.dedup.Create(context.Background(), key, data); err != nil { + t.Fatal(err) + } + cfg := b.cfg + b.Close() + reopened, err := Open(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + out, err := reopened.Publish(context.Background(), "atlas", "shared", in) + if err != nil { + t.Fatal(err) + } + again, err := reopened.Publish(context.Background(), "atlas", "shared", in) + if err != nil || !again.Duplicate || again.Sequence != out.Sequence { + t.Fatalf("pending publish recovery: %+v %v", again, err) + } +} + +func TestDisconnectFailsAndReconnectRestoresDurability(t *testing.T) { + s, opts := testServer(t) + reconnected := make(chan struct{}, 1) + b, err := Open(context.Background(), testConfig(s.ClientURL()), nats.ReconnectHandler(func(*nats.Conn) { + select { + case reconnected <- struct{}{}: + default: + } + })) + if err != nil { + t.Fatal(err) + } + defer b.Close() + publish(t, b, "before", api.LaneEvent) + s.Shutdown() + s.WaitForShutdown() + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100) + _, err = b.Replay(ctx, "atlas", "shared", 0, 1) + cancel() + if err == nil { + t.Fatal("disconnected broker returned success") + } + s = startServer(t, opts) + defer func() { s.Shutdown(); s.WaitForShutdown() }() + select { + case <-reconnected: + case <-time.After(testWait): + t.Fatal("broker did not reconnect") + } + out := publish(t, b, "after", api.LaneEvent) + page, err := b.Replay(context.Background(), "beacon", "shared", 0, MaxReplayBatch) + if err != nil || len(page.Events) != 2 || page.Cursor != out.Sequence { + t.Fatalf("reconnect: %+v %v", page, err) + } +} + +func TestRegistryChangeAndCapacityDoNotOverwriteData(t *testing.T) { + b := testBroker(t, func(c *Config) { c.MaxMessages = 1 }) + first := publish(t, b, "first", api.LaneEvent) + _, err := b.Publish(context.Background(), "atlas", "shared", api.LanePublish{ID: "second", Kind: api.LaneEvent, Schema: "v1", Payload: []byte("value")}) + if err == nil { + t.Fatal("full stream accepted and discarded an event") + } + page, err := b.Replay(context.Background(), "beacon", "shared", 0, 1) + if err != nil || page.Events[0].Sequence != first.Sequence { + t.Fatalf("full stream lost prior message: %+v %v", page, err) + } + cfg := b.cfg + cfg.Lanes[0].Publishers = []string{"atlas"} + if changed, err := Open(context.Background(), cfg); !errors.Is(err, ErrConflict) { + if changed != nil { + changed.Close() + } + t.Fatalf("registry overwritten: %v", err) + } +} diff --git a/src/crux/lanes/retry_isolation_test.go b/src/crux/lanes/retry_isolation_test.go new file mode 100644 index 0000000000..71baca7f1a --- /dev/null +++ b/src/crux/lanes/retry_isolation_test.go @@ -0,0 +1,123 @@ +package lanes + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" +) + +func TestRetryScheduleAndFairTurnSurviveServiceRestart(t *testing.T) { + b := testBroker(t, func(c *Config) { c.RetryDelay, c.MaxRetryDelay = MinRetryDelay, MinRetryDelay }) + ctx := context.Background() + poison := publish(t, b, "poison", api.LaneWork) + first := claim(t, b, "atlas", "workers") + if err := b.Nack(ctx, "atlas", "shared", "workers", receipt(first)); err != nil { + t.Fatal(err) + } + status, err := b.InspectGroup(ctx, "atlas", "shared", "workers") + if err != nil || len(status.Retries) != 1 { + t.Fatalf("scheduled retry: %+v %v", status, err) + } + cfg := b.Config() + b.Close() + peer, err := Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer peer.Close() + restored, err := peer.InspectGroup(ctx, "atlas", "shared", "workers") + if err != nil || len(restored.Retries) != 1 || !restored.Retries[0].AvailableAt.Equal(status.Retries[0].AvailableAt) { + t.Fatal("retry schedule lost on restart", err) + } + due := restored.Retries[0].AvailableAt.Add(DefaultPollForRetryTest) + peer.now = func() time.Time { return due } + fresh := publish(t, peer, "valid", api.LaneWork) + retry := claim(t, peer, "atlas", "workers") + if retry.Sequence != poison.Sequence { + t.Fatal("retry turn lost on restart") + } + if err := peer.Nack(ctx, "atlas", "shared", "workers", receipt(retry)); err != nil { + t.Fatal(err) + } + next, err := Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer next.Close() + due = due.Add(peer.cfg.MaxRetryDelay + DefaultPollForRetryTest) + next.now = func() time.Time { return due } + valid := claim(t, next, "beacon", "workers") + if valid.Sequence != fresh.Sequence { + t.Fatal("poison retry starved fresh work after delay elapsed") + } + if err := next.Ack(ctx, "beacon", "shared", "workers", receipt(valid)); err != nil { + t.Fatal(err) + } + remaining := claim(t, next, "atlas", "workers") + if remaining.Sequence != poison.Sequence || remaining.Attempt != 3 { + t.Fatal("poison work silently dropped or acknowledged") + } +} + +const DefaultPollForRetryTest = time.Second + +func TestRetryBackoffIsBoundedAndNackIsIdempotent(t *testing.T) { + b := testBroker(t, nil) + if b.retryDelay(1) != b.cfg.RetryDelay || b.retryDelay(^uint64(0)) != b.cfg.MaxRetryDelay { + t.Fatal("retry bounds") + } + publish(t, b, "job", api.LaneWork) + c := claim(t, b, "atlas", "workers") + ctx := context.Background() + if err := b.Nack(ctx, "atlas", "shared", "workers", receipt(c)); err != nil { + t.Fatal(err) + } + before, err := b.InspectGroup(ctx, "atlas", "shared", "workers") + if err != nil { + t.Fatal(err) + } + if _, err := b.Claim(ctx, "atlas", "shared", "workers"); !errors.Is(err, ErrEmpty) { + t.Fatal("NACK was immediately retried", err) + } + if err := b.Nack(ctx, "atlas", "shared", "workers", receipt(c)); err != nil { + t.Fatal(err) + } + after, err := b.InspectGroup(ctx, "atlas", "shared", "workers") + if err != nil || !after.Retries[0].AvailableAt.Equal(before.Retries[0].AvailableAt) { + t.Fatal("duplicate NACK moved retry time", err) + } +} + +func TestAmbiguousNamespaceLaneTuplesRemainIsolated(t *testing.T) { + s, _ := testServer(t) + a := testConfig(s.ClientURL()) + a.Namespace, a.Lanes[0].ID = "a", "b_lane_c" + b := testConfig(s.ClientURL()) + b.Namespace, b.Lanes[0].ID = "a_lane_b", "c" + first, err := Open(context.Background(), a) + if err != nil { + t.Fatal(err) + } + defer first.Close() + second, err := Open(context.Background(), b) + if err != nil { + t.Fatal(err) + } + defer second.Close() + if first.streamName(a.Lanes[0].ID) == second.streamName(b.Lanes[0].ID) { + t.Fatal("ambiguous tuple stream name") + } + for _, service := range []*Broker{first, second} { + lane := service.cfg.Lanes[0].ID + if _, err := service.Publish(context.Background(), "atlas", lane, api.LanePublish{ID: "same", Kind: api.LaneEvent, Schema: "v1", Payload: []byte(service.cfg.Namespace)}); err != nil { + t.Fatal(err) + } + page, err := service.Replay(context.Background(), "beacon", lane, 0, 1) + if err != nil || len(page.Events) != 1 || string(page.Events[0].Envelope.Payload) != service.cfg.Namespace { + t.Fatal("namespace leakage", err) + } + } +} diff --git a/src/crux/lanes/retry_rotation_test.go b/src/crux/lanes/retry_rotation_test.go new file mode 100644 index 0000000000..b4f4a85da7 --- /dev/null +++ b/src/crux/lanes/retry_rotation_test.go @@ -0,0 +1,73 @@ +package lanes + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" +) + +func TestRetryRotationCompletesPreviouslyClaimedWorkAcrossRestart(t *testing.T) { + b := testBroker(t, func(c *Config) { c.RetryDelay, c.MaxRetryDelay = MinRetryDelay, MinRetryDelay }) + ctx := context.Background() + poison := publish(t, b, "poison", api.LaneWork) + valid := publish(t, b, "previously-claimed", api.LaneWork) + low := claim(t, b, "atlas", "workers") + lost := claim(t, b, "beacon", "workers") + if low.Sequence != poison.Sequence || lost.Sequence != valid.Sequence { + t.Fatal("regression requires two previously claimed deliveries") + } + if err := b.Nack(ctx, "atlas", "shared", "workers", receipt(low)); err != nil { + t.Fatal(err) + } + // The high sequence's claim response was lost. Both deliveries are ready + // behind the durable cursor; neither can be selected as fresh work. + clock := lost.Expires.Add(DefaultPollForRetryTest) + cfg := b.Config() + b.Close() + restarted, err := Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer restarted.Close() + restarted.now = func() time.Time { return clock } + retry := claim(t, restarted, "atlas", "workers") + if retry.Sequence != poison.Sequence { + t.Fatal("first retry should select low sequence") + } + if err := restarted.Nack(ctx, "atlas", "shared", "workers", receipt(retry)); err != nil { + t.Fatal(err) + } + restarted.Close() + clock = clock.Add(cfg.MaxRetryDelay + DefaultPollForRetryTest) + next, err := Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer next.Close() + next.now = func() time.Time { return clock } + completed := claim(t, next, "beacon", "workers") + if completed.Sequence != valid.Sequence || completed.Attempt != 2 { + t.Fatal("always-ready poison starved previously claimed valid work") + } + if err := next.Ack(ctx, "beacon", "shared", "workers", receipt(completed)); err != nil { + t.Fatal(err) + } + if err := next.Ack(ctx, "beacon", "shared", "workers", receipt(lost)); !errors.Is(err, ErrStale) { + t.Fatal("lost original claim bypassed reassignment fencing", err) + } + clock = clock.Add(cfg.MaxRetryDelay + DefaultPollForRetryTest) + remaining := claim(t, next, "atlas", "workers") + if remaining.Sequence != poison.Sequence || remaining.Attempt != 3 { + t.Fatal("poison was silently acknowledged or discarded") + } + if err := next.Nack(ctx, "atlas", "shared", "workers", receipt(remaining)); err != nil { + t.Fatal(err) + } + status, err := next.InspectGroup(ctx, "atlas", "shared", "workers") + if err != nil || len(status.Retries) != 1 || status.Retries[0].Sequence != poison.Sequence { + t.Fatal("poison retry is not explicit", err) + } +} diff --git a/src/crux/lanes/testdata/config.json b/src/crux/lanes/testdata/config.json new file mode 100644 index 0000000000..2e0c473908 --- /dev/null +++ b/src/crux/lanes/testdata/config.json @@ -0,0 +1,17 @@ +{ + "url": "nats://127.0.0.1:4222", + "principals": [ + {"project": "atlas", "token_env": "ATLAS_TOKEN"}, + {"project": "beacon", "token_env": "BEACON_TOKEN"} + ], + "lanes": [ + { + "id": "shared", + "owner": "atlas", + "publishers": ["atlas", "beacon"], + "subscribers": ["atlas", "beacon"], + "groups": {"workers": ["atlas", "beacon"], "other": ["beacon"]}, + "recoverers": {"workers": ["atlas", "beacon"]} + } + ] +} diff --git a/src/crux/lanes/testdata/unknown-config.json b/src/crux/lanes/testdata/unknown-config.json new file mode 100644 index 0000000000..2623c7082a --- /dev/null +++ b/src/crux/lanes/testdata/unknown-config.json @@ -0,0 +1 @@ +{"unknown": true} diff --git a/src/crux/lanes/work.go b/src/crux/lanes/work.go new file mode 100644 index 0000000000..cd7bd2304f --- /dev/null +++ b/src/crux/lanes/work.go @@ -0,0 +1,331 @@ +package lanes + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/hex" + "encoding/json" + "errors" + "sort" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/nats-io/nats.go/jetstream" +) + +const ( + receiptBytes = 32 + maxCASAttempts = 32 + maxStateBytes = 512 << 10 +) + +type lease struct { + Receipt string `json:"receipt"` + Project string `json:"project"` + Expires time.Time `json:"expires"` + Attempt uint64 `json:"attempt"` + Done bool `json:"done"` + Released bool `json:"released"` + AckedAt time.Time `json:"acked_at,omitempty"` + NextAttempt time.Time `json:"next_attempt,omitempty"` + Expired bool `json:"expired,omitempty"` + RetiredAt time.Time `json:"retired_at,omitempty"` +} + +type groupState struct { + Cursor uint64 `json:"cursor"` + Leases map[uint64]lease `json:"leases"` + RetryTurn bool `json:"retry_turn,omitempty"` + Recoveries map[string]recoveryRecord `json:"recoveries,omitempty"` + MissingThrough uint64 `json:"missing_through,omitempty"` + RetryCursor uint64 `json:"retry_cursor,omitempty"` + Slots int `json:"slots"` + Padding string `json:"padding"` +} + +func (b *Broker) readGroup(ctx context.Context, key string) (groupState, uint64, error) { + entry, err := b.state.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return groupState{Leases: map[uint64]lease{}}, 0, nil + } + if err != nil { + return groupState{}, 0, backend(err) + } + if len(entry.Value()) > maxStateBytes { + return groupState{}, 0, ErrCapacity + } + var state groupState + if err := json.Unmarshal(entry.Value(), &state); err != nil { + return groupState{}, 0, backend(err) + } + if state.Leases == nil { + return groupState{}, 0, ErrConflict + } + state.Padding = "" + if err := validateGroup(state); err != nil { + return groupState{}, 0, err + } + reserved, err := groupReservation(state.Slots) + if err != nil { + return groupState{}, 0, err + } + if reserved > maxStateBytes || state.Slots > b.cfg.MaxLedger || + len(entry.Value()) > reserved || len(entry.Value()) < reserved-revisionHeaderGrowth(^uint64(0)) { + return groupState{}, 0, ErrConflict + } + return state, entry.Revision(), nil +} + +func (b *Broker) saveGroup(ctx context.Context, key string, state groupState, revision uint64) error { + _, err := b.writeGroup(ctx, key, state, revision) + return err +} + +func (b *Broker) writeGroup(ctx context.Context, key string, state groupState, revision uint64) (uint64, error) { + data, err := encodeGroup(state, revision) + if err != nil { + return 0, err + } + if max(state.Slots, len(state.Leases), minGroupSlots) > b.cfg.MaxLedger { + return 0, ErrCapacity + } + var next uint64 + if revision == 0 { + next, err = b.state.Create(ctx, key, data) + } else { + next, err = b.state.Update(ctx, key, data, revision) + } + if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + return 0, ErrConflict + } + return next, backend(err) +} + +func groupKey(lane, group string) string { return "group." + lane + "." + group } + +// Claim's successful KV compare-and-set is the sole ownership linearization +// point. A response lost afterwards leaves a lease, not a second owner. Event +// and work log records remain in the stream for other delivery groups. +func (b *Broker) Claim(ctx context.Context, project, lane, group string) (api.LaneClaim, error) { + ctx, cancel := context.WithTimeout(ctx, b.cfg.Timeout) + defer cancel() + if _, err := b.authorize(project, lane, "work", group); err != nil { + return api.LaneClaim{}, err + } + key := groupKey(lane, group) + for attempt := 0; attempt < maxCASAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return api.LaneClaim{}, err + } + state, revision, err := b.readGroup(ctx, key) + if err != nil { + return api.LaneClaim{}, err + } + now := b.now().UTC() + b.pruneGroup(&state, now) + pending := 0 + ready := []uint64{} + for seq, l := range state.Leases { + if !l.Done && !l.Expired { + if l.Expires.After(now) && !l.Released { + pending++ + } + if !l.Expires.After(now) && !l.NextAttempt.After(now) { + ready = append(ready, seq) + } + } + } + sort.Slice(ready, func(i, j int) bool { return ready[i] < ready[j] }) + var record api.LaneRecord + var previous lease + retried := false + canClaim := pending < b.cfg.MaxPending + canFresh := canClaim && len(state.Leases) < b.cfg.MaxLedger + if canFresh { + reserved, err := groupReservation(max(state.Slots, len(state.Leases)+1)) + if err != nil { + return api.LaneClaim{}, err + } + canFresh = reserved <= maxStateBytes + } + if !canClaim || (!canFresh && len(ready) == 0) { + return api.LaneClaim{}, ErrCapacity + } + freshTurn := canFresh && (len(ready) == 0 || !state.RetryTurn) + if freshTurn && len(ready) != 0 { + // Persist the next retry turn before attempting value growth. + // If global bucket admission fails (or its response is lost), + // the next call can still redeliver already accepted work. + state.RetryTurn = true + revision, err = b.writeGroup(ctx, key, state, revision) + if errors.Is(err, ErrConflict) { + continue + } else if err != nil { + return api.LaneClaim{}, err + } + } + if freshTurn { + page, err := b.scan(ctx, lane, state.Cursor, 1, api.LaneWork) + if err != nil { + return api.LaneClaim{}, err + } + state.Cursor = page.Cursor + if len(page.Events) != 0 { + record = page.Events[0] + } + } + if record.Sequence == 0 && len(ready) != 0 { + // Rotate among already-claimed deliveries as well as alternating + // retries with fresh work. A repeatedly ready low sequence cannot + // monopolize a group's retry turns across process restarts. + index := sort.Search(len(ready), func(i int) bool { return ready[i] > state.RetryCursor }) + if index == len(ready) { + index = 0 + } + seq := ready[index] + msg, err := b.streams[lane].GetMsg(ctx, seq) + if errors.Is(err, jetstream.ErrMsgNotFound) { + return api.LaneClaim{}, ErrCursor + } + if err != nil { + return api.LaneClaim{}, backend(err) + } + record.Sequence = seq + if err := json.Unmarshal(msg.Data, &record.Envelope); err != nil { + return api.LaneClaim{}, backend(err) + } + if record.Envelope.Kind != api.LaneWork || record.Envelope.Lane != lane { + return api.LaneClaim{}, ErrConflict + } + previous = state.Leases[seq] + if previous.Attempt == ^uint64(0) { + return api.LaneClaim{}, ErrCapacity + } + state.RetryCursor = seq + retried = true + } + if record.Sequence == 0 { + if err := b.saveGroup(ctx, key, state, revision); errors.Is(err, ErrConflict) { + continue + } else if err != nil { + return api.LaneClaim{}, err + } + return api.LaneClaim{}, ErrEmpty + } + receipt, err := newReceipt() + if err != nil { + return api.LaneClaim{}, err + } + l := lease{Receipt: receipt, Project: project, + Expires: now.Add(b.cfg.Lease), Attempt: previous.Attempt + 1} + state.Leases[record.Sequence] = l + state.RetryTurn = !retried + if err := b.saveGroup(ctx, key, state, revision); errors.Is(err, ErrConflict) { + continue + } else if err != nil { + return api.LaneClaim{}, err + } + return api.LaneClaim{LaneRecord: record, Receipt: l.Receipt, Expires: l.Expires, Attempt: l.Attempt}, nil + } + return api.LaneClaim{}, ErrConflict +} + +// Receipt operations bind lane, group, authenticated project, sequence and +// random ownership generation. No NATS ACK subject is exposed to clients. +func (b *Broker) receipt(ctx context.Context, project, lane, group, operation string, in api.LaneReceipt) (time.Time, error) { + ctx, cancel := context.WithTimeout(ctx, b.cfg.Timeout) + defer cancel() + if _, err := b.authorize(project, lane, "work", group); err != nil { + return time.Time{}, err + } + if !validReceipt(in.Receipt) || in.Sequence == 0 { + return time.Time{}, ErrInvalid + } + key := groupKey(lane, group) + for attempt := 0; attempt < maxCASAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return time.Time{}, err + } + state, revision, err := b.readGroup(ctx, key) + if err != nil { + return time.Time{}, err + } + l, ok := state.Leases[in.Sequence] + if !ok || subtle.ConstantTimeCompare([]byte(l.Receipt), []byte(in.Receipt)) != 1 || l.Project != project { + return time.Time{}, ErrStale + } + now := b.now().UTC() + if l.Expired { + return time.Time{}, ErrStale + } + if l.Done { + if operation == "ack" && now.Sub(l.AckedAt) < b.cfg.AckRetention { + return l.Expires, nil + } + return time.Time{}, ErrStale + } + if l.Released && operation == "nack" { + return l.Expires, nil + } + if l.Released || !l.Expires.After(now) { + return time.Time{}, ErrStale + } + switch operation { + case "ack": + l.Done, l.AckedAt = true, now + case "nack": + l.Released, l.Expires = true, now + l.NextAttempt = now.Add(b.retryDelay(l.Attempt)) + case "renew": + l.Expires = now.Add(b.cfg.Lease) + default: + return time.Time{}, ErrInvalid + } + + state.Leases[in.Sequence] = l + if err := b.saveGroup(ctx, key, state, revision); errors.Is(err, ErrConflict) { + continue + } else if err != nil { + return time.Time{}, err + } + return l.Expires, nil + } + return time.Time{}, ErrConflict +} + +func newReceipt() (string, error) { + data := make([]byte, receiptBytes) + if _, err := rand.Read(data); err != nil { + return "", err + } + return hex.EncodeToString(data), nil +} + +func validReceipt(receipt string) bool { + data, err := hex.DecodeString(receipt) + return err == nil && len(data) == receiptBytes +} + +func (b *Broker) retryDelay(attempt uint64) time.Duration { + delay := b.cfg.RetryDelay + for n := uint64(1); n < attempt && delay < b.cfg.MaxRetryDelay; n++ { + if delay > b.cfg.MaxRetryDelay/2 { + return b.cfg.MaxRetryDelay + } + delay *= 2 + } + return delay +} + +func (b *Broker) Ack(ctx context.Context, project, lane, group string, in api.LaneReceipt) error { + _, err := b.receipt(ctx, project, lane, group, "ack", in) + return err +} +func (b *Broker) Nack(ctx context.Context, project, lane, group string, in api.LaneReceipt) error { + _, err := b.receipt(ctx, project, lane, group, "nack", in) + return err +} +func (b *Broker) Renew(ctx context.Context, project, lane, group string, in api.LaneReceipt) (time.Time, error) { + return b.receipt(ctx, project, lane, group, "renew", in) +} diff --git a/src/crux/lanes/work_budget.go b/src/crux/lanes/work_budget.go new file mode 100644 index 0000000000..6a52ac7da2 --- /dev/null +++ b/src/crux/lanes/work_budget.go @@ -0,0 +1,169 @@ +package lanes + +import ( + "encoding/json" + "strconv" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux/api" +) + +const ( + groupStateFormat = 3 + minGroupSlots = 1 +) + +type completionSizes struct { + base int + delivery int +} + +var cachedCompletionBudget = sync.OnceValues(func() (completionSizes, error) { + base, delivery, err := buildCompletionBudget() + return completionSizes{base: base, delivery: delivery}, err +}) + +func completionBudget() (int, int, error) { + sizes, err := cachedCompletionBudget() + return sizes.base, sizes.delivery, err +} + +// Reserve the exact serialized maximum of every allowed recovery record and +// every field of every delivery, including terminal/retry timestamps. Padding +// allocates that headroom in the KV bucket at admission, not just in memory. +// An empty poll reserves its first delivery slot. Slot high-water marks never +// shrink: pruning and recovery make slots reusable without growing the value. +// Completion responses occupy already reserved history. +func buildCompletionBudget() (int, int, error) { + worst, worstLease, err := maximumGroupShape() + if err != nil { + return 0, 0, err + } + entry, err := json.Marshal(map[uint64]lease{^uint64(0): worstLease}) + if err != nil { + return 0, 0, err + } + leaseBytes := len(entry) - len("{}") + len(",") + base, err := json.Marshal(worst) + maxHeaderGrowth := revisionHeaderGrowth(^uint64(0)) + return len(base) + maxHeaderGrowth, leaseBytes, err +} + +func maximumGroupShape() (groupState, lease, error) { + longest, err := time.Parse(time.RFC3339Nano, "9999-12-31T23:59:59.999999999Z") + if err != nil { + return groupState{}, lease{}, err + } + maximum := ^uint64(0) + project := strings.Repeat("p", MaxIDLength) + worstLease := lease{Receipt: strings.Repeat("f", receiptBytes*2), Project: project, + Expires: longest, Attempt: maximum, AckedAt: longest, NextAttempt: longest, + Expired: true, RetiredAt: longest} + worst := groupState{Cursor: maximum, MissingThrough: maximum, RetryCursor: maximum, + RetryTurn: true, Slots: MaxLedgerEntries, Leases: map[uint64]lease{}, Recoveries: map[string]recoveryRecord{}} + for n := 0; n < maxRecoveryHistory; n++ { + suffix := strconv.Itoa(n) + id := strings.Repeat("r", MaxIDLength-len(suffix)) + suffix + result := api.LaneRecovered{ID: id, Project: project, Cursor: maximum, + GapFrom: maximum, GapThrough: maximum, MissingThrough: maximum, RecoveredAt: longest} + for item := 0; item < MaxRecoveryBatch; item++ { + seq := maximum - uint64(MaxRecoveryBatch-1-item) + result.Retired = append(result.Retired, seq) + result.Expirations = append(result.Expirations, api.LaneExpiration{ + Sequence: seq, Project: project, Attempt: maximum}) + } + worst.Recoveries[id] = recoveryRecord{RequestHash: strings.Repeat("f", receiptBytes*2), Result: result} + } + return worst, worstLease, nil +} + +func revisionHeaderGrowth(revision uint64) int { + return len(strconv.FormatUint(revision, 10)) - len("0") +} + +func groupReservation(slots int) (int, error) { + base, delivery, err := completionBudget() + if err != nil { + return 0, err + } + if slots < 0 || slots > MaxLedgerEntries { + return 0, ErrCapacity + } + return base + slots*delivery, nil +} + +func encodeGroup(state groupState, revision uint64) ([]byte, error) { + if state.Slots < 0 { + return nil, ErrConflict + } + state.Slots = max(state.Slots, len(state.Leases), minGroupSlots) + state.Padding = "" + if err := validateGroup(state); err != nil { + return nil, err + } + data, err := json.Marshal(state) + if err != nil { + return nil, err + } + if len(data) > maxStateBytes { + return nil, ErrCapacity + } + reserved, err := groupReservation(state.Slots) + if err != nil { + return nil, err + } + if reserved > maxStateBytes { + return nil, ErrCapacity + } + // The pinned KV driver stores the decimal expected-revision CAS header + // on Create and Update. Compensate its growth byte-for-byte in padding, + // so aggregate bucket capacity cannot strand an existing group's ACK. + valueBytes := reserved - revisionHeaderGrowth(revision) + if len(data) > valueBytes { + return nil, ErrConflict + } + state.Padding = strings.Repeat(" ", valueBytes-len(data)) + return json.Marshal(state) +} + +func validateGroup(state groupState) error { + if state.Leases == nil { + return ErrConflict + } + if state.Slots < minGroupSlots || state.Slots < len(state.Leases) { + return ErrConflict + } + if state.Slots > MaxLedgerEntries || len(state.Recoveries) > maxRecoveryHistory { + return ErrCapacity + } + for seq, l := range state.Leases { + if seq == 0 || !ValidID(l.Project) || !validReceipt(l.Receipt) || + l.Attempt == 0 || l.Expires.IsZero() || (l.Done && l.AckedAt.IsZero()) || + (l.Released && l.NextAttempt.IsZero()) || (l.Expired && l.RetiredAt.IsZero()) { + return ErrConflict + } + for _, stamp := range []time.Time{l.Expires, l.AckedAt, l.NextAttempt, l.RetiredAt} { + if stamp.Location() != time.UTC { + return ErrConflict + } + } + } + for id, record := range state.Recoveries { + r := record.Result + if !ValidID(id) || r.ID != id || !validReceipt(record.RequestHash) || !ValidID(r.Project) || + r.RecoveredAt.IsZero() || r.RecoveredAt.Location() != time.UTC || len(r.Retired) > MaxRecoveryBatch || + len(r.Expirations) != len(r.Retired) { + return ErrConflict + } + for index, expiration := range r.Expirations { + if expiration.Sequence == 0 || expiration.Sequence != r.Retired[index] || + !ValidID(expiration.Project) || expiration.Attempt == 0 || + (index > 0 && r.Retired[index-1] >= expiration.Sequence) { + return ErrConflict + } + } + } + return nil +} diff --git a/src/crux/lanes/work_budget_cache_test.go b/src/crux/lanes/work_budget_cache_test.go new file mode 100644 index 0000000000..6de0606f89 --- /dev/null +++ b/src/crux/lanes/work_budget_cache_test.go @@ -0,0 +1,29 @@ +package lanes + +import "testing" + +const budgetCacheReaders = 8 + +func TestCompletionBudgetCachePreservesConcurrentValuesAndError(t *testing.T) { + base, delivery, err := buildCompletionBudget() + if err != nil { + t.Fatal(err) + } + type result struct { + completionSizes + err error + } + results := make(chan result, budgetCacheReaders) + for n := 0; n < budgetCacheReaders; n++ { + go func() { + base, delivery, err := completionBudget() + results <- result{completionSizes: completionSizes{base: base, delivery: delivery}, err: err} + }() + } + for n := 0; n < budgetCacheReaders; n++ { + got := <-results + if got.err != nil || got.base != base || got.delivery != delivery { + t.Fatalf("cached budget differs: %+v", got) + } + } +} diff --git a/src/crux/lanes/work_budget_shape_test.go b/src/crux/lanes/work_budget_shape_test.go new file mode 100644 index 0000000000..33eccd86e0 --- /dev/null +++ b/src/crux/lanes/work_budget_shape_test.go @@ -0,0 +1,94 @@ +package lanes + +import ( + "encoding/json" + "errors" + "testing" + "time" +) + +func TestMaximumCompletionShapeFitsReservedStateAndCASHeader(t *testing.T) { + state, worstLease, err := maximumGroupShape() + if err != nil { + t.Fatal(err) + } + base, delivery, err := completionBudget() + if err != nil { + t.Fatal(err) + } + count := (maxStateBytes - base) / delivery + state.Slots = count + for n := 0; n < count; n++ { + state.Leases[^uint64(0)-uint64(n)] = worstLease + } + for _, revision := range []uint64{0, ^uint64(0)} { + data, err := encodeGroup(state, revision) + if err != nil || len(data) > maxStateBytes { + t.Fatal("full timestamps, IDs, attempts, history and gaps exceed reservation", err) + } + reserved, err := groupReservation(count) + if err != nil || len(data)+revisionHeaderGrowth(revision) != reserved { + t.Fatal("CAS header width is not normalized in physical reservation", err) + } + } + state.Leases[^uint64(0)-uint64(count)] = worstLease + if _, err := encodeGroup(state, 0); !errors.Is(err, ErrCapacity) { + t.Fatal("one more worst-shape delivery was admitted", err) + } +} + +func TestGroupBudgetRejectsShapesOutsideItsValidatedContract(t *testing.T) { + valid, l, err := maximumGroupShape() + if err != nil { + t.Fatal(err) + } + valid.Slots = minGroupSlots + for _, corrupt := range []func(*lease){ + func(l *lease) { l.Project = string(make([]byte, MaxIDLength+1)) }, + func(l *lease) { l.Receipt = "short" }, + func(l *lease) { l.Attempt = 0 }, + func(l *lease) { l.Expires = l.Expires.In(time.FixedZone("non-utc", int(time.Hour/time.Second))) }, + } { + candidate := l + corrupt(&candidate) + valid.Leases = map[uint64]lease{1: candidate} + if _, err := encodeGroup(valid, 0); !errors.Is(err, ErrConflict) { + t.Fatal("unbounded or invalid delivery shape accepted", err) + } + } + +} + +func TestEncodeGroupValidatesBeforeTimestampSerialization(t *testing.T) { + state, l, err := maximumGroupShape() + if err != nil { + t.Fatal(err) + } + l.Expires = l.Expires.In(time.FixedZone("non-utc", int(time.Hour/time.Second))) + state.Slots = minGroupSlots + state.Leases[1] = l + var marshaler *json.MarshalerError + if _, err := json.Marshal(state); !errors.As(err, &marshaler) { + t.Fatal("fixture must fail timestamp serialization before validation", err) + } + if _, err := encodeGroup(state, 0); !errors.Is(err, ErrConflict) { + t.Fatal("invalid persistent shape escaped the typed state contract", err) + } +} + +func TestSlotAllocationRejectsInvalidPersistentShape(t *testing.T) { + state, _, err := maximumGroupShape() + if err != nil { + t.Fatal(err) + } + for _, slots := range []int{-minGroupSlots, 0, MaxLedgerEntries + minGroupSlots} { + state.Slots = slots + if err := validateGroup(state); err == nil { + t.Fatal("invalid persisted slot allocation accepted", slots) + } + } + state.Slots = -minGroupSlots + if _, err := encodeGroup(state, 0); !errors.Is(err, ErrConflict) { + t.Fatal("negative allocation silently normalized", err) + } +} diff --git a/src/crux/lanes/work_budget_test.go b/src/crux/lanes/work_budget_test.go new file mode 100644 index 0000000000..277d011911 --- /dev/null +++ b/src/crux/lanes/work_budget_test.go @@ -0,0 +1,209 @@ +package lanes + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" +) + +const ( + byteCapacityLedgerLimit = 2000 + byteCapacityRetention = time.Minute + byteCapacityGroupCount = 16 +) + +func TestByteBoundAdmissionPreservesCompletionAndRecoveryHeadroom(t *testing.T) { + b := testBroker(t, func(c *Config) { + c.MaxLedger, c.MaxPending = byteCapacityLedgerLimit, byteCapacityLedgerLimit + c.Retention, c.AckRetention, c.Lease = byteCapacityRetention, byteCapacityRetention, MinLease + c.Dedup = MinDedup + c.RetryDelay, c.MaxRetryDelay = MinRetryDelay, MinRetryDelay + }) + ctx := context.Background() + clock := time.Now().UTC().Truncate(time.Second).Add(time.Second - time.Nanosecond) + b.now = func() time.Time { return clock } + base, perLease, err := completionBudget() + if err != nil || base >= maxStateBytes || perLease <= 0 { + t.Fatal("invalid completion budget", base, perLease, err) + } + accepted := (maxStateBytes - base) / perLease + if accepted >= b.cfg.MaxLedger || accepted < MaxRecoveryBatch { + t.Fatal("regression must hit the byte limit, not the count limit") + } + // Bulk seed valid deliveries through the same durable admission encoder + // used by Claim, then exercise the actual final Claim at the byte boundary. + state := groupState{Leases: map[uint64]lease{}} + receipts := []api.LaneReceipt{} + for n := 0; n < accepted-1; n++ { + published := publish(t, b, fmt.Sprintf("seed-%d", n), api.LaneWork) + token, err := newReceipt() + if err != nil { + t.Fatal(err) + } + state.Cursor = published.Sequence + state.Leases[published.Sequence] = lease{Receipt: token, Project: "atlas", + Expires: clock.Add(b.cfg.Lease), Attempt: 1} + receipts = append(receipts, api.LaneReceipt{Sequence: published.Sequence, Receipt: token}) + } + key := groupKey("shared", "workers") + if err := b.saveGroup(ctx, key, state, 0); err != nil { + t.Fatal(err) + } + publish(t, b, "last-admitted", api.LaneWork) + last := claim(t, b, "atlas", "workers") + entry, err := b.state.Get(ctx, key) + if err != nil || maxStateBytes-len(entry.Value()) >= perLease { + t.Fatal("accepted state is not within one delivery of the byte cap", err) + } + reservedBytes := len(entry.Value()) + publish(t, b, "not-admitted", api.LaneWork) + if _, err := b.Claim(ctx, "atlas", "shared", "workers"); !errors.Is(err, ErrCapacity) { + t.Fatal("claim did not reserve worst-case completion bytes", err) + } + if _, err := b.Renew(ctx, "atlas", "shared", "workers", receipt(last)); err != nil { + t.Fatal("accepted near-cap lease cannot renew", err) + } + if err := b.Nack(ctx, "atlas", "shared", "workers", receipt(last)); err != nil { + t.Fatal("accepted near-cap lease cannot NACK", err) + } + clock = clock.Add(b.cfg.RetryDelay + time.Nanosecond) + retried := claim(t, b, "atlas", "workers") + if retried.Sequence != last.Sequence { + t.Fatal("accepted near-cap NACK cannot be redelivered") + } + if err := b.Ack(ctx, "atlas", "shared", "workers", receipt(retried)); err != nil { + t.Fatal("actual near-cap Claim cannot ACK", err) + } + if err := b.Nack(ctx, "atlas", "shared", "workers", receipts[0]); err != nil { + t.Fatal(err) + } + clock = clock.Add(b.cfg.RetryDelay + time.Nanosecond) + // The fresh turn must recognize the byte admission limit and still + // redeliver accepted work instead of repeatedly failing fresh admission. + retried = claim(t, b, "atlas", "workers") + if retried.Sequence != receipts[0].Sequence { + t.Fatal("byte-full fresh turn starved an accepted retry") + } + if err := b.Nack(ctx, "atlas", "shared", "workers", receipt(retried)); err != nil { + t.Fatal(err) + } + // Full-ledger ACKs under the race detector can outlive retention. + // Finish retained-work retries first; expiry and recovery follow below. + for _, in := range receipts[2:] { + if err := b.Ack(ctx, "atlas", "shared", "workers", in); err != nil { + t.Fatal("accepted near-cap lease cannot ACK", err) + } + } + entry, err = b.state.Get(ctx, key) + if err != nil || len(entry.Value()) > reservedBytes { + t.Fatal("completion consumed previously unreserved bucket bytes", err) + } + awaitConditionWithin(t, b.cfg.Retention+testWait, func() (bool, error) { + info, err := b.streams["shared"].Info(ctx) + if err != nil { + return false, err + } + return info.State.FirstSeq > last.Sequence+1, nil + }) + clock = clock.Add(b.cfg.Lease + time.Nanosecond) + fresh := publish(t, b, "after-expiry", api.LaneWork) + in := recoveryRequest(t, b, "atlas", "r") + if len(in.Expired) != 2 { + t.Fatalf("pending near-cap deliveries: %+v", in) + } + out, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in) + if err != nil || len(out.Expirations) != 2 { + t.Fatal("near-cap missing leases cannot recover", err) + } + resumed := claim(t, b, "atlas", "workers") + if resumed.Sequence != fresh.Sequence { + t.Fatal("valid later work did not resume") + } + if err := b.Ack(ctx, "atlas", "shared", "workers", receipt(resumed)); err != nil { + t.Fatal(err) + } + if err := b.Ack(ctx, "atlas", "shared", "workers", receipts[2]); err != nil { + t.Fatal("recovery silently discarded unexpired ACK history", err) + } +} + +func TestNamespaceBucketAdmissionReservesExistingGroupsCompletionBytes(t *testing.T) { + b := testBroker(t, func(c *Config) { + c.MaxKVBytes = 2 * maxStateBytes + c.RetryDelay, c.MaxRetryDelay = MinRetryDelay, MinRetryDelay + for n := 0; n < byteCapacityGroupCount; n++ { + c.Lanes[0].Groups[fmt.Sprintf("bounded-%d", n)] = []string{"atlas"} + } + }) + ctx := context.Background() + publish(t, b, "shared-work", api.LaneWork) + claims := map[string]api.LaneClaim{} + full := false + for n := 0; n < byteCapacityGroupCount; n++ { + group := fmt.Sprintf("bounded-%d", n) + c, err := b.Claim(ctx, "atlas", "shared", group) + if err != nil { + if !errors.Is(err, ErrUnavailable) { + t.Fatal("unexpected bucket admission error", err) + } + full = true + break + } + claims[group] = c + } + if !full || len(claims) < 2 { + t.Fatal("regression did not fill bounded namespace storage") + } + stream, err := b.js.Stream(ctx, "KV_"+b.cfg.Namespace+"_state") + if err != nil { + t.Fatal(err) + } + before, err := stream.Info(ctx) + if err != nil { + t.Fatal(err) + } + pendingGroup := "" + var pendingClaim api.LaneClaim + for group, c := range claims { + if _, err := b.Renew(ctx, "atlas", "shared", group, receipt(c)); err != nil { + t.Fatal("another group's admission consumed renewal headroom", err) + } + if pendingGroup == "" { + pendingGroup, pendingClaim = group, c + continue + } + if err := b.Ack(ctx, "atlas", "shared", group, receipt(c)); err != nil { + t.Fatal("another group's admission consumed completion headroom", err) + } + } + after, err := stream.Info(ctx) + if err != nil || after.State.Bytes > before.State.Bytes { + t.Fatal("completion grew total bucket bytes despite reservation", err) + } + clock := time.Now().UTC() + b.now = func() time.Time { return clock } + if err := b.Nack(ctx, "atlas", "shared", pendingGroup, receipt(pendingClaim)); err != nil { + t.Fatal(err) + } + clock = clock.Add(b.cfg.RetryDelay + time.Nanosecond) + retried := claim(t, b, "atlas", pendingGroup) + if err := b.Nack(ctx, "atlas", "shared", pendingGroup, receipt(retried)); err != nil { + t.Fatal(err) + } + clock = clock.Add(b.cfg.RetryDelay + time.Nanosecond) + publish(t, b, "fresh-while-bucket-full", api.LaneWork) + if _, err := b.Claim(ctx, "atlas", "shared", pendingGroup); !errors.Is(err, ErrUnavailable) { + t.Fatal("regression requires failed fresh value-growth admission", err) + } + retried = claim(t, b, "atlas", pendingGroup) + if retried.Sequence != pendingClaim.Sequence { + t.Fatal("failed fresh admission permanently blocked accepted retry") + } + if err := b.Ack(ctx, "atlas", "shared", pendingGroup, receipt(retried)); err != nil { + t.Fatal("accepted retry could not complete in full bucket", err) + } +} diff --git a/src/crux/lanes/work_expiration_test.go b/src/crux/lanes/work_expiration_test.go new file mode 100644 index 0000000000..d159be63d4 --- /dev/null +++ b/src/crux/lanes/work_expiration_test.go @@ -0,0 +1,121 @@ +package lanes + +import ( + "context" + "errors" + "reflect" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" +) + +const expirationTestRetention = 3 * time.Second + +func TestAbsentForeignOwnerRecoveryReleasesCapacityWithAndWithoutUnreadGap(t *testing.T) { + for _, full := range []bool{true, false} { + name := "no-unread-gap" + if full { + name = "sole-ledger-slot" + } + t.Run(name, func(t *testing.T) { + ctx := context.Background() + s, opts := testServer(t) + cfg := testConfig(s.ClientURL()) + shortRetention(&cfg) + cfg.Retention, cfg.AckRetention = expirationTestRetention, expirationTestRetention + if full { + cfg.MaxLedger, cfg.MaxPending = 1, 1 + } + atlas, err := Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer atlas.Close() + beacon, err := Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + publish(t, atlas, "foreign", api.LaneWork) + old := claim(t, beacon, "beacon", "workers") + beacon.Close() + through := old.Sequence + if full { + through = publish(t, atlas, "unread", api.LaneEvent).Sequence + } + waitExpiredPrefix(t, atlas, through) + awaitCondition(t, func() (bool, error) { return !old.Expires.After(time.Now()), nil }) + fresh := publish(t, atlas, "fresh", api.LaneWork) + status, err := atlas.InspectGroup(ctx, "atlas", "shared", "workers") + if err != nil || len(status.Recoverable) != 0 || len(status.Missing) != 1 { + t.Fatalf("receipt-free inspection: %+v %v", status, err) + } + if !full && status.Cursor != status.FirstRetained-1 { + t.Fatal("regression requires no unread cursor gap") + } + in := recoveryRequest(t, atlas, "atlas", "absent-owner") + if len(in.Expired) != 0 || len(in.Missing) != 1 { + t.Fatalf("foreign recovery requires an unrelated own receipt: %+v", in) + } + out, err := atlas.RecoverGroup(ctx, "atlas", "shared", "workers", in) + if err != nil || len(out.Expirations) != 1 || out.Expirations[0].Project != "beacon" { + t.Fatalf("foreign expiration audit: %+v %v", out, err) + } + atlas.Close() + s.Shutdown() + s.WaitForShutdown() + s = startServer(t, opts) + defer func() { s.Shutdown(); s.WaitForShutdown() }() + atlas, err = Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer atlas.Close() + replayed, err := atlas.RecoverGroup(ctx, "atlas", "shared", "workers", in) + if err != nil || !reflect.DeepEqual(out, replayed) { + t.Fatalf("lost response after service/broker restart: %+v %v", replayed, err) + } + resumed := claim(t, atlas, "atlas", "workers") + if resumed.Sequence != fresh.Sequence { + t.Fatal("absent Beacon permanently stranded shared work") + } + if err := atlas.Ack(ctx, "atlas", "shared", "workers", receipt(resumed)); err != nil { + t.Fatal(err) + } + for _, project := range []string{"atlas", "beacon"} { + if err := atlas.Ack(ctx, project, "shared", "workers", receipt(old)); !errors.Is(err, ErrStale) { + t.Fatal("expiration accepted a foreign or owner ACK", project, err) + } + if _, err := atlas.Renew(ctx, project, "shared", "workers", receipt(old)); !errors.Is(err, ErrStale) { + t.Fatal("expired foreign lease renewed", project, err) + } + } + }) + } +} + +func TestMissingAcceptanceCannotRetireLiveOrRetainedForeignWork(t *testing.T) { + b := testBroker(t, nil) + ctx := context.Background() + publish(t, b, "foreign", api.LaneWork) + foreign := claim(t, b, "beacon", "workers") + status, err := b.InspectGroup(ctx, "atlas", "shared", "workers") + if err != nil { + t.Fatal(err) + } + in := api.LaneRecover{ID: "foreign-expiration", Revision: status.Revision, + CursorThrough: status.Cursor, Missing: []uint64{foreign.Sequence}} + if _, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in); !errors.Is(err, ErrStale) { + t.Fatal("live foreign lease retired", err) + } + b.now = func() time.Time { return foreign.Expires.Add(time.Nanosecond) } + if _, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in); !errors.Is(err, ErrCursor) { + t.Fatal("retained foreign work retired", err) + } + if _, err := b.RecoverGroup(ctx, "atlas", "shared", "other", in); !errors.Is(err, ErrForbidden) { + t.Fatal("foreign group accepted", err) + } + if _, err := b.RecoverGroup(ctx, "outsider", "shared", "workers", in); !errors.Is(err, ErrForbidden) { + t.Fatal("unauthorized expiration accepted", err) + } +} diff --git a/src/crux/lanes/work_recovery.go b/src/crux/lanes/work_recovery.go new file mode 100644 index 0000000000..569a3456c6 --- /dev/null +++ b/src/crux/lanes/work_recovery.go @@ -0,0 +1,206 @@ +package lanes + +import ( + "context" + "crypto/subtle" + "encoding/json" + "errors" + "sort" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/nats-io/nats.go/jetstream" +) + +const maxRecoveryHistory = 64 + +type recoveryRecord struct { + RequestHash string `json:"request_hash"` + Result api.LaneRecovered `json:"result"` +} + +func (b *Broker) pruneGroup(state *groupState, now time.Time) { + for seq, l := range state.Leases { + when := l.AckedAt + if l.Expired { + when = l.RetiredAt + } + if (l.Done || l.Expired) && now.Sub(when) >= b.cfg.AckRetention { + delete(state.Leases, seq) + } + } + for id, record := range state.Recoveries { + if now.Sub(record.Result.RecoveredAt) >= b.cfg.AckRetention { + delete(state.Recoveries, id) + } + } + if state.Recoveries == nil { + state.Recoveries = map[string]recoveryRecord{} + } +} + +// InspectGroup exposes only this principal's recoverable receipts. Missing +// delivery identities are public within the group, never ownership tokens. +func (b *Broker) InspectGroup(ctx context.Context, project, lane, group string) (api.LaneGroupState, error) { + ctx, cancel := context.WithTimeout(ctx, b.cfg.Timeout) + defer cancel() + if _, err := b.authorize(project, lane, "work", group); err != nil { + return api.LaneGroupState{}, err + } + state, revision, err := b.readGroup(ctx, groupKey(lane, group)) + if err != nil { + return api.LaneGroupState{}, err + } + info, err := b.streams[lane].Info(ctx) + if err != nil { + return api.LaneGroupState{}, backend(err) + } + out := api.LaneGroupState{Revision: revision, Cursor: state.Cursor, + FirstRetained: info.State.FirstSeq, LastSequence: info.State.LastSeq, + Recoverable: []api.LaneReceipt{}, Recoveries: []api.LaneRecovered{}, Retries: []api.LaneRetry{}, + Missing: []api.LaneExpiration{}} + out.MissingThrough = state.MissingThrough + out.Slots = state.Slots + now := b.now().UTC() + for seq, l := range state.Leases { + if l.Project == project && !l.Done && !l.Expired && l.Released { + out.Retries = append(out.Retries, api.LaneRetry{Sequence: seq, Attempt: l.Attempt, AvailableAt: l.NextAttempt}) + } + if !l.Done && !l.Expired && !l.Expires.After(now) && seq < out.FirstRetained { + out.Missing = append(out.Missing, api.LaneExpiration{Sequence: seq, Project: l.Project, Attempt: l.Attempt}) + if l.Project == project { + out.Recoverable = append(out.Recoverable, api.LaneReceipt{Sequence: seq, Receipt: l.Receipt}) + } + } + } + for _, r := range state.Recoveries { + if now.Sub(r.Result.RecoveredAt) < b.cfg.AckRetention { + out.Recoveries = append(out.Recoveries, r.Result) + } + } + sort.Slice(out.Recoverable, func(i, j int) bool { return out.Recoverable[i].Sequence < out.Recoverable[j].Sequence }) + sort.Slice(out.Missing, func(i, j int) bool { return out.Missing[i].Sequence < out.Missing[j].Sequence }) + sort.Slice(out.Retries, func(i, j int) bool { return out.Retries[i].Sequence < out.Retries[j].Sequence }) + sort.Slice(out.Recoveries, func(i, j int) bool { return out.Recoveries[i].ID < out.Recoveries[j].ID }) + return out, nil +} + +// RecoverGroup is an explicit gap acceptance, not an ACK or cursor reset. +// The caller's supplied KV revision fences claims, renewals and ACKs. A +// persisted request/result makes response-loss retry safe across restarts. +func (b *Broker) RecoverGroup(ctx context.Context, project, lane, group string, in api.LaneRecover) (api.LaneRecovered, error) { + ctx, cancel := context.WithTimeout(ctx, b.cfg.Timeout) + defer cancel() + if _, err := b.authorize(project, lane, "recover", group); err != nil { + return api.LaneRecovered{}, err + } + if !ValidID(in.ID) || in.Revision == 0 || len(in.Expired)+len(in.Missing) > MaxRecoveryBatch { + return api.LaneRecovered{}, ErrInvalid + } + request, err := json.Marshal(in) + if err != nil { + return api.LaneRecovered{}, err + } + key := groupKey(lane, group) + state, revision, err := b.readGroup(ctx, key) + if err != nil { + return api.LaneRecovered{}, err + } + if previous, ok := state.Recoveries[in.ID]; ok { + if previous.Result.Project != project || previous.RequestHash != digest(request) { + return api.LaneRecovered{}, ErrConflict + } + if b.now().Sub(previous.Result.RecoveredAt) >= b.cfg.AckRetention { + return api.LaneRecovered{}, ErrStale + } + return previous.Result, nil + } + if revision != in.Revision { + return api.LaneRecovered{}, ErrConflict + } + now := b.now().UTC() + b.pruneGroup(&state, now) + if len(state.Recoveries) >= maxRecoveryHistory { + return api.LaneRecovered{}, ErrCapacity + } + info, err := b.streams[lane].Info(ctx) + if err != nil { + return api.LaneRecovered{}, backend(err) + } + first := info.State.FirstSeq + out := api.LaneRecovered{ID: in.ID, Project: project, Cursor: state.Cursor, + Retired: []uint64{}, Expirations: []api.LaneExpiration{}, RecoveredAt: now} + if state.Cursor != 0 && first > 0 && state.Cursor < first-1 { + if in.CursorThrough != first-1 { + return api.LaneRecovered{}, ErrCursor + } + out.GapFrom, out.GapThrough = state.Cursor+1, first-1 + state.Cursor, out.Cursor = first-1, first-1 + } else if in.CursorThrough != state.Cursor { + // Never advance over retained, unclaimed records. + return api.LaneRecovered{}, ErrCursor + } + seen := map[uint64]bool{} + for _, receipt := range in.Expired { + if !validReceipt(receipt.Receipt) || receipt.Sequence == 0 || seen[receipt.Sequence] { + return api.LaneRecovered{}, ErrInvalid + } + seen[receipt.Sequence] = true + l, ok := state.Leases[receipt.Sequence] + if !ok || l.Project != project || subtle.ConstantTimeCompare([]byte(l.Receipt), []byte(receipt.Receipt)) != 1 { + return api.LaneRecovered{}, ErrStale + } + expiration, err := b.retireMissing(ctx, &state, lane, receipt.Sequence, first, now) + if err != nil { + return api.LaneRecovered{}, err + } + out.Retired = append(out.Retired, receipt.Sequence) + out.Expirations = append(out.Expirations, expiration) + } + for _, seq := range in.Missing { + if seq == 0 || seen[seq] { + return api.LaneRecovered{}, ErrInvalid + } + seen[seq] = true + expiration, err := b.retireMissing(ctx, &state, lane, seq, first, now) + if err != nil { + return api.LaneRecovered{}, err + } + out.Retired = append(out.Retired, seq) + out.Expirations = append(out.Expirations, expiration) + } + if len(out.Retired) == 0 && out.GapThrough == 0 { + return api.LaneRecovered{}, ErrInvalid + } + // The prefix checkpoint audits log expiry; it does not ACK or suppress + // unlisted deliveries. Explicit expiration records release ledger slots. + if first > 0 && first-1 > state.MissingThrough { + state.MissingThrough = first - 1 + } + out.MissingThrough = state.MissingThrough + sort.Slice(out.Retired, func(i, j int) bool { return out.Retired[i] < out.Retired[j] }) + sort.Slice(out.Expirations, func(i, j int) bool { return out.Expirations[i].Sequence < out.Expirations[j].Sequence }) + state.Recoveries[in.ID] = recoveryRecord{RequestHash: digest(request), Result: out} + if err := b.saveGroup(ctx, key, state, revision); err != nil { + return api.LaneRecovered{}, err + } + return out, nil +} + +func (b *Broker) retireMissing(ctx context.Context, state *groupState, lane string, seq, first uint64, now time.Time) (api.LaneExpiration, error) { + l, ok := state.Leases[seq] + if !ok || l.Done || l.Expired || l.Expires.After(now) { + return api.LaneExpiration{}, ErrStale + } + if seq >= first { + return api.LaneExpiration{}, ErrCursor + } + if _, err := b.streams[lane].GetMsg(ctx, seq); !errors.Is(err, jetstream.ErrMsgNotFound) { + if err != nil { + return api.LaneExpiration{}, backend(err) + } + return api.LaneExpiration{}, ErrCursor + } + delete(state.Leases, seq) + return api.LaneExpiration{Sequence: seq, Project: l.Project, Attempt: l.Attempt}, nil +} diff --git a/src/crux/lanes/work_recovery_test.go b/src/crux/lanes/work_recovery_test.go new file mode 100644 index 0000000000..149b86f928 --- /dev/null +++ b/src/crux/lanes/work_recovery_test.go @@ -0,0 +1,294 @@ +package lanes + +import ( + "context" + "errors" + "reflect" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/nats-io/nats.go/jetstream" +) + +func shortRetention(c *Config) { + c.Retention, c.Lease, c.AckRetention = time.Second, time.Second, time.Second + c.Dedup = MinDedup +} + +func waitExpiredPrefix(t *testing.T, b *Broker, through uint64) { + t.Helper() + awaitCondition(t, func() (bool, error) { + info, err := b.streams["shared"].Info(context.Background()) + if err != nil { + return false, err + } + return info.State.FirstSeq > through, nil + }) +} + +func recoveryRequest(t *testing.T, b *Broker, project, id string) api.LaneRecover { + t.Helper() + s, err := b.InspectGroup(context.Background(), project, "shared", "workers") + if err != nil { + t.Fatal(err) + } + through := s.Cursor + if s.Cursor != 0 && s.FirstRetained > s.Cursor+1 { + through = s.FirstRetained - 1 + } + in := api.LaneRecover{ID: id, Revision: s.Revision, CursorThrough: through, Expired: s.Recoverable} + for _, missing := range s.Missing { + if missing.Project != project { + in.Missing = append(in.Missing, missing.Sequence) + } + } + return in +} + +func TestExplicitGroupGapRecoverySurvivesResponseLossAndRestart(t *testing.T) { + ctx := context.Background() + s, opts := testServer(t) + cfg := testConfig(s.ClientURL()) + shortRetention(&cfg) + cfg.Retention, cfg.Lease, cfg.AckRetention = 2*time.Second, 2*time.Second, 2*time.Second + b, err := Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(b.Close) + publish(t, b, "expired-work", api.LaneWork) + old := claim(t, b, "atlas", "workers") + gap := publish(t, b, "expired-unread-event", api.LaneEvent) + waitExpiredPrefix(t, b, gap.Sequence) + fresh := publish(t, b, "retained-work", api.LaneWork) + if _, err := b.Claim(ctx, "atlas", "shared", "workers"); !errors.Is(err, ErrCursor) { + t.Fatal("gap was silently skipped", err) + } + in := recoveryRequest(t, b, "atlas", "recover-gap") + if len(in.Expired) != 1 { + t.Fatalf("missing own expired delivery: %+v", in) + } + out, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in) + if err != nil || out.GapFrom != old.Sequence+1 || out.GapThrough != gap.Sequence || len(out.Retired) != 1 { + t.Fatalf("recovery: %+v %v", out, err) + } + cfg = b.Config() + b.Close() + s.Shutdown() + s.WaitForShutdown() + s = startServer(t, opts) + defer func() { s.Shutdown(); s.WaitForShutdown() }() + b, err = Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer b.Close() + duplicate, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in) + if err != nil || !reflect.DeepEqual(out, duplicate) { + t.Fatalf("lost recovery response: %+v %v", duplicate, err) + } + in.CursorThrough++ + if _, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in); !errors.Is(err, ErrConflict) { + t.Fatal("recovery key payload mismatch", err) + } + if err := b.Ack(ctx, "atlas", "shared", "workers", receipt(old)); !errors.Is(err, ErrStale) { + t.Fatal("gap recovery falsely acknowledged expired work", err) + } + resumed := claim(t, b, "beacon", "workers") + if resumed.Sequence != fresh.Sequence { + t.Fatal("retained work remained stranded") + } + status, err := b.InspectGroup(ctx, "atlas", "shared", "workers") + if err != nil || len(status.Recoveries) != 1 || status.Recoveries[0].GapThrough != gap.Sequence { + t.Fatalf("durable gap record: %+v %v", status, err) + } +} + +func TestRecoveryCannotRetireLiveRetainedOrOtherProjectDeliveries(t *testing.T) { + b := testBroker(t, nil) + ctx := context.Background() + publish(t, b, "live", api.LaneWork) + live := claim(t, b, "atlas", "workers") + in := recoveryRequest(t, b, "atlas", "retire-live") + in.Expired = []api.LaneReceipt{receipt(live)} + if _, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in); !errors.Is(err, ErrStale) { + t.Fatal("live lease retired", err) + } + + if _, err := b.RecoverGroup(ctx, "beacon", "shared", "workers", in); !errors.Is(err, ErrStale) { + t.Fatal("another project lease retired", err) + } + if _, err := b.RecoverGroup(ctx, "atlas", "shared", "other", in); !errors.Is(err, ErrForbidden) { + t.Fatal("another group recovered", err) + } + future := live.Expires.Add(time.Nanosecond) + b.now = func() time.Time { return future } + if _, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in); !errors.Is(err, ErrCursor) { + t.Fatal("expired lease with retained work retired", err) + } + in.ID, in.Expired, in.CursorThrough = "skip-retained", nil, live.Sequence+1 + if _, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in); !errors.Is(err, ErrCursor) { + t.Fatal("retained cursor prefix skipped", err) + } + if _, err := b.InspectGroup(ctx, "outsider", "shared", "workers"); !errors.Is(err, ErrForbidden) { + t.Fatal("outsider inspected group", err) + } +} + +func TestRecoveryAuditsForeignExpirationWithoutAcknowledgement(t *testing.T) { + b := testBroker(t, shortRetention) + ctx := context.Background() + publish(t, b, "atlas-old", api.LaneWork) + atlas := claim(t, b, "atlas", "workers") + publish(t, b, "beacon-old", api.LaneWork) + beacon := claim(t, b, "beacon", "workers") + gap := publish(t, b, "unread-gap", api.LaneEvent) + waitExpiredPrefix(t, b, gap.Sequence) + fresh := publish(t, b, "fresh", api.LaneWork) + in := recoveryRequest(t, b, "atlas", "atlas-recovery") + if len(in.Expired) != 1 || in.Expired[0].Sequence != atlas.Sequence { + t.Fatal("recovery exposed foreign receipts") + } + out, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in) + if err != nil || out.MissingThrough != gap.Sequence { + t.Fatal("missing prefix was not durably accepted", err) + } + after, _, err := b.readGroup(ctx, groupKey("shared", "workers")) + if _, exists := after.Leases[beacon.Sequence]; err != nil || exists { + t.Fatal("expired foreign delivery still consumes ledger capacity", err) + } + if len(out.Expirations) != 2 || out.Expirations[1].Project != "beacon" { + t.Fatal("foreign expiration audit missing") + } + resumed := claim(t, b, "atlas", "workers") + if resumed.Sequence != fresh.Sequence { + t.Fatal("foreign missing delivery stranded retained work") + } + if err := b.Ack(ctx, "beacon", "shared", "workers", receipt(beacon)); !errors.Is(err, ErrStale) { + t.Fatal("foreign expiration was mistaken for successful ACK", err) + } + if err := b.Ack(ctx, "atlas", "shared", "workers", receipt(resumed)); err != nil { + t.Fatal("owner recovery changed another live lease", err) + } +} + +func TestRecoveryPreservesStillValidLeasesAndAcknowledgementHistory(t *testing.T) { + b := testBroker(t, shortRetention) + ctx := context.Background() + oldPub := publish(t, b, "old", api.LaneWork) + old := claim(t, b, "atlas", "workers") + msg, err := b.streams["shared"].GetMsg(ctx, oldPub.Sequence) + if err != nil { + t.Fatal(err) + } + timer := time.NewTimer(time.Until(msg.Time.Add(b.cfg.Retention / 2))) + <-timer.C + publish(t, b, "still-live", api.LaneWork) + live := claim(t, b, "beacon", "workers") + publish(t, b, "ack-history", api.LaneWork) + acked := claim(t, b, "atlas", "workers") + if err := b.Ack(ctx, "atlas", "shared", "workers", receipt(acked)); err != nil { + t.Fatal(err) + } + waitExpiredPrefix(t, b, old.Sequence) + awaitCondition(t, func() (bool, error) { return !old.Expires.After(time.Now()), nil }) + before, _, err := b.readGroup(ctx, groupKey("shared", "workers")) + if err != nil { + t.Fatal(err) + } + in := recoveryRequest(t, b, "atlas", "retire-only-expired") + if len(in.Expired) != 1 || in.Expired[0].Sequence != old.Sequence { + t.Fatalf("recoverable scope: %+v", in) + } + if _, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in); err != nil { + t.Fatal(err) + } + after, _, err := b.readGroup(ctx, groupKey("shared", "workers")) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before.Leases[live.Sequence], after.Leases[live.Sequence]) || !reflect.DeepEqual(before.Leases[acked.Sequence], after.Leases[acked.Sequence]) { + t.Fatal("recovery changed live/ACK state") + } + if err := b.Ack(ctx, "beacon", "shared", "workers", receipt(live)); err != nil { + t.Fatal("still-valid lease fenced away", err) + } + if err := b.Ack(ctx, "atlas", "shared", "workers", receipt(acked)); err != nil { + t.Fatal("ACK history lost", err) + } +} + +type heldGroupUpdate struct { + jetstream.KeyValue + reached chan struct{} + release chan struct{} +} + +func (h *heldGroupUpdate) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) { + close(h.reached) + select { + case <-h.release: + case <-ctx.Done(): + return 0, ctx.Err() + } + return h.KeyValue.Update(ctx, key, value, revision) +} + +func TestRecoveryCASFencesConcurrentClaimAndAckAcrossServices(t *testing.T) { + b := testBroker(t, shortRetention) + ctx := context.Background() + publish(t, b, "old", api.LaneWork) + old := claim(t, b, "atlas", "workers") + waitExpiredPrefix(t, b, old.Sequence) + awaitCondition(t, func() (bool, error) { return !old.Expires.After(time.Now()), nil }) + publish(t, b, "fresh", api.LaneWork) + state, revision, err := b.readGroup(ctx, groupKey("shared", "workers")) + if err != nil { + t.Fatal(err) + } + state.RetryTurn = false + if err := b.saveGroup(ctx, groupKey("shared", "workers"), state, revision); err != nil { + t.Fatal(err) + } + peer, err := Open(ctx, b.Config()) + if err != nil { + t.Fatal(err) + } + defer peer.Close() + original := b.state + var fresh api.LaneClaim + for _, operation := range []string{"claim", "ack"} { + in := recoveryRequest(t, b, "atlas", "fenced-"+operation) + h := &heldGroupUpdate{KeyValue: original, reached: make(chan struct{}), release: make(chan struct{})} + b.state = h + result := make(chan error, 1) + go func() { _, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in); result <- err }() + select { + case <-h.reached: + case <-time.After(testWait): + t.Fatal("recovery did not reach CAS") + } + if operation == "claim" { + fresh = claim(t, peer, "beacon", "workers") + } else if err := peer.Ack(ctx, "beacon", "shared", "workers", receipt(fresh)); err != nil { + t.Fatal(err) + } + close(h.release) + if err := <-result; !errors.Is(err, ErrConflict) { + t.Fatal("concurrent state update not fenced", err) + } + b.state = original + s, _, err := b.readGroup(ctx, groupKey("shared", "workers")) + if _, exists := s.Leases[old.Sequence]; err != nil || !exists { + t.Fatal("failed recovery retired a lease", err) + } + } + in := recoveryRequest(t, b, "atlas", "retry-after-fence") + if _, err := b.RecoverGroup(ctx, "atlas", "shared", "workers", in); err != nil { + t.Fatal(err) + } + if err := peer.Ack(ctx, "beacon", "shared", "workers", receipt(fresh)); err != nil { + t.Fatal("ACK history not preserved", err) + } +} diff --git a/src/crux/lanes/work_slots_test.go b/src/crux/lanes/work_slots_test.go new file mode 100644 index 0000000000..94d857bf32 --- /dev/null +++ b/src/crux/lanes/work_slots_test.go @@ -0,0 +1,230 @@ +package lanes + +import ( + "context" + "errors" + "fmt" + "reflect" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/nats-io/nats-server/v2/server" +) + +const ( + reusableGroupCount = 5 + reusableBucketBytes = 1 << 20 +) + +func reusableSlotConfig(c *Config) { + c.MaxKVBytes = reusableBucketBytes + c.MaxLedger, c.MaxPending = 2, 2 + for n := 0; n < reusableGroupCount; n++ { + group := reusableGroup(n) + c.Lanes[0].Groups[group] = []string{"atlas", "beacon"} + c.Lanes[0].Recoverers[group] = []string{"atlas"} + } +} + +func reusableGroup(n int) string { return fmt.Sprintf("reusable-%d", n) } + +func restartSlotFixture(t *testing.T, b *Broker, s *server.Server, opts *server.Options) (*Broker, *server.Server) { + t.Helper() + cfg, clock := b.Config(), b.now + b.Close() + s.Shutdown() + s.WaitForShutdown() + s = startServer(t, opts) + t.Cleanup(func() { s.Shutdown(); s.WaitForShutdown() }) + b, err := Open(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + b.now = clock + t.Cleanup(b.Close) + return b, s +} + +func emptySlotGroups(t *testing.T, b *Broker) { + t.Helper() + for n := 0; n < reusableGroupCount; n++ { + group := reusableGroup(n) + if _, err := b.Claim(context.Background(), "atlas", "shared", group); !errors.Is(err, ErrEmpty) { + t.Fatal("empty polling could not reserve first reusable slot", group, err) + } + status, err := b.InspectGroup(context.Background(), "atlas", "shared", group) + if err != nil || status.Slots != minGroupSlots || status.Cursor != 0 { + t.Fatalf("empty poll reservation: %+v %v", status, err) + } + } + assertSlotBucketPressure(t, b) +} + +func assertSlotBucketPressure(t *testing.T, b *Broker) { + t.Helper() + stream, err := b.js.Stream(context.Background(), "KV_"+b.cfg.Namespace+"_state") + if err != nil { + t.Fatal(err) + } + info, err := stream.Info(context.Background()) + if err != nil { + t.Fatal(err) + } + firstSlot, err := groupReservation(minGroupSlots) + if err != nil || info.State.Bytes > reusableBucketBytes || + info.State.Bytes+uint64(firstSlot) <= reusableBucketBytes { + t.Fatal("regression must reach the pinned broker's replacement-growth boundary", err) + } +} + +func TestFiveEmptyGroupsRetainFirstClaimSlotsAcrossBrokerRestart(t *testing.T) { + ctx := context.Background() + s, opts := testServer(t) + cfg := testConfig(s.ClientURL()) + reusableSlotConfig(&cfg) + b, err := Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(b.Close) + emptySlotGroups(t, b) + // Ignore a repeated empty-poll response, then restart both components. + if _, err := b.Claim(ctx, "atlas", "shared", reusableGroup(0)); !errors.Is(err, ErrEmpty) { + t.Fatal(err) + } + b, _ = restartSlotFixture(t, b, s, opts) + work := publish(t, b, "after-empty-polls", api.LaneWork) + for n := 0; n < reusableGroupCount; n++ { + group := reusableGroup(n) + got := claim(t, b, "atlas", group) + if got.Sequence != work.Sequence { + t.Fatal("empty group permanently pinned its first claim", group) + } + if err := b.Ack(ctx, "atlas", "shared", group, receipt(got)); err != nil { + t.Fatal(err) + } + status, err := b.InspectGroup(ctx, "atlas", "shared", group) + if err != nil || status.Slots != minGroupSlots { + t.Fatal("initial slot changed size at first claim", err) + } + } +} + +func TestForeignRetirementReusesSlotAtFullBucketAfterLostResponse(t *testing.T) { + ctx := context.Background() + s, opts := testServer(t) + cfg := testConfig(s.ClientURL()) + reusableSlotConfig(&cfg) + shortRetention(&cfg) + cfg.Retention, cfg.AckRetention = expirationTestRetention, expirationTestRetention + b, err := Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(b.Close) + emptySlotGroups(t, b) + oldWork := publish(t, b, "absent-owner", api.LaneWork) + group := reusableGroup(0) + old := claim(t, b, "beacon", group) + waitExpiredPrefix(t, b, oldWork.Sequence) + awaitCondition(t, func() (bool, error) { return !old.Expires.After(time.Now()), nil }) + status, err := b.InspectGroup(ctx, "atlas", "shared", group) + if err != nil || len(status.Recoverable) != 0 || len(status.Missing) != 1 { + t.Fatal("foreign recovery must need no own receipt", err) + } + in := api.LaneRecover{ID: "reuse-after-expiration", Revision: status.Revision, + CursorThrough: status.Cursor, Missing: []uint64{old.Sequence}} + out, err := b.RecoverGroup(ctx, "atlas", "shared", group, in) + if err != nil { + t.Fatal(err) + } + state, _, err := b.readGroup(ctx, groupKey("shared", group)) + if err != nil || len(state.Leases) != 0 || state.Slots != minGroupSlots { + t.Fatal("retirement discarded reusable first-claim capacity", err) + } + b, _ = restartSlotFixture(t, b, s, opts) + replayed, err := b.RecoverGroup(ctx, "atlas", "shared", group, in) + if err != nil || !reflect.DeepEqual(out, replayed) { + t.Fatal("lost recovery response or durable expiration audit", err) + } + fresh := publish(t, b, "after-retirement", api.LaneWork) + got := claim(t, b, "atlas", group) + if got.Sequence != fresh.Sequence { + t.Fatal("retired slot cannot be reused at full bucket") + } + if err := b.Ack(ctx, "atlas", "shared", group, receipt(got)); err != nil { + t.Fatal(err) + } + if err := b.Ack(ctx, "beacon", "shared", group, receipt(old)); !errors.Is(err, ErrStale) { + t.Fatal("foreign retirement forged an ACK", err) + } +} + +func TestPrunedHighWaterSlotsRemainReusableAcrossLostClaimAndRestart(t *testing.T) { + ctx := context.Background() + s, opts := testServer(t) + cfg := testConfig(s.ClientURL()) + reusableSlotConfig(&cfg) + b, err := Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(b.Close) + group := reusableGroup(0) + publish(t, b, "first", api.LaneWork) + publish(t, b, "second", api.LaneWork) + first := claim(t, b, "atlas", group) + second := claim(t, b, "atlas", group) + for _, c := range []api.LaneClaim{first, second} { + if err := b.Ack(ctx, "atlas", "shared", group, receipt(c)); err != nil { + t.Fatal(err) + } + } + for n := 1; n < reusableGroupCount; n++ { + c := claim(t, b, "atlas", reusableGroup(n)) + if err := b.Ack(ctx, "atlas", "shared", reusableGroup(n), receipt(c)); err != nil { + t.Fatal(err) + } + } + assertSlotBucketPressure(t, b) + clock := time.Now().UTC().Add(cfg.AckRetention + time.Nanosecond) + b.now = func() time.Time { return clock } + // Persist pruning through an empty poll before publishing anything new. + if _, err := b.Claim(ctx, "atlas", "shared", group); !errors.Is(err, ErrEmpty) { + t.Fatal("pruning failed", err) + } + pruned, _, err := b.readGroup(ctx, groupKey("shared", group)) + if err != nil || len(pruned.Leases) != 0 || pruned.Slots != cfg.MaxLedger { + t.Fatal("pruning shrank the two-slot high-water reservation", err) + } + third := publish(t, b, "third", api.LaneWork) + fourth := publish(t, b, "fourth", api.LaneWork) + lost := claim(t, b, "atlas", group) + if lost.Sequence != third.Sequence { + t.Fatal("first pruned slot was not reusable") + } + b, _ = restartSlotFixture(t, b, s, opts) + clock = lost.Expires.Add(time.Nanosecond) + reassigned := claim(t, b, "beacon", group) + if reassigned.Sequence != lost.Sequence || reassigned.Attempt != lost.Attempt+1 { + t.Fatal("lost-response lease was not durable and fenced") + } + if err := b.Ack(ctx, "atlas", "shared", group, receipt(lost)); !errors.Is(err, ErrStale) { + t.Fatal("lost old receipt bypassed reassignment", err) + } + if err := b.Ack(ctx, "beacon", "shared", group, receipt(reassigned)); err != nil { + t.Fatal(err) + } + next := claim(t, b, "atlas", group) + if next.Sequence != fourth.Sequence { + t.Fatal("second pruned high-water slot was not reusable") + } + if err := b.Ack(ctx, "atlas", "shared", group, receipt(next)); err != nil { + t.Fatal(err) + } + status, err := b.InspectGroup(ctx, "atlas", "shared", group) + if err != nil || status.Slots != cfg.MaxLedger { + t.Fatal("restart lost high-water slot count", err) + } +} diff --git a/src/crux/serve/lanes.go b/src/crux/serve/lanes.go new file mode 100644 index 0000000000..a05e42b5de --- /dev/null +++ b/src/crux/serve/lanes.go @@ -0,0 +1,206 @@ +package serve + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "io" + "net/http" + "strconv" + "strings" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/lanes" +) + +const ( + defaultLaneReplayLimit = 32 + laneRequestOverhead = 8 << 10 +) + +// The lane mux is in the existing server, but outside the agent RPC auth +// domain. Neither the unix transport nor the legacy admin token implies a +// project principal or a lane capability. +func (s *Server) laneHandler(legacy http.Handler) http.Handler { + if s.Lanes == nil || s.LaneAuth == nil { + return legacy + } + mux := http.NewServeMux() + mux.HandleFunc("GET "+api.PathLanes, func(w http.ResponseWriter, r *http.Request) { + s.writeJSON(w, http.StatusOK, s.Lanes.Registry(laneProject(r))) + }) + mux.HandleFunc("POST "+api.PathLanes+"/{lane}/publish", s.handleLanePublish) + mux.HandleFunc("GET "+api.PathLanes+"/{lane}/events", s.handleLaneReplay) + mux.HandleFunc("POST "+api.PathLanes+"/{lane}/groups/{group}/claim", s.handleLaneClaim) + mux.HandleFunc("GET "+api.PathLanes+"/{lane}/groups/{group}", s.handleLaneGroup) + mux.HandleFunc("POST "+api.PathLanes+"/{lane}/groups/{group}/recover", s.handleLaneRecover) + for _, operation := range []string{"ack", "nack", "renew"} { + mux.HandleFunc("POST "+api.PathLanes+"/{lane}/groups/{group}/"+operation, s.handleLaneReceipt(operation)) + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != api.PathLanes && !strings.HasPrefix(r.URL.Path, api.PathLanes+"/") { + legacy.ServeHTTP(w, r) + return + } + token, _ := strings.CutPrefix(r.Header.Get("Authorization"), bearerPrefix) + project, ok := s.LaneAuth.Project(token) + if !ok { + s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "missing or invalid project credential") + return + } + mux.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), lanePrincipalKey{}, project))) + }) +} + +type lanePrincipalKey struct{} + +func laneProject(r *http.Request) string { + project, _ := r.Context().Value(lanePrincipalKey{}).(string) + return project +} + +func (s *Server) readLaneJSON(w http.ResponseWriter, r *http.Request, v any) bool { + r.Body = http.MaxBytesReader(w, r.Body, int64(base64.StdEncoding.EncodedLen(lanes.MaxPayload)+laneRequestOverhead)) + d := json.NewDecoder(r.Body) + d.DisallowUnknownFields() + if err := d.Decode(v); err != nil { + s.writeError(w, http.StatusBadRequest, api.CodeLaneInvalid, "invalid lane JSON body") + return false + } + var extra any + if err := d.Decode(&extra); err == nil || !errors.Is(err, io.EOF) { + s.writeError(w, http.StatusBadRequest, api.CodeLaneInvalid, "expected exactly one JSON object") + return false + } + return true +} + +func (s *Server) handleLanePublish(w http.ResponseWriter, r *http.Request) { + var in api.LanePublish + if !s.readLaneJSON(w, r, &in) { + return + } + out, err := s.Lanes.Publish(r.Context(), laneProject(r), r.PathValue("lane"), in) + if err != nil { + s.writeLaneError(w, err) + return + } + s.writeJSON(w, http.StatusOK, out) +} + +func (s *Server) handleLaneReplay(w http.ResponseWriter, r *http.Request) { + var cursor uint64 + var err error + if text := r.URL.Query().Get("cursor"); text != "" { + cursor, err = strconv.ParseUint(text, 10, 64) + if err != nil { + s.writeLaneError(w, lanes.ErrCursor) + return + } + } + limit := defaultLaneReplayLimit + if text := r.URL.Query().Get("limit"); text != "" { + limit, err = strconv.Atoi(text) + if err != nil { + s.writeLaneError(w, lanes.ErrInvalid) + return + } + } + out, err := s.Lanes.Replay(r.Context(), laneProject(r), r.PathValue("lane"), cursor, limit) + if err != nil { + s.writeLaneError(w, err) + return + } + s.writeJSON(w, http.StatusOK, out) +} + +func (s *Server) handleLaneClaim(w http.ResponseWriter, r *http.Request) { + out, err := s.Lanes.Claim(r.Context(), laneProject(r), r.PathValue("lane"), r.PathValue("group")) + if errors.Is(err, lanes.ErrEmpty) { + w.WriteHeader(http.StatusNoContent) + return + } + + if err != nil { + s.writeLaneError(w, err) + return + } + s.writeJSON(w, http.StatusOK, out) +} + +func (s *Server) handleLaneGroup(w http.ResponseWriter, r *http.Request) { + out, err := s.Lanes.InspectGroup(r.Context(), laneProject(r), r.PathValue("lane"), r.PathValue("group")) + if err != nil { + s.writeLaneError(w, err) + return + } + s.writeJSON(w, http.StatusOK, out) +} + +func (s *Server) handleLaneRecover(w http.ResponseWriter, r *http.Request) { + var in api.LaneRecover + if !s.readLaneJSON(w, r, &in) { + return + } + out, err := s.Lanes.RecoverGroup(r.Context(), laneProject(r), r.PathValue("lane"), r.PathValue("group"), in) + if err != nil { + s.writeLaneError(w, err) + return + } + s.writeJSON(w, http.StatusOK, out) +} + +func (s *Server) handleLaneReceipt(operation string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var in api.LaneReceipt + if !s.readLaneJSON(w, r, &in) { + return + } + project, lane, group := laneProject(r), r.PathValue("lane"), r.PathValue("group") + var err error + switch operation { + case "ack": + err = s.Lanes.Ack(r.Context(), project, lane, group, in) + case "nack": + err = s.Lanes.Nack(r.Context(), project, lane, group, in) + case "renew": + expires, e := s.Lanes.Renew(r.Context(), project, lane, group, in) + if e != nil { + s.writeLaneError(w, e) + return + } + s.writeJSON(w, http.StatusOK, api.LaneRenewed{Expires: expires}) + return + } + if err != nil { + s.writeLaneError(w, err) + return + } + w.WriteHeader(http.StatusNoContent) + } +} + +func (s *Server) writeLaneError(w http.ResponseWriter, err error) { + status, code, message := http.StatusServiceUnavailable, api.CodeLaneUnavailable, "lane broker unavailable" + switch { + case errors.Is(err, lanes.ErrForbidden): + status, code, message = http.StatusForbidden, api.CodeLaneForbidden, "lane capability denied" + case errors.Is(err, lanes.ErrInvalid): + status, code, message = http.StatusBadRequest, api.CodeLaneInvalid, "invalid lane request" + case errors.Is(err, lanes.ErrConflict): + status, code, message = http.StatusConflict, api.CodeLaneConflict, "lane state conflict" + case errors.Is(err, lanes.ErrCursor): + status, code, message = http.StatusConflict, api.CodeLaneCursor, "invalid or expired lane cursor" + case errors.Is(err, lanes.ErrStale): + status, code, message = http.StatusConflict, api.CodeLaneStale, "stale lane receipt" + case errors.Is(err, lanes.ErrCapacity): + status, code, message = http.StatusTooManyRequests, api.CodeLaneCapacity, "lane capacity exhausted" + case errors.Is(err, context.DeadlineExceeded): + status, code, message = http.StatusGatewayTimeout, api.CodeLaneTimeout, "lane operation timed out" + case errors.Is(err, context.Canceled): + status, code, message = http.StatusRequestTimeout, api.CodeLaneCanceled, "lane operation canceled" + } + // Never expose broker connection details or credentials to API clients. + s.writeError(w, status, code, message) +} diff --git a/src/crux/serve/lanes_foreign_recovery_test.go b/src/crux/serve/lanes_foreign_recovery_test.go new file mode 100644 index 0000000000..a2955a18c6 --- /dev/null +++ b/src/crux/serve/lanes_foreign_recovery_test.go @@ -0,0 +1,97 @@ +package serve + +import ( + "context" + "net/http" + "reflect" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/lanes" +) + +const foreignRecoveryWait = 10 * time.Second + +func TestAuthorizedAPIForeignExpirationWithoutReceiptOrGap(t *testing.T) { + s, clients, _ := laneAPIFixtureConfig(t, func(c *lanes.Config) { + c.Retention, c.AckRetention = 3*time.Second, 3*time.Second + c.Lease, c.Dedup = lanes.MinLease, lanes.MinDedup + c.MaxLedger, c.MaxPending = 1, 1 + }) + ctx := context.Background() + if _, err := clients["atlas"].PublishLane(ctx, "exchange", + api.LanePublish{ID: "foreign", Kind: api.LaneWork, Schema: "v1", Payload: []byte("data")}); err != nil { + t.Fatal(err) + } + old, err := clients["beacon"].ClaimLane(ctx, "exchange", "workers") + if err != nil || old == nil { + t.Fatal(err) + } + status, err := clients["atlas"].InspectLaneGroup(ctx, "exchange", "workers") + if err != nil { + t.Fatal(err) + } + in := api.LaneRecover{ID: "foreign-expiry", Revision: status.Revision, + CursorThrough: status.Cursor, Missing: []uint64{old.Sequence}} + _, err = clients["atlas"].RecoverLaneGroup(ctx, "exchange", "workers", in) + requireLaneStatus(t, err, http.StatusConflict) + _, err = clients["beacon"].RecoverLaneGroup(ctx, "exchange", "workers", in) + requireLaneStatus(t, err, http.StatusForbidden) + oversized := in + oversized.Missing = make([]uint64, lanes.MaxRecoveryBatch+1) + _, err = clients["atlas"].RecoverLaneGroup(ctx, "exchange", "workers", oversized) + requireLaneStatus(t, err, http.StatusBadRequest) + ticker := time.NewTicker(lanes.MinRetryDelay) + defer ticker.Stop() + deadline := time.NewTimer(foreignRecoveryWait) + defer deadline.Stop() + for { + status, err = clients["atlas"].InspectLaneGroup(ctx, "exchange", "workers") + if err != nil { + t.Fatal(err) + } + if status.FirstRetained > old.Sequence && len(status.Missing) == 1 { + break + } + select { + case <-ticker.C: + case <-deadline.C: + t.Fatal("missing foreign delivery never became observable") + } + } + if len(status.Recoverable) != 0 || status.Cursor != status.FirstRetained-1 { + t.Fatal("regression requires no own receipt or unread gap") + } + in.Revision = status.Revision + out, err := clients["atlas"].RecoverLaneGroup(ctx, "exchange", "workers", in) + if err != nil || len(out.Expirations) != 1 || out.Expirations[0].Project != "beacon" { + t.Fatalf("foreign expiration: %+v %v", out, err) + } + cfg := s.Lanes.Config() + s.Lanes.Close() + s.Lanes, err = lanes.Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(s.Lanes.Close) + replayed, err := clients["atlas"].RecoverLaneGroup(ctx, "exchange", "workers", in) + if err != nil || !reflect.DeepEqual(out, replayed) { + t.Fatal("foreign recovery lost-response retry failed", err) + } + fresh, err := clients["atlas"].PublishLane(ctx, "exchange", + api.LanePublish{ID: "fresh", Kind: api.LaneWork, Schema: "v1", Payload: []byte("fresh")}) + if err != nil { + t.Fatal(err) + } + resumed, err := clients["atlas"].ClaimLane(ctx, "exchange", "workers") + if err != nil || resumed == nil || resumed.Sequence != fresh.Sequence { + t.Fatal("foreign expired lease still occupies sole capacity", err) + } + requireLaneStatus(t, clients["beacon"].AckLane(ctx, "exchange", "workers", api.LaneReceipt{ + Sequence: old.Sequence, Receipt: old.Receipt}), http.StatusConflict) + if err := clients["atlas"].AckLane(ctx, "exchange", "workers", api.LaneReceipt{ + Sequence: resumed.Sequence, Receipt: resumed.Receipt}); err != nil { + t.Fatal(err) + } +} diff --git a/src/crux/serve/lanes_recovery_test.go b/src/crux/serve/lanes_recovery_test.go new file mode 100644 index 0000000000..90cd907eef --- /dev/null +++ b/src/crux/serve/lanes_recovery_test.go @@ -0,0 +1,98 @@ +package serve + +import ( + "context" + "net/http" + "reflect" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/lanes" +) + +func TestLaneRecoveryAPIAuthFencingAndSameNamespaceRestart(t *testing.T) { + s, clients, _ := laneAPIFixtureConfig(t, func(c *lanes.Config) { + c.Retention, c.Lease, c.AckRetention = 2*time.Second, 2*time.Second, 2*time.Second + c.Dedup = lanes.MinDedup + }) + ctx := context.Background() + if _, err := clients["atlas"].PublishLane(ctx, "exchange", api.LanePublish{ID: "old", Kind: api.LaneWork, Schema: "v1", Payload: []byte("old")}); err != nil { + t.Fatal(err) + } + old, err := clients["atlas"].ClaimLane(ctx, "exchange", "workers") + if err != nil || old == nil { + t.Fatal("old claim", err) + } + gap, err := clients["atlas"].PublishLane(ctx, "exchange", api.LanePublish{ID: "gap", Kind: api.LaneEvent, Schema: "v1", Payload: []byte("gap")}) + if err != nil { + t.Fatal(err) + } + var status api.LaneGroupState + ticker := time.NewTicker(lanes.MinRetryDelay) + defer ticker.Stop() + deadline := time.NewTimer(10 * time.Second) + defer deadline.Stop() + for { + status, err = clients["atlas"].InspectLaneGroup(ctx, "exchange", "workers") + if err != nil { + t.Fatal(err) + } + if status.FirstRetained > gap.Sequence && !old.Expires.After(time.Now()) { + break + } + select { + case <-ticker.C: + case <-deadline.C: + t.Fatal("expired group prefix never became observable") + } + } + in := api.LaneRecover{ID: "recover-gap", Revision: status.Revision, + CursorThrough: status.FirstRetained - 1, Expired: status.Recoverable} + if len(in.Expired) != 1 { + t.Fatalf("own recoverable receipts not reported: %+v", status) + } + foreign, err := clients["beacon"].InspectLaneGroup(ctx, "exchange", "workers") + if err != nil || len(foreign.Recoverable) != 0 { + t.Fatal("other project's recovery receipt exposed", err) + } + _, err = clients["beacon"].RecoverLaneGroup(ctx, "exchange", "workers", in) + requireLaneStatus(t, err, http.StatusForbidden) + _, err = clients["atlas"].RecoverLaneGroup(ctx, "exchange", "beacon-only", in) + requireLaneStatus(t, err, http.StatusForbidden) + _, err = clients["admin"].RecoverLaneGroup(ctx, "exchange", "workers", in) + requireLaneStatus(t, err, http.StatusUnauthorized) + stale := in + stale.Revision++ + _, err = clients["atlas"].RecoverLaneGroup(ctx, "exchange", "workers", stale) + requireLaneStatus(t, err, http.StatusConflict) + fresh, err := clients["atlas"].PublishLane(ctx, "exchange", api.LanePublish{ID: "fresh", Kind: api.LaneWork, Schema: "v1", Payload: []byte("fresh")}) + if err != nil { + t.Fatal(err) + } + skip := in + skip.CursorThrough = fresh.Sequence + _, err = clients["atlas"].RecoverLaneGroup(ctx, "exchange", "workers", skip) + requireLaneStatus(t, err, http.StatusConflict) + requireLaneStatus(t, clients["atlas"].Do(ctx, http.MethodPost, api.PathLanes+"/exchange/groups/workers/recover", map[string]any{"id": in.ID, "revision": in.Revision, "project": "beacon"}, nil), http.StatusBadRequest) + recovered, err := clients["atlas"].RecoverLaneGroup(ctx, "exchange", "workers", in) + if err != nil || recovered.GapThrough != gap.Sequence { + t.Fatalf("group recovery: %+v %v", recovered, err) + } + cfg := s.Lanes.Config() + s.Lanes.Close() + s.Lanes, err = lanes.Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(s.Lanes.Close) + replayed, err := clients["atlas"].RecoverLaneGroup(ctx, "exchange", "workers", in) + if err != nil || !reflect.DeepEqual(recovered, replayed) { + t.Fatal("response loss/restart recovery was not idempotent", err) + } + resumed, err := clients["beacon"].ClaimLane(ctx, "exchange", "workers") + if err != nil || resumed == nil || resumed.Sequence != fresh.Sequence { + t.Fatal("same-namespace work stranded after recovery", err) + } + requireLaneStatus(t, clients["atlas"].AckLane(ctx, "exchange", "workers", api.LaneReceipt{Sequence: old.Sequence, Receipt: old.Receipt}), http.StatusConflict) +} diff --git a/src/crux/serve/lanes_test.go b/src/crux/serve/lanes_test.go new file mode 100644 index 0000000000..91ceea353d --- /dev/null +++ b/src/crux/serve/lanes_test.go @@ -0,0 +1,309 @@ +package serve + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/lanes" + "github.com/nats-io/nats-server/v2/server" +) + +func laneAPIFixtureConfig(t *testing.T, change func(*lanes.Config)) (*Server, map[string]*api.Client, *httptest.Server) { + t.Helper() + dir := filepath.Join(".", fmt.Sprintf(".lane-fixture-%s-%d", strings.ReplaceAll(t.Name(), "/", "-"), time.Now().UnixNano())) + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }) + ns, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: dir, NoLog: true, NoSigs: true}) + if err != nil { + t.Fatal(err) + } + go ns.Start() + if !ns.ReadyForConnections(time.Second * 10) { + ns.Shutdown() + t.Fatal("NATS not ready") + } + t.Cleanup(func() { + ns.Shutdown() + ns.WaitForShutdown() + }) + cfg := lanes.Config{URL: ns.ClientURL(), + Principals: []lanes.Principal{{Project: "atlas", TokenEnv: "ATLAS"}, {Project: "beacon", TokenEnv: "BEACON"}, {Project: "outsider", TokenEnv: "OUTSIDER"}}, + Lanes: []api.Lane{{ID: "exchange", Owner: "atlas", Publishers: []string{"atlas", "beacon"}, Subscribers: []string{"atlas", "beacon"}, + Groups: map[string][]string{"workers": {"atlas", "beacon"}, "beacon-only": {"beacon"}}, + Recoverers: map[string][]string{"workers": {"atlas"}}}}} + cfg.Defaults() + cfg.RetryDelay, cfg.MaxRetryDelay = lanes.MinRetryDelay, lanes.MinRetryDelay + if change != nil { + change(&cfg) + } + broker, err := lanes.Open(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(broker.Close) + tokens := map[string]string{"ATLAS": strings.Repeat("a", 32), "BEACON": strings.Repeat("b", 32), "OUTSIDER": strings.Repeat("c", 32)} + auth, err := lanes.Bind(cfg, func(name string) string { return tokens[name] }, "agent-admin") + if err != nil { + t.Fatal(err) + } + s := New(newFakeRuntime()) + s.Token, s.Lanes, s.LaneAuth = "agent-admin", broker, auth + httpServer := httptest.NewServer(s.Handler()) + t.Cleanup(httpServer.Close) + clients := map[string]*api.Client{} + for project, env := range map[string]string{"atlas": "ATLAS", "beacon": "BEACON", "outsider": "OUTSIDER", "admin": ""} { + token := tokens[env] + if project == "admin" { + token = s.Token + } + c, err := api.NewClient(httpServer.URL, token) + if err != nil { + t.Fatal(err) + } + clients[project] = c + } + return s, clients, httpServer +} + +func laneAPIFixture(t *testing.T) (*Server, map[string]*api.Client, *httptest.Server) { + return laneAPIFixtureConfig(t, nil) +} + +func requireLaneStatus(t *testing.T, err error, status int) { + t.Helper() + var se *api.StatusError + if !errors.As(err, &se) || se.Status != status { + t.Fatalf("wanted HTTP %d, got %v", status, err) + } +} + +func TestTwoForgeFreeProjectsExchangeThroughAPI(t *testing.T) { + _, clients, _ := laneAPIFixture(t) + ctx := context.Background() + for _, project := range []string{"atlas", "beacon"} { + registry, err := clients[project].LaneRegistry(ctx) + if err != nil || len(registry) != 1 { + t.Fatalf("registry: %+v %v", registry, err) + } + if _, err := clients[project].PublishLane(ctx, "exchange", api.LanePublish{ + ID: project + "-event", Kind: api.LaneEvent, Schema: "measurement-v1", Payload: []byte(`{"temperature":21}`), + }); err != nil { + t.Fatal(err) + } + } + for _, project := range []string{"atlas", "beacon"} { + page, err := clients[project].ReplayLane(ctx, "exchange", 0, 8) + if err != nil || len(page.Events) != 2 { + t.Fatalf("fanout %+v %v", page, err) + } + if page.Events[0].Envelope.Project != "atlas" || page.Events[1].Envelope.Project != "beacon" { + t.Fatal("project identity not bound to authentication") + } + } + work := api.LanePublish{ID: "summarize", Kind: api.LaneWork, Schema: "summary-v1", Payload: []byte(`{}`)} + pub, err := clients["atlas"].PublishLane(ctx, "exchange", work) + if err != nil { + t.Fatal(err) + } + dup, err := clients["atlas"].PublishLane(ctx, "exchange", work) + if err != nil || !dup.Duplicate || dup.Sequence != pub.Sequence { + t.Fatalf("publish duplicate %+v %v", dup, err) + } + work.Payload = []byte("changed") + _, err = clients["atlas"].PublishLane(ctx, "exchange", work) + requireLaneStatus(t, err, http.StatusConflict) + c, err := clients["beacon"].ClaimLane(ctx, "exchange", "workers") + if err != nil || c == nil { + t.Fatalf("claim %+v %v", c, err) + } + empty, err := clients["atlas"].ClaimLane(ctx, "exchange", "workers") + if err != nil || empty != nil { + t.Fatalf("204 claim %+v %v", empty, err) + } + r := api.LaneReceipt{Sequence: c.Sequence, Receipt: c.Receipt} + if _, err := clients["beacon"].RenewLane(ctx, "exchange", "workers", r); err != nil { + t.Fatal(err) + } + if err := clients["beacon"].NackLane(ctx, "exchange", "workers", r); err != nil { + t.Fatal(err) + } + state, err := clients["beacon"].InspectLaneGroup(ctx, "exchange", "workers") + if err != nil || len(state.Retries) != 1 { + t.Fatalf("retry state %+v %v", state, err) + } + timer := time.NewTimer(time.Until(state.Retries[0].AvailableAt)) + <-timer.C + redelivery, err := clients["atlas"].ClaimLane(ctx, "exchange", "workers") + if err != nil || redelivery == nil || redelivery.Attempt != 2 { + t.Fatalf("redelivery %+v %v", redelivery, err) + } + requireLaneStatus(t, clients["beacon"].AckLane(ctx, "exchange", "workers", r), http.StatusConflict) + requireLaneStatus(t, clients["beacon"].AckLane(ctx, "exchange", "beacon-only", r), http.StatusConflict) + r = api.LaneReceipt{Sequence: redelivery.Sequence, Receipt: redelivery.Receipt} + for range 2 { + if err := clients["atlas"].AckLane(ctx, "exchange", "workers", r); err != nil { + t.Fatal(err) + } + } + other, err := clients["beacon"].ClaimLane(ctx, "exchange", "beacon-only") + if err != nil || other == nil || other.Sequence != pub.Sequence { + t.Fatalf("per-group delivery %+v %v", other, err) + } +} + +func TestLaneAuthDoesNotInheritAgentOrForgeAuthority(t *testing.T) { + s, clients, hs := laneAPIFixture(t) + ctx := context.Background() + requireLaneStatus(t, clients["atlas"].Do(ctx, http.MethodGet, api.PathAgents, nil, nil), http.StatusUnauthorized) + requireLaneStatus(t, clients["admin"].Do(ctx, http.MethodGet, api.PathLanes, nil, nil), http.StatusUnauthorized) + if err := clients["admin"].Do(ctx, http.MethodGet, api.PathAgents, nil, nil); err != nil { + t.Fatal("legacy admin regressed", err) + } + registry, err := clients["outsider"].LaneRegistry(ctx) + if err != nil || len(registry) != 0 { + t.Fatal("outsider can see registry", err) + } + _, err = clients["outsider"].ReplayLane(ctx, "exchange", 0, 1) + requireLaneStatus(t, err, http.StatusForbidden) + _, err = clients["outsider"].ClaimLane(ctx, "exchange", "workers") + requireLaneStatus(t, err, http.StatusForbidden) + _, err = clients["outsider"].PublishLane(ctx, "exchange", api.LanePublish{ID: "x", Kind: api.LaneEvent, Schema: "v1", Payload: []byte("x")}) + requireLaneStatus(t, err, http.StatusForbidden) + request, err := http.NewRequest(http.MethodPost, hs.URL+api.PathLanes+"/exchange/publish", strings.NewReader(`{"id":"x","kind":"event","schema":"v1","payload":"eA==","project":"atlas"}`)) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+strings.Repeat("c", 32)) + request.Header.Set("X-Project", "atlas") + request.Header.Set("X-GitHub-Repository", "irrelevant/forged") + response, err := hs.Client().Do(request) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != http.StatusBadRequest { + t.Fatalf("forged publisher accepted: %d", response.StatusCode) + } + // A transport-local request without a project token still has no lane principal. + request = httptest.NewRequest(http.MethodGet, api.PathLanes, nil) + request = request.WithContext(context.WithValue(request.Context(), networkKey, networkUnix)) + recorder := httptest.NewRecorder() + s.Handler().ServeHTTP(recorder, request) + if recorder.Code != http.StatusUnauthorized { + t.Fatal("unix socket inherited lane permission") + } + requireLaneStatus(t, clients["atlas"].Do(ctx, http.MethodPost, api.PathLanes, map[string]string{"owner": "atlas"}, nil), http.StatusMethodNotAllowed) +} + +func TestLaneMalformedAndCancellationPaths(t *testing.T) { + _, clients, hs := laneAPIFixture(t) + ctx := context.Background() + for _, query := range []string{"cursor=-1", "cursor=no", "limit=no", "limit=0", "limit=999"} { + err := clients["atlas"].Do(ctx, http.MethodGet, api.PathLanes+"/exchange/events?"+query, nil, nil) + var se *api.StatusError + if !errors.As(err, &se) || (se.Status != http.StatusBadRequest && se.Status != http.StatusConflict) { + t.Fatalf("%s: %v", query, err) + } + } + requireLaneStatus(t, clients["atlas"].AckLane(ctx, "exchange", "workers", api.LaneReceipt{}), http.StatusBadRequest) + _, err := clients["atlas"].PublishLane(ctx, "exchange", api.LanePublish{}) + requireLaneStatus(t, err, http.StatusBadRequest) + err = clients["atlas"].Do(ctx, http.MethodPost, api.PathLanes+"/exchange/publish", json.RawMessage(`{"id":"x"} {}`), nil) + if err == nil { + t.Fatal("malformed request accepted") + } + for _, body := range []string{"{", "{} {}"} { + request, err := http.NewRequest(http.MethodPost, hs.URL+api.PathLanes+"/exchange/publish", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+strings.Repeat("a", 32)) + response, err := hs.Client().Do(request) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != http.StatusBadRequest { + t.Fatal("invalid raw lane JSON accepted") + } + } + ctx, cancel := context.WithCancel(ctx) + cancel() + if _, err := clients["atlas"].ReplayLane(ctx, "exchange", 0, 1); err == nil { + t.Fatal("canceled request succeeded") + } +} + +func TestLaneFailureHTTPClassification(t *testing.T) { + s := New(newFakeRuntime()) + for _, test := range []struct { + err error + status int + code string + message string + }{ + {lanes.ErrInvalid, http.StatusBadRequest, api.CodeLaneInvalid, "invalid lane request"}, + {lanes.ErrForbidden, http.StatusForbidden, api.CodeLaneForbidden, "lane capability denied"}, + {lanes.ErrConflict, http.StatusConflict, api.CodeLaneConflict, "lane state conflict"}, + {lanes.ErrCursor, http.StatusConflict, api.CodeLaneCursor, "invalid or expired lane cursor"}, + {lanes.ErrStale, http.StatusConflict, api.CodeLaneStale, "stale lane receipt"}, + {lanes.ErrCapacity, http.StatusTooManyRequests, api.CodeLaneCapacity, "lane capacity exhausted"}, + {lanes.ErrUnavailable, http.StatusServiceUnavailable, api.CodeLaneUnavailable, "lane broker unavailable"}, + {context.DeadlineExceeded, http.StatusGatewayTimeout, api.CodeLaneTimeout, "lane operation timed out"}, + {context.Canceled, http.StatusRequestTimeout, api.CodeLaneCanceled, "lane operation canceled"}, + {errors.New("dummy-backend-detail"), http.StatusServiceUnavailable, api.CodeLaneUnavailable, "lane broker unavailable"}, + } { + t.Run(test.err.Error(), func(t *testing.T) { + for _, err := range []error{test.err, fmt.Errorf("%w: tls://dummy-user:dummy-password@broker.example:4222", test.err)} { + w := httptest.NewRecorder() + s.writeLaneError(w, err) + want := fmt.Sprintf("{\"error\":%q,\"code\":%q}\n", test.message, test.code) + if w.Code != test.status || w.Header().Get("Content-Type") != api.ContentTypeJSON || w.Body.String() != want { + t.Fatalf("HTTP %d, headers %v, body %q; want HTTP %d, body %q", w.Code, w.Header(), w.Body.String(), test.status, want) + } + } + }) + } +} + +func TestLaneJSONFailureUsesPublicCode(t *testing.T) { + s := New(newFakeRuntime()) + for _, test := range []struct { + body string + message string + }{ + {`{"unknown":"dummy-secret"}`, "invalid lane JSON body"}, + {`not-json`, "invalid lane JSON body"}, + {`{} {}`, "expected exactly one JSON object"}, + {`{} not-json`, "expected exactly one JSON object"}, + } { + t.Run(test.body, func(t *testing.T) { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, api.PathLanes+"/shared/publish", strings.NewReader(test.body)) + var in api.LanePublish + if s.readLaneJSON(w, r, &in) { + t.Fatal("invalid lane JSON accepted") + } + want := fmt.Sprintf("{\"error\":%q,\"code\":%q}\n", test.message, api.CodeLaneInvalid) + if w.Code != http.StatusBadRequest || w.Header().Get("Content-Type") != api.ContentTypeJSON || w.Body.String() != want { + t.Fatalf("HTTP %d, headers %v, body %q; want HTTP 400, body %q", w.Code, w.Header(), w.Body.String(), want) + } + }) + } +} diff --git a/src/crux/serve/serve.go b/src/crux/serve/serve.go index 702e3f3099..a88d328b12 100644 --- a/src/crux/serve/serve.go +++ b/src/crux/serve/serve.go @@ -36,6 +36,7 @@ import ( "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/api" "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/lanes" ) const ( @@ -96,6 +97,11 @@ type Server struct { // endpoint's reachability as `crux serve --hive-callback` sees it. HiveCallback func() api.HiveCallback + // Lanes and LaneAuth enable the separate, project-authenticated durable + // lane surface. They confer no agent-control permissions. + Lanes *lanes.Broker + LaneAuth *lanes.Authenticator + now func() time.Time started time.Time @@ -149,7 +155,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST "+api.PathAgents+"/{name}/stop", s.handleStop) mux.HandleFunc("GET "+api.PathEvents, s.handleEvents) s.registerFamilies(mux) - return s.authenticate(mux) + return s.laneHandler(s.authenticate(mux)) } // agentNamePattern is the CLI's definition-name gate (cmd/crux @@ -164,6 +170,11 @@ var agentNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`) func (s *Server) authenticate(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { network, _ := r.Context().Value(networkKey).(string) + token, _ := strings.CutPrefix(r.Header.Get("Authorization"), bearerPrefix) + if _, projectToken := s.LaneAuth.Project(token); projectToken { + s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "project credentials cannot access agent RPCs") + return + } if s.Token == "" { if network != networkUnix { s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "this server has no bearer token; only unix-socket requests are accepted") diff --git a/src/docs/crux-lanes.md b/src/docs/crux-lanes.md new file mode 100644 index 0000000000..1f6f7faab6 --- /dev/null +++ b/src/docs/crux-lanes.md @@ -0,0 +1,394 @@ +# Crux durable lanes + +Crux owns the lane contract. Projects are peer publishers/subscribers; Hive is +one optional participant, not the broker authority. No repository, issue, pull +request, forge label or forge credential is required. Lane IDs, project IDs, +message IDs, group IDs and schema IDs are plain `^[a-z][a-z0-9_-]*$` identifiers, +at most 64 characters. These lanes are unrelated to trajectory safety lanes. + +## Enable in the existing Crux service + +Use `crux serve --lanes-config ./lanes.json`, or set `CRUX_LANES_CONFIG`. +This adds routes to the **existing** HTTP/JSON Unix-socket/loopback-TCP server; +it does not start another daemon. Without this option, existing behavior is +unchanged. `/v1/events` remains the lossy, in-memory legacy observer SSE stream; +it is **not** the durable lane log. + +Broker initialization failures distinguish typed `state conflict` and +`broker unavailable` categories; other failures remain explicit initialization +errors. These CLI diagnostics never print raw backend errors, broker addresses +or credentials. + +Example administrator-owned `lanes.json` (no secrets): + +```json +{ + "url": "tls://broker.example.net:4222", + "credentials_env": "CRUX_NATS_CREDENTIALS_FILE", + "namespace": "commons", + "principals": [ + {"project": "atlas", "token_env": "ATLAS_LANE_TOKEN"}, + {"project": "beacon", "token_env": "BEACON_LANE_TOKEN"} + ], + "lanes": [{ + "id": "measurements", + "owner": "atlas", + "publishers": ["atlas", "beacon"], + "subscribers": ["atlas", "beacon"], + "groups": {"summarizers": ["beacon"]}, + "recoverers": {"summarizers": ["beacon"]} + }] +} +``` + +Generate distinct project bearer tokens of 32–4,096 bytes in a secret +manager; inject them through the named environment variables. +`CRUX_NATS_CREDENTIALS_FILE` contains the path to a protected NATS credentials +file, not its contents. The existing `CRUX_TOKEN` remains an **agent-control** +credential. It must differ from every project credential. + +The configured project token determines the principal, never an envelope, +`X-Project` header, URI or forge identity. Project tokens cannot call existing +agent-control RPCs; the agent-control token and local socket identity do not +grant lane access. Membership is explicit per capability: ownership alone does +not grant publish, event subscription or work claims. Group membership grants +work consumption, not event subscription. No credential is returned in registry +responses. Provenance is optional, untrusted absolute URI metadata; inline URI +credentials are rejected and it must never contain secrets. + +## Public HTTP and Go API + +All lane routes require a configured project bearer token: + +| Route | Capability / result | +|---|---| +| `GET /v1/lanes` | Registry filtered to the principal's memberships | +| `POST /v1/lanes/{lane}/publish` | Publisher; `LanePublished` | +| `GET /v1/lanes/{lane}/events?cursor=0&limit=32` | Event subscriber; `LaneReplay` | +| `POST /v1/lanes/{lane}/groups/{group}/claim` | Group member; `LaneClaim`, or 204 when empty | +| `GET /v1/lanes/{lane}/groups/{group}` | Group member; revision, reusable slot allocation, retained bounds, missing identities, own receipts/retries and recovery history | +| `POST .../groups/{group}/recover` | Explicitly configured group recoverer; revision-fenced expired-gap acceptance | +| `POST .../groups/{group}/renew` | Current lease holder; renewed expiry | +| `POST .../groups/{group}/ack` | Current lease holder; 204 | +| `POST .../groups/{group}/nack` | Current lease holder; 204 and immediate eligibility for redelivery | + +Publish request: + +```json +{"id":"reading-one","kind":"event","schema":"temperature-v1","payload":"eyJjZWxzaXVzIjoyMX0="} +``` + +`payload` is a bounded byte array encoded as base64 by JSON; here it contains +`{"celsius":21}`. Use `kind:"work"` for claimable work. `provenance` is optional. +The durable envelope adds authenticated `project`, `lane` and `created_at`. +`id` is the idempotency key within `(lane, project)`. A different kind, schema, +payload or provenance with the same retained key fails with `lane_conflict`. + +Ack/nack/renew requests contain only the returned sequence and receipt: + +```json +{"sequence":3,"receipt":"<64 hexadecimal characters returned by claim>"} +``` + +`crux/api.Client` exposes `LaneRegistry`, `PublishLane`, `ReplayLane`, +`ClaimLane`, `RenewLane`, `AckLane`, `NackLane`, `InspectLaneGroup` and +`RecoverLaneGroup`. It uses the existing Crux +address parsing, transport and typed `StatusError`. `ClaimLane` returns +`nil, nil` for 204, not a zero-valued work message. + +Forge-free peer example: + +```go +atlas, err := api.NewClient(address, os.Getenv("ATLAS_LANE_TOKEN")) +if err != nil { return err } +beacon, err := api.NewClient(address, os.Getenv("BEACON_LANE_TOKEN")) +if err != nil { return err } +_, err = atlas.PublishLane(ctx, "measurements", api.LanePublish{ + ID: "summarize-one", Kind: api.LaneWork, + Schema: "summary-v1", Payload: []byte(`{"values":[21,22]}`), +}) +if err != nil { return err } +work, err := beacon.ClaimLane(ctx, "measurements", "summarizers") +if err != nil { return err } +if work == nil { return nil } +// Perform a registered, authorized operation; make its side effects idempotent. +return beacon.AckLane(ctx, "measurements", "summarizers", api.LaneReceipt{ + Sequence: work.Sequence, Receipt: work.Receipt, +}) +``` + +## Durability, cursors and concurrency + +Every accepted publish has a JetStream acknowledgement. Immutable per-lane +file-backed streams retain both events and work; each subscriber stores its +own event cursor. Replay is a bounded pull page, never a dropping channel. +The cursor is the last **examined** stream sequence, including work records +skipped by event replay. Pages examine at most 512 records and return at most +128 events. Save the returned cursor only after processing the events. +Cursor zero explicitly starts at the earliest retained record; a future cursor +or a nonzero cursor with an expired gap fails with `lane_cursor`. Missing +retained messages during a scan also fail rather than silently skipping. + +Work groups have independent persistent cursor/lease ledgers in JetStream KV. +KV compare-and-set is the ownership linearization point, not process memory. +Competing clients in a group claim a work record once per live lease; other +groups receive their own delivery. A lost response or service crash leaves a +lease that expires and redelivers. A random receipt fences each assignment and +binds lane, group, authenticated holder, sequence and ownership generation. +Stale or cross-project/group ack, nack and renew fail, including after +reassignment. No broker ACK subject is exposed to a participant. + +Stream publication and KV ownership are not claimed to be one transaction. +Publication first reserves a key/payload hash in KV, then publishes with a +generation-specific JetStream message ID and a **subject expected-sequence CAS**. After a crash, +retry verifies the persisted subject before completing the publish. This +prevents concurrent mismatched payloads and repeated retained publications, +including outside the broker's finite duplicate window. A reservation has a +random persisted generation; a resumed request reuses it. The broker ACK +sequence (including **normal** and **duplicate** ACKs) is read back and checked +against the exact envelope, subject and generation, and the active KV +reservation is rechecked before success. A message-ID duplicate is never +trusted as proof of this request's payload: NATS deduplicates before subject +CAS. Generation-specific IDs prevent an expired request from inheriting the +ACK of a different payload published under a reused key. + +Reservation and stream writes are separate broker operations, not an atomic +cross-resource transaction. A request fenced by expiry/replacement can fail +even if its own message reached durable storage; unavailable or expired +metadata also fails explicitly. Retry/reconcile rather than treating any +error as proof that nothing was stored. Successful responses always resolve +to matching persisted content. Reservations are +bounded by retention; IDs can be reused once both reservation and log record +have expired. **There is no infinite idempotence guarantee.** + +Acknowledgements are idempotent for the configured acknowledgement-retention +period. Completed receipts occupy bounded ledger slots during that period; +once pruned, an old acknowledgement fails as stale. Unacked work retries until +successful ack or log retention expiry; no automatic dead-letter discard is +performed. Expired pending work fails explicitly until authorized gap recovery. +Receipts are bearer secrets scoped to their project/group: do not log them. + +### Explicit same-namespace group recovery + +An operator configures `recoverers` per group; each recoverer must also be a +group member. Group membership, lane ownership, an agent-control token, Hive +identity and a local socket do **not** implicitly grant this capability. + +Read `InspectLaneGroup`, then submit: + +```json +{ + "id": "accept-expired-gap-one", + "revision": 42, + "cursor_through": 19, + "expired": [{"sequence": 12, "receipt": ""}], + "missing": [13] +} +``` + +The revision must match the entire group state. Concurrent claims, renewal, +ACK or another recovery cause a CAS conflict; refresh the inspection and use +a new request ID if changing the request. `cursor_through` must be exactly the +expired prefix boundary (`first_retained - 1`) when there is a cursor gap, or +the current cursor when there is no gap. It cannot advance across retained +unclaimed work. Each explicitly retired delivery must be missing from retention, +lease-expired and unacknowledged. `expired` additionally requires the caller's +exact original receipt. `missing` lists inspected sequences without needing +another project's secret receipt: the separately configured recovery capability +authorizes expiration of absent owners' deliveries in this group. Neither form +can retire live, retained, completed or foreign-group work. At most 16 combined +deliveries can be accepted per request; inspect and submit successive batches +with refreshed revisions. Duplicate sequences are invalid. + +The result durably records the accepted cursor gap, observed `missing_through` +prefix and expired delivery retirements without marking them acknowledged. +Valid pending leases and ACK history remain intact. The result's `expirations` +records the original project, sequence and attempt of every retired delivery; +its `project` identifies the authenticated recoverer. This is an explicit +expiration audit, **not** a successful ACK. Retired entries release their +ledger slots immediately, including when the owner has disconnected forever. +Recovery requires neither an unrelated unread gap nor an own-project receipt. +Original receipts can no longer ACK, NACK or renew these deliveries. +The missing-prefix checkpoint does not silently suppress unlisted pending +deliveries; those require their own explicit, fenced acceptance. + +The exact request ID/body and result are persisted together in the same group +CAS update. Retrying a lost response, including after service/broker restart, +returns that result without reapplying recovery. A changed body or another +principal using the ID conflicts. Detailed history and recovery idempotence are +bounded by `ack_retention`, at most 64 recovery records per group; overflow is +explicit backpressure. The accepted missing-prefix checkpoint and cursor are +persistent fixed-size state. No new namespace or daemon is required. + +Delivery is **at least once**, not exactly-once side effects or execution. +An operation may finish and crash before ack, or continue after losing its +lease; applications must honor cancellation and use downstream idempotence or +fencing for effects. Renew before expiry. No global FIFO or ordering of work +completion is promised. Stream sequence orders accepted records per lane; +eligible retry selection rotates by a persistent sequence cursor, while a +separate persisted turn alternates retries and fresh work when both exist. +Rotation wraps among ready deliveries, so an always-ready low-sequence poison +item cannot starve already-claimed higher-sequence work after response loss. +When retries exist, the next retry turn is committed before attempting fresh +value growth: even a failed shared-bucket admission cannot pin the group +forever to unsuccessful fresh attempts. +NACKs schedule durable +exponential retry delay capped by `max_retry_delay`; duplicate NACKs do not move +the deadline. Polling slower than the retry delay cannot let one rejected item +starve fresh work. Released/expired retries do not consume live-lease slots, +but remain in the bounded ledger and are never silently ACKed or discarded. + +## Bounds and failure behavior + +Configuration uses integer nanoseconds for duration fields; omitted or zero +values select defaults. Negative/invalid explicit values fail initialization. +Short retention remains supported even when an operation timeout is longer: +expiry produces explicit generation/cursor errors, not a false duplicate ACK. +For normal deployments choose retention comfortably above polling, retry and +operation-deadline requirements. + +| Setting | Default / accepted bounds | +|---|---| +| `retention` | 7 days; 1 second–30 days | +| `dedup` | 2 minutes; 100 milliseconds–retention | +| `lease` | 1 minute; 1 second–1 hour | +| `ack_retention` | 1 hour; lease–retention | +| `retry_delay` | 1 second; 100 milliseconds–`max_retry_delay` | +| `max_retry_delay` | 1 minute; retry delay–1 hour | +| `timeout` | 5 seconds; positive–1 minute | +| `max_payload` | 64 KiB; positive–1 MiB | +| `max_messages` | 10,000 per lane; 1–1,000,000 | +| `max_bytes` | 64 MiB per lane; payload size–1 GiB | +| `max_kv_bytes` | 64 MiB per bucket; positive–1 GiB | +| `max_pending` | 128 live leases per group; positive and at most ledger size | +| `max_ledger` | 1,024 per group; pending size–10,000, additionally limited by worst-case reservations within 512 KiB | +| `replicas` | 1; 1–5 subject to broker topology | + +Registry configuration permits at most 64 lanes, 64 principals, 64 groups per +lane and 64 members per capability/group. The serialized registry is bounded +to 512 KiB and its input file to 1 MiB. Provenance is at most 2,048 bytes. +Streams use `DiscardNew`: exceeding capacity rejects new publications; it +does not evict accepted records before age retention. All request operations +have deadlines. Reconnect buffering is disabled: disconnection returns an +error, never a success-shaped queued publication. Reconnection restores +access to persisted state; retry with the same idempotency key. + +HTTP errors distinguish invalid requests (400), missing credentials (401), +denied capabilities (403), state/cursor/stale-receipt conflicts (409), group +capacity (429), cancellation (408), unavailable broker (503), and deadlines (504). +The existing JSON `code` values have Go constants in `crux/api`: +`CodeLaneInvalid`, `CodeLaneForbidden`, `CodeLaneConflict`, `CodeLaneCursor`, +`CodeLaneStale`, `CodeLaneCapacity`, `CodeLaneCanceled`, `CodeLaneUnavailable` +and `CodeLaneTimeout`. Missing credentials still use `CodeUnauthorized`. +Broker-side capacity/storage failures are unavailable errors, not accepted +publications. A successful duplicate is returned only after verifying a +durably persisted matching record. + +Claim admission reserves the serialized maximum of all fields of every +delivery (including maximum-width attempts, IDs and UTC nanosecond timestamps) +plus all 64 possible recovery responses of 16 deliveries each, gap metadata +and request hashes. Space padding allocates this reservation in the actual +namespace KV value. ACK, NACK and renew do not grow its reserved size; recovery +uses already-reserved history and removes expired delivery entries. Near the +512 KiB byte limit, new claims fail before consuming completion headroom. +The effective delivery count can therefore be below `max_ledger`, even with +short actual IDs. History is pruned only after its documented retention window. +An exhausted recovery-history window applies finite backpressure until expiry. + +Each persisted group has a durable, nonshrinking slot high-water mark. +An initial empty poll reserves **at least one reusable delivery slot**, not +just cursor/history bytes; its later first claim therefore needs no value +growth. Pruning completed records and retiring missing foreign deliveries +free entries **within** those slots without reducing their physical reservation. +Previously allocated slots remain reusable after service/broker restart and +response loss. Inspection reports `slots` (zero for an uninitialized group). +Slot allocation cannot exceed `max_ledger` or the 512 KiB worst-case bound. +Additional slots require successful broker admission before a new lease is +accepted; failed expansion does not remove existing reusable slots. + +The same physical reservation prevents newly admitted groups from consuming +existing groups' value-growth headroom in the shared bucket. Padding also +compensates byte-for-byte for the pinned driver's decimal CAS-header growth. +Bucket exhaustion +rejects admission explicitly. Broker metadata, account limits, replication, +network and disk availability can still fail any operation; reservations are +not a guarantee of infinite broker capacity or availability. The internal +state-format generation 3 rejects incompatible older prototype writers. + +Stream names use a bounded digest of the namespace/lane tuple, not ambiguous +underscore concatenation. Subject, KV and group-key boundaries use validated +identifiers and disjoint fixed delimiters/suffixes. + +The registry and delivery bounds form an immutable namespace generation. +Startup rejects incompatible existing streams, KV buckets or registry; it +does not rewrite another service's permissions. Membership/admin mutations +are **not supported over the project API** (405/404); explicit group recovery +is a separate configured capability, and ownership is not a +remote administrator grant. Operators provision reviewed generations through +configuration, coordinating migration/replay before retiring the old one. + +## Hive adapter + +Set `HIVE_LANES_ADDRESS`, `HIVE_LANES_TOKEN`, `HIVE_LANES_LANE` and +`HIVE_LANES_GROUP` to opt Hive into an ordinary Crux project membership. +`HIVE_LANES_SNAPSHOT_PROJECTS` is an optional comma-separated explicit publisher +allowlist for `hive-governor-snapshot-request-v1` work. Without it no snapshot +handler is registered. Requests must contain `{}`; they can only read governor +state and publish a correlated `hive-governor-snapshot-v1` event, not invoke +tools, prompts, forge writes or merges. At-least-once retries may emit multiple +correlated snapshots if state changes between attempts. + +Committed governor mode changes are additionally published with schema +`hive-governor-mode-change-v1`; existing local hooks remain attached. A broker +failure logs that durable delivery of the transition was **not confirmed**; this optional +adapter does not pretend the governor state commit and remote publish are an +atomic outbox. The public `pkg/laneparticipant` adapter accepts only explicit +schema-and-publisher handler registrations and bounds handlers by their lease. +Unhandled/unauthorized/failed work is nacked, not executed or acknowledged. +Hive's existing trust and forge-write policies remain unchanged; any future +write handler must independently apply them rather than treating lane +membership as write authority. + +## Deployment and verification + +No NATS deployment is installed by this feature. Provision JetStream with +durable storage and backups yourself; use TLS with certificate validation, +NATS account authentication and subject/API ACLs, plus a TLS proxy for remote +Crux HTTP access (the existing listener allows only loopback plain HTTP). +Only Crux service credentials may access this namespace's streams/KV/API. +Participants use Crux HTTP credentials, not direct broker credentials; giving +them direct KV access would bypass fencing and registry policy. +Protect credentials and private Unix sockets with OS ownership/modes. +Configure NATS `max_payload` above the base64-expanded envelope size and +512 KiB ledger ceiling, and provision sufficient account stream/byte limits. +Use synchronized service clocks for lease/expiry decisions; KV CAS remains +the assignment fence. Multi-replica deployments require matching configuration +and supported JetStream topology; single-replica defaults are not an HA claim. + +The named CI integration step tests real JetStream lanes, API and serving in +the standalone Crux module and the Hive participant through the root module. +CI's existing two-module affected-package test matrix also discovers the new +`crux/lanes` and `pkg/laneparticipant` packages, with the normal 90% coverage +gate, and builds both the Hive host and standalone Crux module. Tests start +real bounded embedded JetStream servers in isolated project-relative fixtures +and stop/remove them. They exercise forge-free two-project HTTP exchange, +fan-out, duplicate/mismatched publish, independent/competing groups, +ack/renew/nack, expiry and stale fencing, persisted restart and pending-publish +recovery, broker outage/reconnect, retention/cursors, bounds, cancellation, +auth isolation and the Hive adapter. Independent child Crux processes exercise +delayed publication across reservation expiry and durable retry/fresh fairness. +Recovery regressions include lost responses/restart, concurrent claim/ACK CAS, +live lease and ACK-history preservation, absent foreign-owner expiration with +sole-slot/no-gap recovery, accepted missing prefixes and API +capability isolation. Byte-bound regressions admit a state within one delivery +of 512 KiB, exercise ACK/NACK/renew and retry, recover retained-history gaps, +and resume later work without pruning unexpired ACK history. Maximum-shape +and full-namespace tests cover history, timestamps and CAS-header headroom. +Reusable-slot tests poll five empty groups at a 1 MiB bucket cap before any +publication, then claim through each group; they also exercise pruning and +foreign retirement at that same replacement-growth boundary, with durable +high-water reuse, broker restart and lost-response fencing. +Existing API/SSE tests remain in their normal CI selection. For AI agents, +repository policy prohibits local Go test/build/vet; their executable verdict +comes from CI. Human contributors should follow the +[CONTRIBUTING local development guide](../../CONTRIBUTING.md#local-development). diff --git a/src/docs/crux-serve.md b/src/docs/crux-serve.md index d560c973dc..288494364c 100644 --- a/src/docs/crux-serve.md +++ b/src/docs/crux-serve.md @@ -7,7 +7,9 @@ Stage 3 process boundary of the Crux design record [#13536](https://github.com/hivecommons/hive/issues/13536), stage 2/3 tracker [#14138](https://github.com/hivecommons/hive/issues/14138)): a harness other than hive drives agents through it, and the `crux` CLI has a remote mode that -talks to it. Hive itself does not use the API yet; that is a later decision. +talks to it. Hive's optional [durable lane participant](crux-lanes.md) uses +separate project-authenticated lane RPCs on this server; that does not change +its agent-control hosting mode or grant lane tokens agent-control authority. The wire contract is the stdlib-only package `crux/api` (`src/crux/api`), which a client can import without pulling in the runtime. @@ -31,6 +33,7 @@ runtime and every agent must arrive as a definition on `POST /v1/agents`. | `--socket PATH`, `CRUX_SOCKET` | unix socket to serve on. Default `$XDG_RUNTIME_DIR/crux/crux.sock`, else `/crux-/crux.sock`. The directory is created `0700`, the socket `0600`, and it is never reachable at that path with wider permissions (it is bound under a temporary name, chmod'ed, then renamed). A socket somebody already answers on is refused; a stale one is replaced. | | `--listen HOST:PORT` | also serve on TCP. **Refused without a token.** | | `--token T`, `CRUX_TOKEN` | bearer token every request must carry. When set it gates the socket too. | +| `--lanes-config PATH`, `CRUX_LANES_CONFIG` | opt into [durable event/work lanes](crux-lanes.md) on this same server, backed by NATS JetStream. Lane project credentials are separate from agent-control credentials. | | `--hive-callback ADDR`, `CRUX_HIVE_SOCKET` (+ `CRUX_HIVE_TOKEN`) | dial hive's callback endpoint (`crux/api/callback`) for the capabilities hive owns — scoped GitHub tokens, PR creation, Linear credentials, text/canary scanners, conditions — and feed the runtime's seams from it instead of leaving them nil. Health reports it under `hive_callback`; an unreachable endpoint raises the `hive_callback_unavailable` condition. (#14246) When the runtime is a manager it also polls hive's host policy from the same endpoint; see [Host policy from hive](#host-policy-from-hive-14138-stage-4). | `serve` does not use the per-work-dir control socket that a foreground @@ -54,8 +57,11 @@ runtime and every agent must arrive as a definition on `POST /v1/agents`. ## The API -All routes are under `/v1`. Bodies and responses are JSON; errors are +All routes are under `/v1`. Bodies and responses are JSON; agent-control errors are `{"error": "...", "code": "unauthorized|bad_request|not_found|conflict|runtime_error"}`. +Opt-in lane routes require a separate project bearer credential and capability +ACLs; see the [lane routes and request types](crux-lanes.md#public-http-and-go-api) +and [lane error codes and failure behavior](crux-lanes.md#bounds-and-failure-behavior). | Route | Body | Runtime method | Returns | |---|---|---|---| @@ -68,6 +74,7 @@ All routes are under `/v1`. Bodies and responses are JSON; errors are | `POST /v1/agents/{name}/kick` | `{"message":…}` | `Kicker.SendKick` | `{"agent","process"}` | | `POST /v1/agents/{name}/stop` | | `Lifecycle.Stop` | `{"agent","process"}` | | `GET /v1/events` | | watcher + `Kicker.SetKickObserver` + the `crux.Host` observers | `text/event-stream` of `Event` records | +| `GET /v1/lanes` and lane/group subroutes | Publish/recovery/receipt JSON; [request shapes](crux-lanes.md#public-http-and-go-api) | `lanes.Broker` | Opt-in only; lane-project credential and capability ACLs; JSON or `204`; [full routes and responses](crux-lanes.md#public-http-and-go-api) | ### Runtime families (#14138 stage 3) @@ -287,7 +294,13 @@ link the runtime. Errors from the server are `*api.StatusError`; ## What this is not -It is not tenancy, quotas or hive-as-a-client: the server trusts whoever can -reach the socket or holds the token, and hive still constructs +For agent control it is not tenancy, quotas or hive-as-a-client: those routes +trust whoever can reach the socket or holds the token, and hive still constructs `*agent.Manager` in-process. Those are the rest of Stage 3 in [design/crux.md](design/crux.md#after-the-cut). + +The opt-in [durable lanes](crux-lanes.md) are a separate project-authenticated +surface with explicit capability ACLs, configured per-lane retention/storage +bounds and per-group lease/ledger limits. Hive uses them as an ordinary lane +client. This does not provide general multi-tenant isolation, change Hive's +agent hosting model or grant lane credentials agent-control privileges. diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index 6f599e8700..6bdbafa8ea 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -124,6 +124,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [`bd` beads CLI](beads-cli.md) — work-ledger and knowledge command reference for operators and contributors. - [`crux` CLI](crux-cli.md) — run, list, tail, kick and stop agents through `crux.Runtime` with no governor or dashboard; definition formats, the control socket served by `crux run`, and why the runtime being in-process shapes it. - [`crux serve`: the runtime over a local API](crux-serve.md) — **Developer:** operate the runtime server through its authenticated API, event stream, CLI, or Go client. +- [Crux durable lanes](crux-lanes.md) — **Developer/operator; opt-in:** configure forge-neutral JetStream-backed events and work, project capabilities, replay, leases and explicit recovery through the Crux API; connect Hive as an ordinary participant. - [Publishing Crux](crux-publish.md) — how `hivecommons/crux` is produced from `src/crux` (plain copy, module path rewritten, fast-forward commits), what is preserved there, the `CRUX_SYNC_TOKEN` credential, manual sync and how a `crux/vX.Y.Z` tag becomes a release. ### Backup and API reference diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 007c20d120..59b90e8c08 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -18,6 +18,7 @@ process restart. Never print token or private-key values while diagnosing them. | Rotate dashboard access (deployment admin and affected clients) | [Token generation/rotation](#generating-and-rotating-hive_dashboard_token) and [precedence](#precedence-dashboardauth_token-dashboard_auth_token-hive_dashboard_token) | | Change container/proxy behavior (deployment admin) | [Entrypoint/proxy knobs](#deployment-entrypoint-and-proxy-knobs); read the [credential-injection refusal](#proxy-credential-injection) before enabling it | | Configure inference or CLI launch (operator/deployment admin) | [Inference/backends](#inference-cli-backends-and-agents), [launch controls](#backend-launch-controls), [Linear integration](#linear-agent-integration) | +| Configure Crux durable lanes or Hive's optional participant | [Crux durable lanes](#crux-durable-lanes): separate Crux server and Hive launch environments | | Understand a value already present in an agent terminal | [Inside an agent session](#inside-an-agent-session): **Hive-injected metadata, not operator knobs** | | Configure a local contributor/client command | [Contributor relay/helper scripts](#contributor-relay-and-top-level-helper-scripts) | | Understand platform-provided metadata (cluster admin) | [Kubernetes/platform variables](#kubernetes-downward-api-and-platform-variables): normally injected by the platform | @@ -117,6 +118,24 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_CONTRIBUTOR_KNOWLEDGE_EXPORT_MAX_FACTS` | No | `80` | Caps the number of facts returned by the contributor knowledge export API. Unset, non-numeric, or non-positive values fall back to the default. | | `HIVE_PUBLIC_REPOS` | No | none | Comma-separated, case-insensitive allowlist of additional Gource project names the dashboard's public war-room view may render, beyond the always-allowed default (`hive`). | +## Crux durable lanes + +The `HIVE_LANES_*` variables belong to the **Hive process** and are read at boot. +`CRUX_LANES_CONFIG` belongs to **`crux serve`**, not to Hive or an agent session. +The Hive participant uses the Crux HTTP API as an ordinary project principal; +it does not connect directly to NATS or inherit broker, agent-control or forge +permissions. See [Crux durable lanes](crux-lanes.md) for registry configuration, +project capabilities, deployment requirements and recovery. + +| Variable | Required | Default | Purpose | +|---|---:|---|---| +| `CRUX_LANES_CONFIG` | No | none (durable lanes disabled) | Crux server's lane configuration file, used when `crux serve --lanes-config` is absent. An explicit flag takes precedence. Broker credentials and project tokens are referenced by environment variable name in that file, not embedded as secret values. | +| `HIVE_LANES_ADDRESS` | No | none (participant disabled) | Opts the Hive process into a Crux API endpoint. This is the Crux server address, not the NATS broker URL. | +| `HIVE_LANES_LANE` | Yes when `HIVE_LANES_ADDRESS` is set | none | Registered lane ID for the Hive participant. Missing lane, group or token logs an explicit configuration error and leaves participation disabled. | +| `HIVE_LANES_GROUP` | Yes when `HIVE_LANES_ADDRESS` is set | none | Registered delivery-group ID for Hive's claim loop; the configured project must have that group's work capability. | +| `HIVE_LANES_TOKEN` | Yes when `HIVE_LANES_ADDRESS` is set | none | Secret bearer token bound to Hive's ordinary project principal in the Crux registry. Supply it through Hive's launch environment/secret configuration; never print it or use the agent-control/admin token as an implicit lane grant. | +| `HIVE_LANES_SNAPSHOT_PROJECTS` | No | none (no snapshot handler registered) | Comma-separated explicit publisher-project allowlist for read-only `hive-governor-snapshot-request-v1` work. Use subject-safe IDs without surrounding whitespace. Requests must contain `{}`; this does not mirror events, execute prompts/tools or grant forge writes. | + ## Generating and rotating `HIVE_DASHBOARD_TOKEN` `HIVE_DASHBOARD_TOKEN` (and the `dashboard.auth_token` config key it falls back diff --git a/src/go.mod b/src/go.mod index 2cf5dabd09..342cd0abb3 100644 --- a/src/go.mod +++ b/src/go.mod @@ -32,6 +32,19 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +require ( + github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.3 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.2 // indirect + github.com/nats-io/nats.go v1.55.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.58.0 // indirect + golang.org/x/time v0.16.0 // indirect +) + require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymanbagabas/go-udiff v0.3.1 // indirect @@ -47,6 +60,7 @@ require ( github.com/mattn/go-runewidth v0.0.27 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect + github.com/nats-io/nats-server/v2 v2.15.2 github.com/rivo/uniseg v0.4.7 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.opentelemetry.io/otel/log v1.47.0 // indirect diff --git a/src/go.sum b/src/go.sum index a19f4b6b83..7f74dc3a87 100644 --- a/src/go.sum +++ b/src/go.sum @@ -1,5 +1,7 @@ cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= +github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -53,6 +55,8 @@ github.com/google/go-github/v72 v72.0.0 h1:FcIO37BLoVPBO9igQQ6tStsv2asG4IPcYFi65 github.com/google/go-github/v72 v72.0.0/go.mod h1:WWtw8GMRiL62mvIquf1kO3onRHeWWKmK01qdCY8c5fg= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= @@ -61,6 +65,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePC github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= +github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -75,12 +81,24 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.27 h1:Feg/Oou5zI/wnpgDF6omIU0OokC9GxLC/WRknhVlIR0= github.com/mattn/go-runewidth v0.0.27/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc= +github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc= +github.com/nats-io/nats-server/v2 v2.15.2 h1:yLsgt2kslgymvxvjFq1WGWck5hkat2YYfbXvQ145tzU= +github.com/nats-io/nats-server/v2 v2.15.2/go.mod h1:VXHCZgDglV5wO2B8ZE58yG2HzqxQxYUbQqibUh5Xcr8= +github.com/nats-io/nats.go v1.55.0 h1:Ng8QKHdvdHIWoaiq0+d/aH1BWkxvcJxWJ0Y/9BPKx3M= +github.com/nats-io/nats.go v1.55.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= @@ -136,6 +154,8 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo= @@ -146,10 +166,13 @@ golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= diff --git a/src/pkg/governor/governor.go b/src/pkg/governor/governor.go index a01213af5e..322a919e30 100644 --- a/src/pkg/governor/governor.go +++ b/src/pkg/governor/governor.go @@ -378,6 +378,23 @@ func (g *Governor) SetModeChangeObserver(obs ModeChangeObserver) { g.modeChangeObserver = obs } +// AddModeChangeObserver preserves the existing local observer while attaching +// an optional participant. Both callbacks still run outside the governor lock. +func (g *Governor) AddModeChangeObserver(obs ModeChangeObserver) { + if obs == nil { + return + } + g.mu.Lock() + defer g.mu.Unlock() + previous := g.modeChangeObserver + g.modeChangeObserver = func(change ModeChange) { + if previous != nil { + previous(change) + } + obs(change) + } +} + func (g *Governor) Evaluate(queueIssues, queuePRs, queueHold, slaViolations int) []string { return g.EvaluateWithRepoDepths(queueIssues, queuePRs, queueHold, slaViolations, nil) } diff --git a/src/pkg/governor/lanes_observer_test.go b/src/pkg/governor/lanes_observer_test.go new file mode 100644 index 0000000000..cd196df76d --- /dev/null +++ b/src/pkg/governor/lanes_observer_test.go @@ -0,0 +1,26 @@ +package governor + +import "testing" + +func TestAdditionalModeObserverPreservesHooksAndLockBoundary(t *testing.T) { + g := observerTestGovernor(t) + order := []string{} + g.SetModeChangeObserver(func(ModeChange) { order = append(order, "hooks") }) + g.AddModeChangeObserver(nil) + g.AddModeChangeObserver(func(c ModeChange) { + if g.GetState().Mode != c.To { + t.Fatal("observer did not see committed state") + } + order = append(order, "lanes") + }) + g.Evaluate(20, 0, 0, 0) + if len(order) != 2 || order[0] != "hooks" || order[1] != "lanes" { + t.Fatalf("observer order: %v", order) + } + g.SetModeChangeObserver(nil) + g.AddModeChangeObserver(func(ModeChange) { order = append(order, "new") }) + g.Evaluate(0, 0, 0, 0) + if order[len(order)-1] != "new" { + t.Fatal("observer not installed without predecessor") + } +} diff --git a/src/pkg/laneparticipant/participant.go b/src/pkg/laneparticipant/participant.go new file mode 100644 index 0000000000..ae0ac47ff6 --- /dev/null +++ b/src/pkg/laneparticipant/participant.go @@ -0,0 +1,171 @@ +// Package laneparticipant is Hive's optional Crux lane adapter. It translates +// Hive domain events and dispatches only explicitly registered work schemas. +// It has no forge client, prompt executor, agent-control token or write grant. +package laneparticipant + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/lanes" + "github.com/hivecommons/hive/pkg/governor" +) + +const ( + ModeChangeSchema = "hive-governor-mode-change-v1" + SnapshotRequestSchema = "hive-governor-snapshot-request-v1" + SnapshotSchema = "hive-governor-snapshot-v1" + DefaultTimeout = 5 * time.Second + DefaultPoll = time.Second + eventDigestBytes = 16 +) + +var ErrUnregistered = errors.New("lane work schema or publisher not registered") + +type Handler func(context.Context, api.LaneEnvelope) error +type registration struct { + projects map[string]bool + handler Handler +} + +type Participant struct { + client *api.Client + lane, group string + handlers map[string]registration + logger *slog.Logger +} + +func New(client *api.Client, lane, group string, logger *slog.Logger) (*Participant, error) { + if client == nil || !lanes.ValidID(lane) || !lanes.ValidID(group) { + return nil, errors.New("incomplete Hive lane participant configuration") + } + if logger == nil { + logger = slog.New(slog.DiscardHandler) + } + return &Participant{client: client, lane: lane, group: group, handlers: map[string]registration{}, logger: logger}, nil +} + +// Register is called before Run. A schema alone never grants another project +// execution permission; each handler separately names its trusted publishers. +func (p *Participant) Register(schema string, projects []string, handler Handler) error { + if !lanes.ValidID(schema) || len(projects) == 0 || handler == nil { + return ErrUnregistered + } + if _, exists := p.handlers[schema]; exists { + return errors.New("lane work schema already registered") + } + allowed := map[string]bool{} + for _, project := range projects { + if !lanes.ValidID(project) { + return ErrUnregistered + } + allowed[project] = true + } + p.handlers[schema] = registration{projects: allowed, handler: handler} + return nil +} + +func eventID(prefix string, data []byte) string { + sum := sha256.Sum256(data) + return prefix + "-" + hex.EncodeToString(sum[:eventDigestBytes]) +} + +func (p *Participant) PublishModeChange(ctx context.Context, change governor.ModeChange) error { + data, err := json.Marshal(change) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, DefaultTimeout) + defer cancel() + _, err = p.client.PublishLane(ctx, p.lane, api.LanePublish{ + ID: eventID("mode", data), Kind: api.LaneEvent, Schema: ModeChangeSchema, Payload: data}) + return err +} + +// RegisterSnapshot is the built-in read-only work adapter. Requests are strict +// empty objects, never prompts or forge actions. Responses correlate to the +// neutral work ID and are published using this participant's own credential. +func (p *Participant) RegisterSnapshot(projects []string, read func() governor.State) error { + if read == nil { + return ErrUnregistered + } + return p.Register(SnapshotRequestSchema, projects, func(ctx context.Context, envelope api.LaneEnvelope) error { + var request map[string]json.RawMessage + if err := json.Unmarshal(envelope.Payload, &request); err != nil || request == nil || len(request) != 0 { + return errors.New("snapshot request must be an empty JSON object") + } + data, err := json.Marshal(struct { + RequestID string `json:"request_id"` + State governor.State `json:"state"` + }{envelope.ID, read()}) + if err != nil { + return err + } + _, err = p.client.PublishLane(ctx, p.lane, api.LanePublish{ + ID: eventID("snapshot", append([]byte(envelope.Project+"."), data...)), + Kind: api.LaneEvent, Schema: SnapshotSchema, Payload: data}) + return err + }) +} + +// HandleOnce processes at most one lease. Handler execution is bounded by that +// lease; longer-running handlers should use the explicit Crux renewal API in +// their own adapter rather than treating this helper as an execution guarantee. +func (p *Participant) HandleOnce(ctx context.Context) (bool, error) { + callCtx, cancel := context.WithTimeout(ctx, DefaultTimeout) + claim, err := p.client.ClaimLane(callCtx, p.lane, p.group) + cancel() + if err != nil || claim == nil { + return false, err + } + receipt := api.LaneReceipt{Sequence: claim.Sequence, Receipt: claim.Receipt} + r, ok := p.handlers[claim.Envelope.Schema] + if !ok || !r.projects[claim.Envelope.Project] { + err = ErrUnregistered + } else { + workCtx, stop := context.WithDeadline(ctx, claim.Expires) + err = r.handler(workCtx, claim.Envelope) + if err == nil { + err = workCtx.Err() + } + stop() + } + callCtx, cancel = context.WithTimeout(ctx, DefaultTimeout) + defer cancel() + if err != nil { + return true, errors.Join(err, p.client.NackLane(callCtx, p.lane, p.group, receipt)) + } + return true, p.client.AckLane(callCtx, p.lane, p.group, receipt) +} + +func (p *Participant) Run(ctx context.Context) error { + ticker := time.NewTicker(DefaultPoll) + defer ticker.Stop() + for { + _, err := p.HandleOnce(ctx) + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + p.logger.Warn("Hive lane work failed", "error", err) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +func (p *Participant) ObserveMode(ctx context.Context, change governor.ModeChange) { + if err := p.PublishModeChange(ctx, change); err != nil { + p.logger.Warn("Hive lane event publish failed; durable delivery is not confirmed", "error", fmt.Errorf("mode-change: %w", err)) + } +} diff --git a/src/pkg/laneparticipant/participant_test.go b/src/pkg/laneparticipant/participant_test.go new file mode 100644 index 0000000000..acac305881 --- /dev/null +++ b/src/pkg/laneparticipant/participant_test.go @@ -0,0 +1,247 @@ +package laneparticipant + +import ( + "context" + "errors" + "fmt" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/lanes" + "github.com/hivecommons/hive/crux/serve" + "github.com/hivecommons/hive/pkg/governor" + "github.com/nats-io/nats-server/v2/server" +) + +func fixture(t *testing.T) (*Participant, *api.Client, *lanes.Broker) { + t.Helper() + dir := filepath.Join(".", fmt.Sprintf(".lane-fixture-%s-%d", strings.ReplaceAll(t.Name(), "/", "-"), time.Now().UnixNano())) + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.RemoveAll(dir); err != nil { + t.Error(err) + } + }) + ns, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: dir, NoLog: true, NoSigs: true}) + if err != nil { + t.Fatal(err) + } + go ns.Start() + if !ns.ReadyForConnections(time.Second * 10) { + ns.Shutdown() + t.Fatal("NATS not ready") + } + t.Cleanup(func() { + ns.Shutdown() + ns.WaitForShutdown() + }) + cfg := lanes.Config{URL: ns.ClientURL(), + Principals: []lanes.Principal{{Project: "atlas", TokenEnv: "ATLAS"}, {Project: "hive", TokenEnv: "HIVE"}}, + Lanes: []api.Lane{{ID: "commons", Owner: "atlas", Publishers: []string{"atlas", "hive"}, Subscribers: []string{"atlas"}, + Groups: map[string][]string{"hive-adapter": {"hive"}}}}} + cfg.Defaults() + cfg.RetryDelay, cfg.MaxRetryDelay = lanes.MinRetryDelay, lanes.MinRetryDelay + b, err := lanes.Open(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(b.Close) + tokens := map[string]string{"ATLAS": strings.Repeat("a", 32), "HIVE": strings.Repeat("h", 32)} + auth, err := lanes.Bind(cfg, func(env string) string { return tokens[env] }, "admin") + if err != nil { + t.Fatal(err) + } + s := serve.New(nil) + s.Token, s.Lanes, s.LaneAuth = "admin", b, auth + hs := httptest.NewServer(s.Handler()) + t.Cleanup(hs.Close) + atlas, err := api.NewClient(hs.URL, tokens["ATLAS"]) + if err != nil { + t.Fatal(err) + } + hive, err := api.NewClient(hs.URL, tokens["HIVE"]) + if err != nil { + t.Fatal(err) + } + p, err := New(hive, "commons", "hive-adapter", nil) + if err != nil { + t.Fatal(err) + } + return p, atlas, b +} + +func waitRetry(t *testing.T, b *lanes.Broker) { + t.Helper() + state, err := b.InspectGroup(context.Background(), "hive", "commons", "hive-adapter") + if err != nil || len(state.Retries) == 0 { + t.Fatalf("missing durable retry: %+v %v", state, err) + } + timer := time.NewTimer(time.Until(state.Retries[0].AvailableAt)) + defer timer.Stop() + <-timer.C +} + +func TestHiveIsAnOptInPeerAndSnapshotHandler(t *testing.T) { + p, atlas, _ := fixture(t) + ctx := context.Background() + change := governor.ModeChange{Timestamp: time.Now().UTC(), From: governor.ModeIdle, To: governor.ModeSurge, Reason: "depth"} + p.ObserveMode(ctx, change) + if err := p.PublishModeChange(ctx, change); err != nil { + t.Fatal(err) + } + page, err := atlas.ReplayLane(ctx, "commons", 0, 8) + if err != nil || len(page.Events) != 1 || page.Events[0].Envelope.Project != "hive" || page.Events[0].Envelope.Schema != ModeChangeSchema { + t.Fatalf("Hive event %+v %v", page, err) + } + if err := p.RegisterSnapshot([]string{"atlas"}, func() governor.State { return governor.State{Mode: governor.ModeIdle} }); err != nil { + t.Fatal(err) + } + work := api.LanePublish{ID: "snapshot", Kind: api.LaneWork, Schema: SnapshotRequestSchema, Payload: []byte(`{}`)} + if _, err := atlas.PublishLane(ctx, "commons", work); err != nil { + t.Fatal(err) + } + handled, err := p.HandleOnce(ctx) + if err != nil || !handled { + t.Fatalf("snapshot work: %v %v", handled, err) + } + if handled, err := p.HandleOnce(ctx); err != nil || handled { + t.Fatalf("acked work redelivered: %v %v", handled, err) + } + page, err = atlas.ReplayLane(ctx, "commons", page.Cursor, 8) + if err != nil || len(page.Events) != 1 || page.Events[0].Envelope.Schema != SnapshotSchema { + t.Fatalf("snapshot response %+v %v", page, err) + } +} + +func TestHandlerAllowlistFailureAndCancellation(t *testing.T) { + p, atlas, b := fixture(t) + ctx := context.Background() + work := api.LanePublish{ID: "job", Kind: api.LaneWork, Schema: "tool-execution-v1", Payload: []byte(`{"prompt":"not executable"}`)} + if _, err := atlas.PublishLane(ctx, "commons", work); err != nil { + t.Fatal(err) + } + if handled, err := p.HandleOnce(ctx); !handled || !errors.Is(err, ErrUnregistered) { + t.Fatalf("unknown schema: %v %v", handled, err) + } + waitRetry(t, b) + calls := 0 + if err := p.Register(work.Schema, []string{"not-atlas"}, func(context.Context, api.LaneEnvelope) error { calls++; return nil }); err != nil { + t.Fatal(err) + } + if _, err := p.HandleOnce(ctx); !errors.Is(err, ErrUnregistered) { + t.Fatalf("publisher allowlist: %v", err) + } + if calls != 0 { + t.Fatal("unauthorized project ran a handler") + } + waitRetry(t, b) + delete(p.handlers, work.Schema) + want := errors.New("handler refused") + if err := p.Register(work.Schema, []string{"atlas"}, func(context.Context, api.LaneEnvelope) error { return want }); err != nil { + t.Fatal(err) + } + if _, err := p.HandleOnce(ctx); !errors.Is(err, want) { + t.Fatalf("handler failure hidden: %v", err) + } + waitRetry(t, b) + c, err := b.Claim(ctx, "hive", "commons", "hive-adapter") + if err != nil || c.Attempt < 3 { + t.Fatalf("nack did not redeliver: %+v %v", c, err) + } + canceled, cancel := context.WithCancel(ctx) + cancel() + if err := p.Run(canceled); !errors.Is(err, context.Canceled) { + t.Fatalf("Run cancellation: %v", err) + } + p.ObserveMode(canceled, governor.ModeChange{}) +} + +func TestRegistrationAndSnapshotValidation(t *testing.T) { + p, atlas, _ := fixture(t) + if _, err := New(nil, "", "", nil); err == nil { + t.Fatal("invalid participant accepted") + } + for _, projects := range [][]string{nil, {""}} { + if err := p.Register("schema", projects, func(context.Context, api.LaneEnvelope) error { return nil }); !errors.Is(err, ErrUnregistered) { + t.Fatal("invalid registration", err) + } + } + if err := p.RegisterSnapshot([]string{"atlas"}, nil); !errors.Is(err, ErrUnregistered) { + t.Fatal(err) + } + if err := p.RegisterSnapshot([]string{"atlas"}, func() governor.State { return governor.State{} }); err != nil { + t.Fatal(err) + } + if err := p.RegisterSnapshot([]string{"atlas"}, func() governor.State { return governor.State{} }); err == nil { + t.Fatal("duplicate handler accepted") + } + if _, err := atlas.PublishLane(context.Background(), "commons", api.LanePublish{ID: "bad", Kind: api.LaneWork, Schema: SnapshotRequestSchema, Payload: []byte(`{"prompt":"no"}`)}); err != nil { + t.Fatal(err) + } + if handled, err := p.HandleOnce(context.Background()); !handled || err == nil { + t.Fatal("arbitrary snapshot request accepted") + } +} + +func TestRejectedSnapshotCannotStarveValidWorkAcrossParticipantRestart(t *testing.T) { + for _, kind := range []string{"malformed", "unsupported", "unauthorized-publisher"} { + t.Run(kind, func(t *testing.T) { + p, atlas, b := fixture(t) + read := func() governor.State { return governor.State{Mode: governor.ModeIdle} } + if err := p.RegisterSnapshot([]string{"atlas"}, read); err != nil { + t.Fatal(err) + } + work := api.LanePublish{ID: "poison", Kind: api.LaneWork, Schema: SnapshotRequestSchema, Payload: []byte(`{}`)} + publisher := atlas + switch kind { + case "malformed": + work.Payload = []byte(`{"prompt":"not permitted"}`) + case "unsupported": + work.Schema = "unsupported-v1" + case "unauthorized-publisher": + publisher = p.client + } + if _, err := publisher.PublishLane(context.Background(), "commons", work); err != nil { + t.Fatal(err) + } + if handled, err := p.HandleOnce(context.Background()); !handled || err == nil { + t.Fatal("poison item was not rejected") + } + work.ID, work.Schema, work.Payload = "valid", SnapshotRequestSchema, []byte(`{}`) + if _, err := atlas.PublishLane(context.Background(), "commons", work); err != nil { + t.Fatal(err) + } + // Poll after the persisted delay: a delay alone must not hide starvation. + waitRetry(t, b) + if handled, err := p.HandleOnce(context.Background()); !handled || err == nil { + t.Fatal("expected explicit retry turn") + } + waitRetry(t, b) + restarted, err := New(p.client, "commons", "hive-adapter", nil) + if err != nil { + t.Fatal(err) + } + if err := restarted.RegisterSnapshot([]string{"atlas"}, read); err != nil { + t.Fatal(err) + } + if handled, err := restarted.HandleOnce(context.Background()); !handled || err != nil { + t.Fatalf("valid snapshot starved: %v %v", handled, err) + } + status, err := b.InspectGroup(context.Background(), "hive", "commons", "hive-adapter") + if err != nil || len(status.Retries) != 1 || status.Retries[0].Attempt != 2 { + t.Fatalf("rejected work disappeared: %+v %v", status, err) + } + page, err := atlas.ReplayLane(context.Background(), "commons", 0, 8) + if err != nil || len(page.Events) != 1 || page.Events[0].Envelope.Schema != SnapshotSchema { + t.Fatalf("missing valid result: %+v %v", page, err) + } + }) + } +}